diff --git a/benchmark/benchmark_suite.cpp b/benchmark/benchmark_suite.cpp index 0a48a9173..6eef62371 100644 --- a/benchmark/benchmark_suite.cpp +++ b/benchmark/benchmark_suite.cpp @@ -368,7 +368,7 @@ Status EnsureTable(const std::string& root_path, const std::string& db_name, const std::shared_ptr& schema, const std::vector& primary_keys = {}) { PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr catalog, + std::shared_ptr catalog, AddContext(paimon::Catalog::Create(root_path, options), "create catalog")); PAIMON_RETURN_NOT_OK( AddContext(catalog->CreateDatabase(db_name, options, true), "create database")); @@ -434,25 +434,29 @@ Status WriteSourceDataToWriter(paimon::FileStoreWrite* writer, const SourceDataS Status WriteAndCommit(const std::string& table_path, const std::map& options, const SourceDataSpec& source_spec) { + std::map write_builder_options; paimon::WriteContextBuilder write_builder(table_path, "benchmark-writer"); + write_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_ctx, + AddContext(write_builder.Finish(), "create write context")); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr write_ctx, - AddContext(write_builder.SetOptions(options).Finish(), "create write context")); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, - AddContext(paimon::FileStoreWrite::Create(std::move(write_ctx)), - "create file store writer")); + std::unique_ptr writer, + AddContext(paimon::FileStoreWrite::Create(std::move(write_ctx), write_builder_options), + "create file store writer")); PAIMON_RETURN_NOT_OK(WriteSourceDataToWriter(writer.get(), source_spec)); PAIMON_ASSIGN_OR_RAISE(std::vector> messages, AddContext(writer->PrepareCommit(), "prepare commit")); + std::map commit_builder_options; paimon::CommitContextBuilder commit_builder(table_path, "benchmark-writer"); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr commit_ctx, - AddContext(commit_builder.SetOptions(options).Finish(), "create commit context")); + commit_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_ctx, + AddContext(commit_builder.Finish(), "create commit context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr committer, - AddContext(paimon::FileStoreCommit::Create(std::move(commit_ctx)), "create committer")); + AddContext(paimon::FileStoreCommit::Create(std::move(commit_ctx), commit_builder_options), + "create committer")); PAIMON_RETURN_NOT_OK(AddContext(committer->Commit(messages), "commit write")); return Status::OK(); } @@ -562,20 +566,23 @@ Result GetOrCreateSharedReadTable(const std::string Result ReadRows(const std::string& table_path, const std::map& options, int32_t prefetch_parallel_num) { + std::map scan_builder_options; paimon::ScanContextBuilder scan_builder(table_path); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan_ctx, - AddContext(scan_builder.SetOptions(options).Finish(), "create scan context")); + scan_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_ctx, + AddContext(scan_builder.Finish(), "create scan context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr scanner, - AddContext(paimon::TableScan::Create(std::move(scan_ctx)), "create scanner")); + AddContext(paimon::TableScan::Create(std::move(scan_ctx), scan_builder_options), + "create scanner")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, AddContext(scanner->CreatePlan(), "create plan")); + std::map read_builder_options; paimon::ReadContextBuilder read_builder(table_path); constexpr int32_t kPrefetchBatchCount = 600; - read_builder.SetOptions(options) - .EnablePrefetch(true) + read_builder_options = options; + read_builder.EnablePrefetch(true) .SetPrefetchBatchCount(kPrefetchBatchCount) .SetPrefetchMaxParallelNum(prefetch_parallel_num) .EnableMultiThreadRowToBatch(GetRowToBatchThreadNumber() > 1) @@ -584,7 +591,8 @@ Result ReadRows(const std::string& table_path, AddContext(read_builder.Finish(), "create read context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr reader, - AddContext(paimon::TableRead::Create(std::move(read_ctx)), "create table reader")); + AddContext(paimon::TableRead::Create(std::move(read_ctx), read_builder_options), + "create table reader")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, AddContext(reader->CreateReader(plan->Splits()), "create batch reader")); diff --git a/cmake_modules/SetupCxxFlags.cmake b/cmake_modules/SetupCxxFlags.cmake index 855d8e1d3..cb08e6e84 100644 --- a/cmake_modules/SetupCxxFlags.cmake +++ b/cmake_modules/SetupCxxFlags.cmake @@ -106,7 +106,6 @@ if("${BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN") elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-conversion") - set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated-declarations") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-sign-conversion") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unused-variable") else() @@ -163,6 +162,11 @@ if(MSVC) # (required for protobuf, see https://github.com/protocolbuffers/protobuf/issues/6885) set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4065") elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # Paimon's own deprecated entry points stay exercised by its tests until they are removed, so + # deprecation warnings stay off for every GCC build, as they already are for every Clang build + # below. Downstream consumers compiling the public headers with their own flags see them. + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated-declarations") + if(CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL "7.0" OR CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "7.0") # Without this, gcc >= 7 warns related to changes in C++17 diff --git a/docs/code-style.md b/docs/code-style.md index cb8c648e8..63530fbc8 100644 --- a/docs/code-style.md +++ b/docs/code-style.md @@ -168,10 +168,10 @@ All fallible functions return `Status` (no value) or `Result` (with value). * ```cpp // ✅ Good — explicit type PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - TableScan::Create(std::move(ctx))); + table->NewScan(std::move(ctx))); // ❌ Bad — auto -PAIMON_ASSIGN_OR_RAISE(auto scan, TableScan::Create(std::move(ctx))); +PAIMON_ASSIGN_OR_RAISE(auto scan, table->NewScan(std::move(ctx))); ``` If you need a `shared_ptr` from a function returning `unique_ptr`, write the target type directly — the macro already moves: diff --git a/docs/source/api/format_table.rst b/docs/source/api/format_table.rst index 1ec964375..cec5e848f 100644 --- a/docs/source/api/format_table.rst +++ b/docs/source/api/format_table.rst @@ -35,18 +35,16 @@ and :cpp:class:`paimon::FileStoreCommit`. See :doc:`../user_guide/format_table`. Contexts ======== -Each of those entry points builds its context from a table path, and the schema under that path -says what kind of table it is. A caller that already holds a ``FormatTable`` hands it over instead, -through the constructor each context builder has for one: - -* :cpp:class:`paimon::ScanContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::ReadContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::WriteContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::CommitContextBuilder`\ ``(const std::shared_ptr&)`` - -This is the only way to reach a format table whose schema does not live under its own location, -such as one a REST catalog serves: nothing under the location says that it is a format table, nor -that what sits below it is data rather than metadata. The table knows both, so a setting that would -answer either question again - ``SetTableSchema()``, ``WithFileSystem()`` and a branch among them - -is refused by these builders rather than quietly ignored. Options given at the call still win over -the ones the table carries, as they do everywhere else. +Each of those entry points builds its context the way it does for a data table: from the table's +location, through the builder's usual constructor: + +* :cpp:class:`paimon::ScanContextBuilder`\ ``(const std::string&)`` +* :cpp:class:`paimon::ReadContextBuilder`\ ``(const std::string&)`` +* :cpp:class:`paimon::WriteContextBuilder`\ ``(const std::string& root_path, const std::string& commit_user)`` +* :cpp:class:`paimon::CommitContextBuilder`\ ``(const std::string& root_path, const std::string& commit_user)`` + +The context then goes to the table's own operator - ``NewScan()``, ``NewRead()``, ``NewWrite()`` or +``NewCommit()`` - which serves a table whose schema does not live under its own location as well, +such as one a REST catalog serves: the table carries its schema and the file system it was loaded +through, so nothing under the location has to answer for them. Options given at the call still win +over the ones the table carries, as they do everywhere else. diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 2d789f839..aa8b5a336 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -26,6 +26,53 @@ recommend that you use Catalog to access the Paimon table. Paimon C++ supports two metastores, selected with the catalog option ``metastore``: the filesystem metastore (default) and the REST metastore. +Dynamic options and time travel +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Use ``table->Copy(options)`` to apply dynamic options before creating operators. +Managed table copies retain the current schema and identifier when ``branch`` +changes. ``FileStoreTable::SwitchToBranch`` loads the target branch's latest schema. +Physical layout options such as ``merge-engine``, ``row-tracking.enabled`` and +``data-evolution.enabled`` cannot be changed by a copy. + +``table->GetSchema()`` returns the schema bound to that table instance without +metadata I/O. It does not refresh when the stored schema changes. Load a new +table through ``catalog->GetTable(identifier)`` to obtain the latest schema. +``Copy(options)`` derives a table from the current instance; time-travel options +may trigger snapshot/tag lookup and historical schema loading. These operations +can perform metadata I/O and fail, as indicated by the returned ``Result``. + +Changing ``deletion-vectors.enabled`` requires the table's existing options to +contain ``deletion-vectors.modifiable=true``. Set that option before copying with +a different deletion-vectors mode. + +For time travel, specify one of ``scan.snapshot-id``, ``scan.tag-name``, +``scan.timestamp-millis`` or ``scan.timestamp``. The copy selects the historical +snapshot's schema, preserving runtime options. Create both the scan and read from +that copied table so they use the same historical columns. + +Selectors are mutually exclusive and must match an explicit ``scan.mode``. +Conflicting selectors, including ones accumulated across multiple copies, are +rejected. Data views such as ``$ro``, ``$audit_log`` and ``$binlog`` also retain +the selected historical schema when creating their scan and read operators. + +For example: + +.. code-block:: cpp + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical, + table->Copy({{paimon::Options::SCAN_SNAPSHOT_ID, "1"}})); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + historical->NewScan(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + historical->NewRead(std::move(read_context))); + +If the snapshot cannot be resolved, the copy keeps its current schema and the +scan reports the snapshot error. A snapshot whose schema is missing makes the +copy fail. The string-valued C++ option map cannot express Java's ``null`` option +deletion. ``scan.version`` and watermark-based time travel are not supported. +Existing scan-specific restrictions, including tag and timestamp restrictions +for global-index scans on data-evolution tables, still apply. + Filesystem Catalog ~~~~~~~~~~~~~~~~~~ The filesystem metastore (``metastore=filesystem``, the default) stores both @@ -86,7 +133,7 @@ registered on the REST server. The catalog is configured through the {"token.provider", "bear"}, {"token", ""}, }; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr catalog, paimon::Catalog::Create(/*root_path=*/"my_instance", options)); For DLF, configure one credential source. Static AK/SK credentials, an optional @@ -127,22 +174,19 @@ without an atomic snapshot-file rename: .. code-block:: cpp - // Writers and committers share ownership of the catalog. - std::shared_ptr shared_catalog(std::move(catalog)); - // Load the table's identity before preparing changes. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - shared_catalog->GetTable(paimon::Identifier("db", "tbl"))); + catalog->GetTable(paimon::Identifier("db", "tbl"))); paimon::CommitContextBuilder builder(table_path, "commit-user"); - builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); // CatalogUuid() stays null when the server supplies no id. if (std::optional table_uuid = table->CatalogUuid()) { builder.WithTableId(table_uuid.value()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - paimon::FileStoreCommit::Create(std::move(context))); + table->NewCommit(std::move(context))); PAIMON_RETURN_NOT_OK(commit->Commit(commit_messages)); Configure native writers with the same catalog and identifier. They load the @@ -152,11 +196,13 @@ offsets when a real-time writer is created: .. code-block:: cpp paimon::WriteContextBuilder write_builder(table_path, "commit-user"); - write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + write_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + catalog->GetTable(paimon::Identifier("db", "tbl"))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, - paimon::FileStoreWrite::Create(std::move(write_context))); + table->NewWrite(std::move(write_context))); For real-time writes, also set ``WithStreamingMode(true)`` and ``WithRealtimeContext(...)`` with ``realtime.enabled=true``. After @@ -170,9 +216,9 @@ current snapshot. Both builders use the file system of the table being written or committed for manifests and data, so a catalog that issues temporary credentials per table serves them through ``Catalog::GetTableFileSystem``. ``WithFileSystem`` overrides -it; writers also allow ``WithFileSystemSchemeToIdentifierMap`` to override -file-system selection. For format tables, use ``WriteContextBuilder(FormatTable)`` -instead. +it; both also allow ``WithFileSystemSchemeToIdentifierMap`` to route by URI +scheme instead, which the catalog is then not asked for. A format table is +written through the table's own ``NewWrite`` instead; see :doc:`format_table`. A file system passed to ``WithFileSystem`` is used exactly as it is: neither the built-in file systems nor the table's own credentials are involved, so a custom @@ -188,7 +234,7 @@ returns credentials that are still valid and reloads them before they expire. "oss://bucket/tbl", options)); // MyFileSystem merges provider->GetCredentials() over its own options per access. auto my_fs = std::make_shared(provider); - write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")) + write_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")) .WithFileSystem(my_fs); The snapshot being committed carries a uuid generated on the client, and the @@ -212,15 +258,19 @@ keep it across recovery. .. code-block:: cpp PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - shared_catalog->GetTable(paimon::Identifier("db", "tbl"))); + catalog->GetTable(paimon::Identifier("db", "tbl"))); std::optional table_uuid = table->CatalogUuid(); if (!table_uuid) { return paimon::Status::Invalid("a commit lease needs the table UUID"); } paimon::CommitContextBuilder builder(table_path, "writer-1"); - builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")) - .WithTableId(table_uuid.value()) - .AddOption(paimon::Options::REST_COMMIT_ACQUIRE_LOCK, "true"); + builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")) + .WithTableId(table_uuid.value()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr locked_table, + table->Copy({{paimon::Options::REST_COMMIT_ACQUIRE_LOCK, "true"}})); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, + locked_table->NewCommit(std::move(context))); Each attempt acquires its own lease before reading the latest snapshot, waits while another writer holds it until ``commit.timeout`` without using up @@ -312,40 +362,38 @@ so its restored file references are visible to the expiration operation. Reading through the catalog ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To read a native table with the per-table temporary credentials a catalog issues, -pass the catalog and table identifier to ``ReadContextBuilder::WithCatalog`` or -``ScanContextBuilder::WithCatalog``. This is a shorthand for asking the catalog -for that table's file system and passing it in through ``WithFileSystem``: +Load a table with ``Catalog::GetTable`` to bind its schema, location and file +system, including any per-table temporary credentials. ``WithCatalog`` supplies +catalog services for snapshot operations and dependency reads: .. code-block:: cpp - // Readers and scanners share ownership of the catalog. - std::shared_ptr shared_catalog(std::move(catalog)); - paimon::ScanContextBuilder scan_builder(table_path); - scan_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + scan_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + catalog->GetTable(paimon::Identifier("db", "tbl"))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); paimon::ReadContextBuilder read_builder(table_path); - read_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + read_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - paimon::TableRead::Create(std::move(read_context))); - -When ``Finish()`` builds the context, it asks the catalog for the table's file -system through ``Catalog::GetTableFileSystem`` and uses it as-is for the schema, -snapshots, manifests and data under the table path, so it signs every access with -the table's credentials and reloads them as they expire. An explicit -``WithFileSystem`` takes precedence, so the catalog is not asked. A catalog that -issues no per-table credentials returns its catalog-level file system, so this is -also how you read with the catalog's own object-store credentials. For a format -table, use ``ReadContextBuilder(FormatTable)`` or ``ScanContextBuilder(FormatTable)`` -instead; ``WithCatalog`` is rejected because the table already carries the file -system it was loaded through. + table->NewRead(std::move(read_context))); + +``Finish()`` builds a context without fetching a file system. Operators created +through ``Table::NewScan/NewRead/NewWrite/NewCommit`` use the table's bound path, +schema and file system. Contexts provide execution settings such as filters, +projections, caches and executors; write and commit identity checks still apply. + +The legacy static ``Create(context, options)`` factories load a native table from +the context's loading parameters. They resolve ``Catalog::GetTableFileSystem`` +at that point, unless an explicit ``WithFileSystem`` or scheme map takes +precedence. These loading parameters do not override an already loaded table. +Format tables also use their own ``NewRead`` and ``NewScan``; see :doc:`format_table`. The C++ REST catalog covers the database, table, snapshot and commit operations of the ``Catalog`` API. The parts of the Java REST catalog that have no C++ diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 492b5f0d0..db002f101 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -40,37 +40,46 @@ format table is an insert. Reading and writing ------------------- -A format table is not served through :cpp:func:`Catalog::GetTable`, which describes a managed -table; use :cpp:func:`Catalog::GetFormatTable` instead. - -Reading and writing go through the entry points every other table uses: ``TableScan::Create``, -``TableRead::Create``, ``FileStoreWrite::Create`` and ``FileStoreCommit::Create``, each from its -usual context builder. Each reads the table's schema - from under the table path, or from the one -the context carries - and dispatches to a format table when its ``type`` says so. - -That path needs a schema under the table's own location, which is what every table a file system -catalog serves has. A table loaded from a catalog that keeps the schema elsewhere, such as the -REST catalog, has none: nothing under the location says that it is a format table, nor that -``schema`` and ``branch`` below it are data rather than metadata. Such a table is read and written -by handing the loaded ``FormatTable`` to the context instead of a path - every context builder has -a constructor taking one - which is what Java does through ``FormatTable.newReadBuilder()`` and -``newBatchWriteBuilder()``: +A format table is served through the same :cpp:func:`Catalog::GetTable` as every other table: +its ``type`` option says what it is, and the call hands back the ``FormatTable`` it describes, +through the ``Table`` interface the two kinds of table share. + +Reading and writing go through the table itself: a caller holding one calls ``NewScan()``, +``NewRead()``, ``NewWrite()`` or ``NewCommit()`` on it, with a context its usual builder builds: +``ScanContextBuilder(path)`` and ``ReadContextBuilder(path)`` take the table's location, while +``WriteContextBuilder(root_path, commit_user)`` and ``CommitContextBuilder(root_path, commit_user)`` +also take the commit user that identifies the writer. This is what Java serves through +``FormatTable.newReadBuilder()`` and ``newBatchWriteBuilder()``: .. code-block:: cpp - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - catalog->GetFormatTable(paimon::Identifier("db", "tbl"))); - paimon::ScanContextBuilder scan_builder(table); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded, + catalog->GetTable(paimon::Identifier("db", "tbl"))); + // The table's `type` option is `format-table`, so this is a `FormatTable`. + std::shared_ptr table = + std::dynamic_pointer_cast(loaded); + paimon::ScanContextBuilder scan_builder(table->Location()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); -The table already carries its schema and the file system it was loaded through, so a context built -from one refuses ``SetTableSchema()``, ``WithFileSystem()``, -``WithFileSystemSchemeToIdentifierMap()``, ``WithCatalog()`` and a branch. Options +Because the operator comes from the table itself, this serves a table whose schema does not live +under its own location as well, such as one a REST catalog serves: the table carries its schema and +the file system it was loaded through, so nothing under the location has to answer for them. Options given at the call still win over the ones the schema stored, as they do everywhere else. +The generic entry points - ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create`` +and ``FileStoreCommit::Create`` - are deprecated in favour of the table's own operators above and +serve a data table only: a load from a path is the data table's way in, and a format table the +schema under the path names is refused with a message pointing at the catalog and the table's own +operators above. + +``FormatTable`` is a derived class of ``Table``, so a caller holding one through a +``std::shared_ptr`` calls the same operators on it without the cast: they come back +for the table it is, whatever kind it turns out to be. This is the C++ side of what Java serves +through one ``Table.newReadBuilder()`` for both kinds of table. + A batch comes back in the table's column order, or the projection's when the read names one, behind the ``_VALUE_KIND`` field every ``BatchReader`` carries. Which columns a *file* is asked for and in what order is its own business: they are matched to the table's by name, as Java matches @@ -89,12 +98,10 @@ outside the projection is never read, which is how Java scopes the same checks. named here only to describe what happens. A caller never needs them: a plan comes back as ``Plan``, a split as ``Split``, and a commit message as ``CommitMessage``. -A catalog that can load a format table itself overrides ``Catalog::LoadFormatTable()``, the -protected hook :cpp:func:`Catalog::GetFormatTable` calls. The file system catalog uses it to say -its metadata lives under the table location, and the REST catalog to take the location and the -schema from one response instead of two that could disagree. A catalog that does not override it -still serves format tables, by reading the location and the schema through the virtuals every -catalog has. +Each catalog serves a format table from its own ``GetTable``. The file system catalog reads the +schema under the table location, so it says the location carries the table's own metadata and a +scan leaves the ``schema`` and ``branch`` directories below it alone; the REST catalog takes the +location and the schema from one response, so the two cannot disagree. ``TableScan::ListPartitions()`` lists the partitions a scan can see. A format table answers it by listing directories; every other table type returns ``NotImplemented`` for now. @@ -115,6 +122,10 @@ cannot decide what kind of table this is. The merged result is validated, not th options, so an option a format table refuses - ``metastore.partitioned-table``, a file format nothing here can read - is refused wherever it comes from rather than dropped in silence. +Like Java, ``Copy()`` retains the table's original physical file format even if +``file.format`` changes in the copied option map. Read and write operators +continue using that original format. + A setting the format path cannot act on is refused by name rather than quietly dropped: * ``ReadContextBuilder::SetReadSchema()``. A projected read schema can rename a column, prune a @@ -136,8 +147,7 @@ A setting the format path cannot act on is refused by name rather than quietly d * the ``branch`` option, whatever names it: a format table keeps no metadata to branch, and its data is the files under its one location, which is where a read or a write would go whichever branch was asked for. It is refused wherever it comes from - the schema, a catalog that parsed - ``tbl$branch_dev``, or the options of a single call - and so is ``WithBranch()`` on a context - built from a format table. A branch quietly ignored would be the wrong answer everywhere and a + ``tbl$branch_dev``, or the options passed to ``Copy()``. A branch quietly ignored would be the wrong answer everywhere and a destructive one for an overwrite, which would clear the data it was told not to touch. The value naming the main branch is the default and is left alone. * A scan predicate or bucket filter, as above, and more than one partition filter: a scan descends diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index 897cfebbb..d9e4a6b7c 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -218,9 +218,14 @@ to both the index scan and the data scan: "f0", R"({"match":{"query":"paimon lake"}})", /*limit=*/10))); paimon::ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(paimon::Options::SCAN_MODE, "from-snapshot") - .AddOption(paimon::Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)) - .SetGlobalIndexResult(result); + scan_builder.SetGlobalIndexResult(result); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, + scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + paimon::TableScan::Create(std::move(scan_context), + {{paimon::Options::SCAN_MODE, "from-snapshot"}, + {paimon::Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)}})); **Configuration:** diff --git a/docs/source/user_guide/manifest_entry_cache.rst b/docs/source/user_guide/manifest_entry_cache.rst index 995d77e38..a3c9f7227 100644 --- a/docs/source/user_guide/manifest_entry_cache.rst +++ b/docs/source/user_guide/manifest_entry_cache.rst @@ -83,8 +83,11 @@ under std::unique_ptr scan_context, context_builder .WithCache(cache) - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") .Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + TableScan::Create(std::move(scan_context), + {{Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"}})); Cache entries are scoped by table path, branch, and bucket, so they can be reused across newly created ``TableScan`` and ``FileStoreScan`` instances as diff --git a/examples/clean_demo.cpp b/examples/clean_demo.cpp index b2888a86a..c7e91cb2a 100644 --- a/examples/clean_demo.cpp +++ b/examples/clean_demo.cpp @@ -51,23 +51,27 @@ Status DropPartition(const std::string& table_path, PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr committer, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); PAIMON_RETURN_NOT_OK(committer->DropPartition(partitions, /*commit_identifier=*/10)); return Status::OK(); } Status ExpireSnapshot(const std::string& table_path) { + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, /*commit_user=*/"commit_user_1"); std::map commit_options = { {Options::SNAPSHOT_NUM_RETAINED_MAX, "2"}, {Options::SNAPSHOT_NUM_RETAINED_MIN, "1"}, {Options::SNAPSHOT_TIME_RETAINED, "1ms"}, {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}}; + commit_context_builder_options = commit_options; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_context_builder.SetOptions(commit_options).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr committer, - FileStoreCommit::Create(std::move(commit_context))); + + commit_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr committer, + FileStoreCommit::Create(std::move(commit_context), commit_context_builder_options)); PAIMON_RETURN_NOT_OK(committer->Expire()); return Status::OK(); diff --git a/examples/read_write_demo.cpp b/examples/read_write_demo.cpp index df46fcc54..4113bb83e 100644 --- a/examples/read_write_demo.cpp +++ b/examples/read_write_demo.cpp @@ -61,7 +61,7 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, {paimon::Options::FILE_SYSTEM, "local"}}; // create table - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr catalog, paimon::Catalog::Create(root_path, options)); PAIMON_RETURN_NOT_OK(catalog->CreateDatabase(db_name, options, /*ignore_if_exists=*/false)); arrow::FieldVector fields = { @@ -87,13 +87,16 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, catalog->GetTable(paimon::Identifier(db_name, table_name))); + PAIMON_ASSIGN_OR_RAISE(table, table->Copy(options)); + std::string table_path = root_path + "/" + db_name + ".db/" + table_name; std::string commit_user = "some_commit_user"; // write paimon::WriteContextBuilder context_builder(table_path, commit_user); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + + context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, table->NewWrite(std::move(write_context))); @@ -117,7 +120,8 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, // commit paimon::CommitContextBuilder commit_context_builder(table_path, commit_user); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); + + commit_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr committer, table->NewCommit(std::move(commit_context))); PAIMON_RETURN_NOT_OK(committer->Commit(commit_message)); @@ -125,7 +129,8 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, // scan paimon::ScanContextBuilder scan_context_builder(table_path); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - scan_context_builder.SetOptions(options).Finish()); + + scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scanner, table->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scanner->CreatePlan()); @@ -133,7 +138,8 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, // read paimon::ReadContextBuilder read_context_builder(table_path); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, - read_context_builder.SetOptions(options).Finish()); + + read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, table->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 69eb6187c..9c4aef683 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -40,7 +40,6 @@ namespace paimon { using Instant = std::variant; class Database; -class FormatTable; class Table; class View; class Schema; @@ -69,8 +68,9 @@ class PAIMON_EXPORT Catalog { /// schemes, including the file systems it builds from per-table data /// tokens, keeps routing each scheme to its backend. Ignored when /// `file_system` is supplied, which serves every scheme itself. - /// @return A result containing a unique pointer to a `Catalog` instance, or an error status. - static Result> Create( + /// @return A result containing a shared pointer to a `Catalog` instance, or an error status. + /// @note Global system tables retain shared ownership of the catalog. + static Result> Create( const std::string& root_path, const std::map& options, const std::shared_ptr& file_system = nullptr, const std::map& fs_scheme_to_identifier_map = {}); @@ -146,10 +146,12 @@ class PAIMON_EXPORT Catalog { virtual Status RenameTable(const Identifier& from_table, const Identifier& to_table, bool ignore_if_not_exists) = 0; - /// Gets a table. + /// Gets a managed, format or system table through the common Table interface. + /// + /// Global views that enumerate tables require shared catalog ownership. /// /// @param identifier Identifier of the table to get. - /// @return A result containing the table, or an error status. + /// @return The table, or an error status. virtual Result> GetTable(const Identifier& identifier) const = 0; /// Checks whether a database with the specified name exists in the catalog. @@ -246,33 +248,6 @@ class PAIMON_EXPORT Catalog { /// @note Implementations returning true must also implement the internal /// `VersionManagedCatalog`. virtual bool SupportsVersionManagement() const; - - /// Gets a format table: a directory of data files laid out like a standard Hive table. - /// - /// A format table carries no snapshots and no manifests, so it is loaded through its own - /// method rather than `GetTable()`. Reading and writing it go through the same `TableScan`, - /// `TableRead`, `FileStoreWrite` and `FileStoreCommit` entry points every other table uses: - /// hand the table back to `ScanContextBuilder`, `ReadContextBuilder`, `WriteContextBuilder` or - /// `CommitContextBuilder`, each of which takes one. - /// - /// @param identifier Identifier of the table to get. - /// @return A result containing the format table, or an error status if the table does not - /// exist or its `type` option is not `format-table`. - Result> GetFormatTable(const Identifier& identifier) const; - - protected: - /// Loads `identifier` as a format table, which is what `GetFormatTable()` hands back. - /// - /// The default reads the location and the schema through the virtuals above - two requests - /// that can disagree - and treats every directory below the location as table content. An - /// override answers both questions directly: whether location and schema come back in one - /// round trip, and whether this catalog keeps the table's metadata under the table path. - /// - /// @param identifier Identifier of the table to load. - /// @return A result containing the format table, or an error status if the table does not - /// exist or its `type` option is not `format-table`. - virtual Result> LoadFormatTable( - const Identifier& identifier) const; }; } // namespace paimon diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index e542c9aee..01a08e030 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -33,50 +33,39 @@ namespace paimon { -class TableScan; +class CommitContext; +class FileStoreCommit; +class FileStoreWrite; +class ReadContext; +class ScanContext; class TableRead; +class TableScan; +class WriteContext; -/// A table provides basic abstraction for table type. +/// Common interface for managed, format and system tables. +/// Create operators with NewScan/NewRead/NewWrite/NewCommit; unsupported operations return errors. +/// Operators use the table's bound path, schema and file system. Context loading parameters apply +/// to static Create() factories; write and commit contexts must still identify this table. +/// Contexts supply execution parameters and optional catalog services for snapshot operations. class PAIMON_EXPORT Table { public: - /// @param file_system File system the table's data is accessed through. - /// @param root_path Root path of the table in the file system. - /// @param uuid Metastore UUID, or empty if unavailable. - Table(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = ""); + virtual ~Table() = default; - ~Table() = default; - - /// Returns the file system the table was opened with, which serves the table's - /// files. A catalog that hands out per-table file systems opens a table with the - /// one for that table; it is null when the opener had none. + /// Returns the table file system, or null if none was supplied. std::shared_ptr GetFileSystem() const { return file_system_; } - /// Returns the root path of the table in its file system. It is empty when the - /// opener had no location for the table, which a mock in a test is one case of. + /// Returns the table root path, or an empty string if unavailable. const std::string& GetRootPath() const { return root_path_; } - /// A name to identify this table. - std::string Name() const { - return identifier_.GetObjectName(); - } - - /// Full name of the table, default is database.tableName. - std::string FullName() const { - return identifier_.GetFullName(); - } + /// Returns the object name for data tables, or the system table name. + virtual std::string Name() const = 0; - /// Returns the branch the table was opened for, which its identifier carries; `main` - /// when it names none or a spelling of it, which its managers are bound to. A - /// malformed object name that hides the branch is reported. - Result CurrentBranch() const { - return identifier_.GetBranchNameOrDefault(); - } + /// Returns database.tableName for data tables, or Name() for system tables. + virtual std::string FullName() const = 0; /// Returns the metastore UUID, falling back to `FullName()`. /// @note Use `CatalogUuid()` for commit requests to avoid sending a table name as an ID. @@ -90,42 +79,49 @@ class PAIMON_EXPORT Table { return uuid_.empty() ? std::nullopt : std::optional(uuid_); } - /// Loads the latest schema of table. - std::shared_ptr LatestSchema() const { + /// Returns the schema bound to this table instance without metadata I/O. + /// A time-travel copy exposes its historical schema. Reload through the catalog to obtain + /// the latest schema; this accessor does not refresh it. + std::shared_ptr GetSchema() const { return schema_; } - /// Creates a scan of this table from a finished scan context, the table-first entry point - /// that `TableScan::Create()` also builds through. The scan is derived from the context. - /// @param context The finished scan context. - /// @return The scan, or the error building it reports. - Result> NewScan(std::unique_ptr context) const; - - /// Creates a read of this table from a finished read context, the table-first entry point - /// that `TableRead::Create()` also builds through. The read is derived from the context. - /// @param context The finished read context. - /// @return The read, or the error building it reports. - Result> NewRead(std::unique_ptr context) const; - - /// Creates a write of this table from a finished write context, the table-first entry point - /// that `FileStoreWrite::Create()` also builds through. The write is derived from the context. - /// @param context The finished write context. - /// @return The write, or the error building it reports. - Result> NewWrite(std::unique_ptr context) const; - - /// Creates a commit of this table from a finished commit context, the table-first entry point - /// that `FileStoreCommit::Create()` also builds through. The commit is derived from the - /// context. - /// @param context The finished commit context. - /// @return The commit, or the error building it reports. - Result> NewCommit( - std::unique_ptr context) const; + /// Copies this table with dynamic_options overriding its current options. + /// The table type is structural and cannot be changed by these options. + /// This may perform metadata I/O, including snapshot/tag lookup and schema loading. + virtual Result> Copy( + const std::map& dynamic_options) const = 0; + + /// Creates a scan using this table's schema and options. + virtual Result> NewScan( + std::unique_ptr context) const = 0; + + /// Creates a reader using this table's schema and options. + virtual Result> NewRead( + std::unique_ptr context) const = 0; + + /// Creates a writer using this table's schema and options. + virtual Result> NewWrite( + std::unique_ptr context) const = 0; + + /// Creates a commit using this table's schema and options. + virtual Result> NewCommit( + std::unique_ptr context) const = 0; + + protected: + /// @param file_system File system the table's data is accessed through. + /// @param root_path Root path of the table in the file system. + /// @param uuid Metastore UUID, or empty if unavailable. + Table(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid); - private: std::shared_ptr file_system_; std::string root_path_; std::shared_ptr schema_; Identifier identifier_; + + private: std::string uuid_; }; diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index ffd40ac07..05c95ada5 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -25,6 +25,7 @@ #include #include "paimon/catalog/identifier.h" +#include "paimon/macros.h" #include "paimon/result.h" #include "paimon/type_fwd.h" #include "paimon/visibility.h" @@ -32,7 +33,6 @@ namespace paimon { class Catalog; class Executor; -class FormatTable; class MemoryPool; /// `CommitContext` is some configuration for commit operations. @@ -47,9 +47,7 @@ class PAIMON_EXPORT CommitContext { bool ignore_empty_commit, bool use_rest_catalog_commit, bool append_commit_check_conflict, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table); + const std::shared_ptr& specific_file_system); CommitContext(const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, bool use_rest_catalog_commit, @@ -59,14 +57,19 @@ class PAIMON_EXPORT CommitContext { const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table); + const std::optional& table_schema = std::nullopt, + const std::map& fs_scheme_to_identifier_map = {}); ~CommitContext(); const std::string& GetRootPath() const { return root_path_; } + /// Alias for GetRootPath(). + const std::string& GetPath() const { + return GetRootPath(); + } + const std::string& GetCommitUser() const { return commit_user_; } @@ -79,6 +82,11 @@ class PAIMON_EXPORT CommitContext { return use_rest_catalog_commit_; } + /// Returns the optional schema supplied by the caller. + const std::optional& GetSpecificTableSchema() const { + return table_schema_; + } + /// Returns the configured catalog, or null if unset. const std::shared_ptr& GetCatalog() const { return catalog_; @@ -110,14 +118,8 @@ class PAIMON_EXPORT CommitContext { return specific_file_system_; } - const std::map& GetOptions() const { - return options_; - } - - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; + const std::map& GetFileSystemSchemeToIdentifierMap() const { + return fs_scheme_to_identifier_map_; } private: @@ -132,8 +134,8 @@ class PAIMON_EXPORT CommitContext { std::shared_ptr memory_pool_; std::shared_ptr executor_; std::shared_ptr specific_file_system_; - std::map options_; - std::shared_ptr format_table_; + std::map fs_scheme_to_identifier_map_; + std::optional table_schema_; }; /// `CommitContextBuilder` used to build a `CommitContext`, has input validation. @@ -144,35 +146,8 @@ class PAIMON_EXPORT CommitContextBuilder { /// @param commit_user The user identifier for the commit operation. CommitContextBuilder(const std::string& root_path, const std::string& commit_user); - /// Constructs a `CommitContextBuilder` for a format table that is already loaded: the only way - /// to commit to one whose schema lives in a metastore rather than under its location, such as - /// a table a REST catalog serves. The table carries what such a location does not say, so - /// `WithFileSystem()` is refused here rather than ignored. - /// - /// There is no commit user: a format table keeps no snapshot to record one in. - /// - /// @param table The format table to commit to, as `Catalog::GetFormatTable()` hands it back. - explicit CommitContextBuilder(const std::shared_ptr& table); - ~CommitContextBuilder(); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - CommitContextBuilder& SetOptions(const std::map& options); - - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. - /// - /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` - /// instead. - /// @param key The option key. - /// @param value The option value. - /// @return Reference to this builder for method chaining. - CommitContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Sets whether to ignore empty commits (default is true). /// When set to true, commits that don't contain any actual data changes will be ignored. /// @param ignore_empty_commit True to ignore empty commits, false otherwise. @@ -186,7 +161,8 @@ class PAIMON_EXPORT CommitContextBuilder { /// @return Reference to this builder for method chaining. CommitContextBuilder& UseRESTCatalogCommit(bool use_rest_catalog_commit); - /// Loads the current schema from `catalog`. Catalogs with version management also load and + /// Static Create() loads the table from `catalog`; Table::NewCommit() uses its bound schema + /// and file system. Catalogs with version management also load and /// commit snapshots, rebasing on conflicts. Other catalogs use file-system snapshot commits. /// Mutually exclusive with `UseRESTCatalogCommit()`. /// @@ -228,15 +204,26 @@ class PAIMON_EXPORT CommitContextBuilder { /// @return Reference to this builder for method chaining. CommitContextBuilder& WithExecutor(const std::shared_ptr& executor); - /// Sets a custom file system instance to be used for all file operations in this commit - /// context. - /// This bypasses the global file system registry and uses the provided implementation directly. + /// Sets the file system for table loading through static Create(). + /// Table::NewCommit() uses the table's bound file system. /// /// @param file_system The file system to use. /// @return Reference to this builder for method chaining. /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) CommitContextBuilder& WithFileSystem(const std::shared_ptr& file_system); + /// Maps URI schemes to registered file system identifiers. + /// Mutually exclusive with WithFileSystem(); defaults to Options::FILE_SYSTEM when unset. + /// @param fs_scheme_to_identifier_map Mapping such as {{"oss", "jindo"}, {"file", "local"}}. + /// @return This builder. + CommitContextBuilder& WithFileSystemSchemeToIdentifierMap( + const std::map& fs_scheme_to_identifier_map); + + /// Supplies a serialized table schema to skip schema loading. + /// The caller must ensure it matches the table. Without it, main uses the catalog schema + /// when WithCatalog() is set; other branches load from the table path. + CommitContextBuilder& SetTableSchema(const std::string& table_schema); + /// Build and return a `CommitContext` instance with input validation. /// @return Result containing the constructed `CommitContext` or an error status. Result> Finish(); diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index fcb094e05..7e0c2bbb4 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -28,6 +28,7 @@ #include "paimon/defs.h" #include "paimon/executor.h" +#include "paimon/macros.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" #include "paimon/realtime/realtime_commit_progress.h" @@ -50,12 +51,15 @@ class CommitMessage; /// manages versions. `UseRESTCatalogCommit()` only prepares a request for the caller to send. class PAIMON_EXPORT FileStoreCommit { public: - /// Create an instance of `FileStoreCommit`. - /// - /// @param context A unique pointer to the `CommitContext` used for commit operations. - /// - /// @return A Result containing a unique pointer to the `FileStoreCommit` instance. - static Result> Create(std::unique_ptr context); + /// Creates a managed-table commit from the context and table options. + /// @param context Commit context. + /// @param options Overrides for the stored schema options. Catalog branch commits require a + /// branch-qualified identifier. + /// @return The commit, or an error status. + /// @deprecated Use Table::NewCommit(). + PAIMON_DEPRECATED("Use Table::NewCommit() instead") + static Result> Create( + std::unique_ptr context, const std::map& options); virtual ~FileStoreCommit() = default; diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 8859e4f3f..ecb945e96 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -27,6 +27,7 @@ #include "paimon/commit_message.h" #include "paimon/defs.h" #include "paimon/executor.h" +#include "paimon/macros.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" #include "paimon/realtime/realtime_commit_progress.h" @@ -42,12 +43,15 @@ class WriteContext; /// Interface for write operations in a file store. class PAIMON_EXPORT FileStoreWrite { public: - /// Create an instance of `FileStoreWrite`. - /// - /// @param context A unique pointer to the `WriteContext` used for write operations. - /// - /// @return A Result containing a unique pointer to the `FileStoreWrite` instance. - static Result> Create(std::unique_ptr context); + /// Creates a managed-table writer from the context and table options. + /// @param context Write context. + /// @param options Overrides for the stored schema options. Catalog branch writes require a + /// branch-qualified identifier. + /// @return The writer, or an error status. + /// @deprecated Use Table::NewWrite(). + PAIMON_DEPRECATED("Use Table::NewWrite() instead") + static Result> Create( + std::unique_ptr context, const std::map& options); virtual ~FileStoreWrite() = default; diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index e62386c65..9de014eff 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -28,6 +28,7 @@ #include "arrow/c/abi.h" #include "paimon/cache/cache.h" #include "paimon/catalog/identifier.h" +#include "paimon/macros.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" #include "paimon/type_fwd.h" @@ -38,7 +39,6 @@ namespace paimon { class Catalog; class Executor; struct FullTextSearch; -class FormatTable; class MemoryPool; class Predicate; class FileSystem; @@ -53,8 +53,7 @@ struct VectorSearch; class PAIMON_EXPORT ReadContext { public: ReadContext( - const std::string& path, const std::string& branch, - const std::vector& read_field_names, + const std::string& path, const std::vector& read_field_names, const std::vector& read_field_ids, const std::shared_ptr& predicate, const std::shared_ptr& vector_search, const std::shared_ptr& full_text_search, bool enable_predicate_filter, @@ -64,11 +63,10 @@ class PAIMON_EXPORT ReadContext { const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, const std::map& fs_scheme_to_identifier_map, - const std::shared_ptr& realtime_context, - const std::map& options, bool read_ahead_cache_enabled, + const std::shared_ptr& realtime_context, bool read_ahead_cache_enabled, const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, - const std::shared_ptr& catalog, const std::optional& identifier); + WarmupLevel warmup_level, const std::shared_ptr& catalog, + const std::optional& identifier); ~ReadContext(); @@ -76,18 +74,10 @@ class PAIMON_EXPORT ReadContext { return path_; } - const std::string& GetBranch() const { - return branch_; - } - const std::map& GetFileSystemSchemeToIdentifierMap() const { return fs_scheme_to_identifier_map_; } - const std::map& GetOptions() const { - return options_; - } - const std::vector& GetReadFieldNames() const { return read_field_names_; } @@ -164,12 +154,6 @@ class PAIMON_EXPORT ReadContext { return cache_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - WarmupLevel GetWarmupLevel() const { return warmup_level_; } @@ -184,6 +168,12 @@ class PAIMON_EXPORT ReadContext { return identifier_; } + /// Returns nullopt; read contexts do not carry a table UUID. + const std::optional& GetTableId() const { + static const std::optional kNoTableId; + return kNoTableId; + } + /// Whether a read schema (C ArrowSchema) for nested column pruning was provided. bool HasReadSchema() const { return read_schema_ != nullptr && read_schema_->release != nullptr; @@ -202,7 +192,6 @@ class PAIMON_EXPORT ReadContext { private: std::string path_; - std::string branch_; std::vector read_field_names_; std::vector read_field_ids_; std::shared_ptr predicate_; @@ -221,11 +210,9 @@ class PAIMON_EXPORT ReadContext { std::shared_ptr specific_file_system_; std::map fs_scheme_to_identifier_map_; std::shared_ptr realtime_context_; - std::map options_; bool read_ahead_cache_enabled_; CacheConfig cache_config_; std::shared_ptr cache_; - std::shared_ptr format_table_; WarmupLevel warmup_level_; std::shared_ptr catalog_; std::optional identifier_; @@ -240,15 +227,6 @@ class PAIMON_EXPORT ReadContextBuilder { /// @param path The root path of the table. explicit ReadContextBuilder(const std::string& path); - /// Constructs a `ReadContextBuilder` for a format table that is already loaded: the only way - /// to read one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()`, `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()` and a - /// branch are refused here rather than ignored. - /// - /// @param table The format table to read, as `Catalog::GetFormatTable()` hands it back. - explicit ReadContextBuilder(const std::shared_ptr& table); - ~ReadContextBuilder(); ReadContextBuilder(ReadContextBuilder&&) noexcept; @@ -324,23 +302,6 @@ class PAIMON_EXPORT ReadContextBuilder { /// When set, read_field_ids and read_field_names are ignored. ReadContextBuilder& SetReadSchema(std::unique_ptr read_schema); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - ReadContextBuilder& SetOptions(const std::map& options); - - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. - /// - /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` - /// instead. - /// @param key The option key. - /// @param value The option value. - /// @return Reference to this builder for method chaining. - ReadContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set a predicate for filtering data during reading. /// /// The predicate is used for both partition pruning and data filtering. @@ -490,7 +451,7 @@ class PAIMON_EXPORT ReadContextBuilder { ReadContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// Set the table schema as a string to avoid schema loading I/O operations. + /// Sets the table schema for static Create() to avoid schema loading I/O. /// /// This optimization allows the reader to use a pre-loaded schema instead of /// reading it from the table metadata, which can improve performance especially @@ -502,19 +463,6 @@ class PAIMON_EXPORT ReadContextBuilder { /// @note If not set, the schema will be loaded from the table path. ReadContextBuilder& SetTableSchema(const std::string& table_schema); - /// Set the specific branch to read from in a versioned table. - /// - /// Paimon supports branching for data versioning and time travel queries. - /// This method allows reading from a specific branch instead of the main branch. - /// - /// The `branch` option names the branch too, as it does for a scan of one branch; naming two - /// different branches is refused rather than silently resolved. - /// - /// @param branch Name of the branch to read from. - /// @return Reference to this builder for method chaining. - /// @note Default branch is "main" if not specified. An empty name is the main branch. - ReadContextBuilder& WithBranch(const std::string& branch); - /// Sets a mapping from URI schemes (e.g., "file", "oss") to registered file system /// identifiers. This allows selecting different pre-registered file system implementations /// based on the URI scheme at runtime. @@ -534,19 +482,18 @@ class PAIMON_EXPORT ReadContextBuilder { ReadContextBuilder& WithFileSystemSchemeToIdentifierMap( const std::map& fs_scheme_to_identifier_map); - /// Sets a custom file system instance to be used for all file operations in this read context. - /// This bypasses the global file system registry and uses the provided implementation directly. + /// Sets the file system for table loading through static Create(). + /// Table::NewRead() uses the table's bound file system. /// /// @param file_system The file system to use. /// @return Reference to this builder for method chaining. /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) ReadContextBuilder& WithFileSystem(const std::shared_ptr& file_system); - /// Reads a native table through its own file system - including the per-table temporary - /// credentials a catalog that issues them hands out through `Catalog::GetTableFileSystem`. - /// This is a shorthand for `WithFileSystem(catalog->GetTableFileSystem(identifier))`; an - /// explicit `WithFileSystem()` takes precedence, so the catalog is not asked. - /// @param catalog Non-null catalog, read when `Finish()` builds the context. + /// Sets the catalog used for snapshot operations and table loading. + /// Static Create() resolves per-table credentials when loading the table; explicit file systems + /// and scheme maps take precedence. Table::NewRead() uses the table's bound file system. + /// @param catalog Non-null catalog, retained by the context. /// @param identifier The native table to read. /// @return Reference to this builder for method chaining. ReadContextBuilder& WithCatalog(const std::shared_ptr& catalog, diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 6fbb006f0..b6bbafb8b 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -28,6 +28,7 @@ #include "paimon/cache/cache.h" #include "paimon/catalog/identifier.h" #include "paimon/global_index/global_index_result.h" +#include "paimon/macros.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" #include "paimon/type_fwd.h" @@ -37,7 +38,6 @@ class ScanContextBuilder; class ScanFilter; class Catalog; class Executor; -class FormatTable; class MemoryPool; class Predicate; @@ -55,12 +55,10 @@ class PAIMON_EXPORT ScanContext { const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::optional& table_schema, - const std::map& options, - const std::shared_ptr& cache, - const std::shared_ptr& format_table, + const std::optional& table_schema, const std::shared_ptr& cache, const std::shared_ptr& catalog, - const std::optional& identifier); + const std::optional& identifier, + const std::map& fs_scheme_to_identifier_map = {}); ~ScanContext(); @@ -79,10 +77,6 @@ class PAIMON_EXPORT ScanContext { std::shared_ptr GetScanFilters() const { return scan_filters_; } - const std::map& GetOptions() const { - return options_; - } - std::shared_ptr GetMemoryPool() const { return memory_pool_; } @@ -103,6 +97,10 @@ class PAIMON_EXPORT ScanContext { return specific_file_system_; } + const std::map& GetFileSystemSchemeToIdentifierMap() const { + return fs_scheme_to_identifier_map_; + } + const std::optional& GetSpecificTableSchema() const { return table_schema_; } @@ -111,12 +109,6 @@ class PAIMON_EXPORT ScanContext { return cache_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - /// Returns the catalog supplying table metadata, or null if unset. const std::shared_ptr& GetCatalog() const { return catalog_; @@ -127,6 +119,12 @@ class PAIMON_EXPORT ScanContext { return identifier_; } + /// Returns nullopt; scan contexts do not carry a table UUID. + const std::optional& GetTableId() const { + static const std::optional kNoTableId; + return kNoTableId; + } + private: std::string path_; bool is_streaming_mode_; @@ -137,10 +135,9 @@ class PAIMON_EXPORT ScanContext { std::shared_ptr memory_pool_; std::shared_ptr executor_; std::shared_ptr specific_file_system_; + std::map fs_scheme_to_identifier_map_; std::optional table_schema_; - std::map options_; std::shared_ptr cache_; - std::shared_ptr format_table_; std::shared_ptr catalog_; std::optional identifier_; }; @@ -178,14 +175,6 @@ class PAIMON_EXPORT ScanContextBuilder { /// @param path The root path of the table. explicit ScanContextBuilder(const std::string& path); - /// Constructs a `ScanContextBuilder` for a format table that is already loaded: the only way - /// to scan one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()` and `WithFileSystem()` are refused here rather than ignored. - /// - /// @param table The format table to scan, as `Catalog::GetFormatTable()` hands it back. - explicit ScanContextBuilder(const std::shared_ptr& table); - ~ScanContextBuilder(); /// If limit is not set, it defaults to unlimited. ScanContextBuilder& SetLimit(int32_t limit); @@ -208,16 +197,6 @@ class PAIMON_EXPORT ScanContextBuilder { ScanContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// The options added or set in `ScanContextBuilder` have high priority and will be merged with - /// the options in table schema. - ScanContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - ScanContextBuilder& SetOptions(const std::map& options); - /// Set whether the scan is in streaming mode. /// @note if not set, is_streaming_mode = false ScanContextBuilder& WithStreamingMode(bool is_streaming_mode); @@ -231,24 +210,30 @@ class PAIMON_EXPORT ScanContextBuilder { /// @return Reference to this builder for method chaining. /// @note if not set, executor is default executor ScanContextBuilder& WithExecutor(const std::shared_ptr& executor); - /// Sets a custom file system instance to be used for all file operations in this scan context. - /// This bypasses the global file system registry and uses the provided implementation directly. + /// Sets the file system for table loading through static Create(). + /// Table::NewScan() uses the table's bound file system. /// @param file_system The file system to use. /// @return Reference to this builder for method chaining. /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) ScanContextBuilder& WithFileSystem(const std::shared_ptr& file_system); - /// Plans a native table through its own file system - including the per-table temporary - /// credentials a catalog that issues them hands out through `Catalog::GetTableFileSystem`. - /// This is a shorthand for `WithFileSystem(catalog->GetTableFileSystem(identifier))`; an - /// explicit `WithFileSystem()` takes precedence, so the catalog is not asked. - /// @param catalog Non-null catalog, read when `Finish()` builds the context. + /// Maps URI schemes to registered file system identifiers. + /// Mutually exclusive with WithFileSystem(); defaults to Options::FILE_SYSTEM when unset. + /// @param fs_scheme_to_identifier_map Mapping such as {{"oss", "jindo"}, {"file", "local"}}. + /// @return This builder. + ScanContextBuilder& WithFileSystemSchemeToIdentifierMap( + const std::map& fs_scheme_to_identifier_map); + + /// Sets the catalog used for snapshot operations and table loading. + /// Static Create() resolves per-table credentials when loading the table; explicit file systems + /// and scheme maps take precedence. Table::NewScan() uses the table's bound file system. + /// @param catalog Non-null catalog, retained by the context. /// @param identifier The native table to scan. /// @return Reference to this builder for method chaining. ScanContextBuilder& WithCatalog(const std::shared_ptr& catalog, const Identifier& identifier); - /// Set the table schema as a string to avoid schema loading I/O operations. + /// Sets the table schema for static Create() to avoid schema loading I/O. /// /// This optimization allows the scanner to use a pre-loaded schema instead of /// reading it from the table metadata, which can improve performance especially diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index a4c310254..d10e355b2 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -24,6 +24,7 @@ #include #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/result.h" #include "paimon/schema/schema.h" #include "paimon/status.h" @@ -36,24 +37,17 @@ namespace paimon { class FileSystem; -/// A table that is a directory of data files of one format, laid out like a standard Hive table. +/// A directory of data files in one format, without snapshots or manifests. +/// Partitions use key=value directories, or bare values with +/// format-table.partition-path-only-value. The type option must be format-table; file.format +/// defaults to parquet. /// -/// It carries no snapshots and no manifests: the files under its location are the table, and a -/// partitioned table's partitions are the `key=value` directories below that location, or the -/// bare-value ones under `format-table.partition-path-only-value`. A table is a format table when -/// its `type` option is `format-table`; `file.format` then names the format of every file in it, -/// defaulting to `parquet`. +/// Writes accept inserts only, and reads expose _VALUE_KIND as INSERT. Load through +/// Catalog::GetTable() and create operators through the Table interface. Instances must be owned by +/// shared_ptr. /// -/// Writes only insert - there is nowhere to record an update or a delete - so a read fills -/// `_VALUE_KIND` with inserts throughout. -/// -/// Reading and writing go through the entry points every other table uses: `TableScan`, -/// `TableRead`, `FileStoreWrite` and `FileStoreCommit`. Each recognises a format table from the -/// schema under the table path, or takes one already loaded through the `FormatTable` constructor -/// every context builder has, which is how a table whose schema lives in a metastore is reached. -/// -/// See `docs/source/user_guide/format_table.rst` for what is not supported yet. -class PAIMON_EXPORT FormatTable { +/// See docs/source/user_guide/format_table.rst for supported features. +class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_this { public: /// Formats a format table's files can be in. enum class Format { @@ -68,18 +62,14 @@ class PAIMON_EXPORT FormatTable { /// The identifier of a format, as it appears in `file.format` and as a file extension. static std::string FormatToString(Format format); - /// Loads a format table from its directory, reading the schema stored under it. - /// - /// This needs a schema file under the table directory, which a table created through - /// `SchemaManager` or a file system catalog has. A table whose schema lives in a metastore has - /// none, and is loaded through `Catalog::GetFormatTable()` instead. + /// Loads a format table from a schema stored under its data directory. + /// Use Catalog::GetTable() for tables whose schema is stored in a metastore. /// /// @param file_system File system holding the table directory. - /// @param table_path Root path of the table, which is also its data location. - /// @param identifier Logical table identifier, used for naming and error messages. - /// @param dynamic_options Options given at the call, which win over the ones stored in the - /// schema. Empty when the caller has none. - /// @return A result containing the format table, or an error status. + /// @param table_path Table directory and data location. + /// @param identifier Logical table identifier. + /// @param dynamic_options Overrides for the stored schema options. + /// @return The table, or an error status. static Result> Create( const std::shared_ptr& file_system, const std::string& table_path, const Identifier& identifier, const std::map& dynamic_options); @@ -102,24 +92,23 @@ class PAIMON_EXPORT FormatTable { bool location_carries_paimon_metadata, const std::map& dynamic_options); - /// Copies `table` with `dynamic_options` on top of the options it already carries, which is - /// the precedence every context builder promises for a table it was handed rather than loaded - /// itself. `table` comes back as it is when there is nothing to add. - /// - /// `type` is not overridable: it is structural and is read from the schema alone. - /// - /// @param table The table to copy. - /// @param dynamic_options Options given at the call. - /// @return A result containing the copied table, or an error status. - static Result> Copy( - const std::shared_ptr& table, - const std::map& dynamic_options); + ~FormatTable() override; + + std::string Name() const override { + return identifier_.GetObjectName(); + } + + std::string FullName() const override { + return identifier_.GetFullName(); + } - ~FormatTable(); + /// Returns the same instance when dynamic_options is empty. + Result> Copy( + const std::map& dynamic_options) const override; /// Directory the data files live in. const std::string& Location() const { - return location_; + return GetRootPath(); } /// Format of every data file in the directory. @@ -155,27 +144,14 @@ class PAIMON_EXPORT FormatTable { return options_; } - /// A name to identify this table. - std::string Name() const { - return identifier_.GetObjectName(); - } - - /// Full name of the table, database.tableName. - std::string FullName() const; - - /// Schema of the table, including its partition fields. - std::shared_ptr LatestSchema() const { - return schema_; + /// Returns the bound data schema, including partition columns, without metadata I/O. + std::shared_ptr GetSchema() const { + return std::static_pointer_cast(schema_); } /// Schema of the table as an arrow schema, including its partition fields. Result> GetArrowSchema() const; - /// File system holding the table directory. - std::shared_ptr GetFileSystem() const { - return file_system_; - } - /// Whether this table's own metadata lives under its location, as told by whoever loaded it. /// /// Only then are the `schema` and `branch` directories below the location table metadata @@ -185,6 +161,17 @@ class PAIMON_EXPORT FormatTable { return location_carries_paimon_metadata_; } + /// Creates operators using this table's options and the supplied operation context. + Result> NewScan(std::unique_ptr context) const override; + + Result> NewRead(std::unique_ptr context) const override; + + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; + private: FormatTable(const std::shared_ptr& file_system, const std::string& location, const Identifier& identifier, const std::shared_ptr& schema, @@ -192,10 +179,15 @@ class PAIMON_EXPORT FormatTable { const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata); - std::shared_ptr file_system_; - std::string location_; - Identifier identifier_; - std::shared_ptr schema_; + /// Returns a mutable shared_ptr from this const instance; the table itself is not modified. + std::shared_ptr SharedSelf() const { + return std::const_pointer_cast(shared_from_this()); + } + + /// Typed implementation of Copy() for internal callers. + Result> CopySelf( + const std::map& dynamic_options) const; + std::map options_; Format format_; std::string file_compression_; diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index c77b323da..14f168851 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -19,10 +19,13 @@ #pragma once +#include #include +#include #include #include "paimon/executor.h" +#include "paimon/macros.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" #include "paimon/reader/count_reader.h" @@ -42,11 +45,15 @@ class PAIMON_EXPORT TableRead { public: virtual ~TableRead() = default; - /// Create an instance of `TableRead`. - /// - /// @param context A unique pointer to the `ReadContext` used for read operations. - /// @return A Result containing a unique pointer to the `TableRead` instance. - static Result> Create(std::unique_ptr context); + /// Creates a managed-table reader from the context and table options. + /// System and format tables require Table::NewRead(). + /// @param context Read context. + /// @param options Overrides for the stored schema options. + /// @return The reader, or an error status. + /// @deprecated Use Table::NewRead(). + PAIMON_DEPRECATED("Use Table::NewRead() instead") + static Result> Create( + std::unique_ptr context, const std::map& options); /// Creates a `BatchReader` instance for reading data. /// diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index 8bae74c41..4f68a84c4 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -24,6 +24,7 @@ #include #include +#include "paimon/macros.h" #include "paimon/result.h" #include "paimon/table/source/plan.h" #include "paimon/table/source/scan_metrics.h" @@ -36,11 +37,16 @@ class ScanContext; /// A scanner interface for reading table's meta and create a plan. class PAIMON_EXPORT TableScan { public: - /// Create an instance of `TableScan`. - /// - /// @param context A unique pointer to the `ScanContext` used for scan operations. - /// @return A Result containing a unique pointer to the `TableScan` instance. - static Result> Create(std::unique_ptr context); + /// Creates a managed-table scan from the context and table options. + /// System and format tables require Table::NewScan(). + /// @param context Scan context. + /// @param options Overrides for the stored schema options. The branch option must agree with + /// any branch in the identifier. + /// @return The scan, or an error status. + /// @deprecated Use Table::NewScan(). + PAIMON_DEPRECATED("Use Table::NewScan() instead") + static Result> Create( + std::unique_ptr context, const std::map& options); virtual ~TableScan() = default; diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index e0a940eb0..75dd8662d 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -26,6 +26,7 @@ #include #include "paimon/catalog/identifier.h" +#include "paimon/macros.h" #include "paimon/result.h" #include "paimon/type_fwd.h" #include "paimon/visibility.h" @@ -33,7 +34,6 @@ namespace paimon { class Catalog; class Executor; -class FormatTable; class MemoryPool; /// `WriteContext` is some configuration for write operations. @@ -46,14 +46,13 @@ class PAIMON_EXPORT WriteContext { WriteContext(const std::string& root_path, const std::string& commit_user, bool is_streaming_mode, bool ignore_num_bucket_check, bool ignore_previous_files, bool enable_multi_thread_spill, const std::optional& write_id, - const std::string& branch, const std::vector& write_schema, + const std::vector& write_schema, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::string& temp_directory, const std::shared_ptr& specific_file_system, const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, - const std::map& options, - const std::shared_ptr& format_table, + const std::optional& table_schema = std::nullopt, const std::shared_ptr& catalog = nullptr, const std::optional& identifier = std::nullopt); @@ -63,6 +62,11 @@ class PAIMON_EXPORT WriteContext { return root_path_; } + /// Alias for GetRootPath(). + const std::string& GetPath() const { + return GetRootPath(); + } + const std::string& GetCommitUser() const { return commit_user_; } @@ -71,10 +75,6 @@ class PAIMON_EXPORT WriteContext { return fs_scheme_to_identifier_map_; } - const std::map& GetOptions() const { - return options_; - } - bool IsStreamingMode() const { return is_streaming_mode_; } @@ -91,10 +91,6 @@ class PAIMON_EXPORT WriteContext { return write_id_; } - const std::string& GetBranch() const { - return branch_; - } - const std::vector& GetWriteSchema() const { return write_schema_; } @@ -124,10 +120,9 @@ class PAIMON_EXPORT WriteContext { return realtime_context_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; + /// Returns the optional schema supplied by the caller. + const std::optional& GetSpecificTableSchema() const { + return table_schema_; } /// Returns the catalog supplying table metadata, or null if unset. @@ -140,10 +135,15 @@ class PAIMON_EXPORT WriteContext { return identifier_; } + /// Returns nullopt; write contexts do not carry a table UUID. + const std::optional& GetTableId() const { + static const std::optional kNoTableId; + return kNoTableId; + } + private: std::string root_path_; std::string commit_user_; - std::string branch_; bool is_streaming_mode_; bool ignore_num_bucket_check_; bool ignore_previous_files_; @@ -156,8 +156,7 @@ class PAIMON_EXPORT WriteContext { std::shared_ptr specific_file_system_; std::map fs_scheme_to_identifier_map_; std::shared_ptr realtime_context_; - std::map options_; - std::shared_ptr format_table_; + std::optional table_schema_; std::shared_ptr catalog_; std::optional identifier_; }; @@ -170,36 +169,8 @@ class PAIMON_EXPORT WriteContextBuilder { /// @param commit_user The user identifier for commit operations. WriteContextBuilder(const std::string& root_path, const std::string& commit_user); - /// Constructs a `WriteContextBuilder` for a format table that is already loaded: the only way - /// to write one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()`, `WithCatalog()` and a branch - /// are refused here rather than ignored. - /// - /// There is no commit user: a format table keeps no snapshot to record one in. - /// - /// @param table The format table to write to, as `Catalog::GetFormatTable()` hands it back. - explicit WriteContextBuilder(const std::shared_ptr& table); - ~WriteContextBuilder(); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - WriteContextBuilder& SetOptions(const std::map& options); - - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. - /// - /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` - /// instead. - /// @param key The option key. - /// @param value The option value. - /// @return Reference to this builder for method chaining. - WriteContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set whether to enable streaming mode (default is false) /// @return Reference to this builder for method chaining. WriteContextBuilder& WithStreamingMode(bool is_streaming_mode); @@ -237,22 +208,13 @@ class PAIMON_EXPORT WriteContextBuilder { /// @return Reference to this builder for method chaining. WriteContextBuilder& WithWriteId(int32_t write_id); - /// Write to specific branch, default is main. - /// - /// The identifier of `WithCatalog()` and the `branch` option may name the branch too; naming - /// two different ones is refused rather than silently resolved. An empty name is the main - /// branch. - /// @param branch Name of the branch to write to. - /// @return Reference to this builder for method chaining. - WriteContextBuilder& WithBranch(const std::string& branch); - /// For data evolution, user can write partial specific fields from table schema. /// If not set, write all fields in table. /// @return Reference to this builder for method chaining. WriteContextBuilder& WithWriteSchema(const std::vector& write_schema); - /// Sets a custom file system instance to be used for all file operations in this write context. - /// This bypasses the global file system registry and uses the provided implementation directly. + /// Sets the file system for table loading through static Create(). + /// Table::NewWrite() uses the table's bound file system. /// /// @param file_system The file system to use. /// @return Reference to this builder for method chaining. @@ -267,8 +229,9 @@ class PAIMON_EXPORT WriteContextBuilder { /// for that branch. Its schema is read from the branch directory rather than from the catalog, /// just as a read of that branch reads it. /// Data, manifests and historical metadata remain on the file system. The catalog supplies - /// that file system unless overridden by `WithFileSystem()` or - /// `WithFileSystemSchemeToIdentifierMap()`. + /// that file system when static Create() loads the table, unless overridden by + /// `WithFileSystem()` or `WithFileSystemSchemeToIdentifierMap()`. Table::NewWrite() uses + /// the table's bound schema and file system. /// @param catalog Non-null catalog, kept alive while the writer uses its snapshot loader. /// @param identifier The native table to write to. /// @return Reference to this builder for method chaining. @@ -300,6 +263,11 @@ class PAIMON_EXPORT WriteContextBuilder { WriteContextBuilder& WithFileSystemSchemeToIdentifierMap( const std::map& fs_scheme_to_identifier_map); + /// Supplies a serialized table schema to skip schema loading. + /// The caller must ensure it matches the table. Without it, main uses the catalog schema + /// when WithCatalog() is set; other branches load from the table path. + WriteContextBuilder& SetTableSchema(const std::string& table_schema); + /// Set the thread number for write buffer spill operations. (default is 0) /// If <= 0, threading is disabled for spill IPC read/write. /// If > 0, sets arrow CPU thread pool capacity for spill operations. diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index b4ad6e80f..d2da868ea 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -267,6 +267,7 @@ set(PAIMON_CORE_SRCS core/catalog/catalog_utils.cpp core/catalog/file_system_catalog.cpp core/catalog/identifier.cpp + core/catalog/snapshot_commit.cpp core/core_options.cpp core/deletionvectors/deletion_vector.cpp core/deletionvectors/deletion_file_writer.cpp @@ -443,8 +444,11 @@ set(PAIMON_CORE_SRCS core/stats/simple_stats_converter.cpp core/stats/simple_stats.cpp core/stats/simple_stats_evolution.cpp - core/table/table.cpp + core/table/append_only_file_store_table.cpp core/table/bucket_mode.cpp + core/table/file_store.cpp + core/table/file_store_table.cpp + core/table/file_store_table_factory.cpp core/table/format/format_file_listing.cpp core/table/format/format_file_naming.cpp core/table/format/format_path_validation.cpp @@ -452,11 +456,11 @@ set(PAIMON_CORE_SRCS core/table/format/format_table_commit.cpp core/table/format/format_table_file_store_commit.cpp core/table/format/format_table_file_store_write.cpp - core/table/format/format_table_loader.cpp core/table/format/format_table_read.cpp core/table/format/format_table_scan.cpp core/table/format/format_table_write.cpp core/table/format/lazy_concat_batch_reader.cpp + core/table/primary_key_file_store_table.cpp core/table/sink/commit_message.cpp core/table/sink/commit_message_impl.cpp core/table/sink/commit_message_serializer.cpp @@ -490,6 +494,7 @@ set(PAIMON_CORE_SRCS core/table/system/system_table.cpp core/table/system/system_table_scan.cpp core/table/system/system_table_schema.cpp + core/table/table.cpp core/tag/tag.cpp core/utils/branch_manager.cpp core/utils/blob_view_lookup.cpp diff --git a/src/paimon/core/append/append_compact_coordinator_test.cpp b/src/paimon/core/append/append_compact_coordinator_test.cpp index acdf59cb0..be52d0b07 100644 --- a/src/paimon/core/append/append_compact_coordinator_test.cpp +++ b/src/paimon/core/append/append_compact_coordinator_test.cpp @@ -83,7 +83,7 @@ class AppendCompactCoordinatorTest : public ::testing::Test { write_builder.WithStreamingMode(false); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); auto record_batch = std::make_unique( @@ -101,7 +101,7 @@ class AppendCompactCoordinatorTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); return file_store_commit->Commit(commit_msgs); } diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 978c20e9d..5f2c5da34 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -26,11 +26,9 @@ #include "fmt/format.h" #include "paimon/catalog_options.h" #include "paimon/common/utils/string_utils.h" -#include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/core_options.h" #include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" #ifdef PAIMON_ENABLE_REST #include "paimon/rest/rest_catalog.h" #endif @@ -42,7 +40,7 @@ const char Catalog::SYSTEM_TABLE_SPLITTER[] = "$"; const char Catalog::DB_SUFFIX[] = ".db"; const char Catalog::DB_LOCATION_PROP[] = "location"; -Result> Catalog::Create( +Result> Catalog::Create( const std::string& root_path, const std::map& options, const std::shared_ptr& file_system, const std::map& fs_scheme_to_identifier_map) { @@ -55,8 +53,11 @@ Result> Catalog::Create( } if (metastore == "rest") { #ifdef PAIMON_ENABLE_REST - return RestCatalog::Create(root_path, options, file_system, RestHttpClient::Config(), - fs_scheme_to_identifier_map); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr rest_catalog, + RestCatalog::Create(root_path, options, file_system, RestHttpClient::Config(), + fs_scheme_to_identifier_map)); + return std::shared_ptr(std::move(rest_catalog)); #else return Status::NotImplemented( "the rest catalog requires building paimon with PAIMON_ENABLE_REST=ON"); @@ -67,26 +68,12 @@ Result> Catalog::Create( } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system, fs_scheme_to_identifier_map)); - return std::make_unique(core_options.GetFileSystem(), root_path, options); + return std::shared_ptr(std::make_unique( + core_options.GetFileSystem(), root_path, options, fs_scheme_to_identifier_map)); } bool Catalog::SupportsVersionManagement() const { return false; } -Result> Catalog::GetFormatTable(const Identifier& identifier) const { - // Refused here rather than in each override, so every catalog rejects a system table the same - // way and in the same order: the database first, so that an identifier of "sys" is rejected - // without being parsed as a table name. - PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "getFormatTable")); - return LoadFormatTable(identifier); -} - -Result> Catalog::LoadFormatTable(const Identifier& identifier) const { - // The location and the schema are read separately, and every directory below the location is - // table content: a catalog that knows better overrides this and says so. - return CatalogUtils::LoadFormatTableInTwoRequests(*this, identifier, - /*location_carries_paimon_metadata=*/false); -} - } // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp index 1f2a3800a..4cf128a94 100644 --- a/src/paimon/core/catalog/catalog_utils.cpp +++ b/src/paimon/core/catalog/catalog_utils.cpp @@ -22,16 +22,17 @@ #include #include #include +#include #include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/options/table_type.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/defs.h" #include "paimon/result.h" #include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" namespace paimon { @@ -44,6 +45,21 @@ Status SystemTableError(const Identifier& identifier, const std::string& action) } // namespace +Identifier CatalogUtils::IdentifierFromPath(const std::string& table_path, + const std::string& branch) { + const std::vector paths = StringUtils::Split(table_path, "/"); + if (paths.size() < 2) { + return Identifier(Identifier::kUnknownDatabase, + paths.empty() ? std::string() : paths.front()); + } + const std::string& database = paths[paths.size() - 2]; + const std::string::size_type index = database.rfind(Catalog::DB_SUFFIX); + if (index == std::string::npos) { + return Identifier(Identifier::kUnknownDatabase, paths.back(), branch); + } + return Identifier(database.substr(0, index), paths.back(), branch); +} + bool CatalogUtils::IsSystemDatabase(const std::string& db_name) { return db_name == Catalog::SYSTEM_DATABASE_NAME; } @@ -75,6 +91,16 @@ Result CatalogUtils::BranchIdentifier(const Identifier& identifier, return Identifier(identifier.GetDatabaseName(), data_table_name, branch); } +Result> CatalogUtils::BranchIdentifier( + const std::optional& identifier, const std::string& branch) { + if (!identifier) { + return std::optional(); + } + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + BranchIdentifier(identifier.value(), branch)); + return std::optional(std::move(branch_identifier)); +} + Status CatalogUtils::CheckNotBranch(const Identifier& identifier, const std::string& action) { PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); if (branch) { @@ -117,12 +143,6 @@ Status CatalogUtils::CheckManagedTableType(const Identifier& identifier, } PAIMON_ASSIGN_OR_RAISE(TableType table_type, TableTypeDefine::FromOptions(data_schema->Options())); - if (table_type == TableType::FORMAT_TABLE) { - return Status::Invalid( - fmt::format("Cannot open format table '{}' as a Table in '{}', please use " - "'Catalog::GetFormatTable' or 'FormatTable::Create'.", - identifier.ToString(), action)); - } // A materialized table is a managed table that also carries the SQL it materializes. if (table_type != TableType::TABLE && table_type != TableType::MATERIALIZED_TABLE) { const std::map& options = data_schema->Options(); @@ -136,17 +156,4 @@ Status CatalogUtils::CheckManagedTableType(const Identifier& identifier, return Status::OK(); } -Result> CatalogUtils::LoadFormatTableInTwoRequests( - const Catalog& catalog, const Identifier& identifier, bool location_carries_paimon_metadata) { - PAIMON_ASSIGN_OR_RAISE(std::string location, catalog.GetTableLocation(identifier)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, catalog.LoadTableSchema(identifier)); - std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); - if (data_schema == nullptr) { - return Status::Invalid(fmt::format("{} is not a data table, so it cannot be a format table", - identifier.GetFullName())); - } - return FormatTable::Create(catalog.GetFileSystem(), location, identifier, data_schema, - location_carries_paimon_metadata, /*dynamic_options=*/{}); -} - } // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h index 565a6aef5..3630d4820 100644 --- a/src/paimon/core/catalog/catalog_utils.h +++ b/src/paimon/core/catalog/catalog_utils.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include "paimon/catalog/identifier.h" @@ -31,13 +32,15 @@ class Catalog; class FormatTable; class Schema; -/// Checks shared by the catalog implementations. Every check takes an `action` naming -/// the rejected operation in the error message, e.g. "dropTable". +/// Identifier utilities shared by catalog implementations. class CatalogUtils { public: CatalogUtils() = delete; ~CatalogUtils() = delete; + /// Derives the database from the parent directory's .db suffix; otherwise leaves it unknown. + static Identifier IdentifierFromPath(const std::string& table_path, const std::string& branch); + /// Returns whether `db_name` is the reserved system database "sys". static bool IsSystemDatabase(const std::string& db_name); @@ -58,6 +61,10 @@ class CatalogUtils { static Result BranchIdentifier(const Identifier& identifier, const std::string& branch); + /// Returns an identifier qualified with branch, or nullopt if no identifier was supplied. + static Result> BranchIdentifier( + const std::optional& identifier, const std::string& branch); + /// Fails when `db_name` cannot be used as a single path component, which is required to /// keep the database path under the warehouse. static Status CheckValidDatabaseName(const std::string& db_name); @@ -66,32 +73,12 @@ class CatalogUtils { /// branch name, system table name) cannot be used as a single path component. static Status CheckValidTableName(const Identifier& identifier); - /// Fails when `schema` is not a table a `Table` can describe. - /// - /// Only a managed table is one: a format table is loaded through `GetFormatTable()`, and the - /// remaining table types are stored differently and are not implemented here, so handing one - /// back as a `Table` would promise snapshots it never had. A schema that is not a data - /// table's, such as a system table's, passes. - /// - /// @param action The call being refused, qualified as a caller would write it, e.g. - /// `Catalog::GetTable`. It names the entry point in the message. + /// Rejects unsupported managed-table types; system-table schemas pass through. + /// Format tables must be handled by the caller. + /// @param action Operation name used in the error message, e.g. Catalog::GetTable. static Status CheckManagedTableType(const Identifier& identifier, const std::shared_ptr& schema, const std::string& action); - - /// Loads `identifier` as a format table by reading its location and its schema separately. - /// - /// What every catalog can do, so it is what `Catalog::GetFormatTable()` falls back to. A - /// catalog that can get both from one response should not use it, since two requests can - /// disagree. - /// - /// @param location_carries_paimon_metadata Whether this catalog put the table's own metadata - /// under the table path, which is what tells a `schema` or `branch` directory below the - /// location from a partition directory of the same name. See - /// `FormatTable::LocationCarriesPaimonMetadata()`. - static Result> LoadFormatTableInTwoRequests( - const Catalog& catalog, const Identifier& identifier, - bool location_carries_paimon_metadata); }; } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 87c89de8c..50cc3c395 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -34,7 +34,9 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" @@ -44,6 +46,7 @@ #include "paimon/fs/file_system.h" #include "paimon/logging.h" #include "paimon/result.h" +#include "paimon/table/format/format_table.h" namespace arrow { class Schema; @@ -53,10 +56,12 @@ struct ArrowSchema; namespace paimon { FileSystemCatalog::FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, - const std::map& catalog_options) + const std::map& catalog_options, + const std::map& fs_scheme_map) : fs_(fs), warehouse_(warehouse), catalog_options_(catalog_options), + fs_scheme_map_(fs_scheme_map), logger_(Logger::GetLogger("FileSystemCatalog")) {} Status FileSystemCatalog::CreateDatabase(const std::string& db_name, @@ -285,46 +290,22 @@ Result> FileSystemCatalog::LoadTableSchema( return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } GlobalSystemTableContext context; - context.catalog = this; context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = catalog_options_; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + GlobalSystemTableLoader::LoadForSchema(identifier.GetObjectName(), context)); + return system_table->GetSchema(); } // The system table suffix is dropped when the data table identifier is rebuilt below, so the // identifier is validated as a whole here. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckValidTableName(identifier)); PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, - identifier.GetSystemTableName()); - if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { - return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); - } - PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); - PAIMON_ASSIGN_OR_RAISE( - Identifier data_identifier, - CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - TableSchemaExists(data_identifier)); - if (!latest_schema) { - return Status::NotExist(fmt::format("{} not exist", data_identifier.ToString())); - } - std::map dynamic_options; - if (branch) { - dynamic_options[Options::BRANCH] = branch.value(); - } - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::Load(system_table_name.value(), fs_, table_path, - latest_schema.value(), dynamic_options)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + LoadSystemTable(identifier)); + return system_table->GetSchema(); } PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, TableSchemaExists(identifier)); @@ -335,15 +316,28 @@ Result> FileSystemCatalog::LoadTableSchema( } Result> FileSystemCatalog::GetTable(const Identifier& identifier) const { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported( + identifier.GetObjectName(), catalog_options_)); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::shared_ptr self = weak_from_this().lock(); + if (self == nullptr) { + return Status::Invalid( + "a global system table retains its catalog, so the catalog must be owned by " + "a std::shared_ptr before it is asked for one"); + } + GlobalSystemTableContext context; + context.catalog = std::move(self); + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = catalog_options_; + return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); + } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier)); - // A system table reads the files of the data table it belongs to, so its root path - // is that table's. - PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); - Identifier data_identifier(identifier.GetDatabaseName(), data_table_name); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); - return std::make_shared(fs_, table_path, schema, identifier); + return LoadSystemTable(identifier); } // The schema is read on the table's own branch: `root/branch/branch-` below the // table path, which `main` resolves to itself. `GetBranchNameOrDefault` folds any @@ -353,14 +347,49 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide if (!latest_schema) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); + std::shared_ptr data_schema = checked_pointer_cast(*latest_schema); + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(data_schema->Options())); + if (is_format_table) { + return FormatTable::Create(fs_, table_path, identifier, data_schema, + /*location_carries_paimon_metadata=*/true, + /*dynamic_options=*/{}); + } // Only a managed table is a `Table`; describing another type as one would hand back // snapshots and manifests it never had, so the failure belongs to the call that opens the // table rather than at the first read. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); - return std::make_shared
(fs_, table_path, checked_pointer_cast(*latest_schema), - identifier); + PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); + return FileStoreTable::Create(fs_, table_path, *latest_schema, identifier, /*uuid=*/"", branch, + fs_scheme_map_, /*dynamic_options=*/{}); +} + +Result> FileSystemCatalog::LoadSystemTable( + const Identifier& identifier) const { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + PAIMON_ASSIGN_OR_RAISE( + Identifier data_identifier, + CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + TableSchemaExists(data_identifier)); + if (!latest_schema) { + return Status::NotExist(fmt::format("{} not exist", data_identifier.ToString())); + } + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); + return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, + latest_schema.value(), dynamic_options, + data_identifier.GetObjectName(), fs_scheme_map_); } Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, @@ -579,10 +608,4 @@ Result> FileSystemCatalog::GetLatestSnapshot( return std::optional(snapshot->ToSnapshotInfo()); } -Result> FileSystemCatalog::LoadFormatTable( - const Identifier& identifier) const { - return CatalogUtils::LoadFormatTableInTwoRequests(*this, identifier, - /*location_carries_paimon_metadata=*/true); -} - } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index ccc45f666..4c63add7f 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -37,11 +37,13 @@ class TableSchema; class FileSystem; class Identifier; class Logger; +class SystemTable; -class FileSystemCatalog : public Catalog { +class FileSystemCatalog : public Catalog, public std::enable_shared_from_this { public: FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, - const std::map& catalog_options); + const std::map& catalog_options, + const std::map& fs_scheme_map = {}); Status CreateDatabase(const std::string& db_name, const std::map& options, @@ -71,13 +73,6 @@ class FileSystemCatalog : public Catalog { Result> GetLatestSnapshot(const Identifier& identifier, const std::string& branch) const override; - protected: - /// This catalog keeps a table's schema under the table's own path, so the `schema` and - /// `branch` directories there are metadata rather than table content. Nothing else differs, - /// so the location and the schema are still read separately. - Result> LoadFormatTable( - const Identifier& identifier) const override; - private: /// Fails when `db_name` cannot be used as a single path component, so that the returned /// path always stays under `warehouse`. @@ -92,6 +87,9 @@ class FileSystemCatalog : public Catalog { Result>> TableSchemaExists( const Identifier& identifier) const; + /// Loads a T$system view from the identifier's branch. The base table must have a schema. + Result> LoadSystemTable(const Identifier& identifier) const; + Status CreateDatabaseImpl(const std::string& db_name, const std::map& options); Result TableExistsInFileSystem(const std::string& table_path) const; @@ -110,6 +108,7 @@ class FileSystemCatalog : public Catalog { std::shared_ptr fs_; std::string warehouse_; std::map catalog_options_; + std::map fs_scheme_map_; std::shared_ptr logger_; }; diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 51e7c642d..c1af1e721 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -27,6 +27,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" #include "paimon/commit_context.h" +#include "paimon/commit_message.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/catalog/version_managed_catalog.h" @@ -34,14 +35,25 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" #include "paimon/fs/file_system_factory.h" +#include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/record_batch.h" +#include "paimon/scan_context.h" #include "paimon/snapshot/snapshot_info.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/plan.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/write_context.h" namespace paimon::test { @@ -223,10 +235,12 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_FALSE(loaded_schema->field(1)->nullable()); ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(options_identifier)); - ASSERT_EQ(system_table->Name(), "tbl1$options"); - // A system table reads the files of the data table it belongs to, so the table opened - // for it carries that table's root path. + ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); + ASSERT_TRUE(std::dynamic_pointer_cast(system_table) == nullptr); + ASSERT_EQ("tbl1$options", system_table->Name()); + ASSERT_EQ("tbl1$options", system_table->FullName()); ASSERT_EQ(system_table->GetRootPath(), table_path); + ASSERT_EQ((std::vector{"key", "value"}), system_table->GetSchema()->FieldNames()); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db1", "tbl1$unknown")), "not exist"); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db1", "missing$options")), "not exist"); @@ -242,6 +256,327 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { "Cannot 'renameTable' for system table"); } +TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + + auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false), + arrow::field("dt", arrow::utf8())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog.CreateTable(Identifier("db1", "tbl1"), &schema, /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + ASSERT_OK_AND_ASSIGN(auto managed, catalog.GetTable(Identifier("db1", "tbl1"))); + ASSERT_TRUE(std::dynamic_pointer_cast(managed) == nullptr); + + ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(Identifier("db1", "tbl1$options"))); + ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); + ASSERT_EQ("tbl1$options", system_table->Name()); + + std::map scan_builder_options; + ScanContextBuilder scan_builder(system_table->GetRootPath()); + scan_builder_options = options; + + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_1, + system_table->Copy(scan_builder_options)); + ASSERT_OK_AND_ASSIGN(auto scan, configured_table_1->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); + + std::map read_builder_options; + ReadContextBuilder read_builder(system_table->GetRootPath()); + read_builder_options = options; + + ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_2, + system_table->Copy(read_builder_options)); + ASSERT_OK_AND_ASSIGN(auto read, configured_table_2->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto table_result, reader->NextBatch()); + + auto& [c_array, c_schema] = table_result; + std::shared_ptr batch; + auto maybe_batch = arrow::ImportRecordBatch(c_array.get(), c_schema.get()); + ASSERT_TRUE(maybe_batch.ok()) << maybe_batch.status().ToString(); + batch = std::move(maybe_batch).ValueUnsafe(); + ASSERT_GE(batch->num_rows(), 2); + + std::map write_builder_options; + WriteContextBuilder write_builder(system_table->GetRootPath(), "commit_user"); + write_builder_options = options; + + ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_3, + system_table->Copy(write_builder_options)); + ASSERT_NOK_WITH_MSG(configured_table_3->NewWrite(std::move(write_context)), + "tbl1$options system table is read-only: writing is not supported"); + + std::map commit_builder_options; + CommitContextBuilder commit_builder(system_table->GetRootPath(), "commit_user"); + commit_builder_options = options; + + ASSERT_OK_AND_ASSIGN(auto commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_4, + system_table->Copy(commit_builder_options)); + ASSERT_NOK_WITH_MSG( + configured_table_4->NewCommit(std::move(commit_context)), + "tbl1$options system table is read-only: committing for it is not supported"); +} + +TEST(FileSystemCatalogTest, TestACopyOfACatalogSystemTableAppliesItsOptions) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + + auto typed_schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + const Identifier identifier("db1", "tbl1"); + ASSERT_OK(catalog.CreateTable(identifier, &schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
managed, catalog.GetTable(identifier)); + auto write_and_commit_two_rows = [](const std::shared_ptr
& table, int32_t first_id = 1) { + WriteContextBuilder write_builder(table->GetRootPath(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({first_id, first_id + 1}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = + arrow::StructArray::Make({ids_array}, + {arrow::field("id", arrow::int32(), /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(table->GetRootPath(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + }; + // Snapshot 1 has two rows; snapshot 2 has four. + write_and_commit_two_rows(managed); + write_and_commit_two_rows(managed); + + auto count_rows = [](const std::shared_ptr
& table) -> Result { + ScanContextBuilder scan_builder(table->GetRootPath()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + table->NewScan(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); + ReadContextBuilder read_builder(table->GetRootPath()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + table->NewRead(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + read->CreateReader(plan->Splits())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr rows, + ReadResultCollector::CollectResult(std::move(reader))); + return rows ? rows->length() : 0; + }; + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
ro, catalog.GetTable(Identifier("db1", "tbl1$ro"))); + ASSERT_OK_AND_ASSIGN(int64_t all_rows, count_rows(ro)); + EXPECT_EQ(all_rows, 4); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
selected, + ro->Copy({{Options::SCAN_MODE, "from-snapshot"}, {Options::SCAN_SNAPSHOT_ID, "1"}})); + ASSERT_OK_AND_ASSIGN(int64_t selected_rows, count_rows(selected)); + EXPECT_EQ(selected_rows, 2); + + const std::map pk_options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}}; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog.CreateTable(Identifier("db1", "pk_tbl"), &schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, pk_options, /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
pk_table, + catalog.GetTable(Identifier("db1", "pk_tbl"))); + write_and_commit_two_rows(pk_table, 1); + write_and_commit_two_rows(pk_table, 3); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
audit, + catalog.GetTable(Identifier("db1", "pk_tbl$audit_log"))); + ASSERT_OK_AND_ASSIGN(int64_t audit_rows, count_rows(audit)); + EXPECT_EQ(audit_rows, 4); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
audit_selected, + audit->Copy({{Options::SCAN_MODE, "from-snapshot"}, {Options::SCAN_SNAPSHOT_ID, "1"}})); + EXPECT_EQ(audit_selected->Name(), "pk_tbl$audit_log"); + ASSERT_OK_AND_ASSIGN(int64_t audit_selected_rows, count_rows(audit_selected)); + EXPECT_EQ(audit_selected_rows, 2); + + ASSERT_OK_AND_ASSIGN(int64_t original_audit_rows, count_rows(audit)); + EXPECT_EQ(original_audit_rows, 4); + + SchemaManager dev_schema_manager(core_options.GetFileSystem(), managed->GetRootPath(), "dev"); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr dev_schema, + dev_schema_manager.CreateTable(typed_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_managed, + catalog.GetTable(Identifier("db1", "tbl1$branch_dev"))); + write_and_commit_two_rows(dev_managed); + + // Copy retains the identifier while routing reads to dev. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_selected, ro->Copy({{Options::BRANCH, "dev"}})); + EXPECT_EQ(dev_selected->Name(), ro->Name()); + ASSERT_OK_AND_ASSIGN(int64_t dev_rows, count_rows(dev_selected)); + EXPECT_EQ(dev_rows, 2); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_ro, + catalog.GetTable(Identifier("db1", "tbl1$branch_dev$ro"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_ro_copy, dev_ro->Copy({})); + EXPECT_EQ(dev_ro_copy->Name(), "tbl1$branch_dev$ro"); + ASSERT_OK_AND_ASSIGN(int64_t dev_ro_rows, count_rows(dev_ro_copy)); + EXPECT_EQ(dev_ro_rows, 2); + + // Metadata views also route reads to the copied branch. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
snapshots, + catalog.GetTable(Identifier("db1", "tbl1$snapshots"))); + ASSERT_OK_AND_ASSIGN(int64_t main_snapshot_rows, count_rows(snapshots)); + EXPECT_EQ(main_snapshot_rows, 2); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_snapshots, + snapshots->Copy({{Options::BRANCH, "dev"}})); + ASSERT_OK_AND_ASSIGN(int64_t dev_snapshot_rows, count_rows(dev_snapshots)); + EXPECT_EQ(dev_snapshot_rows, 1); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
missing, + snapshots->Copy({{Options::BRANCH, "missing"}})); + ASSERT_OK_AND_ASSIGN(int64_t missing_rows, count_rows(missing)); + EXPECT_EQ(missing_rows, 0); +} + +TEST(FileSystemCatalogTest, TestGetTableServesTheSysDatabase) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + options[CatalogOptionsSystemTable::kEnabledOption] = "true"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto catalog = + std::make_shared(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK(catalog->CreateDatabase("db1", options, /*ignore_if_exists=*/false)); + + auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false)}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("db1", "tbl1"), &schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + { + std::map disabled = options; + disabled.erase(CatalogOptionsSystemTable::kEnabledOption); + auto disabled_catalog = + std::make_shared(core_options.GetFileSystem(), dir->Str(), disabled); + ASSERT_NOK_WITH_MSG(disabled_catalog->GetTable(Identifier("sys", "catalog_options")), + "not exist"); + } + ASSERT_OK_AND_ASSIGN(auto catalog_options, + catalog->GetTable(Identifier("sys", "catalog_options"))); + ASSERT_TRUE(std::dynamic_pointer_cast(catalog_options) != nullptr); + + // Global system tables ignore dynamic options when copied. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied_catalog_options, + catalog_options->Copy({{"ignored", "on-top"}})); + EXPECT_NE(copied_catalog_options, catalog_options); + std::map copied_scan_builder_options; + ScanContextBuilder copied_scan_builder(copied_catalog_options->GetRootPath()); + copied_scan_builder_options = options; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan_context, + copied_scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_9, + copied_catalog_options->Copy(copied_scan_builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan, + configured_table_9->NewScan(std::move(copied_scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr copied_plan, copied_scan->CreatePlan()); + std::map copied_read_builder_options; + ReadContextBuilder copied_read_builder(copied_catalog_options->GetRootPath()); + copied_read_builder_options = options; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_read_context, + copied_read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_10, + copied_catalog_options->Copy(copied_read_builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_read, + configured_table_10->NewRead(std::move(copied_read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_reader, + copied_read->CreateReader(copied_plan->Splits())); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch copied_result, copied_reader->NextBatch()); + auto& [copied_array, copied_c_schema] = copied_result; + auto maybe_copied_batch = arrow::ImportRecordBatch(copied_array.get(), copied_c_schema.get()); + ASSERT_TRUE(maybe_copied_batch.ok()) << maybe_copied_batch.status().ToString(); + auto copied_batch = std::move(maybe_copied_batch).ValueUnsafe(); + ASSERT_GE(copied_batch->num_rows(), 1); + + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("sys", "nope")), "not exist"); + + ASSERT_OK_AND_ASSIGN(auto tables, catalog->GetTable(Identifier("sys", "tables"))); + std::map scan_builder_options; + ScanContextBuilder scan_builder("sys/tables"); + scan_builder_options = options; + + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_11, + tables->Copy(scan_builder_options)); + ASSERT_OK_AND_ASSIGN(auto scan, configured_table_11->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1); + + std::map read_builder_options; + ReadContextBuilder read_builder("sys/tables"); + read_builder_options = options; + + ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_12, + tables->Copy(read_builder_options)); + ASSERT_OK_AND_ASSIGN(auto read, configured_table_12->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto table_result, reader->NextBatch()); + + auto& [c_array, c_schema] = table_result; + std::shared_ptr batch; + auto maybe_batch = arrow::ImportRecordBatch(c_array.get(), c_schema.get()); + ASSERT_TRUE(maybe_batch.ok()) << maybe_batch.status().ToString(); + batch = std::move(maybe_batch).ValueUnsafe(); + ASSERT_GE(batch->num_rows(), 1); + + std::weak_ptr weak = catalog; + catalog.reset(); + ASSERT_FALSE(weak.expired()) << "the global system table does not retain its catalog"; + + FileSystemCatalog stack_catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_NOK_WITH_MSG(stack_catalog.GetTable(Identifier("sys", "tables")), + "must be owned by a std::shared_ptr"); +} + TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { std::map options; options[Options::FILE_SYSTEM] = "local"; @@ -285,7 +620,7 @@ TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { ASSERT_TRUE(exists); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(branch_identifier)); - data_schema = std::dynamic_pointer_cast(table->LatestSchema()); + data_schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, data_schema); ASSERT_EQ((std::vector{"b1_c0"}), data_schema->FieldNames()); ASSERT_EQ(3, data_schema->Id()); @@ -304,6 +639,10 @@ TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { ASSERT_TRUE(exists); ASSERT_OK_AND_ASSIGN(loaded_schema, catalog.LoadTableSchema(audit_log_identifier)); ASSERT_EQ((std::vector{"rowkind", "b1_c0"}), loaded_schema->FieldNames()); + // The returned name must address the same branch when reloaded. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_view, + catalog.GetTable(audit_log_identifier)); + ASSERT_EQ(branch_view->Name(), "tbl1$branch_b1$audit_log"); for (const auto& missing_identifier : {Identifier("db1", "tbl1$branch_missing"), Identifier("db1", "tbl1$branch_missing$options")}) { @@ -615,7 +954,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { ASSERT_TRUE(typed_schema.Equals(loaded_schema)); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "tbl1"))); - ASSERT_OK_AND_ASSIGN(auto arrow_schema_from_get_table, table->LatestSchema()->GetArrowSchema()); + ASSERT_OK_AND_ASSIGN(auto arrow_schema_from_get_table, table->GetSchema()->GetArrowSchema()); auto schema_from_get_table = arrow::ImportSchema(arrow_schema_from_get_table.get()).ValueOrDie(); ASSERT_TRUE(typed_schema.Equals(schema_from_get_table)); @@ -629,7 +968,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { ArrowSchemaRelease(&schema); } -TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { +TEST(FileSystemCatalogTest, TestGetTableServesAFormatTable) { std::map options; options[Options::FILE_SYSTEM] = "local"; options[Options::FILE_FORMAT] = "orc"; @@ -650,11 +989,9 @@ TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - // A format table has no snapshots and no manifests, so handing it back as a Table would - // describe it as something it is not. - ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db1", "fmt")), "Cannot open format table"); - ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, - catalog.GetFormatTable(Identifier("db1", "fmt"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "fmt"))); + std::shared_ptr format_table = std::dynamic_pointer_cast(table); + ASSERT_NE(nullptr, format_table); ASSERT_EQ(format_table->GetFormat(), FormatTable::Format::PARQUET); ASSERT_EQ(format_table->FullName(), "db1.fmt"); @@ -662,15 +999,6 @@ TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { // `branch` directories there are its metadata and a scan leaves them alone. A catalog that // keeps schemas elsewhere says no, and then every directory below the location is data. ASSERT_TRUE(format_table->LocationCarriesPaimonMetadata()); - - // A system table is refused by `Catalog::GetFormatTable()` itself, so every catalog answers - // alike. The database is checked before the table name is parsed, so a malformed name under - // "sys" is refused as a system table rather than surfacing a parse error. - ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("db1", "fmt$snapshots")), - "for system table"); - ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("sys", "all_table_options")), - "for system table"); - ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("sys", "a$b$c$d")), "for system table"); } TEST(FileSystemCatalogTest, TestInvalidCreateTable) { @@ -1416,13 +1744,15 @@ TEST(FileSystemCatalogTest, TestClaimingVersionManagementWithoutImplementingItIs ASSERT_NOK_WITH_MSG(CheckVersionManagementImplemented(claiming), "does not implement VersionManagedCatalog"); + std::map claiming_builder_options; CommitContextBuilder claiming_builder(dir->Str(), "commit_user_1"); + claiming_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr claiming_context, - claiming_builder.SetOptions(options) - .WithCatalog(claiming, Identifier("db", "tbl")) - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(claiming_context)), - "does not implement VersionManagedCatalog"); + + claiming_builder.WithCatalog(claiming, Identifier("db", "tbl")).Finish()); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(std::move(claiming_context), claiming_builder_options), + "does not implement VersionManagedCatalog"); std::shared_ptr plain = std::make_shared(core_options.GetFileSystem(), dir->Str(), options); diff --git a/src/paimon/core/catalog/snapshot_commit.cpp b/src/paimon/core/catalog/snapshot_commit.cpp new file mode 100644 index 000000000..c9fac382b --- /dev/null +++ b/src/paimon/core/catalog/snapshot_commit.cpp @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/catalog/snapshot_commit.h" + +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/core/catalog/catalog_snapshot_commit.h" +#include "paimon/core/catalog/renaming_snapshot_commit.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/defs.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +Result> SnapshotCommit::Create( + const std::shared_ptr& catalog, const std::optional& catalog_identifier, + const CoreOptions& options, const std::optional& table_id, + const std::string& commit_user, bool use_rest_catalog_commit, + const std::shared_ptr& file_system, + const std::shared_ptr& snapshot_manager) { + if (AsVersionManaged(catalog) != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr lock, + AsVersionManaged(catalog)->CreateCommitLock( + catalog_identifier.value(), table_id, commit_user, options)); + // Reuse the caller's UUID so table recreation cannot redirect this commit. + return std::shared_ptr(std::make_shared( + catalog, catalog_identifier.value(), table_id, std::move(lock))); + } + if (use_rest_catalog_commit) { + if (options.RestCommitAcquireLock()) { + return Status::Invalid(fmt::format( + "'{}' needs a commit through a REST catalog with WithCatalog(); a commit table " + "request built for the caller to send cannot hold a commit lease", + Options::REST_COMMIT_ACQUIRE_LOCK)); + } + return std::shared_ptr(std::make_shared(table_id)); + } + if (table_id && catalog == nullptr) { + return Status::Invalid( + "a commit which publishes the snapshot by writing it to the table directory sends no " + "commit table request, so there is nowhere to put a table id; commit through a " + "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); + } + return std::shared_ptr( + std::make_shared(file_system, snapshot_manager)); +} + +} // namespace paimon diff --git a/src/paimon/core/catalog/snapshot_commit.h b/src/paimon/core/catalog/snapshot_commit.h index 41bd1712a..4df93f9b4 100644 --- a/src/paimon/core/catalog/snapshot_commit.h +++ b/src/paimon/core/catalog/snapshot_commit.h @@ -23,12 +23,18 @@ #include #include +#include "paimon/catalog/identifier.h" #include "paimon/core/catalog/commit_lock.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/snapshot.h" namespace paimon { +class Catalog; +class CoreOptions; +class FileSystem; +class SnapshotManager; + /// Interface to commit snapshot atomically. class SnapshotCommit { public: @@ -41,6 +47,21 @@ class SnapshotCommit { virtual Result> AcquireLock() { return std::unique_ptr(); } + /// Selects catalog-backed or file-system snapshot publication. + /// @param catalog Optional catalog. + /// @param catalog_identifier Branch-qualified identifier required by version-managed catalogs. + /// @param options Effective table options. + /// @param table_id Table UUID captured before preparing changes, or nullopt. + /// @param commit_user Commit user for commit locking. + /// @param use_rest_catalog_commit Build the request without sending it through the catalog. + /// @param file_system File system used for snapshot rename. + /// @param snapshot_manager Manager providing the snapshot paths. + static Result> Create( + const std::shared_ptr& catalog, + const std::optional& catalog_identifier, const CoreOptions& options, + const std::optional& table_id, const std::string& commit_user, + bool use_rest_catalog_commit, const std::shared_ptr& file_system, + const std::shared_ptr& snapshot_manager); /// @param base_snapshot_uuid Base snapshot UUID; null for an absent or legacy snapshot. /// @param snapshot Snapshot to be committed. diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index c5018f6aa..0e9cee757 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "fmt/format.h" diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index 31750da0d..c76332dcf 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -19,6 +19,7 @@ #include "paimon/core/core_options.h" #include +#include #include #include #include diff --git a/src/paimon/core/global_index/global_index_write_task.cpp b/src/paimon/core/global_index/global_index_write_task.cpp index f04d96bb0..3ba9c80bd 100644 --- a/src/paimon/core/global_index/global_index_write_task.cpp +++ b/src/paimon/core/global_index/global_index_write_task.cpp @@ -173,16 +173,18 @@ Result> CreateBatchReader( const std::string& table_path, const std::vector& read_field_names, const std::shared_ptr& indexed_split, const CoreOptions& core_options, const std::shared_ptr& pool) { + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(core_options.ToMap()) - .WithFileSystem(core_options.GetFileSystem()) + read_context_builder_options = core_options.ToMap(); + read_context_builder.WithFileSystem(core_options.GetFileSystem()) .EnablePrefetch(true) .WithMemoryPool(pool) .SetReadFieldNames(read_field_names); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); return table_read->CreateReader(indexed_split); } diff --git a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp index 81803ed90..75a0a8956 100644 --- a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp +++ b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp @@ -298,7 +298,7 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam catalog, + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(directory_->Str(), {})); ASSERT_OK(catalog->CreateDatabase("db", {}, /*ignore_if_exists=*/false)); ASSERT_OK(catalog->CreateTable(Identifier("db", "table"), &c_schema, @@ -320,15 +320,17 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam> CreateWriter(bool with_temp_directory = true, bool write_buffer_spillable = true) const { + std::map builder_options; WriteContextBuilder builder(table_path_, kCommitUser); std::map writer_options = options_; writer_options[Options::WRITE_BUFFER_SPILLABLE] = write_buffer_spillable ? "true" : "false"; - builder.SetOptions(writer_options).WithStreamingMode(true); + builder_options = writer_options; + builder.WithStreamingMode(true); if (with_temp_directory) { builder.WithTempDirectory(directory_->Str()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreWrite::Create(std::move(context)); + return FileStoreWrite::Create(std::move(context), builder_options); } Result>> WriteAndPrepare( @@ -354,11 +356,13 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam>& messages, int64_t commit_identifier) const { + std::map builder_options; CommitContextBuilder builder(table_path_, kCommitUser); - builder.SetOptions(options_); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); return commit->Commit(messages, commit_identifier); } @@ -373,7 +377,7 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsIndexForColumnNamedLikeInter schema_ = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), arrow::field(kColumnName, arrow::utf8(), /*nullable=*/false)}); options_[Options::PK_BTREE_INDEX_COLUMNS] = kColumnName; - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(directory_->Str(), {})); for (bool spillable : {false, true}) { SCOPED_TRACE(spillable); const std::string table_name = spillable ? "row_id_spill" : "row_id_memory"; @@ -454,11 +458,13 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted std::shared_ptr value_predicate = PredicateBuilder::Equal( /*field_index=*/1, "value", FieldType::STRING, Literal(FieldType::STRING, indexed_value.data(), indexed_value.size())); + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path_); - scan_builder.SetPredicate(value_predicate).AddOption(Options::GLOBAL_INDEX_ENABLED, "true"); + scan_builder_options[Options::GLOBAL_INDEX_ENABLED] = "true"; + scan_builder.SetPredicate(value_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); int32_t indexed_split_count = 0; int32_t data_split_count = 0; @@ -472,16 +478,17 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted ASSERT_GT(indexed_split_count, 0); ASSERT_EQ(0, data_split_count); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); // The source build above keeps lazy decoding enabled. Disable it only for the final data-row // assertion, whose predicate reader is outside the maintenance path covered by this test. + read_builder_options["orc.read.enable-lazy-decoding"] = "false"; read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) - .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -686,11 +693,13 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF std::shared_ptr value_predicate = PredicateBuilder::Equal( /*field_index=*/1, "value", FieldType::STRING, Literal(FieldType::STRING, indexed_value.data(), indexed_value.size())); + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path_); - scan_builder.SetPredicate(value_predicate).AddOption(Options::GLOBAL_INDEX_ENABLED, "true"); + scan_builder_options[Options::GLOBAL_INDEX_ENABLED] = "true"; + scan_builder.SetPredicate(value_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); int32_t indexed_split_count = 0; int32_t data_split_count = 0; @@ -704,14 +713,15 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF ASSERT_EQ(0, indexed_split_count); ASSERT_GT(data_split_count, 0); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); + read_builder_options["orc.read.enable-lazy-decoding"] = "false"; read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) - .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -763,13 +773,13 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, DeletesPayloadAfterLastDefinitionI PathUtil::JoinPath(table_path_, "schema/schema-1"), schema_json, /*overwrite=*/false)); + std::map builder_options; WriteContextBuilder builder(table_path_, kCommitUser); - builder.SetOptions(evolved_options) - .WithStreamingMode(true) - .WithTempDirectory(directory_->Str()); + builder_options = evolved_options; + builder.WithStreamingMode(true).WithTempDirectory(directory_->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(context))); + FileStoreWrite::Create(std::move(context), builder_options)); ASSERT_OK(writer->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN(std::vector> cleanup_messages, writer->PrepareCommit(/*wait_compaction=*/true, diff --git a/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp b/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp index 9c3ff443e..b66bafcb8 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp @@ -48,18 +48,20 @@ Result> PkSortedDataFileReader::Create( const std::shared_ptr& executor, const std::shared_ptr& pool) { std::map read_options = options.ToMap(); read_options[Options::BRANCH] = branch; + std::map builder_options; ReadContextBuilder builder(root_path); + builder_options = read_options; + builder_options[Options::BRANCH] = branch; builder.SetReadFieldIds({field_id}) - .SetOptions(read_options) - .WithBranch(branch) .WithFileSystem(options.GetFileSystem()) .WithExecutor(executor) .WithMemoryPool(pool) .EnablePrefetch(false) .EnablePredicateFilter(false); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shared_internal_context, - InternalReadContext::Create(read_context, table_schema, read_options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr shared_internal_context, + InternalReadContext::Create(read_context, table_schema, read_options, builder_options)); return std::unique_ptr( new PkSortedDataFileReader(path_factory, shared_internal_context, pool, executor)); } diff --git a/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp index 9fc0744ef..9de13a39a 100644 --- a/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp @@ -65,18 +65,19 @@ FullChangelogMergeTreeCompactRewriter::Create( std::shared_ptr write_schema = SpecialFields::CompleteSequenceAndValueKindField(data_schema); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(path_factory_cache->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) + read_context_builder_options = options.ToMap(); + read_context_builder.WithFileSystem(options.GetFileSystem()) .EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, path_factory_cache->GetOrCreatePathFactory(options.GetFileFormat()->Identifier())); diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp index 390f95d84..160aeb250 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp @@ -73,18 +73,19 @@ LookupMergeTreeCompactRewriter::Create( auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(data_schema); // TODO(xinyu.lxy): set executor + std::map read_context_builder_options; ReadContextBuilder read_context_builder(path_factory_cache->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) + read_context_builder_options = options.ToMap(); + read_context_builder.WithFileSystem(options.GetFileSystem()) .EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, path_factory_cache->GetOrCreatePathFactory(options.GetFileFormat()->Identifier())); diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp index 223abafed..118e2d931 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp @@ -115,9 +115,10 @@ class MergeTreeCompactManagerFactoryWriteTest : public ::testing::Test { table_options, /*ignore_if_exists=*/false)); + std::map context_builder_options; WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test"); if (!context_options.empty()) { - context_builder.SetOptions(context_options); + context_builder_options = context_options; } if (with_io_manager) { context_builder.WithTempDirectory(dir->Str()); @@ -125,7 +126,7 @@ class MergeTreeCompactManagerFactoryWriteTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, context_builder.Finish()); - return FileStoreWrite::Create(std::move(write_context)); + return FileStoreWrite::Create(std::move(write_context), context_builder_options); } Result> CreateSingleStringFileStoreWrite( diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp index afe50ddfb..bc52d0505 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp @@ -73,18 +73,19 @@ Result> MergeTreeCompactRewriter::Crea auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(data_schema); // TODO(xinyu.lxy): set executor + std::map read_context_builder_options; ReadContextBuilder read_context_builder(path_factory_cache->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) + read_context_builder_options = options.ToMap(); + read_context_builder.WithFileSystem(options.GetFileSystem()) .EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, path_factory_cache->GetOrCreatePathFactory(options.GetFileFormat()->Identifier())); diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp index 27424a1ae..ea7ca2a5f 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp @@ -60,7 +60,7 @@ class MergeTreeCompactRewriterTest : public testing::Test { ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetBucketFilter(bucket).SetPartitionFilter({partition}); PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), {})); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); auto splits = result_plan->Splits(); EXPECT_EQ(1, splits.size()); diff --git a/src/paimon/core/mergetree/lookup_levels.cpp b/src/paimon/core/mergetree/lookup_levels.cpp index 7d7aecedf..88b2e4a5c 100644 --- a/src/paimon/core/mergetree/lookup_levels.cpp +++ b/src/paimon/core/mergetree/lookup_levels.cpp @@ -57,18 +57,19 @@ Result>> LookupLevels::Create( auto partition_schema = DataField::ConvertDataFieldsToArrowSchema(partition_fields); // TODO(xinyu.lxy): set executor + std::map read_context_builder_options; ReadContextBuilder read_context_builder(path_factory->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) + read_context_builder_options = options.ToMap(); + read_context_builder.WithFileSystem(options.GetFileSystem()) .EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_read_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_read_context, + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + read_context_builder_options)); auto split_read = std::make_unique(path_factory, internal_read_context, pool, CreateDefaultExecutor()); diff --git a/src/paimon/core/operation/append_only_file_store_scan_test.cpp b/src/paimon/core/operation/append_only_file_store_scan_test.cpp index b8dfbfced..82ac8800b 100644 --- a/src/paimon/core/operation/append_only_file_store_scan_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_scan_test.cpp @@ -598,13 +598,16 @@ TEST(AppendOnlyFileStoreScanTest, TestReconstructPredicateWithNonCastedFields) { TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) { TimezoneGuard guard("Asia/Shanghai"); std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); auto pool = GetDefaultPool(); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); auto typed_table_scan = dynamic_cast(table_scan.get()); ASSERT_TRUE(typed_table_scan); @@ -644,12 +647,15 @@ TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) { TEST(AppendOnlyFileStoreScanTest, TestScanDurationMetric) { TimezoneGuard guard("Asia/Shanghai"); std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); auto typed_table_scan = dynamic_cast(table_scan.get()); ASSERT_TRUE(typed_table_scan); @@ -745,13 +751,14 @@ std::shared_ptr BuildScan(const std::string& table_path, const std::shared_ptr& predicate = nullptr, bool manifest_entry_lazy_decode_enabled = true, int32_t max_snapshots = 8) { + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, - fmt::format("{}", max_snapshots)) - .AddOption(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED, - manifest_entry_lazy_decode_enabled ? "true" : "false") - .WithCache(cache); + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = + fmt::format("{}", max_snapshots); + context_builder_options[Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED] = + manifest_entry_lazy_decode_enabled ? "true" : "false"; + context_builder.WithCache(cache); if (bucket) { context_builder.SetBucketFilter(bucket.value()); } @@ -759,7 +766,8 @@ std::shared_ptr BuildScan(const std::string& table_path, context_builder.SetPredicate(predicate); } EXPECT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); - EXPECT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + EXPECT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); auto typed_table_scan = dynamic_cast(table_scan.get()); EXPECT_TRUE(typed_table_scan); return typed_table_scan->snapshot_reader_->scan_; diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index 15b89ba90..86f2d8c50 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -369,9 +369,10 @@ Result> AppendOnlyFileStoreWrite::CreateFilesReader const BinaryRow& partition, int32_t bucket, DeletionVector::Factory dv_factory, const std::vector>& files, const std::optional& veto_reason) const { + std::map context_builder_options; ReadContextBuilder context_builder(root_path_); - context_builder.SetOptions(options_.ToMap()) - .WithFileSystem(options_.GetFileSystem()) + context_builder_options = options_.ToMap(); + context_builder.WithFileSystem(options_.GetFileSystem()) .EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) @@ -395,8 +396,9 @@ Result> AppendOnlyFileStoreWrite::CreateFilesReader } options[kParquetReadEnableDictionaryPassthrough] = "false"; } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_read_context, - InternalReadContext::Create(read_context, table_schema_, options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr internal_read_context, + InternalReadContext::Create(read_context, table_schema_, options, context_builder_options)); auto read = std::make_unique(file_store_path_factory_, internal_read_context, pool_, compact_executor_); diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index bfeaefced..0d209ae4a 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -111,11 +111,13 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { const std::string& table_path, const std::shared_ptr& schema, const std::map& options, const std::string& json, int64_t commit_identifier) const { + std::map builder_options; WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + builder_options = options; + EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); EXPECT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), builder_options)); EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( /*wait_compaction=*/false, commit_identifier)); @@ -128,11 +130,13 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { const std::map& options, const std::vector& write_schema, const std::string& json, int64_t commit_identifier) const { + std::map builder_options; WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options).WithWriteSchema(write_schema); + builder_options = options; + builder.WithWriteSchema(write_schema); EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); EXPECT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), builder_options)); EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( /*wait_compaction=*/false, commit_identifier)); @@ -142,11 +146,13 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { void Commit(const std::string& table_path, const std::map& options, const std::vector>& commit_msgs) const { + std::map builder_options; CommitContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); } @@ -228,7 +234,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestWriteWithInvalidBatch) { WriteContextBuilder builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); ASSERT_NOK_WITH_MSG(file_store_write->PrepareCommitWithProgress(/*commit_identifier=*/0), "PrepareCommitWithProgress is only supported by a real-time writer"); ASSERT_NOK_WITH_MSG(file_store_write->Write(nullptr), "batch is null pointer"); @@ -249,7 +255,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestWriteWithInvalidBatch) { commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); auto array = std::make_shared(); arrow::StringBuilder builder; for (size_t j = 0; j < 100; j++) { @@ -290,11 +296,14 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) { ASSERT_OK_AND_ASSIGN(std::shared_ptr batch_realtime_context, RealtimeContext::Create()); + std::map batch_builder_options; WriteContextBuilder batch_builder(table_path, commit_user_); - batch_builder.SetOptions(options).WithRealtimeContext(batch_realtime_context); + batch_builder_options = options; + batch_builder.WithRealtimeContext(batch_realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_context, batch_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_writer, - FileStoreWrite::Create(std::move(batch_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_writer, + FileStoreWrite::Create(std::move(batch_context), batch_builder_options)); ASSERT_NOK_WITH_MSG(batch_writer->PrepareCommitWithProgress(/*commit_identifier=*/0), "PrepareCommitWithProgress requires streaming mode"); ASSERT_OK(batch_writer->Close()); @@ -302,10 +311,13 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); + std::map builder_options; WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context); + builder_options = options; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), builder_options)); ASSERT_OK(file_store_write->Write(MakeBatch(realtime_schema, R"([ [10, 1, "a"], @@ -378,14 +390,18 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) } TEST_F(AppendOnlyFileStoreWriteTest, TestGetMaxSequenceNumberFromMultiPartition) { + std::map builder_options; WriteContextBuilder builder( paimon::test::GetDataDir() + "/orc/multi_partition_append_table.db/multi_partition_append_table/", commit_user_); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr write_context, - builder.AddOption("file.format", "orc").AddOption("manifest.format", "orc").Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + builder_options["file.format"] = "orc"; + builder_options["manifest.format"] = "orc"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + + builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), builder_options)); auto write = dynamic_cast(file_store_write.get()); auto pool = GetDefaultPool(); { @@ -445,10 +461,13 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMapAdaptsAcrossRollingFi std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + std::map builder_options; WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), builder_options)); ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([ [1, [["a", 1], ["b", 2]]] ])"))); @@ -553,10 +572,13 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMultipleMapColumnsAdaptA CreateTable(dir->Str(), logical_schema, options); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + std::map builder_options; WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), builder_options)); ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([ [1, [["a", 1], ["b", 2]], [["c", 3], ["d", 4], ["e", 5], ["f", 6]]] ])"))); diff --git a/src/paimon/core/operation/commit_context.cpp b/src/paimon/core/operation/commit_context.cpp index e61ec950f..fac300109 100644 --- a/src/paimon/core/operation/commit_context.cpp +++ b/src/paimon/core/operation/commit_context.cpp @@ -33,7 +33,6 @@ #include "paimon/memory/memory_pool.h" #include "paimon/result.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { @@ -42,13 +41,11 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co bool append_commit_check_conflict, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table) + const std::shared_ptr& specific_file_system) : CommitContext(root_path, commit_user, ignore_empty_commit, use_rest_catalog_commit, /*catalog=*/nullptr, /*identifier=*/std::nullopt, /*table_id=*/std::nullopt, append_commit_check_conflict, memory_pool, executor, specific_file_system, - options, format_table) {} + /*table_schema=*/std::nullopt, /*fs_scheme_to_identifier_map=*/{}) {} CommitContext::CommitContext(const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, bool use_rest_catalog_commit, @@ -59,8 +56,8 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table) + const std::optional& table_schema, + const std::map& fs_scheme_to_identifier_map) : root_path_(root_path), commit_user_(commit_user), ignore_empty_commit_(ignore_empty_commit), @@ -72,8 +69,8 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co memory_pool_(memory_pool), executor_(executor), specific_file_system_(specific_file_system), - options_(options), - format_table_(format_table) {} + fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), + table_schema_(table_schema) {} CommitContext::~CommitContext() = default; @@ -91,15 +88,12 @@ class CommitContextBuilder::Impl { memory_pool_ = GetDefaultPool(); executor_ = CreateDefaultExecutor(); specific_file_system_.reset(); - options_.clear(); + fs_scheme_to_identifier_map_.clear(); + table_schema_ = std::nullopt; } private: std::string root_path_; - /// Kept across `Reset()`, as `root_path_` is: both name the table this builder builds for, - /// rather than a setting of one commit to it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; std::string commit_user_; bool ignore_empty_commit_ = true; bool use_rest_catalog_commit_ = false; @@ -110,7 +104,8 @@ class CommitContextBuilder::Impl { std::shared_ptr memory_pool_ = GetDefaultPool(); std::shared_ptr executor_ = CreateDefaultExecutor(); std::shared_ptr specific_file_system_; - std::map options_; + std::map fs_scheme_to_identifier_map_; + std::optional table_schema_; }; CommitContextBuilder::CommitContextBuilder(const std::string& root_path, @@ -120,29 +115,8 @@ CommitContextBuilder::CommitContextBuilder(const std::string& root_path, impl_->commit_user_ = commit_user; } -CommitContextBuilder::CommitContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->root_path_ = table->Location(); - } -} - CommitContextBuilder::~CommitContextBuilder() = default; -CommitContextBuilder& CommitContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - -CommitContextBuilder& CommitContextBuilder::SetOptions( - const std::map& opts) { - impl_->options_ = opts; - return *this; -} - CommitContextBuilder& CommitContextBuilder::IgnoreEmptyCommit(bool ignore_empty_commit) { impl_->ignore_empty_commit_ = ignore_empty_commit; return *this; @@ -189,54 +163,36 @@ CommitContextBuilder& CommitContextBuilder::WithFileSystem( return *this; } +CommitContextBuilder& CommitContextBuilder::WithFileSystemSchemeToIdentifierMap( + const std::map& fs_scheme_to_identifier_map) { + impl_->fs_scheme_to_identifier_map_ = fs_scheme_to_identifier_map; + return *this; +} + +CommitContextBuilder& CommitContextBuilder::SetTableSchema(const std::string& table_schema) { + impl_->table_schema_ = table_schema; + return *this; +} + Result> CommitContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot commit with null format table"); - } if (impl_->catalog_ == nullptr && impl_->identifier_) { return Status::Invalid("cannot commit through a null catalog"); } + if (impl_->specific_file_system_ && !impl_->fs_scheme_to_identifier_map_.empty()) { + return Status::Invalid( + "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); + } if (impl_->identifier_) { // Before the branch is read out of the identifier: the branch of `tbl$branch_dev$options` // reads back as `dev`, so a commit built for such a name would be aimed at another branch. PAIMON_RETURN_NOT_OK( CatalogUtils::CheckNotSystemTable(impl_->identifier_.value(), "commit")); } - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - impl_->identifier_, impl_->options_, - /*explicit_branch=*/std::nullopt, "commit")); - if (!BranchManager::IsMainBranch(branch)) { - // A commit which goes to a catalog, or is handed back as a request for one, names its - // branch by an object name, so the branch has to be one that reads back as itself. - // Refused here rather than at the commit, which writes metadata before publishing it. - if (impl_->catalog_ != nullptr || impl_->use_rest_catalog_commit_) { - PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); - } - if (impl_->catalog_ != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::optional identifier_branch, - impl_->identifier_.value().GetBranchName()); - if (!identifier_branch) { - PAIMON_ASSIGN_OR_RAISE(std::string table_name, - impl_->identifier_.value().GetDataTableName()); - return Status::Invalid(fmt::format( - "a commit through a catalog addresses a branch by the table identifier, so " - "name branch '{}' there as '{}$branch_{}' rather than only in the '{}' option", - branch, table_name, branch, Options::BRANCH)); - } - } - } if (impl_->catalog_ != nullptr && impl_->use_rest_catalog_commit_) { return Status::Invalid( "a commit either goes to the catalog of WithCatalog() or is handed back as a request " "for the caller to send, which is what UseRESTCatalogCommit() asks for"); } - // The table already carries the file system it was loaded through, and from a source this - // cannot see behind, so a second answer is refused rather than silently dropped. - if (impl_->format_table_ != nullptr && impl_->specific_file_system_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "cannot be used with one"); - } PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_)); if (impl_->root_path_.empty()) { return Status::Invalid("root path is empty"); @@ -245,7 +201,7 @@ Result> CommitContextBuilder::Finish() { impl_->root_path_, impl_->commit_user_, impl_->ignore_empty_commit_, impl_->use_rest_catalog_commit_, impl_->catalog_, impl_->identifier_, impl_->table_id_, impl_->append_commit_check_conflict_, impl_->memory_pool_, impl_->executor_, - impl_->specific_file_system_, impl_->options_, impl_->format_table_); + impl_->specific_file_system_, impl_->table_schema_, impl_->fs_scheme_to_identifier_map_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/commit_context_test.cpp b/src/paimon/core/operation/commit_context_test.cpp index 51fcab399..fcda83fbe 100644 --- a/src/paimon/core/operation/commit_context_test.cpp +++ b/src/paimon/core/operation/commit_context_test.cpp @@ -38,9 +38,8 @@ TEST(CommitContextTest, TestLegacyConstructor) { auto pool = GetDefaultPool(); std::shared_ptr executor = CreateDefaultExecutor(); auto fs = std::make_shared(); - const std::map options = {{"key", "value"}}; - CommitContext context("table_root_path", "commit_user_1", false, true, true, pool, executor, fs, - options, nullptr); + CommitContext context("table_root_path", "commit_user_1", false, true, true, pool, executor, + fs); ASSERT_EQ(context.GetRootPath(), "table_root_path"); ASSERT_EQ(context.GetCommitUser(), "commit_user_1"); ASSERT_FALSE(context.IgnoreEmptyCommit()); @@ -49,10 +48,10 @@ TEST(CommitContextTest, TestLegacyConstructor) { ASSERT_EQ(context.GetMemoryPool(), pool); ASSERT_EQ(context.GetExecutor(), executor); ASSERT_EQ(context.GetSpecificFileSystem(), fs); - ASSERT_EQ(context.GetOptions(), options); ASSERT_EQ(context.GetCatalog(), nullptr); ASSERT_FALSE(context.GetIdentifier()); ASSERT_FALSE(context.GetTableId()); + ASSERT_TRUE(context.GetFileSystemSchemeToIdentifierMap().empty()); } TEST(CommitContextTest, TestDefaultValue) { @@ -72,7 +71,28 @@ TEST(CommitContextTest, TestDefaultValue) { ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); - ASSERT_TRUE(ctx->GetOptions().empty()); + ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); + ASSERT_FALSE(ctx->GetSpecificTableSchema()); + ASSERT_FALSE(ctx->GetTableId()); +} + +TEST(CommitContextTest, TestFileSystemAndSchemeMapConflict) { + CommitContextBuilder builder("table_root_path", "commit_user_1"); + auto fs = std::make_shared(); + builder.WithFileSystem(fs); + builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); + ASSERT_NOK_WITH_MSG( + builder.Finish(), + "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); +} + +TEST(CommitContextTest, TestSchemeMapWithoutFileSystem) { + CommitContextBuilder builder("table_root_path", "commit_user_1"); + builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); + std::map expected_fs_map = {{"file", "local"}}; + ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap()); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); } TEST(CommitContextTest, TestSetContent) { @@ -89,7 +109,7 @@ TEST(CommitContextTest, TestSetContent) { .WithMemoryPool(memory_pool) .WithExecutor(executor) .WithFileSystem(fs) - .AddOption("key", "value") + .SetTableSchema("table-schema-json") .Finish()); ASSERT_OK_AND_ASSIGN(auto expected_root_path, PathUtil::NormalizePath("table_root_path")); @@ -102,9 +122,10 @@ TEST(CommitContextTest, TestSetContent) { ASSERT_EQ(ctx->GetMemoryPool(), memory_pool); ASSERT_EQ(ctx->GetExecutor(), executor); ASSERT_EQ(ctx->GetSpecificFileSystem(), fs); + ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); + ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); std::map expected_options = {{"key", "value"}}; - ASSERT_EQ(ctx->GetOptions(), expected_options); } TEST(CommitContextTest, TestWithCatalog) { @@ -117,6 +138,8 @@ TEST(CommitContextTest, TestWithCatalog) { ASSERT_TRUE(ctx->GetIdentifier().has_value()); ASSERT_EQ(ctx->GetIdentifier().value(), Identifier("db1", "t1")); ASSERT_EQ(ctx->GetTableId(), std::optional("table-uuid")); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); CommitContextBuilder null_catalog_builder("table_root_path", "commit_user_1"); ASSERT_NOK_WITH_MSG(null_catalog_builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), @@ -134,100 +157,4 @@ TEST(CommitContextTest, TestWithCatalog) { ASSERT_EQ(next_ctx->GetTableId(), std::nullopt); } -TEST(CommitContextTest, TestCatalogAddressesBranchByIdentifier) { - auto catalog = std::make_shared(); - - CommitContextBuilder branch_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto branch_ctx, - branch_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")).Finish()); - ASSERT_EQ(branch_ctx->GetIdentifier().value(), Identifier("db1", "t1$branch_dev")); - CommitContextBuilder agreeing_builder("table_root_path", "commit_user_1"); - ASSERT_OK(agreeing_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .AddOption(Options::BRANCH, "dev") - .Finish()); - CommitContextBuilder constructed_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto constructed_ctx, - constructed_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).Finish()); - ASSERT_EQ(constructed_ctx->GetIdentifier().value(), Identifier("db1", "t1$branch_dev")); - CommitContextBuilder main_builder("table_root_path", "commit_user_1"); - ASSERT_OK(main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) - .AddOption(Options::BRANCH, "main") - .Finish()); - - CommitContextBuilder mixed_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(mixed_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .AddOption(Options::BRANCH, "main") - .Finish(), - "but both 'dev' and 'main' were named"); - CommitContextBuilder mixed_option_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - mixed_option_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) - .AddOption(Options::BRANCH, "dev") - .Finish(), - "but both 'main' and 'dev' were named"); - - CommitContextBuilder option_only_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(option_only_builder.WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "dev") - .Finish(), - "name branch 'dev' there as 't1$branch_dev'"); - - CommitContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - upper_main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_MAIN")).Finish(), - "a catalog names branch 'MAIN' as it names the main branch"); - - CommitContextBuilder splitter_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(splitter_builder.WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "dev$options") - .Finish(), - "a branch a catalog addresses cannot contain '$'"); - - CommitContextBuilder system_table_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - system_table_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev$options")).Finish(), - "Cannot 'commit' for system table"); -} - -TEST(CommitContextTest, TestBranch) { - CommitContextBuilder file_system_builder("table_root_path", "commit_user_1"); - ASSERT_OK(file_system_builder.AddOption(Options::BRANCH, "dev").Finish()); - - CommitContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto upper_main_ctx, - upper_main_builder.AddOption(Options::BRANCH, "MAIN").Finish()); - ASSERT_EQ(upper_main_ctx->GetOptions().at(Options::BRANCH), "MAIN"); - CommitContextBuilder splitter_builder("table_root_path", "commit_user_1"); - ASSERT_OK(splitter_builder.AddOption(Options::BRANCH, "dev$options").Finish()); - - CommitContextBuilder rest_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto rest_ctx, - rest_builder.UseRESTCatalogCommit(true).AddOption(Options::BRANCH, "dev").Finish()); - ASSERT_TRUE(rest_ctx->UseRESTCatalogCommit()); - ASSERT_EQ(rest_ctx->GetOptions().at(Options::BRANCH), "dev"); - CommitContextBuilder rest_upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(rest_upper_main_builder.UseRESTCatalogCommit(true) - .AddOption(Options::BRANCH, "MAIN") - .Finish(), - "a catalog names branch 'MAIN' as it names the main branch"); - - CommitContextBuilder escaping_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(escaping_builder.AddOption(Options::BRANCH, "dev/../../outside").Finish(), - "branch name cannot contain path separators"); -} - -TEST(CommitContextTest, TestSetOptionsOverridesAddedOptions) { - CommitContextBuilder builder("table_root_path", "commit_user_1"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(ctx->GetOptions(), expected_options); -} - } // namespace paimon::test diff --git a/src/paimon/core/operation/data_evolution_split_read_test.cpp b/src/paimon/core/operation/data_evolution_split_read_test.cpp index 4bd0e0c29..89f983c72 100644 --- a/src/paimon/core/operation/data_evolution_split_read_test.cpp +++ b/src/paimon/core/operation/data_evolution_split_read_test.cpp @@ -962,7 +962,7 @@ TEST_F(DataEvolutionSplitReadTest, TestSequentialVectorReadReusesMainReader) { builder.WithFileSystem(counting_fs).EnablePrefetch(false).SetReadFieldNames(projection); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + InternalReadContext::Create(read_context, table_schema, options, {})); auto split_read = std::make_unique(path_factory, context, GetDefaultPool(), executor); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, split_read->CreateReader(split)); diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index 4e0747437..c5bf07d9d 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -18,3 +18,20 @@ */ #include "paimon/file_store_commit.h" + +#include "paimon/catalog/table.h" +#include "paimon/commit_context.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" + +namespace paimon { + +Result> FileStoreCommit::Create( + std::unique_ptr ctx, const std::map& options) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + FileStoreTableFactory::Create(*ctx, options)); + return table->NewCommit(std::move(ctx)); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index 96c6de34d..1f8d94147 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -323,11 +323,14 @@ class FileStoreCommitImplTest : public testing::Test { } void CommitAsCompetitor(int32_t round, int64_t commit_identifier) { + std::map builder_options; CommitContextBuilder builder(table_path_, "competitor"); + builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); + + builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr competitor, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); ASSERT_OK(competitor->Commit(CommitMessagesOfRound(round), commit_identifier)); } @@ -343,19 +346,20 @@ class FileStoreCommitImplTest : public testing::Test { Result> CreateCatalogCommit( const CatalogCommitSpec& spec) const { + std::map builder_options; CommitContextBuilder builder(spec.root_path.empty() ? table_path_ : spec.root_path, spec.commit_user); - builder.AddOption(Options::FILE_SYSTEM, "local") - .WithCatalog(spec.catalog, spec.identifier.value()) + builder_options[Options::FILE_SYSTEM] = "local"; + builder.WithCatalog(spec.catalog, spec.identifier.value()) .AppendCommitCheckConflict(spec.append_commit_check_conflict); if (spec.table_id) { builder.WithTableId(spec.table_id.value()); } for (const auto& [key, value] : spec.options) { - builder.AddOption(key, value); + builder_options[key] = value; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreCommit::Create(std::move(context)); + return FileStoreCommit::Create(std::move(context), builder_options); } Result> CreateCatalogCommit( @@ -412,16 +416,18 @@ class FileStoreCommitImplTest : public testing::Test { void CheckExpireIsRefusedWithoutDeleting(const std::string& branch, const std::string& expected_msg) { + std::map builder_options; CommitContextBuilder builder(table_path_, "commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::BRANCH] = branch; + builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; + builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, branch) - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .Finish()); + + builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); ASSERT_OK(PrepareFirstCommitMessageFiles()); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); ASSERT_OK(commit->DropPartition({{{"f1", "10"}}}, /*commit_identifier=*/2)); @@ -610,13 +616,16 @@ class FileStoreCommitImplTest : public testing::Test { }; TEST_F(FileStoreCommitImplTest, TestCommit) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -638,15 +647,17 @@ TEST_F(FileStoreCommitImplTest, TestCommit) { TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { TimezoneGuard guard("Asia/Shanghai"); + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_context, - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.UseRESTCatalogCommit(true).WithTableId("table-uuid").Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = CommitMessagesOfRound(1); ASSERT_NOK_WITH_MSG(commit->GetLastCommitTableRequest(), @@ -696,10 +707,14 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { TEST_F(FileStoreCommitImplTest, TestCommitCarriesBaseSnapshotUuid) { std::vector> msgs = CommitMessagesOfRound(1); + std::map base_builder_options; CommitContextBuilder base_builder(table_path_, "commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + + base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(std::move(base_context), base_builder_options)); ASSERT_OK(base_commit->Commit(msgs, 1)); std::string snapshot_path = PathUtil::JoinPath(table_path_, "snapshot/snapshot-1"); @@ -707,13 +722,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitCarriesBaseSnapshotUuid) { ASSERT_TRUE(base_snapshot.Uuid().has_value()); const std::string base_uuid = base_snapshot.Uuid().value(); + std::map rest_builder_options; CommitContextBuilder rest_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_context, - rest_builder.AddOption(Options::FILE_SYSTEM, "local") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto rest_commit, FileStoreCommit::Create(std::move(rest_context))); + rest_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr rest_context, + + rest_builder.UseRESTCatalogCommit(true).WithTableId("table-uuid").Finish()); + ASSERT_OK_AND_ASSIGN(auto rest_commit, + FileStoreCommit::Create(std::move(rest_context), rest_builder_options)); ASSERT_OK(rest_commit->Commit(msgs, 2)); ASSERT_OK_AND_ASSIGN(std::string request_str, rest_commit->GetLastCommitTableRequest()); @@ -773,25 +790,30 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitTakesTheSnapshot) { } TEST_F(FileStoreCommitImplTest, TestCommitToBranch) { + std::map missing_builder_options; CommitContextBuilder missing_builder(table_path_, "commit_user_1"); + missing_builder_options[Options::FILE_SYSTEM] = "local"; + missing_builder_options[Options::BRANCH] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr missing_context, - missing_builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(missing_context)), - "not found latest schema in branch dev"); + + missing_builder.Finish()); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(std::move(missing_context), missing_builder_options), + "not found latest schema in branch dev"); SchemaManager branch_schema_manager(file_system_, table_path_, "dev"); ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); + std::map builder_options; CommitContextBuilder builder(table_path_, "commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::BRANCH] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .Finish()); + + builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); std::string branch_snapshots = PathUtil::JoinPath(table_path_, "branch/branch-dev/snapshot"); @@ -817,15 +839,15 @@ TEST_F(FileStoreCommitImplTest, TestRequestOnlyCommitToBranch) { ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); + std::map builder_options; CommitContextBuilder builder(table_path_, "commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::BRANCH] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); + + builder.UseRESTCatalogCommit(true).WithTableId("table-uuid").Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); ASSERT_OK_AND_ASSIGN(std::string request_str, commit->GetLastCommitTableRequest()); @@ -936,13 +958,15 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitToBranchTakesTheBranchSnapshot) ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); + std::map base_builder_options; CommitContextBuilder base_builder(table_path_, "commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; + base_builder_options[Options::BRANCH] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .Finish()); + + base_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr base_commit, - FileStoreCommit::Create(std::move(base_context))); + FileStoreCommit::Create(std::move(base_context), base_builder_options)); ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); SnapshotManager branch_snapshot_manager(file_system_, table_path_, "dev"); ASSERT_OK_AND_ASSIGN(std::optional base_snapshot, @@ -1111,10 +1135,14 @@ TEST_F(FileStoreCommitImplTest, TestCatalogSchemaWinsOverTheOneUnderTheTablePath ASSERT_EQ(catalog->CommitCalls().size(), 1u); ASSERT_EQ(catalog->CommitCalls().front().snapshot.SchemaId(), 7); + std::map path_builder_options; CommitContextBuilder path_builder(table_path_, "commit_user_2"); + path_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr path_context, - path_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto path_commit, FileStoreCommit::Create(std::move(path_context))); + + path_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto path_commit, + FileStoreCommit::Create(std::move(path_context), path_builder_options)); std::vector> other_msgs = CommitMessagesOfRound(2); ASSERT_OK(path_commit->Commit(other_msgs, 2)); ASSERT_OK_AND_ASSIGN( @@ -1148,12 +1176,13 @@ TEST_F(FileStoreCommitImplTest, TestCatalogSchemaWhichIsNotADataTableSchemaIsRef } TEST_F(FileStoreCommitImplTest, TestTableIdWithNothingToSendItInIsRefused) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local") - .WithTableId("table-uuid") - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), + + context_builder.WithTableId("table-uuid").Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context), context_builder_options), "publishes the snapshot by writing it to the table directory"); auto catalog = CreateCatalog(); @@ -1325,14 +1354,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithoutAUsableLockIsNotCreated) { ASSERT_NOK_WITH_MSG(CreateCatalogCommit(catalog), "no commit lease for this table"); ASSERT_TRUE(catalog->CommitCalls().empty()); + std::map builder_options; CommitContextBuilder builder(table_path_, "commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::REST_COMMIT_ACQUIRE_LOCK] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::REST_COMMIT_ACQUIRE_LOCK, "true") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(context)), "cannot hold a commit lease"); + + builder.UseRESTCatalogCommit(true).WithTableId("table-uuid").Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(context), builder_options), + "cannot hold a commit lease"); } TEST_F(FileStoreCommitImplTest, TestUnknownCatalogOutcomeKeepsTheManifests) { @@ -1631,7 +1661,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitReadsThroughTheCatalogsFileSystem) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), {})); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1))); ASSERT_EQ(catalog->CommitCalls().size(), 1u); @@ -1669,7 +1699,7 @@ TEST_F(FileStoreCommitImplTest, TestAGivenFileSystemWinsOverTheCatalogs) { context_builder.WithCatalog(catalog, Identifier("db", "tbl")) .WithFileSystem(given_fs) .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), {})); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1))); ASSERT_EQ(catalog->CommitCalls().size(), 1u); @@ -1843,14 +1873,17 @@ TEST_F(FileStoreCommitImplTest, TestCatalogSnapshotLoaderFailurePropagates) { } TEST_F(FileStoreCommitImplTest, TestSnapshotSequenceMaxPropertyMergedOnCommit) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::WRITE_SEQUENCE_NUMBER_INIT_MODE] = "snapshot"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -1890,16 +1923,18 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes) ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path, {})); + std::map context_builder_options; CommitContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::COMMIT_MAX_RETRIES] = "10"; + context_builder_options[Options::COMMIT_MIN_RETRY_WAIT] = "1ms"; + context_builder_options[Options::COMMIT_MAX_RETRY_WAIT] = "1ms"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::COMMIT_MAX_RETRIES, "10") - .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") - .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.WithFileSystem(fs).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::string latest_hint = PathUtil::JoinPath(table_path, "snapshot/LATEST"); auto* mock_fs = dynamic_cast(fs.get()); @@ -1931,15 +1966,17 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path, {})); + std::map context_builder_options; CommitContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::COMMIT_MIN_RETRY_WAIT] = "1ms"; + context_builder_options[Options::COMMIT_MAX_RETRY_WAIT] = "1ms"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") - .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.WithFileSystem(fs).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::string latest_hint = PathUtil::JoinPath(table_path, "snapshot/LATEST"); auto* mock_fs = dynamic_cast(fs.get()); EXPECT_CALL(*mock_fs, ReadFile(testing::_, testing::_)) @@ -2003,17 +2040,19 @@ TEST_F(FileStoreCommitImplTest, TestRealtimeCommitRebasesAfterSnapshotConflict) ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path_, {})); + std::map realtime_builder_options; CommitContextBuilder realtime_builder(table_path_, "realtime_commit_user"); + realtime_builder_options[Options::REALTIME_ENABLED] = "true"; + realtime_builder_options[Options::BUCKET] = "2"; + realtime_builder_options[Options::COMMIT_MAX_RETRIES] = "1"; + realtime_builder_options[Options::COMMIT_MIN_RETRY_WAIT] = "1ms"; + realtime_builder_options[Options::COMMIT_MAX_RETRY_WAIT] = "1ms"; ASSERT_OK_AND_ASSIGN(std::unique_ptr realtime_context, - realtime_builder.AddOption(Options::REALTIME_ENABLED, "true") - .AddOption(Options::BUCKET, "2") - .AddOption(Options::COMMIT_MAX_RETRIES, "1") - .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") - .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr realtime_commit, - FileStoreCommit::Create(std::move(realtime_context))); + + realtime_builder.WithFileSystem(fs).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr realtime_commit, + FileStoreCommit::Create(std::move(realtime_context), realtime_builder_options)); auto realtime_commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(realtime_commit))); @@ -2034,7 +2073,7 @@ TEST_F(FileStoreCommitImplTest, TestRealtimeCommitRebasesAfterSnapshotConflict) ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_context, concurrent_builder.WithFileSystem(file_system_).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_commit, - FileStoreCommit::Create(std::move(concurrent_context))); + FileStoreCommit::Create(std::move(concurrent_context), {})); auto* mock_fs = dynamic_cast(fs.get()); ASSERT_NE(nullptr, mock_fs); @@ -2078,13 +2117,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path, {})); + std::map context_builder_options; CommitContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.WithFileSystem(fs).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::string new_snapshot_6 = PathUtil::JoinPath(table_path, "snapshot/snapshot-6"); auto* mock_fs = dynamic_cast(fs.get()); EXPECT_CALL(*mock_fs, AtomicStore(testing::StrEq(new_snapshot_6), testing::_)) @@ -2105,13 +2146,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua bool exist, file_system_->Exists(PathUtil::JoinPath(table_path, "snapshot/snapshot-6"))); ASSERT_TRUE(exist); + std::map context_builder_2_options; CommitContextBuilder context_builder_2(table_path, "commit_user_1"); + context_builder_2_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context_2, - context_builder_2.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit_2, FileStoreCommit::Create(std::move(commit_context_2))); + context_builder_2.WithFileSystem(fs).Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit_2, FileStoreCommit::Create(std::move(commit_context_2), + context_builder_2_options)); ASSERT_OK_AND_ASSIGN(int32_t num_committed, commit_2->FilterAndCommit({{1, msgs}})); ASSERT_EQ(0, num_committed); std::string new_snapshot_7 = PathUtil::JoinPath(table_path, "snapshot/snapshot-7"); @@ -2129,14 +2172,17 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua } TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "5kb"; + context_builder_options[Options::MANIFEST_MERGE_MIN_COUNT] = "2"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "5kb") - .AddOption(Options::MANIFEST_MERGE_MIN_COUNT, "2") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); { std::vector> msgs = @@ -2192,13 +2238,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { } TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); { std::vector> msgs = @@ -2263,12 +2312,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2320,10 +2372,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestThroughCatalog) { + std::map base_builder_options; CommitContextBuilder base_builder(table_path_, "commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + + base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(std::move(base_context), base_builder_options)); for (int32_t round = 1; round <= 2; ++round) { ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(round), round)); } @@ -2347,10 +2403,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestThroughCatalog) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestHoldsALockScope) { + std::map base_builder_options; CommitContextBuilder base_builder(table_path_, "commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + + base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(std::move(base_context), base_builder_options)); for (int32_t round = 1; round <= 2; ++round) { ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(round), round)); } @@ -2379,10 +2439,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestHoldsALockScope) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNeedsTheTargetUnderTheTablePath) { + std::map base_builder_options; CommitContextBuilder base_builder(table_path_, "commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + + base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(std::move(base_context), base_builder_options)); for (int32_t round = 1; round <= 2; ++round) { ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(round), round)); } @@ -2396,23 +2460,29 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNeedsTheTargetUnderTheTabl } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNoLatestSnapshotReturnsError) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); ASSERT_NOK_WITH_MSG(commit->RollbackToAsLatest(/*target_snapshot_id=*/1), "Latest snapshot is null, can not roll back."); } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestTargetNotExistReturnsError) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", @@ -2441,14 +2511,17 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestDeletionVectorOnlyChange) /*ignore_if_exists=*/false)); std::string dv_table_path = PathUtil::JoinPath(test_root_, "foo.db/dv_bar"); + std::map context_builder_options; CommitContextBuilder context_builder(dv_table_path, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2518,12 +2591,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF // before writing. A stateful Exists() mock reproduces exactly that ordering. ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path_, {})); + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.WithFileSystem(fs).Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2560,12 +2635,15 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF } TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); { std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -2605,12 +2683,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) { } TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -2662,13 +2743,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { } TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); { std::vector> msgs = @@ -2780,14 +2864,16 @@ TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { } TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs; ASSERT_OK(commit->Commit(msgs)); std::shared_ptr metrics = commit->GetCommitMetrics(); @@ -2798,14 +2884,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { } TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -2828,14 +2916,16 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { } TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); std::vector changes; @@ -2862,14 +2952,16 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { } TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2889,14 +2981,16 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { } TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); std::vector changes; @@ -3221,18 +3315,20 @@ TEST_F(FileStoreCommitImplTest, TestCatalogExpireWithoutPublishedSnapshots) { } TEST_F(FileStoreCommitImplTest, TestDropPartitionAndExpireSnapshot) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; + context_builder_options[Options::SNAPSHOT_EXPIRE_LIMIT] = "30"; + context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .AddOption(Options::SNAPSHOT_EXPIRE_LIMIT, "30") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3293,18 +3389,20 @@ TEST_F(FileStoreCommitImplTest, TestExpireOnMainOfTableWithBranchesIsNotSupporte } TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; + context_builder_options[Options::SNAPSHOT_EXPIRE_LIMIT] = "30"; + context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .AddOption(Options::SNAPSHOT_EXPIRE_LIMIT, "30") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3340,14 +3438,16 @@ TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { } TEST_F(FileStoreCommitImplTest, TestTruncateTable) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3373,10 +3473,14 @@ TEST_F(FileStoreCommitImplTest, TestTruncateTable) { } TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3425,10 +3529,14 @@ TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { } TEST_F(FileStoreCommitImplTest, AbortIgnoresMissingFilesAndFailsForNonImplMessage) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3449,10 +3557,14 @@ TEST_F(FileStoreCommitImplTest, AbortIgnoresDeleteFailures) { // A delete that fails with an IO error (e.g. permission denied) must be swallowed by // best-effort cleanup, mirroring Java deleteQuietly: Abort still returns OK and does not // propagate the failure. + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -3488,13 +3600,15 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { // Truncating a table that has never been committed succeeds and produces an OVERWRITE snapshot // with no added and no deleted files: there is nothing to overwrite, but the overwrite is still // materialized. + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.IgnoreEmptyCommit(true).Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); ASSERT_OK(commit->TruncateTable(/*commit_identifier=*/1)); @@ -3513,14 +3627,16 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { } TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); std::vector> msgs; @@ -3533,14 +3649,17 @@ TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { } TEST_F(FileStoreCommitImplTest, TestCollectChanges) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::BUCKET] = "10"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BUCKET, "10") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3576,13 +3695,16 @@ TEST_F(FileStoreCommitImplTest, TestCollectChanges) { } TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3607,13 +3729,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { } TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3647,13 +3772,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { } TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3678,13 +3806,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) } TEST_F(FileStoreCommitImplTest, FilterAndCommit) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); std::vector data_files = { @@ -3725,13 +3856,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommit) { } TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3747,13 +3881,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { } TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3781,13 +3918,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { } TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3814,13 +3954,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { } TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3853,13 +3996,16 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { } TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -3884,15 +4030,18 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::ROW_TRACKING_ENABLED] = "true"; + context_builder_options[Options::DATA_EVOLUTION_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::ROW_TRACKING_ENABLED, "true") - .AddOption(Options::DATA_EVOLUTION_ENABLED, "true") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -3916,14 +4065,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { } TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -3949,13 +4100,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -3987,14 +4141,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4027,13 +4183,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4061,13 +4220,16 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); std::vector data_files = { @@ -4116,13 +4278,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { } TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4155,13 +4320,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4189,13 +4357,16 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { } TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4219,14 +4390,17 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { } TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::COMMIT_DISCARD_DUPLICATE_FILES] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::COMMIT_DISCARD_DUPLICATE_FILES, "true") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4251,12 +4425,15 @@ TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -4283,12 +4460,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { std::string tmp_table_path = tmp_dir->Str(); ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + std::map context_builder2_options; CommitContextBuilder context_builder2(tmp_table_path, "commit_user_1"); + context_builder2_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder2_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context2, - context_builder2.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - auto commit2 = FileStoreCommit::Create(std::move(commit_context2)); + + context_builder2.Finish()); + auto commit2 = + FileStoreCommit::Create(std::move(commit_context2), context_builder2_options); CHECK_HOOK_STATUS(commit2.status(), i); CHECK_HOOK_STATUS(commit2.value()->Commit(msgs), i); commit_run_complete = true; @@ -4314,7 +4494,7 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { // REST commit with local path should work without the object store flag CommitContextBuilder builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.UseRESTCatalogCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx), {})); std::vector> msgs = CommitMessagesOfRound(1); ASSERT_OK(commit->Commit(msgs)); @@ -4325,7 +4505,7 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { CommitContextBuilder catalog_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(auto catalog_ctx, catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_OK_AND_ASSIGN(auto catalog_commit, FileStoreCommit::Create(std::move(catalog_ctx))); + ASSERT_OK_AND_ASSIGN(auto catalog_commit, FileStoreCommit::Create(std::move(catalog_ctx), {})); ASSERT_OK(catalog_commit->Commit(msgs)); ASSERT_EQ(catalog->CommitCalls().size(), 1u); @@ -4338,11 +4518,11 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { ASSERT_OK_AND_ASSIGN( auto object_store_ctx, object_store_builder.WithCatalog(object_store_catalog, Identifier("db", "tbl")).Finish()); - ASSERT_OK(FileStoreCommit::Create(std::move(object_store_ctx))); + ASSERT_OK(FileStoreCommit::Create(std::move(object_store_ctx), {})); CommitContextBuilder plain_builder(object_store_path, "commit_user_1"); ASSERT_OK_AND_ASSIGN(auto plain_ctx, plain_builder.WithFileSystem(file_system_).Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(plain_ctx)), + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(plain_ctx), {}), "invalid scheme oss for local file system"); } @@ -4364,10 +4544,13 @@ TEST_F(FileStoreCommitImplTest, TestFixedBucketPKTableCommitAllowed) { std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); + std::map builder_options; CommitContextBuilder builder(pk_table_path, "test_user"); - builder.AddOption(Options::FILE_SYSTEM, "local").UseRESTCatalogCommit(true); + builder_options[Options::FILE_SYSTEM] = "local"; + builder.UseRESTCatalogCommit(true); ASSERT_OK_AND_ASSIGN(auto commit_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto committer, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN(auto committer, + FileStoreCommit::Create(std::move(commit_context), builder_options)); ASSERT_TRUE(committer != nullptr); } @@ -4397,14 +4580,17 @@ TEST_F(FileStoreCommitImplTest, ValidateCommitOptionsAllowsManifestDeleteFileDro } TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::MANIFEST_DELETE_FILE_DROP_STATS] = "true"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::MANIFEST_DELETE_FILE_DROP_STATS, "true") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -4426,14 +4612,17 @@ TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) { } TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::MANIFEST_DELETE_FILE_DROP_STATS] = "true"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::MANIFEST_DELETE_FILE_DROP_STATS, "true") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -4501,21 +4690,28 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) { } TEST_F(FileStoreCommitImplTest, DropPartitionWithEmptyPartitionsFails) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); ASSERT_NOK_WITH_MSG(commit->DropPartition({}, /*commit_identifier=*/1), "partitions list cannot be empty"); } TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4549,10 +4745,14 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) } TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4563,10 +4763,14 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) } TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4590,13 +4794,16 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) { } TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::DYNAMIC_PARTITION_OVERWRITE] = "false"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::DYNAMIC_PARTITION_OVERWRITE, "false") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4618,12 +4825,15 @@ TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) } TEST_F(FileStoreCommitImplTest, OverwriteWithChangelogFilesLogsWarning) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4663,12 +4873,15 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) /*ignore_if_exists=*/false)); std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); + std::map builder_options; CommitContextBuilder builder(pk_table_path, "test_user"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::OVERWRITE_UPGRADE] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::OVERWRITE_UPGRADE, "true") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, + FileStoreCommit::Create(std::move(commit_context), builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4713,13 +4926,15 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) } TEST_F(FileStoreCommitImplTest, CommitWithAppendCommitCheckConflict) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AppendCommitCheckConflict(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.AppendCommitCheckConflict(true).Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -4736,12 +4951,15 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { // First snapshot is committed in the default (scan) mode, so it does NOT carry the // max-sequence-number property. { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + context_builder_options)); std::vector> msgs1 = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", @@ -4751,13 +4969,16 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { // Second commit uses snapshot-init mode; since the previous snapshot lacks the property, // the max sequence number is recomputed by scanning the base manifests. + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::WRITE_SEQUENCE_NUMBER_INIT_MODE] = "snapshot"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4780,12 +5001,15 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { } TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4806,12 +5030,15 @@ TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) { } TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenOverwriteUpgradeDisabled) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::OVERWRITE_UPGRADE] = "false"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::OVERWRITE_UPGRADE, "false") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4839,12 +5066,15 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { /*ignore_if_exists=*/false)); std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); + std::map builder_options; CommitContextBuilder builder(pk_table_path, "test_user"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::OVERWRITE_UPGRADE] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::OVERWRITE_UPGRADE, "true") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, + FileStoreCommit::Create(std::move(commit_context), builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4859,12 +5089,15 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { } TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelta) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4882,10 +5115,14 @@ TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelt } TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4895,10 +5132,14 @@ TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests } TEST_F(FileStoreCommitImplTest, RowIdCheckConflictSetsCheckSnapshotAndReturnsSelf) { + std::map context_builder_options; CommitContextBuilder context_builder(table_path_, "commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp index a7f39628d..aa61dfb46 100644 --- a/src/paimon/core/operation/file_store_commit_test.cpp +++ b/src/paimon/core/operation/file_store_commit_test.cpp @@ -41,6 +41,7 @@ #include "paimon/core/operation/file_store_commit_impl.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" #include "paimon/executor.h" @@ -80,12 +81,15 @@ TEST(FileStoreCommitTest, TestCreate) { /*ignore_if_exists=*/false)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + std::map context_builder_options; CommitContextBuilder context_builder(table_path, "commit_user"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); } @@ -99,9 +103,8 @@ TEST(FileStoreCommitTest, TestCreateWithCatalogRequiresIdentifier) { dir->Str(), "commit_user", /*ignore_empty_commit=*/true, /*use_rest_catalog_commit=*/false, catalog, /*identifier=*/std::nullopt, /*table_id=*/std::nullopt, /*append_commit_check_conflict=*/false, GetDefaultPool(), - CreateDefaultExecutor(), /*specific_file_system=*/nullptr, options, - /*format_table=*/nullptr); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), + CreateDefaultExecutor(), /*specific_file_system=*/nullptr); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context), {}), "a catalog commit requires a table identifier"); } @@ -126,7 +129,7 @@ TEST(FileStoreCommitTest, TestCatalogCommitUsesPerTableFileSystem) { CommitContextBuilder builder(table_path, "commit_user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx), {})); // The file system was resolved per table, not from the catalog-wide one. const std::vector& requests = catalog->TableFileSystemRequests(); @@ -161,12 +164,15 @@ TEST(FileStoreCommitTest, TestAppendDvIndexShouldUseOverwriteCommitKind) { /*ignore_if_exists=*/false)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + std::map context_builder_options; CommitContextBuilder context_builder(table_path, "commit_user"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(std::move(commit_context), context_builder_options)); LinkedHashMap dv_ranges; dv_ranges.insert_or_assign("data-file-1", diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index c1b889473..2cd3bc97e 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -18,8 +18,21 @@ #include "paimon/file_store_write.h" +#include "paimon/catalog/table.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/write_context.h" + namespace paimon { +Result> FileStoreWrite::Create( + std::unique_ptr ctx, const std::map& options) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + FileStoreTableFactory::Create(*ctx, options)); + return table->NewWrite(std::move(ctx)); +} + Result> FileStoreWrite::PrepareCommitWithProgress(int64_t) { return Status::Invalid("prepare commit with progress requires a real-time writer"); } diff --git a/src/paimon/core/operation/file_store_write_test.cpp b/src/paimon/core/operation/file_store_write_test.cpp index 520e850e5..ad1c664ae 100644 --- a/src/paimon/core/operation/file_store_write_test.cpp +++ b/src/paimon/core/operation/file_store_write_test.cpp @@ -88,14 +88,16 @@ TEST(FileStoreWriteTest, TestRealtimeCatalogCommitRefreshAndRecovery) { }; auto create_writer = [&](const std::shared_ptr& realtime_context) -> Result> { + std::map builder_options; WriteContextBuilder builder(table_path, "writer"); + builder_options[Options::FILE_SYSTEM] = "unregistered-file-system"; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr ctx, + builder.WithCatalog(catalog, identifier) .WithStreamingMode(true) .WithRealtimeContext(realtime_context) - .AddOption(Options::FILE_SYSTEM, "unregistered-file-system") .Finish()); - return FileStoreWrite::Create(std::move(ctx)); + return FileStoreWrite::Create(std::move(ctx), builder_options); }; ASSERT_OK_AND_ASSIGN(auto realtime_context, RealtimeContext::Create()); ASSERT_OK_AND_ASSIGN(auto writer, create_writer(realtime_context)); @@ -106,7 +108,7 @@ TEST(FileStoreWriteTest, TestRealtimeCatalogCommitRefreshAndRecovery) { CommitContextBuilder commit_builder(table_path, "writer"); ASSERT_OK_AND_ASSIGN(auto commit_ctx, commit_builder.WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_ctx))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_ctx), {})); ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, commit->CommitWithProgress(progress, 1, std::nullopt)); ASSERT_EQ(snapshot_id, 1); @@ -149,11 +151,11 @@ TEST(FileStoreWriteTest, TestCreateWithInvalidInput) { WriteContextBuilder builder(dir->Str(), "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx1, builder.WithMemoryPool(nullptr).Finish()); - ASSERT_NOK(FileStoreWrite::Create(std::move(ctx1))); + ASSERT_NOK(FileStoreWrite::Create(std::move(ctx1), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx2, builder.WithExecutor(nullptr).Finish()); - ASSERT_NOK(FileStoreWrite::Create(std::move(ctx2))); - ASSERT_NOK(FileStoreWrite::Create(/*context=*/nullptr)); + ASSERT_NOK(FileStoreWrite::Create(std::move(ctx2), {})); + ASSERT_NOK(FileStoreWrite::Create(/*context=*/nullptr, {})); } TEST(FileStoreWriteTest, TestCreateAppendTable) { @@ -170,18 +172,17 @@ TEST(FileStoreWriteTest, TestCreateAppendTable) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{"f0", "f3"}, /*primary_keys=*/{}, /*options=*/{}, /*ignore_if_exists=*/false)); - std::shared_ptr shared_catalog(std::move(catalog)); - ASSERT_FALSE(shared_catalog->SupportsVersionManagement()); + ASSERT_FALSE(catalog->SupportsVersionManagement()); for (bool use_catalog : {false, true}) { SCOPED_TRACE(use_catalog); WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "commit_user_1"); if (use_catalog) { - context_builder.WithCatalog(shared_catalog, Identifier("foo", "bar")); + context_builder.WithCatalog(catalog, Identifier("foo", "bar")); } ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); ASSERT_OK(file_store_write->Close()); } } @@ -203,7 +204,7 @@ TEST(FileStoreWriteTest, TestCatalogWriteAsksTheCatalogOnlyForTheMainBranchSchem std::unique_ptr main_context, main_builder.WithCatalog(catalog, Identifier("db", "table$branch_main")).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_write, - FileStoreWrite::Create(std::move(main_context))); + FileStoreWrite::Create(std::move(main_context), {})); ASSERT_OK(main_write->Close()); ASSERT_EQ(catalog->LoadTableSchemaIdentifiers().size(), 1u); ASSERT_EQ(catalog->LoadTableSchemaIdentifiers().front(), Identifier("db", "table")); @@ -217,7 +218,7 @@ TEST(FileStoreWriteTest, TestCatalogWriteAsksTheCatalogOnlyForTheMainBranchSchem std::unique_ptr branch_context, branch_builder.WithCatalog(catalog, Identifier("db", "table$branch_dev")).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_write, - FileStoreWrite::Create(std::move(branch_context))); + FileStoreWrite::Create(std::move(branch_context), {})); ASSERT_OK(branch_write->Close()); ASSERT_EQ(catalog->LoadTableSchemaIdentifiers().size(), 1u); } @@ -244,7 +245,7 @@ TEST(FileStoreWriteTest, TestCatalogWriteUsesPerTableFileSystem) { ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.WithCatalog(catalog, identifier).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(ctx))); + FileStoreWrite::Create(std::move(ctx), {})); ASSERT_OK(writer->Close()); // The file system was resolved per table, not from the catalog-wide one. @@ -268,11 +269,14 @@ TEST(FileStoreWriteTest, TestCreateAppendTableWithInvalidBucket) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{"f0", "f3"}, /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + std::map context_builder_options; WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "commit_user_1"); + context_builder_options[Options::BUCKET] = "-2"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::BUCKET, "-2").Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + + context_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context), context_builder_options), "not support bucket -2 in append table"); } @@ -294,13 +298,14 @@ TEST(FileStoreWriteTest, TestCreateAppendTableWithInvalidWriteType) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + std::map context_builder_options; WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "commit_user_1"); + context_builder_options[Options::BUCKET] = "-1"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::BUCKET, "-1") - .WithWriteSchema({"field_non_exist"}) - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + + context_builder.WithWriteSchema({"field_non_exist"}).Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context), context_builder_options), "write field field_non_exist does not exist in table schema"); } @@ -320,12 +325,15 @@ TEST(FileStoreWriteTest, TestCreatePrimaryKeyTable) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{"f0"}, /*primary_keys=*/{"f0", "f1", "f4"}, options, /*ignore_if_exists=*/false)); + std::map context_builder_options; WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "commit_user_1"); + context_builder_options[Options::BUCKET] = "2"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::BUCKET, "2").Finish()); + + context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); auto fs = std::make_shared(); SchemaManager schema_manager(fs, PathUtil::JoinPath(dir->Str(), "foo.db/bar")); @@ -357,7 +365,7 @@ TEST(FileStoreWriteTest, TestCreatePrimaryKeyTableWithInvalidBucket) { WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context), {}), "not support bucket -1 in key value table"); } diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index f36a15d71..956dacff8 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -19,6 +19,7 @@ #include "paimon/core/operation/internal_read_context.h" #include +#include #include #include #include @@ -44,7 +45,11 @@ #include "paimon/common/utils/url_utils.h" #include "paimon/core/options/map_storage_layout.h" #include "paimon/core/schema/arrow_schema_validator.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/core/utils/nested_projection_utils.h" +#include "paimon/defs.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/predicate_utils.h" @@ -91,7 +96,7 @@ std::shared_ptr InternalReadContext::CreateDependencyCatalogCont for (const auto& [key, value] : catalog_options) { dependency_options[key] = value; } - const auto& read_options = read_context_->GetOptions(); + const auto& read_options = dynamic_options_; for (const auto& [key, value] : read_options) { dependency_options[key] = value; } @@ -273,13 +278,85 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( return std::nullopt; } +Result> InternalReadContext::CreateForTable( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + CoreOptions core_options, const std::string& root_path, + const std::map& dynamic_options, + const std::map& fs_scheme_map) { + core_options.WithCache(context->GetCache()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr result, + CreateFromCoreOptions(context, table_schema, std::move(core_options), root_path)); + result->dynamic_options_ = dynamic_options; + result->fs_scheme_map_ = fs_scheme_map; + return result; +} + Result> InternalReadContext::Create( const std::shared_ptr& context, const std::shared_ptr& table_schema, - const std::map& options) { - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); + const std::map& options, + const std::map& dynamic_options) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr result, + CreateFromOptions(context, table_schema, options, /*file_system=*/nullptr, + /*path=*/std::nullopt)); + result->dynamic_options_ = dynamic_options; + return result; +} + +Result> InternalReadContext::CreateFromOptions( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + const std::map& options, + const std::shared_ptr& file_system, std::optional path) { + std::shared_ptr resolved_file_system = file_system; + if (resolved_file_system == nullptr) { + PAIMON_ASSIGN_OR_RAISE(resolved_file_system, + ContextUtils::ResolveFileSystemForLoad(*context)); + } + PAIMON_ASSIGN_OR_RAISE( + CoreOptions core_options, + CoreOptions::FromMap(options, resolved_file_system, + file_system ? std::map{} + : context->GetFileSystemSchemeToIdentifierMap())); core_options.WithCache(context->GetCache()); + return CreateFromCoreOptions(context, table_schema, std::move(core_options), std::move(path)); +} + +Result> InternalReadContext::CreateForFallbackBranch( + const std::shared_ptr& context, const std::string& branch, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& table_dynamic_options, + const std::map& fs_scheme_map) { + SchemaManager schema_manager(file_system, root_path, branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + schema_manager.Latest()); + if (!latest_schema) { + return Status::Invalid( + fmt::format("schema file not found in path {}, branch {}", root_path, branch)); + } + std::shared_ptr table_schema = latest_schema.value(); + auto options = table_schema->Options(); + for (const auto& [key, value] : table_dynamic_options) { + if (key != Options::TYPE) { + options[key] = value; + } + } + if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { + options[Options::BRANCH] = branch; + } else { + options.erase(Options::BRANCH); + } + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr result, + CreateFromOptions(context, table_schema, options, file_system, root_path)); + result->dynamic_options_ = table_dynamic_options; + result->fs_scheme_map_ = fs_scheme_map; + return result; +} + +Result> InternalReadContext::CreateFromCoreOptions( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + CoreOptions core_options, std::optional path) { if (context->HasVectorOrFullTextSearch()) { if (!core_options.FileIndexReadEnabled()) { return Status::Invalid( @@ -390,8 +467,8 @@ Result> InternalReadContext::Create( return Status::Invalid("memory pool is null"); } std::shared_ptr arrow_pool = GetArrowPool(context->GetMemoryPool()); - auto internal_context = std::unique_ptr( - new InternalReadContext(context, table_schema, read_schema, core_options, arrow_pool)); + auto internal_context = std::unique_ptr(new InternalReadContext( + context, table_schema, read_schema, core_options, std::move(path), arrow_pool)); PAIMON_RETURN_NOT_OK(internal_context->BindIndexScores()); return internal_context; } @@ -471,12 +548,15 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, const CoreOptions& options, + std::optional path, const std::shared_ptr& arrow_pool) : read_context_(read_context), table_schema_(table_schema), read_schema_(read_schema), arrow_pool_(arrow_pool), - options_(options) {} + options_(options), + fs_scheme_map_(read_context->GetFileSystemSchemeToIdentifierMap()), + path_(std::move(path)) {} Result> InternalReadContext::CreateWithSchema( const std::shared_ptr& original, @@ -488,9 +568,12 @@ Result> InternalReadContext::CreateWithSche } // Create a new InternalReadContext sharing all properties except read_schema. // The new read_schema is the minimal column set for COUNT(*). - return std::shared_ptr( + auto result = std::shared_ptr( new InternalReadContext(original->read_context_, original->table_schema_, new_read_schema, - original->options_, original->arrow_pool_)); + original->options_, original->path_, original->arrow_pool_)); + result->dynamic_options_ = original->dynamic_options_; + result->fs_scheme_map_ = original->fs_scheme_map_; + return result; } } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 9de415d5f..da6e75351 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,7 @@ class Schema; namespace paimon { struct CatalogContext; class Executor; +class FileSystem; class MemoryPool; class Predicate; class RealtimeContext; @@ -56,10 +58,30 @@ class InternalReadContext { std::string field_name; }; + /// Builds a read context using the table's resolved options and the operation's cache. + /// @param root_path Table root path, overriding the context's path. + static Result> CreateForTable( + const std::shared_ptr& read_context, + const std::shared_ptr& table_schema, CoreOptions core_options, + const std::string& root_path, const std::map& dynamic_options, + const std::map& fs_scheme_map); + + /// Builds an internal write/lookup read from effective options and the context's file system. static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, - const std::map& options); + const std::map& options, + const std::map& dynamic_options); + + /// Loads the fallback branch schema and applies the table's accumulated dynamic options. + /// @param file_system File system used to open the table. + /// @param root_path Table root path, overriding the context's path. + /// @param table_dynamic_options Overrides accumulated when loading or copying the table. + static Result> CreateForFallbackBranch( + const std::shared_ptr& context, const std::string& branch, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& table_dynamic_options, + const std::map& fs_scheme_map); /// Maps score output positions in the read schema to their logical search sources. const std::map& GetIndexScoreBindings() const { @@ -79,7 +101,7 @@ class InternalReadContext { return table_schema_; } const std::string& GetPath() const { - return read_context_->GetPath(); + return path_ ? *path_ : read_context_->GetPath(); } std::shared_ptr CreateDependencyCatalogContext() const; @@ -138,7 +160,7 @@ class InternalReadContext { /// The scheme map the file system of this read was resolved with, which a catalog reading a /// dependency table on its behalf needs to resolve that table's location the same way. const std::map& GetFileSystemSchemeToIdentifierMap() const { - return read_context_->GetFileSystemSchemeToIdentifierMap(); + return fs_scheme_map_; } bool ReadAheadCacheEnabled() const { @@ -163,9 +185,22 @@ class InternalReadContext { InternalReadContext(const std::shared_ptr& read_context, const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, - const CoreOptions& options, + const CoreOptions& options, std::optional path, const std::shared_ptr& arrow_pool); + /// Resolves option maps using file_system, or the context's file system when null. + static Result> CreateFromOptions( + const std::shared_ptr& read_context, + const std::shared_ptr& table_schema, + const std::map& options, + const std::shared_ptr& file_system, std::optional path); + + /// Validates the resolved options and builds the read schema. + static Result> CreateFromCoreOptions( + const std::shared_ptr& read_context, + const std::shared_ptr& table_schema, CoreOptions core_options, + std::optional path); + static std::optional TryResolveSpecialFieldById(int32_t field_id, const CoreOptions& core_options, bool has_vector_or_full_text_search); @@ -183,7 +218,11 @@ class InternalReadContext { std::shared_ptr read_schema_; std::shared_ptr arrow_pool_; CoreOptions options_; + std::map dynamic_options_; + std::map fs_scheme_map_; std::map index_score_bindings_; + /// Optional table root path; unset for reads using the context's path. + std::optional path_; }; } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context_test.cpp b/src/paimon/core/operation/internal_read_context_test.cpp index 6e4957f4e..3e07ae54d 100644 --- a/src/paimon/core/operation/internal_read_context_test.cpp +++ b/src/paimon/core/operation/internal_read_context_test.cpp @@ -48,6 +48,50 @@ namespace paimon::test { +TEST(InternalReadContext, TestBoundReadIgnoresContextLoadingParameters) { + const std::string path = GetDataDir() + "/orc/append_09.db/append_09"; + auto file_system = std::make_shared(); + SchemaManager schema_manager(file_system, path); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, schema_manager.ReadSchema(0)); + auto catalog = std::make_shared(); + const std::map table_scheme_map = {{"file", "local"}}; + const std::map dynamic_options = {{Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ReadContextBuilder("another-table") + .WithCatalog(catalog, Identifier("db", "tbl")) + .WithFileSystemSchemeToIdentifierMap({{"file", "missing"}}) + .SetTableSchema("invalid schema") + .Finish()); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap(table_schema->Options(), file_system)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr bound, + InternalReadContext::CreateForTable(std::move(context), table_schema, core_options, path, + dynamic_options, table_scheme_map)); + ASSERT_EQ(bound->GetTableSchema(), table_schema); + ASSERT_EQ(bound->GetPath(), path); + ASSERT_EQ(bound->GetCoreOptions().GetFileSystem(), file_system); + ASSERT_EQ(bound->GetFileSystemSchemeToIdentifierMap(), table_scheme_map); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr replaced, + InternalReadContext::CreateWithSchema(bound, bound->GetReadSchema())); + ASSERT_EQ(replaced->GetFileSystemSchemeToIdentifierMap(), table_scheme_map); +} + +TEST(InternalReadContext, TestFallbackToMainIgnoresContextSchema) { + const std::string path = GetDataDir() + "/orc/append_09.db/append_09"; + auto file_system = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr context, + ReadContextBuilder("another-table").SetTableSchema("invalid schema").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fallback, + InternalReadContext::CreateForFallbackBranch(std::move(context), "main", + file_system, path, {}, {})); + ASSERT_EQ(fallback->GetPath(), path); + ASSERT_EQ(fallback->GetTableSchema()->Fields().size(), 4U); + ASSERT_EQ(fallback->GetCoreOptions().GetFileSystem(), file_system); +} + TEST(InternalReadContext, TestReadWithUnspecifiedSchema) { // no read schema is specified, read all fields std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; @@ -57,7 +101,7 @@ TEST(InternalReadContext, TestReadWithUnspecifiedSchema) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); std::vector read_fields = {DataField(0, arrow::field("f0", arrow::utf8())), DataField(1, arrow::field("f1", arrow::int32())), DataField(2, arrow::field("f2", arrow::int32())), @@ -75,7 +119,7 @@ TEST(InternalReadContext, TestReadWithSpecifiedSchema) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); std::vector read_fields = {DataField(3, arrow::field("f3", arrow::float64())), DataField(0, arrow::field("f0", arrow::utf8()))}; auto expected_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); @@ -99,10 +143,12 @@ TEST(InternalReadContext, TestDependencyReadMergesTableCatalogAndReadOptions) { {CatalogOptions::URI, "http://read"}, {CatalogOptions::TOKEN, "read-token"}, {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}}; + std::map builder_options; ReadContextBuilder builder(path); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr read_context, - builder.WithCatalog(catalog, Identifier("db1", "t1")).SetOptions(read_options).Finish()); + builder_options = read_options; + ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, + + builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); std::map options = table_schema->Options(); options[Options::READ_BATCH_SIZE] = "64"; options[CatalogOptions::DATA_TOKEN_ENABLED] = "false"; @@ -110,8 +156,9 @@ TEST(InternalReadContext, TestDependencyReadMergesTableCatalogAndReadOptions) { for (const auto& [key, value] : read_options) { options[key] = value; } - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create(read_context, table_schema, options, builder_options)); auto dependency_context = internal_context->CreateDependencyCatalogContext(); auto expected_options = options; @@ -119,7 +166,6 @@ TEST(InternalReadContext, TestDependencyReadMergesTableCatalogAndReadOptions) { ASSERT_EQ(dependency_context->root_path, catalog->GetRootPath()); ASSERT_EQ(dependency_context->options, expected_options); ASSERT_EQ(dependency_context->options.at("header.X-Paimon-Read-Via"), "table-view"); - ASSERT_EQ(read_context->GetOptions(), read_options); ASSERT_EQ(catalog->GetOptions(), catalog_options); ASSERT_EQ(internal_context->GetCoreOptions().ToMap(), options); } @@ -148,20 +194,22 @@ TEST(InternalReadContext, TestDependencyReadKeepsTheOutermostView) { if (test_case.read_header) { read_options["header.X-Paimon-Read-Via"] = test_case.read_header; } + std::map builder_options; ReadContextBuilder builder(path); - ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, - builder.WithCatalog(catalog, Identifier("db1", "nested-view")) - .SetOptions(read_options) - .Finish()); + builder_options = read_options; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr read_context, + + builder.WithCatalog(catalog, Identifier("db1", "nested-view")).Finish()); auto options = table_schema->Options(); options["header.X-Paimon-Read-Via"] = test_case.read_header ? test_case.read_header : "table-view"; - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create(read_context, table_schema, options, builder_options)); auto dependency_context = internal_context->CreateDependencyCatalogContext(); ASSERT_EQ(dependency_context->options.at("header.X-Paimon-Read-Via"), test_case.expected_header); - ASSERT_EQ(read_context->GetOptions(), read_options); ASSERT_EQ(catalog->GetOptions(), catalog_options); ASSERT_EQ(internal_context->GetCoreOptions().ToMap(), options); } @@ -205,13 +253,15 @@ TEST(InternalReadContext, TestDependencyReadHeaderUsesFinalMetastore) { read_options[CatalogOptions::METASTORE] = test_case.read_metastore; options[CatalogOptions::METASTORE] = test_case.read_metastore; } + std::map builder_options; ReadContextBuilder builder(path); + builder_options = read_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, - builder.WithCatalog(catalog, Identifier("db1", "t1")) - .SetOptions(read_options) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(read_context, table_schema, options)); + + builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create(read_context, table_schema, options, builder_options)); auto dependency_context = internal_context->CreateDependencyCatalogContext(); if (test_case.expected_metastore) { ASSERT_EQ(dependency_context->options.at(CatalogOptions::METASTORE), @@ -224,7 +274,6 @@ TEST(InternalReadContext, TestDependencyReadHeaderUsesFinalMetastore) { } else { ASSERT_EQ(dependency_context->options.count("header.X-Paimon-Read-Via"), 0u); } - ASSERT_EQ(read_context->GetOptions(), read_options); ASSERT_EQ(catalog->GetOptions(), catalog_options); ASSERT_EQ(internal_context->GetCoreOptions().ToMap(), options); } @@ -234,14 +283,18 @@ TEST(InternalReadContext, TestDependencyReadHeaderNeedsAnIdentifier) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; SchemaManager schema_manager(std::make_shared(), path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); + std::map builder_options; ReadContextBuilder builder(path); + builder_options[CatalogOptions::METASTORE] = "REST"; ASSERT_OK_AND_ASSIGN(auto read_context, - builder.AddOption(CatalogOptions::METASTORE, "REST").Finish()); + + builder.Finish()); ASSERT_FALSE(read_context->GetIdentifier()); auto options = table_schema->Options(); options[CatalogOptions::METASTORE] = "REST"; - ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), - table_schema, options)); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create(std::move(read_context), table_schema, options, {})); auto dependency_context = internal_context->CreateDependencyCatalogContext(); ASSERT_EQ(dependency_context->options.count("header.X-Paimon-Read-Via"), 0u); } @@ -258,7 +311,7 @@ TEST(InternalReadContext, TestDependencyReadHeaderReportsTheBranchOfTheStartingT builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).Finish()); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); auto dependency_context = internal_context->CreateDependencyCatalogContext(); ASSERT_EQ(dependency_context->options.at("header.X-Paimon-Read-Via"), "%7B%22database%22%3A%22db1%22%2C%22object%22%3A%22t1%24branch_dev%22%7D"); @@ -279,7 +332,7 @@ TEST(InternalReadContext, TestFileSystemSchemeMapReachesTheDependencyRead) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); ASSERT_EQ(internal_context->GetFileSystemSchemeToIdentifierMap(), scheme_map); ASSERT_EQ(internal_context->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map, scheme_map); @@ -294,7 +347,7 @@ TEST(InternalReadContext, TestReadWithSpecifiedFieldId) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); std::vector read_fields = {DataField(3, arrow::field("f3", arrow::float64())), DataField(0, arrow::field("f0", arrow::utf8()))}; auto expected_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); @@ -313,7 +366,7 @@ TEST(InternalReadContext, TestReadWithSpecifiedFieldIdAndSchema) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); std::vector read_fields = {DataField(3, arrow::field("f3", arrow::float64())), DataField(0, arrow::field("f0", arrow::utf8()))}; auto expected_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); @@ -335,7 +388,7 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { new_options[Options::DATA_EVOLUTION_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(std::move(read_context), table_schema, new_options)); + InternalReadContext::Create(std::move(read_context), table_schema, new_options, {})); std::vector read_fields = { DataField(3, arrow::field("f3", arrow::float64())), DataField(0, arrow::field("f0", arrow::utf8())), SpecialFields::RowId(), @@ -352,7 +405,7 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_NOK_WITH_MSG(InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options()), + table_schema->Options(), {}), "Get field _ROW_ID failed: not exist in table schema"); } { @@ -364,7 +417,7 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_NOK_WITH_MSG(InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options()), + table_schema->Options(), {}), "Get field _INDEX_SCORE failed: not exist in table schema"); } } @@ -383,7 +436,7 @@ TEST(InternalReadContext, TestVectorSearchIndexScoreProjection) { ASSERT_OK_AND_ASSIGN(auto read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); std::shared_ptr expected = DataField::ConvertDataFieldsToArrowSchema( {DataField(0, arrow::field("f0", arrow::utf8())), SpecialFields::IndexScore()}); EXPECT_TRUE(internal_context->GetReadSchema()->Equals(expected)); @@ -394,7 +447,7 @@ TEST(InternalReadContext, TestVectorSearchIndexScoreProjection) { ASSERT_OK_AND_ASSIGN(auto read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); EXPECT_EQ(arrow::Type::FLOAT, internal_context->GetReadSchema()->field(1)->type()->id()); } { @@ -407,7 +460,7 @@ TEST(InternalReadContext, TestVectorSearchIndexScoreProjection) { ASSERT_OK_AND_ASSIGN(auto read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); EXPECT_EQ(std::vector({"_INDEX_SCORE", "f0"}), internal_context->GetReadSchema()->field_names()); } @@ -426,12 +479,12 @@ TEST(InternalReadContext, TestPrimaryKeyVectorSearchRequiresDeletionVectors) { ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr internal, - InternalReadContext::Create(context, table_schema, table_schema->Options())); + InternalReadContext::Create(context, table_schema, table_schema->Options(), {})); EXPECT_TRUE(internal->HasVectorOrFullTextSearch()); std::map options = table_schema->Options(); options[Options::DELETION_VECTORS_ENABLED] = "false"; - ASSERT_NOK_WITH_MSG(InternalReadContext::Create(context, table_schema, options), + ASSERT_NOK_WITH_MSG(InternalReadContext::Create(context, table_schema, options, {}), "primary-key tables requires deletion vectors"); } @@ -444,8 +497,9 @@ TEST(InternalReadContext, TestFileFullTextSearchScoreIsNotSupported) { ReadContextBuilder builder(path); builder.SetReadFieldNames({"f0", "_INDEX_SCORE"}).SetFullTextSearch(search); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); - ASSERT_NOK_WITH_MSG(InternalReadContext::Create(context, table_schema, table_schema->Options()), - "does not support score output"); + ASSERT_NOK_WITH_MSG( + InternalReadContext::Create(context, table_schema, table_schema->Options(), {}), + "does not support score output"); } TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { @@ -477,7 +531,7 @@ TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { if (test_case.expected_error.empty()) { ASSERT_OK_AND_ASSIGN( std::shared_ptr internal, - InternalReadContext::Create(context, table_schema, table_schema->Options())); + InternalReadContext::Create(context, table_schema, table_schema->Options(), {})); ASSERT_TRUE(internal->GetIndexScoreBindings().empty()); ASSERT_FALSE(internal->NeedsScores( {InternalReadContext::IndexScoreSource::Type::FULL_TEXT_MATCH, match->field_name})); @@ -486,7 +540,7 @@ TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { "CreateWithSchema does not support"); } else { ASSERT_NOK_WITH_MSG( - InternalReadContext::Create(context, table_schema, table_schema->Options()), + InternalReadContext::Create(context, table_schema, table_schema->Options(), {}), test_case.expected_error); } } @@ -521,7 +575,7 @@ TEST(InternalReadContext, TestPerFieldTopKScoreBindings) { ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr internal, - InternalReadContext::Create(context, table_schema, table_schema->Options())); + InternalReadContext::Create(context, table_schema, table_schema->Options(), {})); ASSERT_EQ(internal->GetReadSchema()->field_names(), projection); ASSERT_TRUE(internal->NeedsScores(source)); ASSERT_FALSE(internal->NeedsScores( @@ -557,7 +611,7 @@ class InternalReadContextScoreTest : public testing::Test { Result> Create(ReadContextBuilder* builder) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr context, builder->Finish()); - return InternalReadContext::Create(context, table_schema_, table_schema_->Options()); + return InternalReadContext::Create(context, table_schema_, table_schema_->Options(), {}); } std::shared_ptr table_schema_; @@ -783,7 +837,7 @@ TEST(InternalReadContext, TestReadWithValueKindField) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + table_schema->Options(), {})); std::vector read_fields = {DataField(3, arrow::field("f3", arrow::float64())), SpecialFields::ValueKind(), DataField(0, arrow::field("f0", arrow::utf8()))}; @@ -809,7 +863,7 @@ TEST(InternalReadContext, TestReadWithFieldIdsAndSpecialFields) { new_options[Options::DATA_EVOLUTION_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(std::move(read_context), table_schema, new_options)); + InternalReadContext::Create(std::move(read_context), table_schema, new_options, {})); std::vector read_fields = { DataField(3, arrow::field("f3", arrow::float64())), DataField(0, arrow::field("f0", arrow::utf8())), SpecialFields::RowId(), @@ -838,7 +892,7 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaAndSpecialFields) { ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); std::shared_ptr read_context = std::move(unique_read_context); ASSERT_NOK_WITH_MSG( - InternalReadContext::Create(read_context, table_schema, table_schema->Options()), + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {}), "not exist in table schema"); } @@ -857,7 +911,7 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaAndSpecialFields) { std::shared_ptr read_context = std::move(unique_read_context); ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, enabled_options)); + InternalReadContext::Create(read_context, table_schema, enabled_options, {})); auto expected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields); ASSERT_TRUE(internal_context->GetReadSchema()->Equals(expected_schema)); } @@ -881,7 +935,7 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaWithoutFieldIds) { ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {})); std::vector expected_fields = { DataField(3, arrow::field("f3", arrow::float64())), @@ -913,7 +967,7 @@ TEST(InternalReadContext, TestProjectedSchemaMetadataWhitelist) { ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {})); auto aligned_field = internal_context->GetReadSchema()->GetFieldByName("f0"); ASSERT_TRUE(aligned_field); @@ -954,7 +1008,7 @@ TEST(InternalReadContext, TestMapSharedShreddingAccessRequiresSharedShreddingLay std::shared_ptr read_context = std::move(unique_read_context); ASSERT_NOK_WITH_MSG( - InternalReadContext::Create(read_context, table_schema, table_schema->Options()), + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {}), "Selected-key MAP pushdown only supports top-level shared-shredding MAP field: tags"); } diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp index c0e94e236..94ea4f9c2 100644 --- a/src/paimon/core/operation/key_value_file_store_write_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp @@ -142,7 +142,7 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, context_builder.Finish()); - return FileStoreWrite::Create(std::move(write_context)); + return FileStoreWrite::Create(std::move(write_context), {}); } Status WriteSingleStringRow(FileStoreWrite* file_store_write, int32_t bucket, @@ -196,11 +196,13 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { const std::string& table_path, const std::shared_ptr& schema, const std::map& options, const std::string& json, int64_t commit_identifier) const { + std::map builder_options; WriteContextBuilder builder(table_path, "test"); - builder.SetOptions(options); + builder_options = options; + EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); EXPECT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), builder_options)); EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( /*wait_compaction=*/false, commit_identifier)); @@ -219,11 +221,13 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { void Commit(const std::string& table_path, const std::map& options, const std::vector>& commit_msgs) const { + std::map builder_options; CommitContextBuilder builder(table_path, "test"); - builder.SetOptions(options); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); } @@ -340,7 +344,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { WriteContextBuilder builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); ASSERT_NOK_WITH_MSG(file_store_write->Write(nullptr), "batch is null pointer"); } { @@ -359,7 +363,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); auto array = std::make_shared(); arrow::StringBuilder builder; for (size_t j = 0; j < 100; j++) { @@ -391,7 +395,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); auto array = std::make_shared(); arrow::StringBuilder builder; for (size_t j = 0; j < 100; j++) { @@ -442,14 +446,15 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) { ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); + std::map builder_options; WriteContextBuilder builder(table_path, "test"); - builder.SetOptions(options) - .WithStreamingMode(true) + builder_options = options; + builder.WithStreamingMode(true) .WithRealtimeContext(realtime_context) .WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), builder_options)); std::unique_ptr batch = MakeBatch(realtime_schema, R"([ @@ -505,15 +510,17 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) { SCOPED_TRACE(expected_error); ASSERT_OK_AND_ASSIGN(std::shared_ptr invalid_context, RealtimeContext::Create()); + std::map invalid_builder_options; WriteContextBuilder invalid_builder(table_path, "invalid"); - invalid_builder.SetOptions(options) - .WithStreamingMode(true) + invalid_builder_options = options; + invalid_builder.WithStreamingMode(true) .WithRealtimeContext(invalid_context) .WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_write_context, invalid_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_writer, - FileStoreWrite::Create(std::move(invalid_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr invalid_writer, + FileStoreWrite::Create(std::move(invalid_write_context), invalid_builder_options)); ASSERT_OK(invalid_writer->Write(MakeBatch(realtime_schema, R"([[30, 1, "existing"]])"))); ASSERT_OK_AND_ASSIGN(std::vector prepared, invalid_writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); @@ -544,14 +551,13 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) { std::shared_ptr pool = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); + std::map builder_options; WriteContextBuilder builder(table_path, "test"); - builder.SetOptions(options) - .WithStreamingMode(true) - .WithRealtimeContext(realtime_context) - .WithMemoryPool(pool); + builder_options = options; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context).WithMemoryPool(pool); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), builder_options)); const int64_t allocations_before_write = pool->allocation_count; ASSERT_OK(writer->Write(MakeBatch(realtime_schema, R"([[0, 1, "one"]])"))); @@ -567,15 +573,17 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) { std::shared_ptr rejecting_pool = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr rejecting_context, RealtimeContext::Create()); + std::map rejecting_builder_options; WriteContextBuilder rejecting_builder(table_path, "rejecting"); - rejecting_builder.SetOptions(options) - .WithStreamingMode(true) + rejecting_builder_options = options; + rejecting_builder.WithStreamingMode(true) .WithRealtimeContext(rejecting_context) .WithMemoryPool(rejecting_pool); ASSERT_OK_AND_ASSIGN(std::unique_ptr rejecting_write_context, rejecting_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr rejecting_writer, - FileStoreWrite::Create(std::move(rejecting_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr rejecting_writer, + FileStoreWrite::Create(std::move(rejecting_write_context), rejecting_builder_options)); ASSERT_OK(rejecting_writer->Write(MakeBatch(realtime_schema, "[]"))); const int64_t rejecting_allocations_before_write = rejecting_pool->allocation_count; rejecting_pool->reject_allocations = true; @@ -605,13 +613,15 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) { ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_context, RealtimeContext::Create()); + std::map initial_builder_options; WriteContextBuilder initial_builder(table_path, "initial"); - initial_builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext( - initial_context); + initial_builder_options = options; + initial_builder.WithStreamingMode(true).WithRealtimeContext(initial_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr initial_write_context, initial_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr initial_writer, - FileStoreWrite::Create(std::move(initial_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr initial_writer, + FileStoreWrite::Create(std::move(initial_write_context), initial_builder_options)); ASSERT_OK(initial_writer->Write(MakeBatch(realtime_schema, R"([[0, 0, "initial"]])"))); ASSERT_OK_AND_ASSIGN(std::vector initial_progress, initial_writer->PrepareCommitWithProgress(0)); @@ -623,22 +633,27 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) { initial_message->GetNewFilesIncrement().NewFiles()[0]->AssignSequenceNumber(max - 2, max - 2); initial_progress[0].offset_range = OffsetRange(0, max - 1); + std::map commit_builder_options; CommitContextBuilder commit_builder(table_path, "initial"); - commit_builder.SetOptions(options); + commit_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr committer, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr committer, + FileStoreCommit::Create(std::move(commit_context), commit_builder_options)); ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, committer->CommitWithProgress(initial_progress, 0, std::nullopt)); ASSERT_OK(initial_writer->Close()); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); + std::map builder_options; WriteContextBuilder builder(table_path, "boundary"); - builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context); + builder_options = options; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), builder_options)); ASSERT_OK( writer->Write(MakeBatch(realtime_schema, fmt::format(R"([[{}, 1, "legal"]])", max - 1)))); ASSERT_OK_AND_ASSIGN(auto store_rows, ReadRealtimePrimaryKeyStoreRows(realtime_context)); @@ -751,7 +766,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestSpillSimple) { context_builder.WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); auto key_value_file_store_write = dynamic_cast(file_store_write.get()); auto get_writer = [&](int32_t bucket) -> std::shared_ptr { auto partition_iter = key_value_file_store_write->writers_.find(BinaryRow::EmptyRow()); @@ -806,11 +822,14 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriterRestoreKeepsValueStats) { /*ignore_if_exists=*/false)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + std::map first_context_builder_options; WriteContextBuilder first_context_builder(table_path, "first-writer"); - first_context_builder.SetOptions(options); + first_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr first_context, first_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto first_write, FileStoreWrite::Create(std::move(first_context))); + ASSERT_OK_AND_ASSIGN(auto first_write, FileStoreWrite::Create(std::move(first_context), + first_context_builder_options)); ASSERT_OK(WriteSingleStringRow(first_write.get(), /*bucket=*/0, "alice")); ASSERT_OK_AND_ASSIGN(auto commit_messages, first_write->PrepareCommit(/*wait_compaction=*/false, 1)); @@ -823,11 +842,15 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriterRestoreKeepsValueStats) { SimpleStats::EmptyStats()); Commit(table_path, options, commit_messages); + std::map restored_context_builder_options; WriteContextBuilder restored_context_builder(table_path, "restored-writer"); - restored_context_builder.SetOptions(options); + restored_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr restored_context, restored_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto restored_write, FileStoreWrite::Create(std::move(restored_context))); + ASSERT_OK_AND_ASSIGN( + auto restored_write, + FileStoreWrite::Create(std::move(restored_context), restored_context_builder_options)); auto key_value_write = dynamic_cast(restored_write.get()); ASSERT_NE(nullptr, key_value_write); ASSERT_OK_AND_ASSIGN(std::shared_ptr restore_files, @@ -860,7 +883,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestSpillDiskQuotaExhaustedFallsBackToFlushDa context_builder.WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); // Disk quota is 1 byte, so spill will exhaust quota immediately and fall back to // FlushWriteBuffer (writing data files directly instead of spill temp files). @@ -908,7 +932,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestMultiRoundSpillWithSameKeyDeduplication) context_builder.WithTempDirectory(dir->Str()).WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); // Round 1: alice, bob, alice (duplicate key) → after dedup: alice + bob = 2 rows ASSERT_OK(WriteSingleStringRow(file_store_write.get(), /*bucket=*/0, "alice")); diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp index 2096c2aa9..7fc553f99 100644 --- a/src/paimon/core/operation/merge_file_split_read_test.cpp +++ b/src/paimon/core/operation/merge_file_split_read_test.cpp @@ -91,23 +91,28 @@ class MergeFileSplitReadTest : public ::testing::Test, } std::shared_ptr CreateInternalReadContext( - const std::shared_ptr& read_context, int32_t schema_id = 0) { + const std::shared_ptr& read_context, int32_t schema_id, + const std::map& dynamic_options) { SchemaManager schema_manager(fs_, read_context->GetPath()); EXPECT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(schema_id)); + auto options = ReadOptions(); + for (const auto& [key, value] : dynamic_options) { + options[key] = value; + } EXPECT_OK_AND_ASSIGN(auto context, InternalReadContext::Create(read_context, table_schema, - read_context->GetOptions())); + options, dynamic_options)); return context; } + std::map ReadOptions() const { + auto [use_min_heap, enable_io_prefetch, enable_multi_thread_row_to_batch] = GetParam(); + return {{Options::SORT_ENGINE, use_min_heap ? "min-heap" : "loser-tree"}, + {"test.enable-adaptive-prefetch-strategy", "false"}}; + } + void AddOptions(ReadContextBuilder* context_builder) const { auto [use_min_heap, enable_io_prefetch, enable_multi_thread_row_to_batch] = GetParam(); - if (use_min_heap) { - context_builder->AddOption(Options::SORT_ENGINE, "min-heap"); - } else { - context_builder->AddOption(Options::SORT_ENGINE, "loser-tree"); - } if (enable_io_prefetch) { - context_builder->AddOption("test.enable-adaptive-prefetch-strategy", "false"); context_builder->EnablePrefetch(true); context_builder->SetPrefetchBatchCount(/*batch_count=*/3); } else { @@ -609,6 +614,7 @@ TEST_F(MergeFileSplitReadTest, TestGenerateKeyPredicates2) { TEST_P(MergeFileSplitReadTest, TestSimple) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), @@ -620,13 +626,14 @@ TEST_P(MergeFileSplitReadTest, TestSimple) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN( std::unique_ptr split_read, @@ -675,6 +682,7 @@ TEST_P(MergeFileSplitReadTest, TestSimple) { TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), DataField(1, arrow::field("k1", arrow::int32())), @@ -685,11 +693,13 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"}); - context_builder.SetOptions( - {{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - std::shared_ptr internal_context = CreateInternalReadContext(read_context); + std::shared_ptr internal_context = + CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr split_read, CreateMergeFileSplitRead(internal_context)); @@ -742,6 +752,7 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) TEST_P(MergeFileSplitReadTest, TestLookUp) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), @@ -753,14 +764,15 @@ TEST_P(MergeFileSplitReadTest, TestLookUp) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}, - {Options::FORCE_LOOKUP, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}, + {Options::FORCE_LOOKUP, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -790,15 +802,17 @@ TEST_P(MergeFileSplitReadTest, TestLookUp) { TEST_P(MergeFileSplitReadTest, TestReadWithLimits) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}, - {Options::READ_BATCH_SIZE, "1"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}, + {Options::READ_BATCH_SIZE, "1"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); @@ -816,6 +830,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithLimits) { TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), @@ -827,11 +842,12 @@ TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::MERGE_ENGINE, "deduplicate"}}); + context_builder_options = {{Options::MERGE_ENGINE, "deduplicate"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit2())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -856,6 +872,7 @@ TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), @@ -868,9 +885,10 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "s0", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + context_builder.EnableLateMaterializing(false); AddOptions(&context_builder); @@ -890,7 +908,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { context_builder.SetPredicate(predicate_result); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -919,6 +937,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), @@ -931,9 +950,10 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "s0", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); context_builder.EnableLateMaterializing(true); // key predicate, always pushed down into the data files @@ -948,7 +968,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { context_builder.EnablePredicateFilter(true).EnableLateMaterializing(true); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -976,6 +996,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(0, arrow::field("k1", arrow::int32())), @@ -991,14 +1012,16 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "k0", "p0", "p1", "s1", "s0", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context, /*schema_id=*/7); + auto internal_context = + CreateInternalReadContext(read_context, /*schema_id=*/7, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1028,6 +1051,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(8, arrow::field("v2", arrow::int32())), @@ -1040,14 +1064,16 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"v2", "p1", "k0", "p0", "s0", "v0"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context, /*schema_id=*/7); + auto internal_context = + CreateInternalReadContext(read_context, /*schema_id=*/7, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1077,6 +1103,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), @@ -1088,14 +1115,15 @@ TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "aggregation"}, - {"fields.v1.aggregate-function", "bool_and"}, - {"fields.v0.aggregate-function", "sum"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "aggregation"}, + {"fields.v1.aggregate-function", "bool_and"}, + {"fields.v0.aggregate-function", "sum"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1125,6 +1153,7 @@ TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), @@ -1135,14 +1164,15 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "partial-update"}, - {"fields.v1.sequence-group", "v0"}, - {"fields.v0.aggregate-function", "first_value"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "partial-update"}, + {"fields.v1.sequence-group", "v0"}, + {"fields.v0.aggregate-function", "first_value"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1172,6 +1202,7 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), @@ -1183,11 +1214,12 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions( - {{Options::MERGE_ENGINE, "partial-update"}, {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::MERGE_ENGINE, "partial-update"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit2())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1212,6 +1244,7 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDelete) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), @@ -1223,11 +1256,12 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDel ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::MERGE_ENGINE, "partial-update"}, - {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}); + context_builder_options = {{Options::MERGE_ENGINE, "partial-update"}, + {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit2())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1252,6 +1286,7 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDel TEST_P(MergeFileSplitReadTest, TestEmptyPlan) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), @@ -1263,11 +1298,12 @@ TEST_P(MergeFileSplitReadTest, TestEmptyPlan) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::MERGE_ENGINE, "partial-update"}, - {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}); + context_builder_options = {{Options::MERGE_ENGINE, "partial-update"}, + {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); std::vector> empty_data_split; ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, empty_data_split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_result, @@ -1285,18 +1321,20 @@ TEST_P(MergeFileSplitReadTest, TestIOException) { DataField(6, arrow::field("v0", arrow::float64())), DataField(7, arrow::field("v1", arrow::boolean()))}; + std::map context_builder_options; ReadContextBuilder context_builder(path); auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); auto fields_with_row_kind = read_schema->fields(); fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -1336,6 +1374,7 @@ TEST_P(MergeFileSplitReadTest, TestIOException) { TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { std::string path = paimon::test::GetDataDir() + "/orc/pk_09.db/pk_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); std::vector raw_read_fields = {DataField(3, arrow::field("f3", arrow::float64())), DataField(2, arrow::field("f2", arrow::int32())), @@ -1344,9 +1383,10 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"f3", "f2", "f0", "f1"}); - context_builder.SetOptions({{Options::FILE_FORMAT, "orc"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {"orc.read.enable-metrics", "true"}}); + context_builder_options = {{Options::FILE_FORMAT, "orc"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {"orc.read.enable-metrics", "true"}}; + AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto meta1 = std::make_shared( @@ -1391,7 +1431,7 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { paimon::test::GetDataDir() + "/orc/pk_09.db/pk_09/f1=10/bucket-1/", {meta1, meta2}); ASSERT_OK_AND_ASSIGN(auto data_split, builder.WithSnapshot(5).IsStreaming(false).RawConvertible(false).Build()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = CreateInternalReadContext(read_context, 0, context_builder_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, {data_split})); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, ReadResultCollector::CollectResult(batch_reader.get())); diff --git a/src/paimon/core/operation/orphan_files_cleaner.cpp b/src/paimon/core/operation/orphan_files_cleaner.cpp index 397ba68ea..8fae06e2c 100644 --- a/src/paimon/core/operation/orphan_files_cleaner.cpp +++ b/src/paimon/core/operation/orphan_files_cleaner.cpp @@ -33,6 +33,7 @@ #include "paimon/core/operation/orphan_files_cleaner_impl.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" @@ -150,15 +151,7 @@ Result> CleanContextBuilder::Finish() { Result> OrphanFilesCleaner::Create( std::unique_ptr&& ctx) { - if (ctx == nullptr) { - return Status::Invalid("clean context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "clean")); if (ctx->GetOlderThanMs() < 0) { return Status::Invalid("older than time needs to be greater than or equal to 0."); } diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp index 77f92109c..b2b8f6f92 100644 --- a/src/paimon/core/operation/raw_file_split_read_test.cpp +++ b/src/paimon/core/operation/raw_file_split_read_test.cpp @@ -145,14 +145,18 @@ class RawFileSplitReadTest : public ::testing::Test { Result> ReadSearchWithDeletionVector( const std::shared_ptr& read_context, const std::string& index_type, - const RoaringBitmap32& deleted_rows, std::shared_ptr table_schema) const { + const RoaringBitmap32& deleted_rows, std::shared_ptr table_schema, + const std::map& dynamic_options) const { SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); if (!table_schema) { PAIMON_ASSIGN_OR_RAISE(table_schema, schema_manager.ReadSchema(0)); } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr effective_schema, + table_schema->MergeOptions(dynamic_options)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, effective_schema->Options(), + dynamic_options)); auto split = std::dynamic_pointer_cast(PrepareDataSplits().front()); if (!split) { return Status::Invalid("Raw search test split must be a DataSplitImpl"); @@ -197,10 +201,12 @@ class RawFileSplitReadTest : public ::testing::Test { std::string path = paimon::test::GetDataDir() + "/orc/multi_partition_append_table.db/" "multi_partition_append_table"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames(read_schema->field_names()); if (dv_factory) { - context_builder.AddOption(Options::FILE_INDEX_READ_ENABLED, "true"); + context_builder_options[Options::FILE_INDEX_READ_ENABLED] = "true"; + context_builder.SetPredicate(PredicateBuilder::IsNotNull(0, "f0", FieldType::STRING)); } ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); @@ -209,7 +215,8 @@ class RawFileSplitReadTest : public ::testing::Test { ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), + context_builder_options)); auto data_splits = PrepareDataSplits(); const auto& core_options = internal_context->GetCoreOptions(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); @@ -468,7 +475,7 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {})); auto data_splits = PrepareDataSplits(); auto data_split = std::dynamic_pointer_cast(data_splits.front()); @@ -514,9 +521,9 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { without_score_builder.SetReadFieldNames({"f0"}).SetVectorSearch(vector_search); ASSERT_OK_AND_ASSIGN(std::shared_ptr without_score_context, without_score_builder.Finish()); - ASSERT_OK_AND_ASSIGN( - auto without_score_internal_context, - InternalReadContext::Create(without_score_context, table_schema, table_schema->Options())); + ASSERT_OK_AND_ASSIGN(auto without_score_internal_context, + InternalReadContext::Create(without_score_context, table_schema, + table_schema->Options(), {})); auto without_score_split_read = std::make_unique( path_factory, std::move(without_score_internal_context), pool_, executor); ASSERT_OK_AND_ASSIGN(std::unique_ptr without_score_reader, @@ -545,7 +552,7 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchFiltersDeletionVectorBeforeTopK) { ASSERT_OK_AND_ASSIGN( std::shared_ptr result, ReadSearchWithDeletionVector(read_context, MockSearchFileIndex::VECTOR_IDENTIFIER, - RoaringBitmap32::From({0}), /*table_schema=*/nullptr)); + RoaringBitmap32::From({0}), /*table_schema=*/nullptr, {})); std::shared_ptr expected; ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), @@ -570,7 +577,7 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchFiltersDeletionVectorBeforeLimit) ASSERT_OK_AND_ASSIGN( std::shared_ptr result, ReadSearchWithDeletionVector(read_context, MockSearchFileIndex::FULL_TEXT_IDENTIFIER, - RoaringBitmap32::From({0}), /*table_schema=*/nullptr)); + RoaringBitmap32::From({0}), /*table_schema=*/nullptr, {})); std::shared_ptr expected; ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), @@ -609,8 +616,10 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { auto search = std::make_shared("renamed_f3", 2, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); + std::map builder_options; ReadContextBuilder builder(path); - builder.SetOptions(options).SetReadFieldNames({"f0", "_INDEX_SCORE"}).SetVectorSearch(search); + builder_options = options; + builder.SetReadFieldNames({"f0", "_INDEX_SCORE"}).SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), @@ -621,8 +630,9 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { {RoaringBitmap32::From({0}), R"([[0,"Tony",0.75]])"}}; for (const auto& [deleted, expected_json] : cases) { SCOPED_TRACE(deleted.ToString()); - ASSERT_OK_AND_ASSIGN(auto actual, ReadSearchWithDeletionVector(context, identifier, deleted, - current_schema)); + ASSERT_OK_AND_ASSIGN( + auto actual, ReadSearchWithDeletionVector(context, identifier, deleted, current_schema, + builder_options)); auto expected_result = arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_json); ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString(); @@ -638,13 +648,14 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { // The old file's f3 is field ID 3; the current table's f3 is a different field with ID 4. auto missing_search = std::make_shared(*search); missing_search->field_name = "f3"; + std::map missing_field_builder_options; ReadContextBuilder missing_field_builder(path); - missing_field_builder.SetOptions(options).SetReadFieldNames({"f0"}).SetVectorSearch( - missing_search); + missing_field_builder_options = options; + missing_field_builder.SetReadFieldNames({"f0"}).SetVectorSearch(missing_search); ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_context, missing_field_builder.Finish()); ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, identifier, RoaringBitmap32(), - current_schema), + current_schema, missing_field_builder_options), "has no matching field ID"); } @@ -676,8 +687,10 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) original_schema->PartitionKeys(), {}, options)); auto search = std::make_shared("renamed_f3", R"({"match":{"query":"match"}})", 2); + std::map builder_options; ReadContextBuilder builder(path); - builder.SetOptions(options).SetReadFieldNames({"f0"}).SetFullTextSearch(search); + builder_options = options; + builder.SetReadFieldNames({"f0"}).SetFullTextSearch(search); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); auto expected_type = arrow::struct_( @@ -687,8 +700,9 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) {RoaringBitmap32::From({0}), R"([[0,"Tony"]])"}}; for (const auto& [deleted, expected_json] : cases) { SCOPED_TRACE(deleted.ToString()); - ASSERT_OK_AND_ASSIGN(auto actual, ReadSearchWithDeletionVector(context, identifier, deleted, - current_schema)); + ASSERT_OK_AND_ASSIGN( + auto actual, ReadSearchWithDeletionVector(context, identifier, deleted, current_schema, + builder_options)); auto expected_result = arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_json); ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString(); @@ -704,13 +718,14 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) // The old file's f3 is field ID 3; the current table's f3 is a different field with ID 4. auto missing_search = std::make_shared(*search); missing_search->field_name = "f3"; + std::map missing_field_builder_options; ReadContextBuilder missing_field_builder(path); - missing_field_builder.SetOptions(options).SetReadFieldNames({"f0"}).SetFullTextSearch( - missing_search); + missing_field_builder_options = options; + missing_field_builder.SetReadFieldNames({"f0"}).SetFullTextSearch(missing_search); ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_context, missing_field_builder.Finish()); ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, identifier, RoaringBitmap32(), - current_schema), + current_schema, missing_field_builder_options), "has no matching field ID"); } @@ -725,7 +740,7 @@ TEST_F(RawFileSplitReadTest, TestEmptyPlan) { ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {})); const auto& core_options = internal_context->GetCoreOptions(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); ASSERT_OK_AND_ASSIGN(std::vector external_paths, @@ -775,7 +790,7 @@ TEST_F(RawFileSplitReadTest, TestMatch) { ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(/*thread_count=*/2)); auto split_read = std::make_unique( diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index bc65b4661..186dc0943 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -32,15 +32,14 @@ #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/vector_search.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { class Predicate; ReadContext::ReadContext( - const std::string& path, const std::string& branch, - const std::vector& read_field_names, const std::vector& read_field_ids, - const std::shared_ptr& predicate, const std::shared_ptr& vector_search, + const std::string& path, const std::vector& read_field_names, + const std::vector& read_field_ids, const std::shared_ptr& predicate, + const std::shared_ptr& vector_search, const std::shared_ptr& full_text_search, bool enable_predicate_filter, bool enable_prefetch, bool enable_late_materializing, uint32_t prefetch_batch_count, uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch, @@ -48,13 +47,10 @@ ReadContext::ReadContext( const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, const std::map& fs_scheme_to_identifier_map, - const std::shared_ptr& realtime_context, - const std::map& options, bool read_ahead_cache_enabled, - const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, + const std::shared_ptr& realtime_context, bool read_ahead_cache_enabled, + const CacheConfig& cache_config, const std::shared_ptr& cache, WarmupLevel warmup_level, const std::shared_ptr& catalog, const std::optional& identifier) : path_(path), - branch_(branch), read_field_names_(read_field_names), read_field_ids_(read_field_ids), predicate_(predicate), @@ -73,11 +69,9 @@ ReadContext::ReadContext( specific_file_system_(specific_file_system), fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), realtime_context_(realtime_context), - options_(options), read_ahead_cache_enabled_(read_ahead_cache_enabled), cache_config_(cache_config), cache_(cache), - format_table_(format_table), warmup_level_(warmup_level), catalog_(catalog), identifier_(identifier) {} @@ -104,12 +98,10 @@ class ReadContextBuilder::Impl { public: friend class ReadContextBuilder; void Reset() { - branch_.reset(); read_field_names_.clear(); read_field_ids_.clear(); read_schema_.reset(); fs_scheme_to_identifier_map_.clear(); - options_.clear(); predicate_.reset(); vector_search_.reset(); full_text_search_.reset(); @@ -135,18 +127,10 @@ class ReadContextBuilder::Impl { private: std::string path_; - /// Kept across `Reset()`, as `path_` is: both name the table this builder builds for, rather - /// than a setting of one read of it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; - /// Unset until `WithBranch()` names one, so that a branch named only by the `branch` - /// option is not overruled by the default. - std::optional branch_; std::vector read_field_names_; std::vector read_field_ids_; std::unique_ptr read_schema_; std::map fs_scheme_to_identifier_map_; - std::map options_; std::shared_ptr predicate_; std::shared_ptr vector_search_; std::shared_ptr full_text_search_; @@ -175,31 +159,11 @@ ReadContextBuilder::ReadContextBuilder(const std::string& path) impl_->path_ = path; } -ReadContextBuilder::ReadContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->path_ = table->Location(); - } -} - ReadContextBuilder::~ReadContextBuilder() = default; ReadContextBuilder::ReadContextBuilder(ReadContextBuilder&&) noexcept = default; ReadContextBuilder& ReadContextBuilder::operator=(ReadContextBuilder&&) noexcept = default; -ReadContextBuilder& ReadContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - -ReadContextBuilder& ReadContextBuilder::SetOptions(const std::map& opts) { - impl_->options_ = opts; - return *this; -} - ReadContextBuilder& ReadContextBuilder::SetReadFieldNames( const std::vector& read_field_names) { impl_->read_field_names_ = read_field_names; @@ -293,11 +257,6 @@ ReadContextBuilder& ReadContextBuilder::SetTableSchema(const std::string& table_ return *this; } -ReadContextBuilder& ReadContextBuilder::WithBranch(const std::string& branch) { - impl_->branch_ = branch; - return *this; -} - ReadContextBuilder& ReadContextBuilder::WithFileSystemSchemeToIdentifierMap( const std::map& fs_scheme_to_identifier_map) { impl_->fs_scheme_to_identifier_map_ = fs_scheme_to_identifier_map; @@ -359,34 +318,6 @@ Result> ReadContextBuilder::Finish() { return Status::Invalid("FullTextSearch field name and query must not be empty"); } } - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot read with null format table"); - } - if (impl_->format_table_ != nullptr) { - // The table already answers each of these, and from a source this cannot see behind, so a - // second answer is refused rather than silently dropped. - if (impl_->catalog_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithCatalog() " - "cannot be used with one"); - } - if (impl_->table_schema_) { - return Status::Invalid( - "a format table carries its own schema, so SetTableSchema() cannot be used with " - "one"); - } - if (impl_->specific_file_system_ != nullptr || - !impl_->fs_scheme_to_identifier_map_.empty()) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "and WithFileSystemSchemeToIdentifierMap() cannot be used with one"); - } - if (impl_->branch_ && - !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(impl_->branch_.value()))) { - return Status::Invalid( - "a format table has no branches, so WithBranch() cannot be used with one"); - } - } PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_)); if (impl_->path_.empty()) { return Status::Invalid("cannot read with empty table path"); @@ -394,15 +325,6 @@ Result> ReadContextBuilder::Finish() { if (impl_->catalog_ == nullptr && impl_->identifier_) { return Status::Invalid("cannot read through a null catalog"); } - if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr) { - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the read uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, - impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); - } - PAIMON_ASSIGN_OR_RAISE( - std::string branch, - BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, impl_->branch_, "read")); if (impl_->enable_prefetch_ && impl_->prefetch_max_parallel_num_ == 0) { return Status::Invalid("prefetch max parallel num should be greater than 0"); } @@ -429,15 +351,14 @@ Result> ReadContextBuilder::Finish() { } auto ctx = std::make_unique( - impl_->path_, branch, impl_->read_field_names_, impl_->read_field_ids_, impl_->predicate_, + impl_->path_, impl_->read_field_names_, impl_->read_field_ids_, impl_->predicate_, impl_->vector_search_, impl_->full_text_search_, impl_->enable_predicate_filter_, impl_->enable_prefetch_, impl_->enable_late_materializing_, impl_->prefetch_batch_count_, impl_->prefetch_max_parallel_num_, impl_->enable_multi_thread_row_to_batch_, impl_->row_to_batch_thread_number_, impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, - impl_->realtime_context_, impl_->options_, impl_->read_ahead_cache_enabled_, - impl_->cache_config_, impl_->cache_, impl_->format_table_, impl_->warmup_level_, - impl_->catalog_, impl_->identifier_); + impl_->realtime_context_, impl_->read_ahead_cache_enabled_, impl_->cache_config_, + impl_->cache_, impl_->warmup_level_, impl_->catalog_, impl_->identifier_); if (impl_->read_schema_ && impl_->read_schema_->release) { ctx->SetReadSchema(std::move(impl_->read_schema_)); } diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index 4fb5e00b4..d268ba9ca 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -40,28 +40,24 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { -TEST(ReadContextTest, TestWithCatalogResolvesTableFileSystem) { - // WithCatalog is a shorthand for WithFileSystem(catalog->GetTableFileSystem(identifier)); it - // resolves the per-table file system when Finish() builds the context, which also keeps the - // catalog and identifier for the operators to resolve the table through. +TEST(ReadContextTest, TestWithCatalogDefersTableFileSystem) { auto table_fs = std::make_shared(); auto catalog = std::make_shared(); catalog->SetTableFileSystem(table_fs); ReadContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); ASSERT_EQ(ctx->GetCatalog(), catalog); ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); - ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); // Finish() resets the builder, so the next context reads without the catalog. ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); ASSERT_FALSE(next_ctx->GetSpecificFileSystem()); ASSERT_FALSE(next_ctx->GetCatalog()); ASSERT_FALSE(next_ctx->GetIdentifier()); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); } TEST(ReadContextTest, TestWithFileSystemOverridesCatalog) { @@ -85,42 +81,15 @@ TEST(ReadContextTest, TestWithCatalogNullRejected) { "cannot read through a null catalog"); } -TEST(ReadContextTest, TestWithCatalogKeepsReadOptionsSeparate) { - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(std::make_shared()); - catalog->SetOptions({{CatalogOptions::METASTORE, "rest"}, - {CatalogOptions::URI, "http://catalog"}, - {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}}); - - ReadContextBuilder builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - // Finishing a read neither imports catalog defaults nor generates a dependency header. - ASSERT_TRUE(ctx->GetOptions().empty()); - - catalog->SetOptions({{CatalogOptions::METASTORE, "rest"}, - {CatalogOptions::URI, "http://catalog"}, - {"header.X-Paimon-Read-Via", "catalog-view"}}); - ASSERT_OK_AND_ASSIGN(auto nested_ctx, - builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_TRUE(nested_ctx->GetOptions().empty()); - const std::map read_options = { - {Options::BLOB_VIEW_RESOLVE_ENABLED, "true"}, - {"header.X-Paimon-Read-Via", "explicit-view"}}; - ASSERT_OK_AND_ASSIGN( - auto explicit_ctx, - builder.WithCatalog(catalog, Identifier("db1", "t1")).SetOptions(read_options).Finish()); - ASSERT_EQ(explicit_ctx->GetOptions(), read_options); -} - TEST(ReadContextTest, TestDefaultValue) { ReadContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); ASSERT_EQ(ctx->GetPath(), "table_root_path"); + ASSERT_FALSE(ctx->GetTableId()); ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); ASSERT_TRUE(ctx->GetReadFieldNames().empty()); ASSERT_TRUE(ctx->GetReadFieldIds().empty()); - ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_FALSE(ctx->GetPredicate()); ASSERT_FALSE(ctx->HasVectorOrFullTextSearch()); ASSERT_FALSE(ctx->EnablePredicateFilter()); @@ -131,7 +100,6 @@ TEST(ReadContextTest, TestDefaultValue) { ASSERT_EQ(3, ctx->GetPrefetchMaxParallelNum()); ASSERT_FALSE(ctx->EnableMultiThreadRowToBatch()); ASSERT_EQ(1, ctx->GetRowToBatchThreadNumber()); - ASSERT_EQ("main", ctx->GetBranch()); ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); } @@ -145,7 +113,6 @@ TEST(ReadContextTest, TestSetContent) { cache_config.SetHoleSizeLimit(128); cache_config.SetPreBufferLimit(2048); - builder.AddOption("key", "value"); builder.SetReadFieldNames({"f1", "f2"}); builder.SetReadFieldIds({0, 1}); auto predicate = @@ -162,7 +129,7 @@ TEST(ReadContextTest, TestSetContent) { builder.WithMemoryPool(memory_pool); builder.WithExecutor(executor); builder.SetTableSchema("table-schema-json"); - builder.WithBranch("rt"); + builder.WithCacheConfig(cache_config); auto fs = std::make_shared(); builder.WithFileSystem(fs); @@ -189,13 +156,11 @@ TEST(ReadContextTest, TestSetContent) { ASSERT_EQ(executor, ctx->GetExecutor()); ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); - ASSERT_EQ("rt", ctx->GetBranch()); ASSERT_EQ(512U, ctx->GetCacheConfig().GetRangeSizeLimit()); ASSERT_EQ(128U, ctx->GetCacheConfig().GetHoleSizeLimit()); ASSERT_EQ(2048U, ctx->GetCacheConfig().GetPreBufferLimit()); ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); std::map expected_options = {{"key", "value"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); ASSERT_EQ(ctx->GetSpecificFileSystem(), fs); ASSERT_TRUE(ctx->GetCache()); } @@ -218,73 +183,6 @@ TEST(ReadContextTest, TestSetWarmupLevel) { ASSERT_EQ(WarmupLevel::RAW, second_ctx->GetWarmupLevel()); } -TEST(ReadContextTest, TestSetOptionsOverridesAddedOptions) { - ReadContextBuilder builder("table_root_path"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); -} - -TEST(ReadContextTest, TestBranch) { - ReadContextBuilder option_builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto option_ctx, option_builder.AddOption(Options::BRANCH, "rt").Finish()); - ASSERT_EQ("rt", option_ctx->GetBranch()); - - ReadContextBuilder agreeing_builder("table_root_path"); - ASSERT_OK_AND_ASSIGN( - auto agreeing_ctx, - agreeing_builder.WithBranch("rt").AddOption(Options::BRANCH, "rt").Finish()); - ASSERT_EQ("rt", agreeing_ctx->GetBranch()); - - ReadContextBuilder mixed_builder("table_root_path"); - ASSERT_NOK_WITH_MSG(mixed_builder.WithBranch("rt").AddOption(Options::BRANCH, "dev").Finish(), - "but both 'dev' and 'rt' were named"); - - // An empty branch selects the main branch and stays accepted. - ReadContextBuilder main_builder("table_root_path"); - main_builder.WithBranch(""); - ASSERT_OK_AND_ASSIGN(auto ctx, main_builder.Finish()); - ASSERT_EQ(BranchManager::DEFAULT_MAIN_BRANCH, ctx->GetBranch()); - ReadContextBuilder empty_builder("table_root_path"); - ASSERT_NOK_WITH_MSG(empty_builder.WithBranch("").AddOption(Options::BRANCH, "dev").Finish(), - "but both 'dev' and 'main' were named"); - - // The branch names a directory under the table path, so a value that is not a single path - // component is rejected when the context is built. - ReadContextBuilder escaping_builder("table_root_path"); - escaping_builder.WithBranch("rt/../../../../../outside"); - ASSERT_NOK_WITH_MSG(escaping_builder.Finish(), "branch name cannot contain path separators"); -} - -TEST(ReadContextTest, TestBranchAgreesWithIdentifier) { - // A read through a catalog names its table with an Identifier that can itself carry a branch, - // so that branch and the one WithBranch() names must agree, exactly as the write and commit - // builders already require: one read cannot be aimed at two branches at once. - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(std::make_shared()); - - // The identifier's branch is honored when WithBranch() stays silent. - ReadContextBuilder identifier_builder("table_root_path"); - identifier_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")); - ASSERT_OK_AND_ASSIGN(auto identifier_ctx, identifier_builder.Finish()); - ASSERT_EQ("dev", identifier_ctx->GetBranch()); - - // An agreeing WithBranch() is accepted. - ReadContextBuilder agreeing_builder("table_root_path"); - agreeing_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).WithBranch("dev"); - ASSERT_OK_AND_ASSIGN(auto agreeing_ctx, agreeing_builder.Finish()); - ASSERT_EQ("dev", agreeing_ctx->GetBranch()); - - // A disagreeing WithBranch() is refused, and both branches are named. - ReadContextBuilder mixed_builder("table_root_path"); - mixed_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).WithBranch("rt"); - ASSERT_NOK_WITH_MSG(mixed_builder.Finish(), "but both 'dev' and 'rt' were named"); -} - TEST(ReadContextTest, TestFileSystemAndSchemeMapConflict) { ReadContextBuilder builder("table_root_path"); auto fs = std::make_shared(); diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 89cccaa69..16d78da75 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -22,10 +22,10 @@ #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" +#include "paimon/core/utils/branch_manager.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { class Predicate; @@ -37,10 +37,9 @@ ScanContext::ScanContext( const std::shared_ptr& realtime_context, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::optional& table_schema, - const std::map& options, const std::shared_ptr& cache, - const std::shared_ptr& format_table, const std::shared_ptr& catalog, - const std::optional& identifier) + const std::optional& table_schema, const std::shared_ptr& cache, + const std::shared_ptr& catalog, const std::optional& identifier, + const std::map& fs_scheme_to_identifier_map) : path_(path), is_streaming_mode_(is_streaming_mode), limit_(limit), @@ -50,10 +49,9 @@ ScanContext::ScanContext( memory_pool_(memory_pool), executor_(executor), specific_file_system_(specific_file_system), + fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), table_schema_(table_schema), - options_(options), cache_(cache), - format_table_(format_table), catalog_(catalog), identifier_(identifier) {} @@ -74,8 +72,8 @@ class ScanContextBuilder::Impl { memory_pool_ = GetDefaultPool(); executor_.reset(); specific_file_system_.reset(); + fs_scheme_to_identifier_map_.clear(); table_schema_ = std::nullopt; - options_.clear(); cache_.reset(); catalog_.reset(); identifier_.reset(); @@ -83,10 +81,6 @@ class ScanContextBuilder::Impl { private: std::string path_; - /// Kept across `Reset()`, as `path_` is: both name the table this builder builds for, rather - /// than a setting of one scan of it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; bool is_streaming_mode_ = false; std::optional limit_; std::optional bucket_filter_; @@ -98,8 +92,8 @@ class ScanContextBuilder::Impl { // Resolved in Finish(); a builder never owns an executor of its own. std::shared_ptr executor_; std::shared_ptr specific_file_system_; + std::map fs_scheme_to_identifier_map_; std::optional table_schema_; - std::map options_; std::shared_ptr cache_; std::shared_ptr catalog_; std::optional identifier_; @@ -110,15 +104,6 @@ ScanContextBuilder::ScanContextBuilder(const std::string& path) impl_->path_ = path; } -ScanContextBuilder::ScanContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->path_ = table->Location(); - } -} - ScanContextBuilder::~ScanContextBuilder() = default; ScanContextBuilder& ScanContextBuilder::WithStreamingMode(bool is_streaming_mode) { impl_->is_streaming_mode_ = is_streaming_mode; @@ -158,18 +143,6 @@ ScanContextBuilder& ScanContextBuilder::WithRealtimeContext( return *this; } -ScanContextBuilder& ScanContextBuilder::SetOptions( - const std::map& options) { - impl_->options_ = options; - return *this; -} - -ScanContextBuilder& ScanContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - ScanContextBuilder& ScanContextBuilder::WithMemoryPool( const std::shared_ptr& memory_pool) { impl_->memory_pool_ = memory_pool; @@ -187,6 +160,12 @@ ScanContextBuilder& ScanContextBuilder::WithFileSystem( return *this; } +ScanContextBuilder& ScanContextBuilder::WithFileSystemSchemeToIdentifierMap( + const std::map& fs_scheme_to_identifier_map) { + impl_->fs_scheme_to_identifier_map_ = fs_scheme_to_identifier_map; + return *this; +} + ScanContextBuilder& ScanContextBuilder::WithCatalog(const std::shared_ptr& catalog, const Identifier& identifier) { impl_->catalog_ = catalog; @@ -205,28 +184,6 @@ ScanContextBuilder& ScanContextBuilder::WithCache(const std::shared_ptr& } Result> ScanContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot scan with null format table"); - } - if (impl_->format_table_ != nullptr) { - // The table already answers both, and from a source this cannot see behind, so a second - // answer is refused rather than silently dropped. - if (impl_->catalog_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithCatalog() " - "cannot be used with one"); - } - if (impl_->table_schema_) { - return Status::Invalid( - "a format table carries its own schema, so SetTableSchema() cannot be used with " - "one"); - } - if (impl_->specific_file_system_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "cannot be used with one"); - } - } PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_)); if (impl_->path_.empty()) { return Status::Invalid("cannot scan with empty table path"); @@ -234,11 +191,9 @@ Result> ScanContextBuilder::Finish() { if (impl_->catalog_ == nullptr && impl_->identifier_) { return Status::Invalid("cannot scan through a null catalog"); } - if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr) { - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the scan uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, - impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); + if (impl_->specific_file_system_ && !impl_->fs_scheme_to_identifier_map_.empty()) { + return Status::Invalid( + "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); } std::shared_ptr executor = impl_->executor_ ? impl_->executor_ : CreateDefaultExecutor(); @@ -247,8 +202,8 @@ Result> ScanContextBuilder::Finish() { std::make_shared(impl_->predicates_, impl_->partition_filters_, impl_->bucket_filter_), impl_->global_index_result_, impl_->realtime_context_, impl_->memory_pool_, executor, - impl_->specific_file_system_, impl_->table_schema_, impl_->options_, impl_->cache_, - impl_->format_table_, impl_->catalog_, impl_->identifier_); + impl_->specific_file_system_, impl_->table_schema_, impl_->cache_, impl_->catalog_, + impl_->identifier_, impl_->fs_scheme_to_identifier_map_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/scan_context_test.cpp b/src/paimon/core/operation/scan_context_test.cpp index 4ce2395b3..e1fad6b4c 100644 --- a/src/paimon/core/operation/scan_context_test.cpp +++ b/src/paimon/core/operation/scan_context_test.cpp @@ -31,28 +31,24 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { -TEST(ScanContextTest, TestWithCatalogResolvesTableFileSystem) { - // WithCatalog is a shorthand for WithFileSystem(catalog->GetTableFileSystem(identifier)); it - // resolves the per-table file system when Finish() builds the context, which also keeps the - // catalog and identifier for the operators to resolve the table through. +TEST(ScanContextTest, TestWithCatalogDefersTableFileSystem) { auto table_fs = std::make_shared(); auto catalog = std::make_shared(); catalog->SetTableFileSystem(table_fs); ScanContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); ASSERT_EQ(ctx->GetCatalog(), catalog); ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); - ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); // Finish() resets the builder, so the next context scans without the catalog. ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); ASSERT_FALSE(next_ctx->GetSpecificFileSystem()); ASSERT_FALSE(next_ctx->GetCatalog()); ASSERT_FALSE(next_ctx->GetIdentifier()); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); } TEST(ScanContextTest, TestWithFileSystemOverridesCatalog) { @@ -70,6 +66,40 @@ TEST(ScanContextTest, TestWithFileSystemOverridesCatalog) { ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); } +TEST(ScanContextTest, TestFileSystemAndSchemeMapConflict) { + ScanContextBuilder builder("table_root_path"); + auto fs = std::make_shared(); + builder.WithFileSystem(fs); + builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); + ASSERT_NOK_WITH_MSG( + builder.Finish(), + "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); +} + +TEST(ScanContextTest, TestSchemeMapWithoutFileSystem) { + ScanContextBuilder builder("table_root_path"); + builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); + std::map expected_fs_map = {{"file", "local"}}; + ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap()); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); +} + +TEST(ScanContextTest, TestSchemeMapOverridesCatalogFileSystem) { + auto table_fs = std::make_shared(); + auto catalog = std::make_shared(); + catalog->SetTableFileSystem(table_fs); + + ScanContextBuilder builder("table_root_path"); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")) + .WithFileSystemSchemeToIdentifierMap({{"file", "local"}}) + .Finish()); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); + std::map expected_fs_map = {{"file", "local"}}; + ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap()); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); +} + TEST(ScanContextTest, TestWithCatalogNullRejected) { ScanContextBuilder builder("table_root_path"); ASSERT_NOK_WITH_MSG(builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), @@ -80,6 +110,7 @@ TEST(ScanContextTest, TestDefaultValue) { ScanContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); ASSERT_EQ(ctx->GetPath(), "table_root_path"); + ASSERT_FALSE(ctx->GetTableId()); ASSERT_FALSE(ctx->IsStreamingMode()); ASSERT_FALSE(ctx->GetLimit()); ASSERT_TRUE(ctx->GetMemoryPool()); @@ -88,9 +119,9 @@ TEST(ScanContextTest, TestDefaultValue) { ASSERT_FALSE(ctx->GetScanFilters()->GetBucketFilter()); ASSERT_FALSE(ctx->GetScanFilters()->GetPredicate()); ASSERT_TRUE(ctx->GetScanFilters()->GetPartitionFilters().empty()); - ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_FALSE(ctx->GetGlobalIndexResult()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); + ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); } TEST(ScanContextTest, TestSetContent) { @@ -108,7 +139,7 @@ TEST(ScanContextTest, TestSetContent) { auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); builder.SetGlobalIndexResult(global_index_result); builder.SetLimit(1000); - builder.AddOption("key", "value"); + builder.WithStreamingMode(true); builder.WithMemoryPool(memory_pool); builder.WithExecutor(executor); @@ -131,22 +162,10 @@ TEST(ScanContextTest, TestSetContent) { ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); std::map expected_options = {{"key", "value"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); ASSERT_EQ(fs, ctx->GetSpecificFileSystem()); ASSERT_TRUE(ctx->GetCache()); } -TEST(ScanContextTest, TestSetOptionsOverridesAddedOptions) { - ScanContextBuilder builder("table_root_path"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); -} - TEST(ScanContextTest, TestDefaultExecutorIsCreatedPerContext) { // A builder without WithExecutor() gives every context a default executor // of its own; nothing is shared across contexts. diff --git a/src/paimon/core/operation/write_context.cpp b/src/paimon/core/operation/write_context.cpp index e31d64ac4..93d4f340a 100644 --- a/src/paimon/core/operation/write_context.cpp +++ b/src/paimon/core/operation/write_context.cpp @@ -22,6 +22,7 @@ #include "arrow/util/thread_pool.h" #include "fmt/format.h" +#include "paimon/catalog/catalog.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/catalog/catalog_utils.h" @@ -30,25 +31,21 @@ #include "paimon/memory/memory_pool.h" #include "paimon/result.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { WriteContext::WriteContext( const std::string& root_path, const std::string& commit_user, bool is_streaming_mode, bool ignore_num_bucket_check, bool ignore_previous_files, bool enable_multi_thread_spill, - const std::optional& write_id, const std::string& branch, - const std::vector& write_schema, const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, const std::string& temp_directory, - const std::shared_ptr& specific_file_system, + const std::optional& write_id, const std::vector& write_schema, + const std::shared_ptr& memory_pool, const std::shared_ptr& executor, + const std::string& temp_directory, const std::shared_ptr& specific_file_system, const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, - const std::map& options, - const std::shared_ptr& format_table, const std::shared_ptr& catalog, + const std::optional& table_schema, const std::shared_ptr& catalog, const std::optional& identifier) : root_path_(root_path), commit_user_(commit_user), - branch_(branch), is_streaming_mode_(is_streaming_mode), ignore_num_bucket_check_(ignore_num_bucket_check), ignore_previous_files_(ignore_previous_files), @@ -61,8 +58,7 @@ WriteContext::WriteContext( specific_file_system_(specific_file_system), fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), realtime_context_(realtime_context), - options_(options), - format_table_(format_table), + table_schema_(table_schema), catalog_(catalog), identifier_(identifier) {} @@ -81,26 +77,18 @@ class WriteContextBuilder::Impl { memory_pool_ = GetDefaultPool(); executor_ = CreateDefaultExecutor(); temp_directory_.clear(); - branch_.reset(); write_schema_.clear(); fs_scheme_to_identifier_map_.clear(); specific_file_system_.reset(); realtime_context_.reset(); catalog_.reset(); identifier_.reset(); - options_.clear(); + table_schema_ = std::nullopt; } private: std::string root_path_; - /// Kept across `Reset()`, as `root_path_` is: both name the table this builder builds for, - /// rather than a setting of one write to it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; std::string commit_user_; - /// Unset until `WithBranch()` names one, so that a branch named only by the identifier or - /// the `branch` option is not overruled by the default. - std::optional branch_; std::optional write_id_; bool is_streaming_mode_ = false; bool ignore_num_bucket_check_ = false; @@ -115,7 +103,7 @@ class WriteContextBuilder::Impl { std::shared_ptr realtime_context_; std::shared_ptr catalog_; std::optional identifier_; - std::map options_; + std::optional table_schema_; }; WriteContextBuilder::WriteContextBuilder(const std::string& root_path, @@ -125,29 +113,8 @@ WriteContextBuilder::WriteContextBuilder(const std::string& root_path, impl_->commit_user_ = commit_user; } -WriteContextBuilder::WriteContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->root_path_ = table->Location(); - } -} - WriteContextBuilder::~WriteContextBuilder() = default; -WriteContextBuilder& WriteContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - -WriteContextBuilder& WriteContextBuilder::SetOptions( - const std::map& opts) { - impl_->options_ = opts; - return *this; -} - WriteContextBuilder& WriteContextBuilder::WithStreamingMode(bool is_streaming_mode) { impl_->is_streaming_mode_ = is_streaming_mode; return *this; @@ -184,11 +151,6 @@ WriteContextBuilder& WriteContextBuilder::WithWriteId(int32_t write_id) { return *this; } -WriteContextBuilder& WriteContextBuilder::WithBranch(const std::string& branch) { - impl_->branch_ = branch; - return *this; -} - WriteContextBuilder& WriteContextBuilder::WithWriteSchema( const std::vector& write_schema) { impl_->write_schema_ = write_schema; @@ -206,6 +168,11 @@ WriteContextBuilder& WriteContextBuilder::SetWriteBufferSpillThreadNumber(int32_ return *this; } +WriteContextBuilder& WriteContextBuilder::SetTableSchema(const std::string& table_schema) { + impl_->table_schema_ = table_schema; + return *this; +} + WriteContextBuilder& WriteContextBuilder::WithFileSystem( const std::shared_ptr& file_system) { impl_->specific_file_system_ = file_system; @@ -226,27 +193,6 @@ WriteContextBuilder& WriteContextBuilder::WithCatalog(const std::shared_ptr> WriteContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot write with null format table"); - } - if (impl_->format_table_ != nullptr) { - if (impl_->catalog_ != nullptr) { - return Status::Invalid("WithCatalog() requires a native table"); - } - // The table already answers each of these, and from a source this cannot see behind, so a - // second answer is refused rather than silently dropped. - if (impl_->specific_file_system_ != nullptr || - !impl_->fs_scheme_to_identifier_map_.empty()) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "and WithFileSystemSchemeToIdentifierMap() cannot be used with one"); - } - if (impl_->branch_ && - !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(impl_->branch_.value()))) { - return Status::Invalid( - "a format table has no branches, so WithBranch() cannot be used with one"); - } - } PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_)); if (impl_->root_path_.empty()) { return Status::Invalid("root path is empty"); @@ -260,22 +206,6 @@ Result> WriteContextBuilder::Finish() { PAIMON_RETURN_NOT_OK( CatalogUtils::CheckNotSystemTable(impl_->identifier_.value(), "write")); } - PAIMON_ASSIGN_OR_RAISE( - std::string branch, - BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, impl_->branch_, "write")); - if (impl_->catalog_ != nullptr && !BranchManager::IsMainBranch(branch)) { - // Refused here rather than at the commit of what is written to the branch. - PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); - PAIMON_ASSIGN_OR_RAISE(std::optional identifier_branch, - impl_->identifier_->GetBranchName()); - if (!identifier_branch) { - PAIMON_ASSIGN_OR_RAISE(std::string table_name, impl_->identifier_->GetDataTableName()); - return Status::Invalid(fmt::format( - "a write through a catalog addresses a branch by the table identifier, so name " - "branch '{}' there as '{}$branch_{}'", - branch, table_name, branch)); - } - } bool enable_multi_thread_spill = impl_->spill_thread_number_ > 0; if (enable_multi_thread_spill) { PAIMON_RETURN_NOT_OK_FROM_ARROW( @@ -284,10 +214,9 @@ Result> WriteContextBuilder::Finish() { auto ctx = std::make_unique( impl_->root_path_, impl_->commit_user_, impl_->is_streaming_mode_, impl_->ignore_num_bucket_check_, impl_->ignore_previous_files_, enable_multi_thread_spill, - impl_->write_id_, branch, impl_->write_schema_, impl_->memory_pool_, impl_->executor_, + impl_->write_id_, impl_->write_schema_, impl_->memory_pool_, impl_->executor_, impl_->temp_directory_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, - impl_->realtime_context_, impl_->options_, impl_->format_table_, impl_->catalog_, - impl_->identifier_); + impl_->realtime_context_, impl_->table_schema_, impl_->catalog_, impl_->identifier_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/write_context_test.cpp b/src/paimon/core/operation/write_context_test.cpp index 5e777c351..d4119862b 100644 --- a/src/paimon/core/operation/write_context_test.cpp +++ b/src/paimon/core/operation/write_context_test.cpp @@ -43,16 +43,16 @@ TEST(WriteContextTest, TestDefaultValue) { ASSERT_FALSE(ctx->IgnorePreviousFiles()); ASSERT_FALSE(ctx->EnableMultiThreadSpill()); ASSERT_EQ(ctx->GetWriteId(), std::nullopt); - ASSERT_EQ(ctx->GetBranch(), "main"); ASSERT_TRUE(ctx->GetWriteSchema().empty()); ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); ASSERT_TRUE(ctx->GetTempDirectory().empty()); - ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); + ASSERT_FALSE(ctx->GetSpecificTableSchema()); ASSERT_FALSE(ctx->GetCatalog()); ASSERT_FALSE(ctx->GetIdentifier()); + ASSERT_FALSE(ctx->GetTableId()); } TEST(WriteContextTest, TestWithCatalog) { @@ -69,85 +69,22 @@ TEST(WriteContextTest, TestWithCatalog) { "cannot write through a null catalog"); } -TEST(WriteContextTest, TestCatalogAddressesBranchByIdentifier) { +TEST(WriteContextTest, TestWithCatalogDefersTableFileSystem) { + auto table_fs = std::make_shared(); auto catalog = std::make_shared(); - WriteContextBuilder builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .WithBranch("dev") - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_EQ(ctx->GetBranch(), "dev"); - - WriteContextBuilder identifier_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto identifier_ctx, - identifier_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")).Finish()); - ASSERT_EQ(identifier_ctx->GetBranch(), "dev"); - - for (int32_t branch_source = 0; branch_source < 2; ++branch_source) { - SCOPED_TRACE(branch_source); - WriteContextBuilder option_only_builder("table_root_path", "commit_user_1"); - option_only_builder.WithCatalog(catalog, Identifier("db1", "t1")); - if (branch_source == 0) { - option_only_builder.WithBranch("dev"); - } else { - option_only_builder.AddOption(Options::BRANCH, "dev"); - } - ASSERT_NOK_WITH_MSG(option_only_builder.Finish(), - "name branch 'dev' there as 't1$branch_dev'"); - } - - WriteContextBuilder mixed_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(mixed_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .WithBranch("release") - .Finish(), - "but both 'dev' and 'release' were named"); - - WriteContextBuilder main_builder("table_root_path", "commit_user_1"); - ASSERT_OK(main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) - .WithBranch("") - .AddOption(Options::BRANCH, "main") - .Finish()); - - WriteContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - upper_main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_MAIN")).Finish(), - "a catalog names branch 'MAIN' as it names the main branch"); - - WriteContextBuilder constructed_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto constructed_ctx, - constructed_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).Finish()); - ASSERT_EQ(constructed_ctx->GetBranch(), "dev"); - WriteContextBuilder system_table_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - system_table_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev$options")).Finish(), - "Cannot 'write' for system table"); -} + catalog->SetTableFileSystem(table_fs); -TEST(WriteContextTest, TestBranch) { - WriteContextBuilder option_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto option_ctx, option_builder.AddOption(Options::BRANCH, "rt").Finish()); - ASSERT_EQ("rt", option_ctx->GetBranch()); - - WriteContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto upper_main_ctx, upper_main_builder.WithBranch("MAIN").Finish()); - ASSERT_EQ("MAIN", upper_main_ctx->GetBranch()); - - // An empty branch selects the main branch and stays accepted. - WriteContextBuilder main_builder("table_root_path", "commit_user_1"); - main_builder.WithBranch(""); - ASSERT_OK_AND_ASSIGN(auto ctx, main_builder.Finish()); - ASSERT_EQ(BranchManager::DEFAULT_MAIN_BRANCH, ctx->GetBranch()); - WriteContextBuilder empty_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(empty_builder.WithBranch("").AddOption(Options::BRANCH, "dev").Finish(), - "but both 'dev' and 'main' were named"); - - // The branch names a directory under the root path, so a value that is not a single path - // component is rejected when the context is built. - WriteContextBuilder escaping_builder("table_root_path", "commit_user_1"); - escaping_builder.WithBranch("rt/../../../../../outside"); - ASSERT_NOK_WITH_MSG(escaping_builder.Finish(), "branch name cannot contain path separators"); + WriteContextBuilder builder("table_root_path", "commit_user_1"); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); + + WriteContextBuilder map_builder("table_root_path", "commit_user_1"); + ASSERT_OK_AND_ASSIGN(auto map_ctx, map_builder.WithCatalog(catalog, Identifier("db1", "t1")) + .WithFileSystemSchemeToIdentifierMap({{"file", "local"}}) + .Finish()); + ASSERT_FALSE(map_ctx->GetSpecificFileSystem()); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); } TEST(WriteContextTest, TestSetContent) { @@ -161,6 +98,7 @@ TEST(WriteContextTest, TestSetContent) { {"oss", "jindo"}}; ASSERT_OK_AND_ASSIGN(auto ctx, + builder.WithStreamingMode(true) .WithIgnoreNumBucketCheck(true) .WithIgnorePreviousFiles(true) @@ -168,11 +106,10 @@ TEST(WriteContextTest, TestSetContent) { .WithExecutor(executor) .WithTempDirectory("/tmp/with-all") .WithWriteId(123) - .WithBranch("test_branch") .WithWriteSchema(write_schema) .WithFileSystemSchemeToIdentifierMap(fs_scheme_to_identifier_map) .WithFileSystem(file_system) - .AddOption("key", "value") + .SetTableSchema("table-schema-json") .Finish()); ASSERT_TRUE(ctx->IsStreamingMode()); @@ -183,23 +120,12 @@ TEST(WriteContextTest, TestSetContent) { ASSERT_EQ(ctx->GetExecutor(), executor); ASSERT_EQ(ctx->GetTempDirectory(), "/tmp/with-all"); ASSERT_EQ(ctx->GetWriteId(), 123); - ASSERT_EQ(ctx->GetBranch(), "test_branch"); ASSERT_EQ(ctx->GetWriteSchema(), write_schema); ASSERT_EQ(ctx->GetFileSystemSchemeToIdentifierMap(), fs_scheme_to_identifier_map); ASSERT_EQ(ctx->GetSpecificFileSystem(), file_system); + ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); + ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); std::map expected_options = {{"key", "value"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); -} - -TEST(WriteContextTest, TestSetOptionsOverridesAddedOptions) { - WriteContextBuilder builder("table_root_path", "commit_user_1"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); } TEST(WriteContextTest, TestSetWriteBufferSpillThreadNumber) { diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 3a73838d9..9640a4173 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -75,6 +75,39 @@ namespace paimon { namespace { constexpr char kDeletionVectorsMergeOnRead[] = "deletion-vectors.merge-on-read"; +constexpr char kDeletionVectorsModifiable[] = "deletion-vectors.modifiable"; + +// Layout options protected by both operator factories and table copies. +const char* const kImmutableOptions[] = { + Options::BUCKET_KEY, + Options::BUCKET_FUNCTION_TYPE, + Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP, + Options::FORCE_LOOKUP, + Options::INDEX_FILE_IN_DATA_FILE_DIR, + Options::BLOB_FIELD, + Options::FALLBACK_BLOB_DESCRIPTOR_FIELD, + Options::BLOB_VIEW_FIELD, + Options::PK_CLUSTERING_OVERRIDE, +}; + +// Copy-only immutable options; low-level factories retain their initialization semantics. +const char* const kCopyImmutableOptions[] = { + Options::BLOB_DESCRIPTOR_FIELD, + Options::MERGE_ENGINE, + Options::ROW_TRACKING_ENABLED, + Options::DATA_EVOLUTION_ENABLED, + "bucket-function.provider", + "data-file.path-directory", + "sequence.snapshot-ordering", + "rowkind.field", + "primary-key", + "primary-key.nullable", + "partition", + "dynamic-bucket.initial-buckets", + "video-frame-field", +}; bool ContainsBlobField(const std::shared_ptr& field) { if (BlobUtils::IsBlobField(field)) { @@ -366,7 +399,7 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { PAIMON_RETURN_NOT_OK(ValidateBucket(schema, options)); PAIMON_RETURN_NOT_OK(ValidateRealtime(schema, options)); // PAIMON_RETURN_NOT_OK(ValidateDefaultValues(schema)); - // PAIMON_RETURN_NOT_OK(ValidateStartupMode(options)); + PAIMON_RETURN_NOT_OK(ValidateStartupMode(options)); PAIMON_RETURN_NOT_OK(ValidateFieldsPrefix(schema, options)); PAIMON_RETURN_NOT_OK(ValidateSequenceField(schema, options)); PAIMON_RETURN_NOT_OK(ValidateSequenceGroup(schema, options)); @@ -402,6 +435,122 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { return Status::OK(); } +Status SchemaValidation::CheckImmutableOptions( + const TableSchema& schema, const std::map& dynamic_options) { + const auto& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + auto old_value = schema_options.find(key); + if (old_value != schema_options.end() && old_value->second == value) { + continue; + } + if (std::find(std::begin(kImmutableOptions), std::end(kImmutableOptions), key) != + std::end(kImmutableOptions)) { + return Status::Invalid(fmt::format("Change '{}' is not supported yet.", key)); + } + if (key == Options::IGNORE_DELETE) { + PAIMON_ASSIGN_OR_RAISE(bool old_ignore_delete, + OptionsUtils::GetValueFromMap(schema_options, key, false)); + PAIMON_ASSIGN_OR_RAISE(bool new_ignore_delete, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (old_ignore_delete && !new_ignore_delete) { + return Status::Invalid("Cannot change ignore-delete from true to false."); + } + } + } + return Status::OK(); +} + +Status SchemaValidation::CheckCopyOptions( + const TableSchema& schema, const std::map& dynamic_options) { + PAIMON_RETURN_NOT_OK(CheckImmutableOptions(schema, dynamic_options)); + const std::map& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + auto schema_option = schema_options.find(key); + if (schema_option != schema_options.end() && schema_option->second == value) { + continue; + } + if (schema_option == schema_options.end() && (key == "primary-key" || key == "partition")) { + std::vector keys; + for (const auto& item : StringUtils::Split(value, ",")) { + std::string trimmed = item; + StringUtils::Trim(&trimmed); + if (!trimmed.empty()) { + keys.push_back(trimmed); + } + } + if (keys == (key == "primary-key" ? schema.PrimaryKeys() : schema.PartitionKeys())) { + continue; + } + } + if (std::find(std::begin(kCopyImmutableOptions), std::end(kCopyImmutableOptions), key) != + std::end(kCopyImmutableOptions)) { + return Status::Invalid(fmt::format("Change '{}' is not supported yet.", key)); + } + if (key == Options::BUCKET) { + PAIMON_ASSIGN_OR_RAISE(int32_t old_bucket, + OptionsUtils::GetValueFromMap(schema_options, key, -1)); + PAIMON_ASSIGN_OR_RAISE(int32_t new_bucket, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (old_bucket == -1) { + return Status::Invalid("Cannot change bucket when it is -1."); + } + if (new_bucket == -1) { + return Status::Invalid("Cannot change bucket to -1."); + } + } + if (key == Options::DELETION_VECTORS_ENABLED) { + PAIMON_ASSIGN_OR_RAISE(bool modifiable, + OptionsUtils::GetValueFromMap( + schema_options, kDeletionVectorsModifiable, false)); + PAIMON_ASSIGN_OR_RAISE(bool old_enabled, + OptionsUtils::GetValueFromMap(schema_options, key, false)); + PAIMON_ASSIGN_OR_RAISE(bool new_enabled, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (!modifiable && old_enabled != new_enabled) { + return Status::Invalid(fmt::format( + "Cannot change deletion vectors mode from {} to {}. Set '{}' to true on the " + "table before changing deletion vectors mode.", + old_enabled, new_enabled, kDeletionVectorsModifiable)); + } + } + if (key == "ignore-update-before") { + PAIMON_ASSIGN_OR_RAISE(bool old_ignore_delete, + OptionsUtils::GetValueFromMap(schema_options, key, false)); + PAIMON_ASSIGN_OR_RAISE(bool new_ignore_delete, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (old_ignore_delete && !new_ignore_delete) { + return Status::Invalid(fmt::format("Cannot change {} from true to false.", key)); + } + } + } + return Status::OK(); +} + +Status SchemaValidation::ValidateStartupMode(const CoreOptions& options) { + const auto& values = options.ToMap(); + const bool snapshot = values.count(Options::SCAN_SNAPSHOT_ID) != 0; + const bool tag = values.count(Options::SCAN_TAG_NAME) != 0; + const bool timestamp = values.count(Options::SCAN_TIMESTAMP) != 0; + const bool timestamp_millis = values.count(Options::SCAN_TIMESTAMP_MILLIS) != 0; + const int32_t selectors = snapshot + tag + timestamp + timestamp_millis; + const StartupMode mode = options.GetStartupMode(); + bool valid = selectors == 0; + if (mode == StartupMode::FromSnapshot()) { + valid = selectors == 1 && (snapshot || tag); + } else if (mode == StartupMode::FromSnapshotFull()) { + valid = selectors == 1 && snapshot; + } else if (mode == StartupMode::FromTimestamp()) { + valid = selectors == 1 && (timestamp || timestamp_millis); + } + if (!valid) { + return Status::Invalid(fmt::format( + "Invalid time travel options for scan.mode '{}': set exactly one compatible selector " + "for from-snapshot/from-snapshot-full/from-timestamp, and none for latest/latest-full.", + mode.ToString())); + } + return Status::OK(); +} + Status SchemaValidation::ValidateNoDuplicateField(const std::vector& field_names, const std::string& error_message_intro) { auto duplicate_field_names = ObjectUtils::DuplicateItems(field_names); diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h index 813aaebb4..3e69ee0f9 100644 --- a/src/paimon/core/schema/schema_validation.h +++ b/src/paimon/core/schema/schema_validation.h @@ -51,6 +51,17 @@ class SchemaValidation { static Status ValidateTableSchema(const TableSchema& schema); + /// Checks layout options protected by low-level factories and Copy alike. + static Status CheckImmutableOptions(const TableSchema& schema, + const std::map& dynamic_options); + + /// Checks Java table Copy immutability, including bucket-mode transitions. + static Status CheckCopyOptions(const TableSchema& schema, + const std::map& dynamic_options); + + /// Checks selector exclusivity and compatibility with the effective scan.mode. + static Status ValidateStartupMode(const CoreOptions& options); + /// Validates real-time options after operation-specific options have been merged with the /// table schema options. static Status ValidateRealtime(const TableSchema& schema, const CoreOptions& options); @@ -121,9 +132,6 @@ class SchemaValidation { static Status ValidateDefaultValues(const TableSchema& schema) { return Status::NotImplemented("validate default values not implemented"); } - static Status ValidateStartupMode(const CoreOptions& options) { - return Status::NotImplemented("validate startup mode not implemented"); - } static Status ValidateFieldsPrefix(const TableSchema& schema, const CoreOptions& options); static Status ValidateSequenceField(const TableSchema& schema, const CoreOptions& options); static Status ValidateSequenceGroup(const TableSchema& schema, const CoreOptions& options); diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index 1d69c033a..1c0cfd333 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -194,6 +194,36 @@ TableSchema::TableSchema(int32_t version, int64_t id, const std::vector> TableSchema::MergeOptions( + const std::map& dynamic_options) const { + std::map options = options_; + for (const auto& [key, value] : dynamic_options) { + options[key] = value; + } + return Copy(options); +} + +Result> TableSchema::Copy( + const std::map& options) const { + auto merged = std::unique_ptr( + new TableSchema(version_, id_, fields_, highest_field_id_, partition_keys_, primary_keys_, + options, comment_, time_millis_)); + PAIMON_RETURN_NOT_OK(merged->DeriveBucketInfo()); + return merged; +} + +Status TableSchema::DeriveBucketInfo() { + PAIMON_ASSIGN_OR_RAISE(std::vector keys, TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, OriginalBucketKeys()); + if (bucket_keys.empty()) { + bucket_keys = keys; + } + bucket_keys_ = bucket_keys; + PAIMON_ASSIGN_OR_RAISE(num_bucket_, + OptionsUtils::GetValueFromMap(Options(), Options::BUCKET, -1)); + return Status::OK(); +} + bool TableSchema::operator==(const TableSchema& other) const { return version_ == other.version_ && fields_ == other.fields_ && partition_keys_ == other.partition_keys_ && primary_keys_ == other.primary_keys_ && @@ -332,18 +362,7 @@ Result> TableSchema::InitSchema( auto table_schema = std::unique_ptr( new TableSchema(TableSchema::CURRENT_VERSION, schema_id, fields, highest_field_id, partition_keys, primary_keys, options, comment, time_millis)); - PAIMON_ASSIGN_OR_RAISE(std::vector keys, table_schema->TrimmedPrimaryKeys()); - - // Try to validate bucket keys - PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, - table_schema->OriginalBucketKeys()); - if (bucket_keys.empty()) { - bucket_keys = keys; - } - table_schema->bucket_keys_ = bucket_keys; - PAIMON_ASSIGN_OR_RAISE( - table_schema->num_bucket_, - OptionsUtils::GetValueFromMap(table_schema->Options(), Options::BUCKET, -1)); + PAIMON_RETURN_NOT_OK(table_schema->DeriveBucketInfo()); return table_schema; } diff --git a/src/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h index eab109cf1..ee817c030 100644 --- a/src/paimon/core/schema/table_schema.h +++ b/src/paimon/core/schema/table_schema.h @@ -101,6 +101,15 @@ class TableSchema : public DataSchema, public Jsonizable { const std::map& Options() const override { return options_; } + + /// Copies the schema with dynamic_options overriding its current options. + /// Other schema attributes are preserved. + Result> MergeOptions( + const std::map& dynamic_options) const; + + /// Copies this schema with the complete replacement option map. + Result> Copy( + const std::map& options) const; const std::vector& Fields() const { return fields_; } @@ -142,6 +151,9 @@ class TableSchema : public DataSchema, public Jsonizable { Result> OriginalBucketKeys() const; + /// Derives cached bucket keys and count after construction or option changes. + Status DeriveBucketInfo(); + static Result> AssignFieldIdsRecursively( const std::shared_ptr& field, bool set_field_id, int32_t* field_id); diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp new file mode 100644 index 000000000..77df95336 --- /dev/null +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/append_only_file_store_table.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/operation/append_only_file_store_scan.h" +#include "paimon/core/operation/append_only_file_store_write.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/data_evolution_file_store_scan.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/source/append_only_split_generator.h" +#include "paimon/core/table/source/data_evolution_split_generator.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { + +AppendOnlyFileStoreTable::AppendOnlyFileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, + const Identifier& identifier, + const std::string& uuid) + : FileStoreTable(store, schema, identifier, uuid) {} + +Result> AppendOnlyFileStoreTable::CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const { + const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); + std::shared_ptr schema = std::static_pointer_cast(schema_); + bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); + if (options.GetBucket() == -1) { + need_dv_maintainer_factory = false; + ignore_previous_files = true; + } else if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in append table", options.GetBucket())); + } + if (ctx.GetRealtimeContext()) { + if (!ctx.GetWriteSchema().empty()) { + return Status::Invalid("real-time append write does not support a custom schema"); + } + if (!realtime_deduplicate) { + PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx.GetRealtimeContext(), + snapshot_manager, options)); + } + } else if (realtime_deduplicate) { + return Status::Invalid("real-time deduplicate write requires a real-time context"); + } + std::shared_ptr write_schema = arrow_schema; + const auto& write_field_names = ctx.GetWriteSchema(); + if (!write_field_names.empty()) { + arrow::FieldVector write_fields; + write_fields.reserve(write_field_names.size()); + for (const auto& field_name : write_field_names) { + auto field = arrow_schema->GetFieldByName(field_name); + if (!field) { + // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER + return Status::Invalid( + fmt::format("write field {} does not exist in table schema", field_name)); + } + write_fields.push_back(field); + } + write_schema = arrow::schema(write_fields); + } + + std::shared_ptr realtime_schema_layout; + if (ctx.GetRealtimeContext()) { + const RealtimeStoreMode mode = + realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE : RealtimeStoreMode::APPEND_ONLY; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, + RealtimeSchemaLayout::Create(mode, write_schema)); + realtime_schema_layout = std::move(layout); + } + + std::shared_ptr dv_maintainer_factory; + if (need_dv_maintainer_factory) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_file_handler, + store_->NewIndexFileHandler(options, path_factory, ctx.GetMemoryPool())); + dv_maintainer_factory = + std::make_shared(std::move(index_file_handler)); + } + + auto file_store_write = std::make_unique( + path_factory, snapshot_manager, GetSchemaManager(), ctx.GetCommitUser(), root_path_, schema, + arrow_schema, write_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, + io_manager, options, ignore_previous_files, ctx.IsStreamingMode(), + ctx.IgnoreNumBucketCheck(), ctx.GetRealtimeContext(), ctx.GetExecutor(), + ctx.GetMemoryPool()); + if (ctx.GetRealtimeContext() && realtime_deduplicate) { + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_RETURN_NOT_OK(file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); + } + } + return std::unique_ptr(std::move(file_store_write)); +} + +Result> AppendOnlyFileStoreTable::NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const { + // TODO(liancheng.lsz): support fallback branch in scan + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + std::unique_ptr scan; + if (core_options.DataEvolutionEnabled()) { + PAIMON_ASSIGN_OR_RAISE( + scan, DataEvolutionFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), + manifest_list, manifest_file, table_schema, + arrow_schema, context->GetScanFilters(), + core_options, executor, memory_pool)); + } else { + PAIMON_ASSIGN_OR_RAISE( + scan, AppendOnlyFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), + manifest_list, manifest_file, table_schema, + arrow_schema, context->GetScanFilters(), + core_options, executor, memory_pool)); + } + return WithTablePath(std::move(scan), root_path_); +} + +Result> AppendOnlyFileStoreTable::NewSplitGenerator( + const CoreOptions& core_options) const { + auto source_split_target_size = core_options.GetSourceSplitTargetSize(); + auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); + if (core_options.DataEvolutionEnabled()) { + return std::make_unique(source_split_target_size, + source_split_open_file_cost, + core_options.BlobSplitByFileSize()); + } + BucketMode bucket_mode = + (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE : BucketMode::HASH_FIXED); + return std::make_unique(source_split_target_size, + source_split_open_file_cost, bucket_mode); +} + +CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) const { + const std::shared_ptr& schema_manager = GetSchemaManager(); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h new file mode 100644 index 000000000..43e29bda1 --- /dev/null +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class FileStoreScan; +class IOManager; +class SplitGenerator; +class TableSchema; + +/// Managed table without primary keys; uses the shared FileStoreTable operator flows. +class AppendOnlyFileStoreTable : public FileStoreTable { + public: + AppendOnlyFileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid); + + private: + Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const override; + + Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const override; + + Result> NewSplitGenerator( + const CoreOptions& core_options) const override; + + CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp new file mode 100644 index 000000000..157b807f4 --- /dev/null +++ b/src/paimon/core/table/file_store.cpp @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/file_store.h" + +#include +#include +#include +#include +#include + +#include "paimon/catalog/catalog.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/binary_row_partition_computer.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/manifest/manifest_file.h" +#include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/index_file_path_factories.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +std::shared_ptr FileStore::Create(const std::string& root_path, + CoreOptions core_options) { + return std::shared_ptr(new FileStore(root_path, std::move(core_options))); +} + +FileStore::FileStore(const std::string& root_path, CoreOptions core_options) + : file_system_(core_options.GetFileSystem()), + root_path_(root_path), + branch_(BranchManager::NormalizeBranch(core_options.GetBranch())), + schema_manager_(std::make_shared(file_system_, root_path_, branch_)), + snapshot_manager_(std::make_shared(file_system_, root_path_, branch_)), + core_options_(std::move(core_options)) {} + +Result> FileStore::NewPathFactory( + const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, + const std::string& file_format_identifier, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + options.CreateGlobalIndexExternalPath()); + return FileStorePathFactory::Create( + root_path_, arrow_schema, partition_keys, options.GetPartitionDefaultName(), + file_format_identifier, options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, options.IndexFileInDataFileDir(), memory_pool); +} + +Result> FileStore::NewSnapshotManager( + const CoreOptions& options, const std::shared_ptr& catalog, + const std::optional& catalog_identifier) const { + auto snapshot_manager = + std::make_shared(options.GetFileSystem(), root_path_, branch_); + if (VersionManagedCatalog* versioned = AsVersionManaged(catalog)) { + if (!catalog_identifier) { + return Status::Invalid("a version-managed catalog requires a table identifier"); + } + Identifier identifier = catalog_identifier.value(); + snapshot_manager->SetSnapshotLoader( + [catalog, versioned, identifier]() -> Result> { + return versioned->LoadSnapshot(identifier); + }); + } + return snapshot_manager; +} + +Result> FileStore::NewManifestList( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_list, + ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, options.GetCache(), + memory_pool)); + return manifest_list; +} + +Result> FileStore::NewManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& partition_schema, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_file, + ManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, + options.GetManifestTargetFileSize(), memory_pool, options, + partition_schema)); + return manifest_file; +} + +Result> FileStore::NewIndexManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const { + return IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, + options.GetBucket(), memory_pool, options); +} + +Result> FileStore::NewIndexFileHandler( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_manifest_file, + NewIndexManifestFile(options, path_factory, memory_pool)); + return std::make_unique( + options.GetFileSystem(), std::move(index_manifest_file), + std::make_shared(path_factory), options.DeletionVectorsBitmap64(), + memory_pool); +} + +Result> FileStore::NewPartitionComputer( + const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, + const std::shared_ptr& memory_pool) const { + return BinaryRowPartitionComputer::Create(partition_keys, arrow_schema, + options.GetPartitionDefaultName(), + options.LegacyPartitionNameEnabled(), memory_pool); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h new file mode 100644 index 000000000..12d200091 --- /dev/null +++ b/src/paimon/core/table/file_store.h @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/core_options.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class BinaryRowPartitionComputer; +class Catalog; +class IndexFileHandler; +class IndexManifestFile; +class ManifestFile; +class ManifestList; + +/// Storage state and factories for managed-table operators. +/// Operator factories create fresh instances so catalog snapshot loaders are not shared +/// between scans, writes and commits. +class FileStore { + public: + /// Builds storage from already resolved CoreOptions. + /// @param root_path Table root path. + /// @param core_options Effective options, including the file system and branch. + static std::shared_ptr Create(const std::string& root_path, + CoreOptions core_options); + + const std::string& CurrentBranch() const { + return branch_; + } + + const std::shared_ptr& GetSchemaManager() const { + return schema_manager_; + } + + const std::shared_ptr& GetSnapshotManager() const { + return snapshot_manager_; + } + + const std::shared_ptr& GetFileSystem() const { + return file_system_; + } + + const std::string& GetRootPath() const { + return root_path_; + } + + const CoreOptions& GetOptions() const { + return core_options_; + } + + /// Uses the caller's partition keys and format identifier, which may differ for fallback + /// reads or per-level writes. + Result> NewPathFactory( + const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, + const std::string& file_format_identifier, + const std::shared_ptr& memory_pool) const; + + /// Creates an operator-local snapshot manager with an optional catalog loader. + /// Version-managed catalogs require catalog_identifier. + Result> NewSnapshotManager( + const CoreOptions& options, const std::shared_ptr& catalog, + const std::optional& catalog_identifier) const; + + Result> NewManifestList( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const; + + Result> NewManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& partition_schema, + const std::shared_ptr& memory_pool) const; + + Result> NewIndexManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const; + + Result> NewIndexFileHandler( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const; + + Result> NewPartitionComputer( + const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, + const std::shared_ptr& memory_pool) const; + + private: + FileStore(const std::string& root_path, CoreOptions core_options); + + std::shared_ptr file_system_; + std::string root_path_; + std::string branch_; + std::shared_ptr schema_manager_; + std::shared_ptr snapshot_manager_; + CoreOptions core_options_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp new file mode 100644 index 000000000..90fca78ee --- /dev/null +++ b/src/paimon/core/table/file_store_table.cpp @@ -0,0 +1,680 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/file_store_table.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" +#include "paimon/common/predicate/predicate_validator.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/binary_row_partition_computer.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/sensitive_config_utils.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/snapshot_commit.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/disk/io_manager.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/operation/commit/realtime_commit_properties.h" +#include "paimon/core/operation/expire_snapshots.h" +#include "paimon/core/operation/file_store_commit_impl.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/internal_read_context.h" +#include "paimon/core/options/table_type.h" +#include "paimon/core/realtime/realtime_context_impl.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/append_only_file_store_table.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/primary_key_file_store_table.h" +#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/source/data_evolution_batch_scan.h" +#include "paimon/core/table/source/data_table_batch_scan.h" +#include "paimon/core/table/source/data_table_stream_scan.h" +#include "paimon/core/table/source/fallback_table_read.h" +#include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/table/source/primary_key_index_batch_scan.h" +#include "paimon/core/table/source/read_optimized_scan_options.h" +#include "paimon/core/table/source/realtime_table_scan.h" +#include "paimon/core/table/source/snapshot/snapshot_reader.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/core/utils/tag_manager.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/format/file_format.h" +#include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/read_context.h" +#include "paimon/realtime/realtime_context.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +namespace { + +/// Allows object-store commits in integration tests. +constexpr const char kObjectStoreCommitInteTestOption[] = "enable-object-store-commit-in-inte-test"; + +} // namespace + +class Executor; +class MemoryPool; + +FileStoreTable::FileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid) + : Table(store->GetFileSystem(), store->GetRootPath(), schema, identifier, uuid), + store_(store) {} + +Result> FileStoreTable::Create( + const std::shared_ptr& specified_file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid, const std::string& branch, + const std::map& fs_scheme_map, + const std::map& dynamic_options) { + if (schema == nullptr) { + return Status::Invalid( + fmt::format("not found latest schema of {}", identifier.GetFullName())); + } + std::map options = schema->Options(); + const std::string normalized_branch = BranchManager::NormalizeBranch(branch); + if (BranchManager::IsMainBranch(normalized_branch)) { + options.erase(Options::BRANCH); + } else { + options[Options::BRANCH] = normalized_branch; + } + std::shared_ptr effective_schema = schema; + if (options != schema->Options()) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr normalized_schema, + schema->Copy(options)); + effective_schema = std::shared_ptr(std::move(normalized_schema)); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(options, specified_file_system, fs_scheme_map)); + std::shared_ptr store = FileStore::Create(root_path, std::move(core_options)); + std::shared_ptr table; + if (!schema->PrimaryKeys().empty()) { + table = + std::make_shared(store, effective_schema, identifier, uuid); + } else { + table = + std::make_shared(store, effective_schema, identifier, uuid); + } + table->dynamic_options_ = dynamic_options; + table->specified_file_system_ = specified_file_system; + table->fs_scheme_map_ = fs_scheme_map; + return table; +} + +Result> FileStoreTable::CopyTableSchema( + const std::shared_ptr& fs, const std::string& root_path, + const std::shared_ptr& schema, + const std::map& applied_options, + const std::map& dynamic_options) { + std::map overrides = dynamic_options; + auto type_option = overrides.find(Options::TYPE); + if (type_option != overrides.end()) { + PAIMON_ASSIGN_OR_RAISE(TableType schema_type, + TableTypeDefine::FromOptions(schema->Options())); + PAIMON_ASSIGN_OR_RAISE(TableType named_type, TableTypeDefine::FromOptions( + {{Options::TYPE, type_option->second}})); + if (named_type != schema_type) { + return Status::Invalid("Change 'type' is not supported yet."); + } + overrides.erase(type_option); + } + PAIMON_RETURN_NOT_OK(SchemaValidation::CheckCopyOptions(*schema, overrides)); + for (const std::string& key : {std::string("primary-key"), std::string("partition")}) { + if (schema->Options().count(key) == 0) { + // The immutable check accepts these only when they normalize to the schema keys. + overrides.erase(key); + } + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged, schema->MergeOptions(overrides)); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(merged->Options(), fs)); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateStartupMode(options)); + const std::string target_branch = BranchManager::NormalizeBranch(options.GetBranch()); + + // Snapshot lookup failures defer to scanning; a resolved snapshot with a missing schema + // fails Copy. + const std::shared_ptr resolved_fs = options.GetFileSystem(); + SnapshotManager snapshots(resolved_fs, root_path, target_branch); + auto resolve_snapshot = [&]() -> Result> { + const int32_t selectors = options.GetScanSnapshotId().has_value() + + options.GetScanTagName().has_value() + + options.GetScanTimestampMillis().has_value(); + if (selectors == 0) { + return std::optional(); + } + if (options.GetScanSnapshotId()) { + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, + snapshots.LoadSnapshot(*options.GetScanSnapshotId())); + return std::optional(std::move(snapshot)); + } + if (options.GetScanTagName()) { + TagManager tags(resolved_fs, root_path, target_branch); + PAIMON_ASSIGN_OR_RAISE(Tag tag, tags.GetOrThrow(*options.GetScanTagName())); + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, tag.TrimToSnapshot()); + return std::optional(std::move(snapshot)); + } + return snapshots.EarlierOrEqualTimeMillis(*options.GetScanTimestampMillis()); + }; + auto snapshot = resolve_snapshot(); + if (snapshot.ok() && snapshot.value()) { + SchemaManager schemas(resolved_fs, root_path, target_branch); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical, + schemas.ReadSchema(snapshot.value()->SchemaId())); + auto historical_options = merged->Options(); + // Restore historical column declarations and aliases unless explicitly overridden. + for (const std::vector& keys : + {std::vector{"vector-field"}, + {Options::BLOB_FIELD}, + {Options::BLOB_DESCRIPTOR_FIELD, Options::FALLBACK_BLOB_DESCRIPTOR_FIELD}, + {Options::BLOB_VIEW_FIELD}, + {"video-frame-field"}}) { + bool explicitly_set = false; + for (const auto& key : keys) { + explicitly_set |= applied_options.count(key) != 0 || overrides.count(key) != 0; + } + if (explicitly_set) { + continue; + } + for (const auto& key : keys) { + historical_options.erase(key); + auto value = historical->Options().find(key); + if (value != historical->Options().end()) { + historical_options.emplace(*value); + } + } + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr historical_copy, + historical->Copy(historical_options)); + merged = std::move(historical_copy); + } + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateTableSchema(*merged)); + return std::shared_ptr(std::move(merged)); +} + +Result> FileStoreTable::Copy( + const std::map& dynamic_options) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr copied_schema, + CopyTableSchema(file_system_, root_path_, std::static_pointer_cast(schema_), + dynamic_options_, dynamic_options)); + auto accumulated = dynamic_options_; + for (const auto& [key, value] : dynamic_options) { + if (key != Options::TYPE) { + accumulated.insert_or_assign(key, value); + } + } + auto branch = copied_schema->Options().find(Options::BRANCH); + const std::string target_branch = branch == copied_schema->Options().end() + ? BranchManager::DEFAULT_MAIN_BRANCH + : BranchManager::NormalizeBranch(branch->second); + return FileStoreTable::Create(specified_file_system_, root_path_, copied_schema, identifier_, + CatalogUuid().value_or(""), target_branch, fs_scheme_map_, + accumulated); +} + +Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { + const std::string target_branch = BranchManager::NormalizeBranch(branch); + if (target_branch == CurrentBranch()) { + return FileStoreTable::Create(specified_file_system_, root_path_, + std::static_pointer_cast(schema_), identifier_, + CatalogUuid().value_or(""), CurrentBranch(), fs_scheme_map_, + dynamic_options_); + } + SchemaManager schema_manager(file_system_, root_path_, target_branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (!latest) { + return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); + } + std::map overridable; + if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { + overridable[Options::BRANCH] = target_branch; + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, + (*latest)->MergeOptions(overridable)); + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + CatalogUtils::BranchIdentifier(identifier_, target_branch)); + return FileStoreTable::Create(specified_file_system_, root_path_, + std::shared_ptr(std::move(branch_schema)), + branch_identifier, CatalogUuid().value_or(""), target_branch, + fs_scheme_map_, /*dynamic_options=*/{}); +} + +Status FileStoreTable::RestoreRealtimeCommittedProgress( + const std::shared_ptr& realtime_context, + const std::shared_ptr& snapshot_manager, const CoreOptions& options) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, + RealtimeContextImpl::Cast(realtime_context)); + PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_ASSIGN_OR_RAISE( + RealtimeOffsetMap realtime_committed_offsets, + RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); + PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( + latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); + } + return Status::OK(); +} + +Status FileStoreTable::CheckContextNamesThisTable( + const std::string& kind, const std::string& context_root_path, + const std::optional& context_identifier, + const std::optional& context_table_id) const { + if (!root_path_.empty()) { + PAIMON_ASSIGN_OR_RAISE(std::string context_path, + PathUtil::NormalizePath(context_root_path)); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, PathUtil::NormalizePath(root_path_)); + if (context_path != table_path) { + return Status::Invalid(fmt::format( + "the {} context names root path '{}' but this table serves '{}': the store below " + "reads and writes this table's files, so a context naming another path would " + "leave this table's metadata there and publish it as the other's", + kind, context_root_path, root_path_)); + } + } + if (context_identifier) { + PAIMON_ASSIGN_OR_RAISE(std::optional context_branch, + context_identifier.value().GetBranchName()); + if (context_branch && context_branch.value() != CurrentBranch()) { + return Status::Invalid(fmt::format( + "the {} context names branch '{}' of '{}' but this table serves branch '{}', " + "which is the one a {} of this table writes and publishes", + kind, context_branch.value(), context_identifier.value().ToString(), + CurrentBranch(), kind)); + } + PAIMON_ASSIGN_OR_RAISE( + Identifier context_table, + CatalogUtils::BranchIdentifier(context_identifier.value(), CurrentBranch())); + PAIMON_ASSIGN_OR_RAISE(Identifier served_table, + CatalogUtils::BranchIdentifier(identifier_, CurrentBranch())); + const bool names_this_table = + (served_table.GetDatabaseName() == Identifier::kUnknownDatabase || + context_table.GetDatabaseName() == served_table.GetDatabaseName()) && + context_table.GetObjectName() == served_table.GetObjectName(); + if (!names_this_table) { + return Status::Invalid(fmt::format( + "the {} context names table '{}' but this table is '{}': the store below serves " + "this table's files, so a context naming another table would write this table's " + "metadata and publish it as the other's", + kind, context_identifier.value().ToString(), identifier_.ToString())); + } + } + if (context_table_id && context_table_id != CatalogUuid()) { + return Status::Invalid(fmt::format( + "the {} context carries table id '{}' but this table's is '{}': the id tells a " + "recreated table from the one the changes were prepared for, so one this table does " + "not carry would publish this table's metadata as the recreation's", + kind, context_table_id.value(), CatalogUuid().value_or("none"))); + } + return Status::OK(); +} + +std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr&& scan, + const std::string& root_path) { + scan->WithTablePath(root_path); + return std::move(scan); +} + +Result> FileStoreTable::NewWrite( + std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("write", ctx->GetRootPath(), + ctx->GetIdentifier(), + /*context_table_id=*/std::nullopt)); + if (AsVersionManaged(ctx->GetCatalog()) != nullptr) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && + ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); + std::shared_ptr schema = std::static_pointer_cast(schema_); + CoreOptions options = store_->GetOptions(); + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); + if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) { + return Status::NotImplemented( + "Dedicated VECTOR storage does not yet support custom write schemas or partial writes"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr file_store_path_factory, + store_->NewPathFactory(arrow_schema, schema->PartitionKeys(), options, + options.GetWriteFileFormat(/*level=*/0)->Identifier(), + ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr snapshot_manager, + store_->NewSnapshotManager(options, ctx->GetCatalog(), catalog_identifier)); + std::shared_ptr io_manager; + const auto& io_temp_dir = ctx->GetTempDirectory(); + if (!io_temp_dir.empty()) { + io_manager = std::make_shared(io_temp_dir, options.GetFileSystem()); + } + + bool ignore_previous_files = ctx->IgnorePreviousFiles(); + if (ctx->GetRealtimeContext() && !options.RealtimeEnabled()) { + return Status::Invalid("real-time write requires realtime.enabled=true"); + } + if (ctx->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, + RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); + PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); + } + return CreateWrite(*ctx, arrow_schema, partition_schema, options, file_store_path_factory, + snapshot_manager, io_manager, ignore_previous_files); +} + +Result> FileStoreTable::NewScan( + std::unique_ptr context) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + PAIMON_ASSIGN_OR_RAISE(bool read_optimized, + OptionsUtils::GetValueFromMap(store_->GetOptions().ToMap(), + kReadOptimizedScanOption, false)); + CoreOptions core_options = store_->GetOptions(); + core_options.WithCache(context->GetCache()); + + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); + + PAIMON_RETURN_NOT_OK( + RealtimeTableScan::ValidateScan(*table_schema, core_options, *context, read_optimized)); + if (core_options.GetBucket() == -1) { + if (!table_schema->PrimaryKeys().empty()) { + return Status::NotImplemented(fmt::format( + "do not support pk table bucket={} in scan process", core_options.GetBucket())); + } + } else if (core_options.GetBucket() < 1 && + !SchemaValidation::IsPostponeBucketTable(*table_schema, core_options.GetBucket())) { + return Status::Invalid( + fmt::format("do not support bucket={} in scan process", core_options.GetBucket())); + } + + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { + PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( + *arrow_schema, context->GetScanFilters()->GetPredicate(), + /*validate_field_idx=*/false)); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + store_->NewPathFactory(arrow_schema, table_schema->PartitionKeys(), core_options, + core_options.GetFileFormat()->Identifier(), + context->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_list, + store_->NewManifestList(core_options, path_factory, context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_file, + store_->NewManifestFile(core_options, path_factory, partition_schema, + context->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr file_store_scan, + NewFileStoreScan(manifest_list, manifest_file, arrow_schema, core_options, + context->GetExecutor(), context->GetMemoryPool(), context.get())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr split_generator, + NewSplitGenerator(core_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_file_handler, + store_->NewIndexFileHandler(core_options, path_factory, context->GetMemoryPool())); + auto snapshot_reader = std::make_shared( + file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler)); + const bool pk_table = !table_schema->PrimaryKeys().empty(); + if (read_optimized && pk_table && context->IsStreamingMode()) { + return Status::NotImplemented( + "read-optimized system table does not support streaming scan for primary key table"); + } + if (context->IsStreamingMode()) { + return std::make_unique(core_options, snapshot_reader); + } + auto batch_scan = std::make_unique( + /*pk_table=*/pk_table, core_options, snapshot_reader, read_optimized, context->GetLimit(), + /*realtime_pk_scan=*/pk_table && context->GetRealtimeContext() != nullptr); + if (context->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, + RealtimeContextImpl::Cast(context->GetRealtimeContext())); + return std::make_unique( + std::move(batch_scan), pk_table, realtime_context, path_factory, + snapshot_reader->GetSnapshotManager(), core_options.GetFileSystem(), + context->GetScanFilters(), core_options.GetRealtimeReadViewTtlMillis()); + } + if (core_options.DataEvolutionEnabled()) { + return std::make_unique( + root_path_, snapshot_reader, std::move(batch_scan), table_schema, + context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), + context->GetExecutor()); + } + if (pk_table && !read_optimized && core_options.GlobalIndexEnabled()) { + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*table_schema)); + if (!definitions.ScalarDefinitions().empty()) { + return PrimaryKeyIndexBatchScan::Create(snapshot_reader, std::move(batch_scan), + table_schema, path_factory, core_options, + context->GetMemoryPool()); + } + } + return batch_scan; +} + +Result> FileStoreTable::CreateTableRead( + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const { + if (PredicateUtils::ContainsFullTextMatch(internal_context->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } + if (!internal_context->GetIndexScoreBindings().empty()) { + return Status::NotImplemented("Per-field index score output is not implemented yet"); + } + const auto& core_options = internal_context->GetCoreOptions(); + const auto& table_schema = internal_context->GetTableSchema(); + if (internal_context->GetRealtimeContext() && !core_options.RealtimeEnabled()) { + return Status::Invalid("real-time read requires realtime.enabled=true"); + } + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + store_->NewPathFactory(arrow_schema, table_schema->PartitionKeys(), core_options, + core_options.GetFileFormat()->Identifier(), memory_pool)); + if (internal_context->GetPrimaryKeys().empty()) { + return std::make_unique(path_factory, internal_context, memory_pool, + executor); + } + return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); +} + +Result> FileStoreTable::NewRead( + std::unique_ptr context) const { + std::shared_ptr read_context = std::move(context); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); + std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr internal_context, + InternalReadContext::CreateForTable(read_context, loaded_schema, store_->GetOptions(), + root_path_, dynamic_options_, fs_scheme_map_)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + CreateTableRead(internal_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); + + std::optional scan_fallback_branch = + internal_context->GetCoreOptions().GetScanFallbackBranch(); + if (!scan_fallback_branch || + StringUtils::IsNullOrWhitespaceOnly(scan_fallback_branch.value())) { + return std::move(table_read); + } + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fallback_context, + InternalReadContext::CreateForFallbackBranch( + read_context, /*branch=*/scan_fallback_branch.value(), file_system_, + root_path_, dynamic_options_, fs_scheme_map_)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, + CreateTableRead(fallback_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); + const std::shared_ptr& arrow_pool = internal_context->GetArrowMemoryPool(); + return std::make_unique(std::move(table_read), + std::move(fallback_table_read), arrow_pool); +} + +Result> FileStoreTable::NewCommit( + std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("commit", ctx->GetRootPath(), + ctx->GetIdentifier(), ctx->GetTableId())); + if (ctx->GetCatalog() != nullptr || ctx->UseRESTCatalogCommit()) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && + ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); + FileStoreCommitImpl::SchemaIdLoader schema_id_loader; + if (ctx->GetCatalog() != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + std::shared_ptr catalog = ctx->GetCatalog(); + Identifier identifier = catalog_identifier.value(); + schema_id_loader = [catalog, identifier]() -> Result { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + catalog->LoadTableSchema(identifier)); + std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); + if (data_schema == nullptr) { + return Status::Invalid( + "the catalog did not hand back a data table schema, which has the id a " + "snapshot records"); + } + return data_schema->Id(); + }; + } + std::shared_ptr schema = std::static_pointer_cast(schema_); + std::shared_ptr arrow_schema = + DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); + CoreOptions options = store_->GetOptions(); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); + assert(options.GetFileSystem()); + assert(options.GetFileFormat()); + PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); + PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); + if (is_object_store && !ctx->UseRESTCatalogCommit() && + AsVersionManaged(ctx->GetCatalog()) == nullptr && + options.ToMap().count(kObjectStoreCommitInteTestOption) == 0) { + return Status::NotImplemented( + "commit operation does not support object store file system for now"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr partition_computer, + store_->NewPartitionComputer(arrow_schema, schema->PartitionKeys(), + options, ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + store_->NewPathFactory(arrow_schema, schema->PartitionKeys(), options, + options.GetFileFormat()->Identifier(), ctx->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr snapshot_manager, + store_->NewSnapshotManager(options, ctx->GetCatalog(), catalog_identifier)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, + store_->NewManifestList(options, path_factory, ctx->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_file, + store_->NewManifestFile(options, path_factory, partition_schema, ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr index_manifest_file, + store_->NewIndexManifestFile(options, path_factory, ctx->GetMemoryPool())); + + auto expire_snapshots = std::make_shared( + snapshot_manager, path_factory, manifest_list, manifest_file, index_manifest_file, + options.GetFileSystem(), options.GetExpireConfig(), options.RealtimeEnabled(), + ctx->GetExecutor()); + + CommitScanner::ScanSupplier scan_supplier = + NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, arrow_schema, options, + ctx->GetExecutor(), ctx->GetMemoryPool()); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr snapshot_commit, + SnapshotCommit::Create(ctx->GetCatalog(), catalog_identifier, options, ctx->GetTableId(), + ctx->GetCommitUser(), ctx->UseRESTCatalogCommit(), + options.GetFileSystem(), snapshot_manager)); + + return std::make_unique( + ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path_, ctx->GetCommitUser(), + options, path_factory, std::move(partition_computer), snapshot_manager, snapshot_commit, + ctx->IgnoreEmptyCommit(), ctx->AppendCommitCheckConflict(), schema, + std::move(schema_id_loader), manifest_file, manifest_list, index_manifest_file, + expire_snapshots, GetSchemaManager(), std::move(scan_supplier)); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h new file mode 100644 index 000000000..f6b403a62 --- /dev/null +++ b/src/paimon/core/table/file_store_table.h @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class InternalReadContext; +class IOManager; +class ManifestFile; +class ManifestList; +class RealtimeContext; +class SplitGenerator; + +/// Managed table backed by snapshots and manifests. +/// Shares operator setup across append-only and primary-key tables, selected from the schema. +class FileStoreTable : public Table { + public: + /// Creates an append-only or primary-key table according to the schema's primary keys. + /// @param specified_file_system Explicit file system, overriding option-based resolution. + /// @param schema Loaded table schema; must not be null. + /// @param branch Branch used by all operators and metadata managers. + /// @param fs_scheme_map URI scheme routing for option-based file system resolution. + /// @param dynamic_options Overrides accumulated by Copy and reused for fallback reads. + static Result> Create( + const std::shared_ptr& specified_file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid, const std::string& branch, + const std::map& fs_scheme_map, + const std::map& dynamic_options); + + const std::string& CurrentBranch() const { + return store_->CurrentBranch(); + } + + const std::shared_ptr& GetSchemaManager() const { + return store_->GetSchemaManager(); + } + + const std::shared_ptr& GetSnapshotManager() const { + return store_->GetSnapshotManager(); + } + + std::string Name() const override { + return identifier_.GetObjectName(); + } + + std::string FullName() const override { + return identifier_.GetFullName(); + } + + /// Applies dynamic options to the current schema, retaining the table's identifier. Time + /// travel selects the schema of the requested snapshot. Use SwitchToBranch to load another + /// branch's latest schema. + /// Snapshot/tag lookup and historical schema loading may perform metadata I/O. + Result> Copy( + const std::map& dynamic_options) const override; + + /// Shared schema-copy semantics for managed tables and their system-table views. + static Result> CopyTableSchema( + const std::shared_ptr& fs, const std::string& root_path, + const std::shared_ptr& schema, + const std::map& applied_options, + const std::map& dynamic_options); + + /// Loads the latest schema from branch and discards this table's dynamic options. + /// Switching to the current branch preserves its effective schema and options. + Result> SwitchToBranch(const std::string& branch) const; + + Result> NewScan(std::unique_ptr context) const override; + + Result> NewRead(std::unique_ptr context) const override; + + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; + + private: + /// Rejects paths, identifiers or UUIDs that contradict this table's identity. + /// Missing context fields are allowed. + /// @param kind Operation name used in errors. + /// @param context_table_id Catalog commit UUID; nullopt for writes. + Status CheckContextNamesThisTable(const std::string& kind, const std::string& context_root_path, + const std::optional& context_identifier, + const std::optional& context_table_id) const; + + /// Creates a reader for the internal context's schema and primary keys. + Result> CreateTableRead( + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const; + + protected: + FileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid); + + /// Restores real-time write progress from the latest snapshot. + static Status RestoreRealtimeCommittedProgress( + const std::shared_ptr& realtime_context, + const std::shared_ptr& snapshot_manager, const CoreOptions& options); + + static std::unique_ptr WithTablePath(std::unique_ptr&& scan, + const std::string& root_path); + + /// Creates the table-specific writer. + /// @param io_manager Optional spill manager. + /// @param ignore_previous_files Whether to start over instead of appending. + virtual Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; + + virtual Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const = 0; + + virtual Result> NewSplitGenerator( + const CoreOptions& core_options) const = 0; + + virtual CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const = 0; + + std::shared_ptr store_; + + /// Accumulated overrides kept separately so fallback reads can apply them to another schema. + std::map dynamic_options_; + + /// Original explicit file system, or null for option-based resolution; reused by Copy and + /// SwitchToBranch. + std::shared_ptr specified_file_system_; + + /// Original URI scheme routing, reused by Copy and SwitchToBranch. + std::map fs_scheme_map_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp new file mode 100644 index 000000000..e99b3df78 --- /dev/null +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/file_store_table_factory.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" +#include "paimon/commit_context.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/defs.h" +#include "paimon/fs/file_system.h" +#include "paimon/read_context.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { + +namespace { + +Result> ManagedTableOf( + const std::shared_ptr& file_system, + const std::shared_ptr& explicit_file_system, const std::string& path, + const std::string& branch, const std::map& options, + const std::optional& schema_json, const std::optional& identifier, + const std::string& uuid, const std::map& fs_scheme_map) { + SchemaManager schema_manager(file_system, path, branch); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { + PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); + } else { + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; + } + } + if (latest_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema in branch {}", + BranchManager::NormalizeBranch(branch))); + } + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(latest_schema->Options())); + if (is_format_table) { + return Status::Invalid(fmt::format( + "the schema under {} names a format table, which is not loaded from a path: a " + "catalog hands the table back through GetTable, and its operators come from " + "table->NewScan/NewRead/NewWrite/NewCommit on it", + path)); + } + Identifier table_identifier = + identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)); + + std::map dynamic_options = options; + dynamic_options.erase(Options::TYPE); + dynamic_options.erase(Options::BRANCH); + PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*latest_schema, dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, + latest_schema->MergeOptions(dynamic_options)); + return FileStoreTable::Create(explicit_file_system, path, + std::shared_ptr(std::move(merged_schema)), + table_identifier, uuid, branch, fs_scheme_map, dynamic_options); +} + +Result> CatalogMainBranchSchemaJson( + const std::shared_ptr& catalog, const std::optional& catalog_identifier, + const std::string& branch, const char* kind) { + if (catalog == nullptr) { + return std::optional(); + } + if (!catalog_identifier) { + return Status::Invalid(fmt::format("a catalog {} requires a table identifier", kind)); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + catalog->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(std::string schema_json, schema->GetJsonSchema()); + return std::optional(std::move(schema_json)); + } + return std::optional(); +} + +template +Result> ManagedTable(const Context& ctx, + const std::map& options, + const char* kind, bool catalog_write) { + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + if (ctx.GetCatalog() && !ctx.GetIdentifier()) { + return Status::Invalid(fmt::format("a catalog {} requires a table identifier", kind)); + } + PAIMON_ASSIGN_OR_RAISE(std::string branch, + BranchManager::ResolveBranch(ctx.GetIdentifier(), options, kind)); + if (catalog_write && ctx.GetCatalog() && !BranchManager::IsMainBranch(branch)) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); + PAIMON_ASSIGN_OR_RAISE(std::optional identifier_branch, + ctx.GetIdentifier()->GetBranchName()); + if (!identifier_branch) { + return Status::Invalid( + fmt::format("a {} through a catalog addresses a branch by the table identifier; " + "name branch '{}' in the identifier", + kind, branch)); + } + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_system, + ContextUtils::ResolveFileSystemForLoad(ctx)); + PAIMON_ASSIGN_OR_RAISE( + CoreOptions tmp_options, + CoreOptions::FromMap(options, file_system, ctx.GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + return Status::Invalid(fmt::format( + "the path {} names a system table, which is not loaded from a path: a catalog " + "hands the table back through GetTable, and its operators come from " + "table->NewScan/NewRead/NewWrite/NewCommit on it", + ctx.GetPath())); + } + std::optional catalog_table_schema = ctx.GetSpecificTableSchema(); + if (!catalog_table_schema) { + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), branch)); + PAIMON_ASSIGN_OR_RAISE( + catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, kind)); + } + return ManagedTableOf(tmp_options.GetFileSystem(), file_system, ctx.GetPath(), branch, options, + catalog_table_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or(""), + ctx.GetFileSystemSchemeToIdentifierMap()); +} + +} // namespace + +Result> FileStoreTableFactory::Create( + const ScanContext& ctx, const std::map& options) { + return ManagedTable(ctx, options, "scan", false); +} + +Result> FileStoreTableFactory::Create( + const ReadContext& ctx, const std::map& options) { + return ManagedTable(ctx, options, "read", false); +} + +Result> FileStoreTableFactory::Create( + const WriteContext& ctx, const std::map& options) { + return ManagedTable(ctx, options, "write", true); +} + +Result> FileStoreTableFactory::Create( + const CommitContext& ctx, const std::map& options) { + return ManagedTable(ctx, options, "commit", true); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table_factory.h b/src/paimon/core/table/file_store_table_factory.h new file mode 100644 index 000000000..480a09418 --- /dev/null +++ b/src/paimon/core/table/file_store_table_factory.h @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/result.h" + +namespace paimon { + +class CommitContext; +class ReadContext; +class ScanContext; +class Table; +class WriteContext; + +/// Loads managed tables for the static operator Create methods. +/// System and format tables must be loaded through Catalog::GetTable(). +class FileStoreTableFactory { + public: + FileStoreTableFactory() = delete; + ~FileStoreTableFactory() = delete; + + static Result> Create(const ScanContext& context, + const std::map& options); + + static Result> Create(const ReadContext& context, + const std::map& options); + + static Result> Create(const WriteContext& context, + const std::map& options); + + static Result> Create(const CommitContext& context, + const std::map& options); +}; + +} // namespace paimon diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index a039baa4b..4340ac145 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -18,9 +18,15 @@ #include "paimon/table/format/format_table.h" +#include +#include #include +#include +#include +#include #include "fmt/format.h" +#include "paimon/commit_context.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/string_utils.h" @@ -29,8 +35,22 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/format/format_table_file_store_commit.h" +#include "paimon/core/table/format/format_table_file_store_write.h" +#include "paimon/core/table/format/format_table_read.h" +#include "paimon/core/table/format/format_table_scan.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/status.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" namespace paimon { @@ -45,10 +65,7 @@ FormatTable::FormatTable(const std::shared_ptr& file_system, const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata) - : file_system_(file_system), - location_(location), - identifier_(identifier), - schema_(schema), + : Table(file_system, location, schema, identifier, /*uuid=*/""), options_(options), format_(format), file_compression_(file_compression), @@ -177,35 +194,159 @@ Result> FormatTable::Create( core_options.FormatTablePartitionOnlyValueInPath(), location_carries_paimon_metadata)); } -Result> FormatTable::Copy( - const std::shared_ptr& table, - const std::map& dynamic_options) { - if (table == nullptr) { - return Status::Invalid("format table requires a table to copy"); - } +Result> FormatTable::CopySelf( + const std::map& dynamic_options) const { if (dynamic_options.empty()) { - return table; + return SharedSelf(); } // The options this table already carries go back in as dynamic ones, so an option an earlier // call set is not lost to the schema's own value underneath it. - std::map options = table->options_; + std::map options = options_; for (const auto& [key, value] : dynamic_options) { options[key] = value; } - return Create(table->file_system_, table->location_, table->identifier_, table->schema_, - table->location_carries_paimon_metadata_, options); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr copied, + Create(file_system_, root_path_, identifier_, std::static_pointer_cast(schema_), + location_carries_paimon_metadata_, options)); + // Java merges file.format into the option map but retains the physical format of the table. + copied->format_ = format_; + return copied; } -const std::vector& FormatTable::PartitionKeys() const { - return schema_->PartitionKeys(); +Result> FormatTable::Copy( + const std::map& dynamic_options) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr copied, CopySelf(dynamic_options)); + return std::shared_ptr
(std::move(copied)); } -std::string FormatTable::FullName() const { - return identifier_.GetFullName(); +const std::vector& FormatTable::PartitionKeys() const { + return GetSchema()->PartitionKeys(); } Result> FormatTable::GetArrowSchema() const { return schema_->GetArrowSchema(); } +Result> FormatTable::NewScan( + std::unique_ptr context) const { + std::shared_ptr shared_context = std::move(context); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(shared_context, "scan")); + std::shared_ptr table = SharedSelf(); + if (shared_context->IsStreamingMode()) { + return Status::Invalid( + "a format table has no snapshots, so there is nothing for a streaming scan to follow"); + } + if (shared_context->GetRealtimeContext() != nullptr) { + return Status::Invalid( + "a format table has no real-time store to union with what is on disk"); + } + if (shared_context->GetGlobalIndexResult() != nullptr) { + return Status::Invalid("a format table carries no global index"); + } + std::map partition_filter; + std::shared_ptr filters = shared_context->GetScanFilters(); + if (filters != nullptr) { + if (filters->GetPredicate() != nullptr) { + return Status::NotImplemented( + "a format table scan does not take a predicate: its files carry no statistics to " + "skip by, and pruning by the partition columns a predicate names is not " + "implemented yet; give the partition values as a partition filter instead"); + } + if (filters->GetBucketFilter()) { + return Status::Invalid("a format table has no buckets to filter by"); + } + const std::vector>& partition_filters = + filters->GetPartitionFilters(); + if (partition_filters.size() > 1) { + return Status::Invalid("a format table scan takes at most one partition filter"); + } + if (partition_filters.size() == 1) { + partition_filter = partition_filters.front(); + } + } + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + FormatTableScan::Create(table, partition_filter, shared_context->GetLimit())); + return std::unique_ptr(std::move(scan)); +} + +Result> FormatTable::NewRead( + std::unique_ptr context) const { + std::shared_ptr read_context = std::move(context); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); + if (read_context->HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch is not supported for format tables"); + } + if (PredicateUtils::ContainsFullTextMatch(read_context->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + FormatTableRead::Create(SharedSelf(), read_context)); + return std::unique_ptr(std::move(read)); +} + +Result> FormatTable::NewWrite( + std::unique_ptr context) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "write")); + std::shared_ptr table = SharedSelf(); + if (context->GetCatalog() != nullptr) { + return Status::Invalid( + "WithCatalog() requires a native table; a format table takes a WriteContextBuilder " + "built from its location instead"); + } + if (context->IsStreamingMode()) { + return Status::Invalid( + "a format table has no snapshots, so there is nothing a streaming write could " + "commit against"); + } + if (context->GetRealtimeContext() != nullptr) { + return Status::Invalid("a format table has no real-time store to write into"); + } + if (!context->GetWriteSchema().empty()) { + return Status::NotImplemented( + "a format table write takes the table's own columns; a write schema naming a " + "subset of them is not supported yet"); + } + if (context->GetWriteId().has_value()) { + return Status::Invalid("a format table has no buckets, so a write id would name nothing"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, + FormatTableFileStoreWrite::Create(table, context->GetMemoryPool())); + return std::unique_ptr(std::move(format_write)); +} + +Result> FormatTable::NewCommit( + std::unique_ptr context) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "commit")); + std::shared_ptr given_table = SharedSelf(); + if (!context->IgnoreEmptyCommit()) { + return Status::Invalid( + "a format table cannot record an empty commit: keeping one means writing a " + "snapshot that adds no files, and there are no snapshots here"); + } + if (context->UseRESTCatalogCommit()) { + return Status::Invalid( + "a format table commits by renaming files into place, not by sending a snapshot " + "to a rest catalog"); + } + if (context->GetTableId()) { + return Status::Invalid( + "a format table sends no commit table request, so there is nowhere to put a table id"); + } + if (context->GetCatalog() != nullptr) { + return Status::Invalid( + "a format table keeps no snapshot for a catalog to take, so it commits by renaming " + "files into place rather than through the catalog it was loaded from"); + } + if (context->AppendCommitCheckConflict()) { + return Status::Invalid( + "a format table has no manifests to check a concurrent commit against"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, + FormatTableFileStoreCommit::Create(given_table)); + return std::unique_ptr(std::move(format_commit)); +} + } // namespace paimon diff --git a/src/paimon/core/table/format/format_table_loader.cpp b/src/paimon/core/table/format/format_table_loader.cpp deleted file mode 100644 index 797bc3351..000000000 --- a/src/paimon/core/table/format/format_table_loader.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "paimon/core/table/format/format_table_loader.h" - -#include -#include - -#include "fmt/format.h" -#include "paimon/catalog/identifier.h" -#include "paimon/common/utils/checked_cast.h" -#include "paimon/common/utils/path_util.h" -#include "paimon/core/options/table_type.h" -#include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" - -namespace paimon { - -Result> FormatTableLoader::TryLoad( - const std::shared_ptr& file_system, const std::string& table_path, - const std::string& branch, const std::map& options, - const std::optional& specific_table_schema, const SchemaManager* schema_manager, - std::shared_ptr* table_schema_out) { - assert(table_schema_out != nullptr); - table_schema_out->reset(); - - std::shared_ptr table_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - // Through the caller's manager when it has one, so that the read warms the cache it goes - // on to use rather than a cache that dies with this call. - SchemaManager own_schema_manager(file_system, table_path, branch); - const SchemaManager& reader = - schema_manager != nullptr ? *schema_manager : own_schema_manager; - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - reader.Latest()); - if (!latest_schema) { - return std::shared_ptr(); - } - table_schema = *latest_schema; - } - *table_schema_out = table_schema; - - // From the schema alone: see `FormatTable::Create()`. - PAIMON_ASSIGN_OR_RAISE(bool is_format_table, - TableTypeDefine::IsFormatTable(table_schema->Options())); - if (!is_format_table) { - return std::shared_ptr(); - } - // A format table has no branches, so it is refused rather than served from the table path - // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. - if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { - return Status::Invalid( - fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", - table_path, branch)); - } - // A context names a path, never a catalog identifier, so the table is known by the directory - // it sits in. - Identifier identifier(Identifier::kUnknownDatabase, PathUtil::GetName(table_path)); - return FormatTable::Create(file_system, table_path, identifier, - checked_pointer_cast(table_schema), - /*location_carries_paimon_metadata=*/true, options); -} - -} // namespace paimon diff --git a/src/paimon/core/table/format/format_table_loader.h b/src/paimon/core/table/format/format_table_loader.h deleted file mode 100644 index 290ee94a7..000000000 --- a/src/paimon/core/table/format/format_table_loader.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include - -#include "paimon/result.h" - -namespace paimon { - -class FileSystem; -class FormatTable; -class SchemaManager; -class TableSchema; - -/// Recognises a format table behind a table path. -/// -/// The scan, read, write and commit entry points all take a path and have to answer the same -/// question before they can dispatch, so one place answers it and they cannot disagree. -class FormatTableLoader { - public: - FormatTableLoader() = delete; - ~FormatTableLoader() = delete; - - /// Loads `table_path` as a format table, or returns null when it is not one. - /// - /// Null is also the answer when there is no schema under the path at all: what to say about a - /// table that is not there is the managed path's to decide, and it says it in more detail. - /// - /// The table type is read from the schema, never from `options`: `type` is structural, and one - /// call's options must not decide what kind of table this is. - /// - /// @param branch Branch the schema is read from. Take it from the same place the managed path - /// in the same entry point takes it, or the two dispatch on different schemas. Anything - /// but the main branch is refused once the schema turns out to be a format table's, - /// which has no branches to read. - /// @param options Options given at the call, which win over the ones the schema stored. - /// @param specific_table_schema A schema the caller already holds, as json, which saves - /// reading one from under the path. Used only on the main branch, as on the managed - /// read path. - /// @param schema_manager The manager to read the schema through, or null to use one of this - /// call's own. A caller that has one must pass it: a manager caches what it read. - /// @param table_schema_out Set to the schema this read, or null when there is none, so that a - /// caller falling back to the managed path does not read it twice. - static Result> TryLoad( - const std::shared_ptr& file_system, const std::string& table_path, - const std::string& branch, const std::map& options, - const std::optional& specific_table_schema, - const SchemaManager* schema_manager, std::shared_ptr* table_schema_out); -}; - -} // namespace paimon diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index e8391f68c..f77887ba5 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -34,6 +34,7 @@ #include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/cache/cache.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" #include "paimon/common/table/special_fields.h" @@ -58,6 +59,7 @@ #include "paimon/memory/memory_pool.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/record_batch.h" #include "paimon/scan_context.h" @@ -752,48 +754,24 @@ TEST(FormatTableTest, TestCatalogManagedPartitionsAreRejected) { // And the same refusal when it arrives at the call rather than in the schema. Validating the // schema's own options alone would let this one through and then drop it, leaving a caller // who asked for catalog-managed partitions with a scan that read the directories anyway. - Result> dynamic_table = - FormatTable::Create(off_dir->GetFileSystem(), off_dir->Str(), Identifier("db", "tbl"), - off_table->LatestSchema(), - /*location_carries_paimon_metadata=*/true, - {{Options::METASTORE_PARTITIONED_TABLE, "true"}}); + Result> dynamic_table = FormatTable::Create( + off_dir->GetFileSystem(), off_dir->Str(), Identifier("db", "tbl"), off_table->GetSchema(), + /*location_carries_paimon_metadata=*/true, + {{Options::METASTORE_PARTITIONED_TABLE, "true"}}); ASSERT_FALSE(dynamic_table.ok()); ASSERT_TRUE(dynamic_table.status().IsNotImplemented()) << dynamic_table.status().ToString(); } -TEST(FormatTableTest, TestTheGenericEntryPointsValidateOptionsGivenAtTheCall) { - // An option a format table refuses has to be refused wherever it comes from. Every generic - // entry point merges what the call gave it over what the schema stored, so each runs the same - // checks over the merged result; otherwise a caller could set through a context what the - // schema would have rejected and have it silently dropped. - std::unique_ptr dir = UniqueTestDirectory::Create(); +TEST(FormatTableTest, TestCopyValidatesDynamicOptions) { + auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - const std::map refused = { - {Options::METASTORE_PARTITIONED_TABLE, "true"}}; - - ScanContextBuilder scan_builder(dir->Str()); - scan_builder.SetOptions(refused); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "metastore.partitioned-table"); - - ReadContextBuilder read_builder(dir->Str()); - read_builder.SetOptions(refused); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "metastore.partitioned-table"); - - WriteContextBuilder write_builder(dir->Str(), "test-user"); - write_builder.SetOptions(refused); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), - "metastore.partitioned-table"); - - CommitContextBuilder commit_builder(dir->Str(), "test-user"); - commit_builder.SetOptions(refused); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), - "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->Copy({{Options::METASTORE_PARTITIONED_TABLE, "true"}}), + "is not supported by paimon-cpp yet"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK(table->NewScan(std::move(context)).status()); } TEST(FormatTableTest, TestValueOnlyPartitionCannotBeNamedAfterThisTablesMetadata) { @@ -2665,19 +2643,16 @@ TEST(FormatTableTest, TestScanRejectsUnknownPartitionField) { ASSERT_TRUE(scan.status().IsInvalid()); } -TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { - // A caller holding a table path uses the interfaces it uses for every other table. Each one - // recognises a format table from its schema and dispatches to it, the way Java Paimon serves - // both kinds through one `ReadBuilder` and one `BatchWriteBuilder`. +TEST(FormatTableTest, TestAFormatTableIsWrittenCommittedAndReadBack) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); WriteContextBuilder write_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + table->NewWrite(std::move(write_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({1, 2}, {"alice", "bob"}, "20240101", {{"dt", "20240101"}})); ASSERT_OK(write->Write(std::move(batch))); @@ -2694,47 +2669,47 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { WriteContextBuilder write_id_builder(dir->Str(), "test-user"); write_id_builder.WithWriteId(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id_context, write_id_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id_context)), + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_id_context)), "a write id would name nothing"); CommitContextBuilder empty_commit_builder(dir->Str(), "test-user"); empty_commit_builder.IgnoreEmptyCommit(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr empty_commit_context, empty_commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(empty_commit_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(empty_commit_context)), "cannot record an empty commit"); CommitContextBuilder rest_builder(dir->Str(), "test-user"); rest_builder.UseRESTCatalogCommit(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_context, rest_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(rest_context)), "rest catalog"); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(rest_context)), "rest catalog"); CommitContextBuilder table_id_builder(dir->Str(), "test-user"); table_id_builder.WithTableId("table-uuid"); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_id_context, table_id_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(table_id_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(table_id_context)), "a format table sends no commit table request"); auto format_catalog = std::make_shared(); - format_catalog->SetTableSchema(table->LatestSchema()); + format_catalog->SetTableSchema(table->GetSchema()); CommitContextBuilder catalog_builder(dir->Str(), "test-user"); catalog_builder.WithCatalog(format_catalog, Identifier("db", "fmt")); ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_context, catalog_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(catalog_context)), "keeps no snapshot for a catalog to take"); CommitContextBuilder conflict_builder(dir->Str(), "test-user"); conflict_builder.AppendCommitCheckConflict(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr conflict_context, conflict_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(conflict_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(conflict_context)), "no manifests to check a concurrent commit against"); CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); ASSERT_OK(commit->Commit(messages)); // The same for the snapshot half of the commit interface. ASSERT_NOK_WITH_MSG(commit->Expire(), "no snapshots to expire"); @@ -2743,15 +2718,13 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { ScanContextBuilder scan_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1u); ReadContextBuilder read_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch()); ASSERT_FALSE(BatchReader::IsEofBatch(read_batch)); @@ -2764,11 +2737,32 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { reader->Close(); } -TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { - // A table a catalog keeps the schema for, the way a rest catalog does. Nothing under its - // location says that it is a format table, nor that everything below the location is data, so - // a path is not enough to reach it: the loaded table is handed to the context instead, which - // is what `FormatTable.newReadBuilder()` and `newBatchWriteBuilder()` do in Java. +TEST(FormatTableTest, TestACatalogInTheContextIsRefusedAlikeByWriteAndCommit) { + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + + auto catalog = std::make_shared(); + catalog->SetTableSchema(table->GetSchema()); + Identifier identifier("db", "fmt"); + + WriteContextBuilder write_builder(dir->Str(), "test-user"); + write_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + Result> write = table->NewWrite(std::move(write_context)); + ASSERT_FALSE(write.ok()); + ASSERT_TRUE(write.status().IsInvalid()); + + CommitContextBuilder commit_builder(dir->Str(), "test-user"); + commit_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + Result> commit = table->NewCommit(std::move(commit_context)); + ASSERT_FALSE(commit.ok()); + ASSERT_TRUE(commit.status().IsInvalid()); +} + +TEST(FormatTableTest, TestALoadedFormatTableIsServedThroughItsOwnOperators) { std::unique_ptr schema_dir = UniqueTestDirectory::Create(); ASSERT_TRUE(schema_dir); SchemaManager schema_manager(schema_dir->GetFileSystem(), schema_dir->Str()); @@ -2791,15 +2785,14 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { // what kind of table it is. ScanContextBuilder path_builder(data_dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr path_context, path_builder.Finish()); - ASSERT_NOK(TableScan::Create(std::move(path_context))); + ASSERT_NOK(TableScan::Create(std::move(path_context), {})); - // Options given at the call still win over the ones the schema stored, so a file is rolled - // per row rather than one file holding all three. - WriteContextBuilder write_builder(external); - write_builder.SetOptions({{Options::TARGET_FILE_ROW_NUM, "1"}}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
rolled, + external->Copy({{Options::TARGET_FILE_ROW_NUM, "1"}})); + WriteContextBuilder write_builder(external->Location(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + rolled->NewWrite(std::move(write_context))); for (int32_t i = 1; i <= 3; i++) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({i}, {"name"}, "20240101", {{"dt", "20240101"}})); @@ -2809,25 +2802,25 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { write->PrepareCommit()); ASSERT_EQ(messages.size(), 3u) << "target-file-row-num given at the call was ignored"; - CommitContextBuilder commit_builder(external); + CommitContextBuilder commit_builder(external->Location(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + external->NewCommit(std::move(commit_context))); ASSERT_OK(commit->Commit(messages)); ASSERT_OK(write->Close()); - ScanContextBuilder scan_builder(external); + ScanContextBuilder scan_builder(external->Location()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + external->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(PartitionList partitions, scan->ListPartitions()); ASSERT_EQ(partitions, (PartitionList{{{"dt", "20240101"}}})); - ReadContextBuilder read_builder(external); + ReadContextBuilder read_builder(external->Location()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + external->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); int64_t rows = 0; while (true) { @@ -2843,70 +2836,174 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { ASSERT_EQ(rows, 3); } -TEST(FormatTableTest, TestAContextBuiltFromAFormatTableRefusesASecondAnswer) { - // The table already carries its schema and the file system it was loaded through, from a - // source the context cannot see behind, so a second answer is refused rather than dropped. +TEST(FormatTableTest, TestATableServesAFormatTableThroughItsOwnEntryPoints) { + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::shared_ptr
table; + ASSERT_OK_AND_ASSIGN(table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + + WriteContextBuilder write_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + MakeBatch({1, 2}, {"alice", "bob"}, "20240101", {{"dt", "20240101"}})); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + + CommitContextBuilder commit_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + ScanContextBuilder scan_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1u); + + ReadContextBuilder read_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); + std::vector rows; + while (true) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(read_batch)) { + break; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr record_batch, + ImportBatch(read_batch)); + auto ids = checked_pointer_cast(record_batch->column(1)); + auto names = checked_pointer_cast(record_batch->column(2)); + auto dts = checked_pointer_cast(record_batch->column(3)); + for (int64_t i = 0; i < record_batch->num_rows(); i++) { + rows.push_back(std::to_string(ids->Value(i)) + "|" + names->GetString(i) + "|" + + dts->GetString(i)); + } + } + reader->Close(); + ASSERT_EQ(rows, (std::vector{"1|alice|20240101", "2|bob|20240101"})); +} + +TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) { + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + std::shared_ptr
table = format_table; + + ASSERT_NOK_WITH_MSG(TableScan::Create(nullptr, {}), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(TableRead::Create(nullptr, {}), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(nullptr, {}), "write context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewWrite(nullptr), "write context is null pointer"); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(nullptr, {}), "commit context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewCommit(nullptr), "commit context is null pointer"); + + auto streaming_scan_context = [&dir]() -> Result> { + ScanContextBuilder builder(dir->Str()); + builder.WithStreamingMode(true); + return builder.Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context, streaming_scan_context()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context), {}), + "names a format table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context_again, + streaming_scan_context()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(streaming_context_again)), + "nothing for a streaming scan to follow"); + + auto vector_read_context = [&dir]() -> Result> { + ReadContextBuilder builder(dir->Str()); + builder.SetVectorSearch(std::make_shared( + /*field_name=*/"name", /*limit=*/1, std::vector{0.0f}, /*pre_filter=*/nullptr, + /*predicate=*/nullptr, /*distance_type=*/std::nullopt, + /*options=*/std::map{})); + return builder.Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context, vector_read_context()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(vector_context), {}), "names a format table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context_again, vector_read_context()); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(vector_context_again)), + "not supported for format tables"); + + auto catalog = std::make_shared(); + catalog->SetTableSchema(format_table->GetSchema()); + + auto catalog_write_context = [&dir, &catalog]() -> Result> { + WriteContextBuilder builder(dir->Str(), "test-user"); + return builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write, catalog_write_context()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(catalog_write), {}), + "names a format table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write_again, + catalog_write_context()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(catalog_write_again)), + "WithCatalog() requires a native table"); + + auto catalog_commit_context = [&dir, &catalog]() -> Result> { + CommitContextBuilder builder(dir->Str(), "test-user"); + return builder.WithCatalog(catalog, Identifier("db", "fmt")).Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit, catalog_commit_context()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_commit), {}), + "names a format table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit_again, + catalog_commit_context()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(catalog_commit_again)), + "keeps no snapshot for a catalog to take"); + + auto write_id_context = [&dir]() -> Result> { + WriteContextBuilder builder(dir->Str(), "test-user"); + builder.WithWriteId(3); + return builder.Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id, write_id_context()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id), {}), "names a format table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id_again, write_id_context()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_id_again)), + "a write id would name nothing"); +} + +TEST(FormatTableTest, TestTheGenericEntryPointsRefuseAFormatTablePath) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - SchemaManager schema_manager(dir->GetFileSystem(), dir->Str()); - ASSERT_OK_AND_ASSIGN(std::optional> latest, - schema_manager.Latest()); - ASSERT_TRUE(latest.has_value()); - ASSERT_OK_AND_ASSIGN(std::string schema_json, (*latest)->GetJsonSchema()); - ScanContextBuilder schema_builder(table); - schema_builder.SetTableSchema(schema_json); - ASSERT_NOK_WITH_MSG(schema_builder.Finish(), "carries its own schema"); + ScanContextBuilder scan_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), {}), "names a format table"); - ScanContextBuilder fs_builder(table); - fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(fs_builder.Finish(), "carries the file system"); + ReadContextBuilder read_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), {}), "names a format table"); - ReadContextBuilder branch_builder(table); - branch_builder.WithBranch("other"); - ASSERT_NOK_WITH_MSG(branch_builder.Finish(), "has no branches"); + WriteContextBuilder write_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context), {}), + "names a format table"); - WriteContextBuilder write_fs_builder(table); - write_fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(write_fs_builder.Finish(), "carries the file system"); + CommitContextBuilder commit_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context), {}), + "names a format table"); - auto catalog = std::make_shared(); - WriteContextBuilder write_catalog_builder(table); - ASSERT_NOK_WITH_MSG( - write_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() requires a native table"); - ReadContextBuilder read_catalog_builder(table); - ASSERT_NOK_WITH_MSG(read_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() cannot be used with one"); - ScanContextBuilder scan_catalog_builder(table); - ASSERT_NOK_WITH_MSG(scan_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() cannot be used with one"); - catalog->SetTableSchema(latest.value()); - WriteContextBuilder path_catalog_builder(dir->Str(), "test-user"); - ASSERT_OK_AND_ASSIGN( - auto path_context, - path_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(path_context)), - "use WriteContextBuilder(FormatTable)"); - - CommitContextBuilder commit_fs_builder(table); - commit_fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(commit_fs_builder.Finish(), "carries the file system"); - - // A null table is the caller's mistake, and says so rather than complaining about a path it - // was never given. - ScanContextBuilder null_builder{std::shared_ptr()}; - ASSERT_NOK_WITH_MSG(null_builder.Finish(), "null format table"); -} - -TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) { - // `WithBranch()` is refused on a context built from a format table, and the `branch` option - // says the same thing by another name: a format table keeps no metadata to branch, and a read - // or a write would go to its one location whatever the option named. Refusing it is the only - // answer that is not silently the wrong one - an overwrite would delete the main branch's - // data while claiming to replace another branch's. + ScanContextBuilder served_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr served_context, served_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr served_scan, + table->NewScan(std::move(served_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, served_scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 0u); +} + +TEST(FormatTableTest, TestBranchOptionsAreRefusedByFormatTables) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -2916,33 +3013,11 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) ASSERT_OK(WriteAndCommit(table, std::move(batch))); const std::map branch = {{Options::BRANCH, "dev"}}; - ASSERT_NOK_WITH_MSG(FormatTable::Copy(table, branch), "has no branches"); + ASSERT_NOK_WITH_MSG(table->Copy(branch), "has no branches"); ASSERT_NOK_WITH_MSG( FormatTable::Create(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"), branch), "has no branches"); - ScanContextBuilder scan_builder(table); - scan_builder.SetOptions(branch); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "has no branches"); - - ReadContextBuilder read_builder(table); - read_builder.SetOptions(branch); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "has no branches"); - - WriteContextBuilder write_builder(table); - write_builder.SetOptions(branch); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "has no branches"); - - // The commit is the one that matters most: an overwrite clears what it replaces before it - // publishes anything, so the refusal has to come before the deleting does. - CommitContextBuilder commit_builder(table); - commit_builder.SetOptions(branch); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "has no branches"); - ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt)); @@ -2953,15 +3028,64 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) // The branch that names no branch at all is the main one, and is left alone: refusing it // would refuse the default every context carries. for (const char* main_branch : {"main", ""}) { - ASSERT_OK(FormatTable::Copy(table, {{Options::BRANCH, main_branch}})) << main_branch; + ASSERT_OK(table->Copy({{Options::BRANCH, main_branch}})) << main_branch; + } +} + +TEST(FormatTableTest, TestATableCopiesItselfWithTheOptionsGivenAtTheCall) { + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, + table->Copy({{Options::TYPE, "table"}, {"probe", "on-top"}})); + std::shared_ptr copied_table = std::static_pointer_cast(copied); + EXPECT_EQ(copied_table->Options().at("probe"), "on-top"); + EXPECT_EQ(copied_table->Options().at(Options::TYPE), "format-table"); + EXPECT_EQ(copied_table->FullName(), table->FullName()); + EXPECT_EQ(copied_table->Location(), table->Location()); + EXPECT_EQ(copied_table->GetFileSystem(), table->GetFileSystem()); + + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BRANCH, "dev"}}), "has no branches"); +} + +TEST(FormatTableTest, TestCopyRetainsPhysicalFileFormat) { + for (const std::string& format : {std::string("parquet"), std::string("orc")}) { + SCOPED_TRACE(format); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr original, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"}, + {{Options::FILE_FORMAT, format}})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr first, + MakeBatch({1}, {"alice"}, "20240101", {{"dt", "20240101"}})); + ASSERT_OK(WriteAndCommit(original, std::move(first))); + const std::string hinted_format = format == "orc" ? "parquet" : "orc"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, + original->Copy({{Options::FILE_FORMAT, hinted_format}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
another, + copy->Copy({{Options::READ_BATCH_SIZE, "1"}})); + auto copied = std::dynamic_pointer_cast(another); + ASSERT_TRUE(copied); + EXPECT_EQ(copied->GetFormat(), original->GetFormat()); + EXPECT_EQ(copied->Options().at(Options::FILE_FORMAT), hinted_format); + ASSERT_OK_AND_ASSIGN(std::unique_ptr second, + MakeBatch({2}, {"bob"}, "20240101", {{"dt", "20240101"}})); + ASSERT_OK(WriteAndCommit(copied, std::move(second))); + for (const auto& table : {original, copied}) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(std::vector rows, ReadAll(table, plan->Splits())); + std::sort(rows.begin(), rows.end()); + EXPECT_EQ(rows, (std::vector{"1|alice|20240101", "2|bob|20240101"})); + } } } TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFromTheTableRoot) { - // A context that names a path takes its branch from `WithBranch()`, and the schema of that - // branch is what the entry point dispatches on. A format table's data is the files under the - // table path whichever branch named it, so serving one from there under another branch's name - // would answer a question about `dev` with the rows of `main`. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -2977,32 +3101,34 @@ TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFr MakeSchema(), /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {{Options::TYPE, "format-table"}, {Options::FILE_FORMAT, "parquet"}})); + std::map read_builder_options; ReadContextBuilder read_builder(dir->Str()); - read_builder.WithBranch("dev"); + read_builder_options[Options::BRANCH] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), read_builder_options), + "names a format table"); // A scan takes its branch from the option rather than from a `WithBranch()` of its own, which // is the same branch by another name and gets the same answer. + std::map scan_builder_options; ScanContextBuilder scan_builder(dir->Str()); - scan_builder.SetOptions({{Options::BRANCH, "dev"}}); + scan_builder_options = {{Options::BRANCH, "dev"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), scan_builder_options), + "names a format table"); - // The main branch is untouched by any of it. ScanContextBuilder main_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_context, main_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan, - TableScan::Create(std::move(main_context))); + table->NewScan(std::move(main_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, main_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::vector rows, ReadAll(table, plan->Splits())); ASSERT_EQ(rows, (std::vector{"1|alice|20240101"})); } -TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { - // A context promises that options given at the call win over the ones the schema stored, and - // that a branch and a caller-held schema are honoured. Dispatching to a format table must not - // quietly drop any of that. +TEST(FormatTableTest, TestTheTableOperatorsCarryTheContextThrough) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -3010,11 +3136,15 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { // `target-file-row-num` is stored nowhere in this table's schema, so a file would hold every // row. Given at the call it has to roll a file per row, as it does for a managed table. + std::map write_builder_options; WriteContextBuilder write_builder(dir->Str(), "test-user"); - write_builder.SetOptions({{Options::TARGET_FILE_ROW_NUM, "1"}}); + write_builder_options = {{Options::TARGET_FILE_ROW_NUM, "1"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_23, + table->Copy(write_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + configured_table_23->NewWrite(std::move(write_context))); for (int32_t i = 0; i < 3; i++) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({i}, {"name"}, "20240101", {})); @@ -3024,49 +3154,33 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { write->PrepareCommit()); ASSERT_EQ(messages.size(), 3u) << "target-file-row-num given at the call was ignored"; - // Published through the generic commit, or nothing below would see the rows: until the commit - // renames them the files sit in the hidden `_temporary` directory a scan skips. CommitContextBuilder generic_commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr generic_commit_context, generic_commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr generic_commit, - FileStoreCommit::Create(std::move(generic_commit_context))); + table->NewCommit(std::move(generic_commit_context))); ASSERT_OK(generic_commit->Commit(messages)); ASSERT_OK(write->Close()); - // A branch this table has no schema on is a managed table as far as dispatch is concerned, so - // the format branch must not answer for it. + std::map branch_builder_options; ReadContextBuilder branch_builder(dir->Str()); - branch_builder.WithBranch("nosuchbranch"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_context, branch_builder.Finish()); - ASSERT_NOK(TableRead::Create(std::move(branch_context))); + branch_builder_options[Options::BRANCH] = "nosuchbranch"; - // A caller-held schema is used instead of reading one from under the path. - SchemaManager schema_manager(dir->GetFileSystem(), dir->Str()); - ASSERT_OK_AND_ASSIGN(std::optional> latest, - schema_manager.Latest()); - ASSERT_TRUE(latest.has_value()); - ASSERT_OK_AND_ASSIGN(std::string schema_json, (*latest)->GetJsonSchema()); - ScanContextBuilder seeded_builder(dir->Str()); - seeded_builder.SetTableSchema(schema_json); - ASSERT_OK_AND_ASSIGN(std::unique_ptr seeded_context, seeded_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr seeded_scan, - TableScan::Create(std::move(seeded_context))); - // Planning, not just dispatch: handing the schema over must not turn the `schema` directory - // the table keeps under its own path into data, which a plan would then read as a data file. - ASSERT_OK_AND_ASSIGN(std::shared_ptr seeded_plan, seeded_scan->CreatePlan()); - ASSERT_OK_AND_ASSIGN(std::vector seeded_rows, - ReadAll(table, seeded_plan->Splits())); - ASSERT_EQ(seeded_rows.size(), 3u); + ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_context, branch_builder.Finish()); + ASSERT_NOK(TableRead::Create(std::move(branch_context), branch_builder_options)); // `type` is structural, so an option given at the call must not decide what kind of table // this is. Checked by reading the rows back rather than by counting splits: three small files // are packed into one split, so a split count says nothing about how many there are. + std::map retyped_builder_options; ScanContextBuilder retyped_builder(dir->Str()); - retyped_builder.SetOptions({{Options::TYPE, "table"}}); + retyped_builder_options = {{Options::TYPE, "table"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_context, retyped_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_24, + table->Copy(retyped_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_scan, - TableScan::Create(std::move(retyped_context))); + configured_table_24->NewScan(std::move(retyped_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr retyped_plan, retyped_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::vector retyped_rows, ReadAll(table, retyped_plan->Splits())); @@ -3101,7 +3215,7 @@ class CountingCache : public Cache { } // namespace -TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { +TEST(FormatTableTest, TestTheReadCarriesPrefetchAndTheCacheThrough) { // A format table opens its files through the same component the managed table path opens its // own with, so what a `ReadContext` asks for about opening a file applies here too. std::unique_ptr dir = UniqueTestDirectory::Create(); @@ -3115,13 +3229,11 @@ TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { ScanContextBuilder scan_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - auto read_rows = [&plan](std::unique_ptr context) -> Result { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - TableRead::Create(std::move(context))); + auto read_rows = [&plan, &table](std::unique_ptr context) -> Result { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, table->NewRead(std::move(context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); int64_t rows = 0; @@ -3156,7 +3268,7 @@ TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { ASSERT_GT(cache->gets, 0) << "the cache the read context carries never reached the file reader"; } -TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { +TEST(FormatTableTest, TestTheOperatorsRefuseWhatAFormatTableCannotHonour) { // Anything a context carries that a format table cannot do is a refusal naming the setting, // never a read or a write that quietly did something else. std::unique_ptr dir = UniqueTestDirectory::Create(); @@ -3171,14 +3283,14 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { read_schema_builder.SetReadSchema(std::move(projected_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_schema_context, read_schema_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_schema_context)), + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_schema_context)), "does not take a projected read schema"); ScanContextBuilder streaming_builder(dir->Str()); streaming_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context, streaming_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context)), + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(streaming_context)), "nothing for a streaming scan to follow"); ScanContextBuilder predicate_builder(dir->Str()); @@ -3186,13 +3298,12 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(1))); ASSERT_OK_AND_ASSIGN(std::unique_ptr predicate_context, predicate_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(predicate_context)), - "does not take a predicate"); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(predicate_context)), "does not take a predicate"); CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); // A commit takes batch writes only. The rest of the snapshot half is // `TestTheGenericCommitRefusesEveryCallAboutSnapshots`. ASSERT_NOK_WITH_MSG(commit->Commit({}, /*commit_identifier=*/7), "commit identifier"); @@ -3213,12 +3324,12 @@ TEST(FormatTableTest, TestTheGenericCommitRefusesEveryCallAboutSnapshots) { // types then finds out at the call rather than from a table that did not change. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress({}, BATCH_WRITE_COMMIT_IDENTIFIER, /*watermark=*/std::nullopt), @@ -3262,7 +3373,7 @@ TEST(FormatTableTest, TestTheGenericCommitOverwritesOnePartition) { WriteContextBuilder write_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + table->NewWrite(std::move(write_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr replacement, MakeBatch({3}, {"carol"}, "20240101", {{"dt", "20240101"}})); ASSERT_OK(write->Write(std::move(replacement))); @@ -3275,7 +3386,7 @@ TEST(FormatTableTest, TestTheGenericCommitOverwritesOnePartition) { CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); // Batch writes only, and refused before anything is cleared: an overwrite that failed on its // arguments must not have deleted the partition it named. ASSERT_NOK_WITH_MSG(commit->Overwrite({{"dt", "20240101"}}, messages, /*commit_identifier=*/7), diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp new file mode 100644 index 000000000..355ebf148 --- /dev/null +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/primary_key_file_store_table.h" + +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/global_index/btree/btree_defs.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/mergetree/compact/lookup_merge_function.h" +#include "paimon/core/mergetree/compact/merge_function.h" +#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_write.h" +#include "paimon/core/options/merge_engine.h" +#include "paimon/core/postpone/postpone_bucket_file_store_write.h" +#include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/source/merge_tree_split_generator.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/utils/primary_key_table_utils.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { + +struct KeyValue; +template +class MergeFunctionWrapper; + +namespace { + +Result HasHistoricalPrimaryKeyBTreeDefinition( + const std::shared_ptr& schema_manager, int64_t current_schema_id) { + if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); + for (int64_t schema_id : schema_ids) { + if (schema_id >= current_schema_id) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, + schema_manager->ReadSchema(schema_id)); + const auto& historical_options = historical_schema->Options(); + auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); + if (option != historical_options.end() && + !StringUtils::IsNullOrWhitespaceOnly(option->second)) { + return true; + } + } + return false; +} + +} // namespace + +PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, + const Identifier& identifier, + const std::string& uuid) + : FileStoreTable(store, schema, identifier, uuid) {} + +Result> PrimaryKeyFileStoreTable::CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const { + std::shared_ptr schema = std::static_pointer_cast(schema_); + if (ctx.GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); + if (ignore_previous_files) { + return Status::NotImplemented("PK realtime requires restore from the latest snapshot"); + } + if (!ctx.GetWriteSchema().empty()) { + return Status::NotImplemented("PK realtime does not support a custom write schema"); + } + PAIMON_RETURN_NOT_OK( + RestoreRealtimeCommittedProgress(ctx.GetRealtimeContext(), snapshot_manager, options)); + } + if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { + return PostponeBucketFileStoreWrite::Create( + snapshot_manager, GetSchemaManager(), ctx.GetCommitUser(), root_path_, schema, + arrow_schema, partition_schema, io_manager, options, ctx.IsStreamingMode(), + ctx.IgnoreNumBucketCheck(), ctx.GetWriteId(), ctx.GetFileSystemSchemeToIdentifierMap(), + ctx.GetExecutor(), ctx.GetMemoryPool(), file_system_); + } + if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in key value table", options.GetBucket())); + } + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, + schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_primary_key_fields, + options.SequenceFieldSortOrderIsAscending())); + auto primary_keys = schema->PrimaryKeys(); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, + PrimaryKeyTableUtils::CreateMergeFunction(arrow_schema, primary_keys, + options, ctx.GetMemoryPool())); + if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { + merge_function = std::make_unique(std::move(merge_function)); + } + std::shared_ptr> merge_function_wrapper = + std::make_shared(std::move(merge_function)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr sequence_fields_comparator, + PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); + + std::shared_ptr dv_maintainer_factory; + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + bool has_btree_index = false; + for (const PrimaryKeyIndexDefinition& definition : + primary_key_index_definitions.Definitions()) { + if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { + has_btree_index = true; + break; + } + } + std::optional latest_snapshot; + bool has_existing_index_manifest = false; + if (!has_btree_index) { + PAIMON_ASSIGN_OR_RAISE( + bool has_historical_btree_index, + HasHistoricalPrimaryKeyBTreeDefinition(GetSchemaManager(), schema->Id())); + if (has_historical_btree_index) { + PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); + has_existing_index_manifest = + latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); + } + } + std::shared_ptr index_file_handler; + if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE(index_file_handler, store_->NewIndexFileHandler( + options, path_factory, ctx.GetMemoryPool())); + } + bool has_restored_btree_index = false; + if (!has_btree_index && has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE( + std::vector restored_btree_entries, + index_file_handler->Scan( + latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { + return entry.index_file->IndexType() == BtreeDefs::kIdentifier && + IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); + })); + has_restored_btree_index = !restored_btree_entries.empty(); + } + if (options.DeletionVectorsEnabled()) { + dv_maintainer_factory = std::make_shared(index_file_handler); + } + std::shared_ptr + primary_key_index_maintainer_factory; + if (has_btree_index || has_restored_btree_index) { + primary_key_index_maintainer_factory = + std::make_shared( + root_path_, CurrentBranch(), schema, primary_key_index_definitions.Definitions(), + path_factory, index_file_handler, options, io_manager, ctx.EnableMultiThreadSpill(), + ctx.GetExecutor(), ctx.GetMemoryPool()); + } + + std::shared_ptr realtime_schema_layout; + if (ctx.GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr layout, + RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); + realtime_schema_layout = std::move(layout); + } + + return std::make_unique( + path_factory, snapshot_manager, GetSchemaManager(), ctx.GetCommitUser(), root_path_, schema, + arrow_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, + primary_key_index_maintainer_factory, io_manager, key_comparator, + sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, + ctx.IsStreamingMode(), ctx.IgnoreNumBucketCheck(), ctx.EnableMultiThreadSpill(), + ctx.GetRealtimeContext(), ctx.GetExecutor(), ctx.GetMemoryPool()); +} + +Result> PrimaryKeyFileStoreTable::NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const { + // TODO(liancheng.lsz): support fallback branch in scan + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + std::unique_ptr scan; + PAIMON_ASSIGN_OR_RAISE( + scan, KeyValueFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), manifest_list, + manifest_file, table_schema, arrow_schema, + context->GetScanFilters(), core_options, executor, + memory_pool)); + return WithTablePath(std::move(scan), root_path_); +} + +Result> PrimaryKeyFileStoreTable::NewSplitGenerator( + const CoreOptions& core_options) const { + auto source_split_target_size = core_options.GetSourceSplitTargetSize(); + auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); + // TODO(liancheng.lsz): support evolution + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + table_schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, + table_schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); + return std::make_unique( + source_split_target_size, source_split_open_file_cost, + core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), key_comparator); +} + +CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) const { + const std::shared_ptr& schema_manager = GetSchemaManager(); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h new file mode 100644 index 000000000..8ab8c05cf --- /dev/null +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class FileStoreScan; +class IOManager; +class SplitGenerator; +class TableSchema; + +/// Managed table with primary keys; uses the shared FileStoreTable operator flows. +class PrimaryKeyFileStoreTable : public FileStoreTable { + public: + PrimaryKeyFileStoreTable(const std::shared_ptr& store, + const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid); + + private: + Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const override; + + Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const override; + + Result> NewSplitGenerator( + const CoreOptions& core_options) const override; + + CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/sink/commit_message_test.cpp b/src/paimon/core/table/sink/commit_message_test.cpp index 8fd31c4c8..de6d5da23 100644 --- a/src/paimon/core/table/sink/commit_message_test.cpp +++ b/src/paimon/core/table/sink/commit_message_test.cpp @@ -1230,16 +1230,18 @@ TEST(CommitMessageTest, TestSerialize) { /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + std::map context_builder_options; WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "commit_user1"); + context_builder_options[Options::FILE_FORMAT] = "mock_format"; + context_builder_options[Options::TARGET_FILE_SIZE] = "1024"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::FILE_FORMAT, "mock_format") - .AddOption(Options::TARGET_FILE_SIZE, "1024") - .Finish()); + + context_builder.Finish()); std::string root_path = write_context->GetRootPath(); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); for (size_t i = 0; i < 10240; i++) { auto f0_array = std::make_shared(); diff --git a/src/paimon/core/table/source/append_count_reader_test.cpp b/src/paimon/core/table/source/append_count_reader_test.cpp index 3b191a3d0..82bf53eef 100644 --- a/src/paimon/core/table/source/append_count_reader_test.cpp +++ b/src/paimon/core/table/source/append_count_reader_test.cpp @@ -51,14 +51,16 @@ class AppendCountReaderTest : public testing::Test { protected: Result>> CreateSplits( const std::string& table_path, const std::optional& snapshot_id) { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); if (snapshot_id.has_value()) { - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, - std::to_string(snapshot_id.value())); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = + std::to_string(snapshot_id.value()); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); if (snapshot_id.has_value() && (!plan->SnapshotId().has_value() || plan->SnapshotId().value() != snapshot_id)) { @@ -136,7 +138,7 @@ TEST_F(AppendCountReaderTest, TestAppendOnlyTableReadCreateCountReaderPredicateN ReadContextBuilder read_context_builder(table_path); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/5)); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader(splits), diff --git a/src/paimon/core/table/source/pk_count_reader_test.cpp b/src/paimon/core/table/source/pk_count_reader_test.cpp index 39c0b6b50..feee28344 100644 --- a/src/paimon/core/table/source/pk_count_reader_test.cpp +++ b/src/paimon/core/table/source/pk_count_reader_test.cpp @@ -51,14 +51,16 @@ class PKCountReaderTest : public testing::Test { protected: Result>> CreateSplits( const std::string& table_path, const std::optional& snapshot_id) { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); if (snapshot_id.has_value()) { - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, - std::to_string(snapshot_id.value())); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = + std::to_string(snapshot_id.value()); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); if (snapshot_id.has_value()) { if (!plan->SnapshotId().has_value() || plan->SnapshotId().value() != snapshot_id) { @@ -78,7 +80,7 @@ class PKCountReaderTest : public testing::Test { PAIMON_ASSIGN_OR_RAISE(auto table_schema, schema_manager.ReadSchema(0)); PAIMON_ASSIGN_OR_RAISE( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create(read_context, table_schema, table_schema->Options(), {})); return std::shared_ptr(std::move(internal_context)); } diff --git a/src/paimon/core/table/source/realtime_table_scan.cpp b/src/paimon/core/table/source/realtime_table_scan.cpp index 42a7171b3..cfb61c17d 100644 --- a/src/paimon/core/table/source/realtime_table_scan.cpp +++ b/src/paimon/core/table/source/realtime_table_scan.cpp @@ -27,17 +27,21 @@ #include "fmt/format.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/core_options.h" #include "paimon/core/operation/commit/realtime_commit_properties.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/plan_impl.h" #include "paimon/core/table/source/realtime_split.h" #include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/primary_key_table_utils.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/realtime/realtime_context.h" #include "paimon/realtime/realtime_store.h" #include "paimon/scan_context.h" #include "paimon/status.h" +#include "paimon/table/source/startup_mode.h" namespace paimon { @@ -203,6 +207,49 @@ Result>> RealtimeTableScan::CreateRealtimeSpl return result; } +Status RealtimeTableScan::ValidateScan(const TableSchema& table_schema, + const CoreOptions& core_options, const ScanContext& context, + bool read_optimized) { + if (!context.GetRealtimeContext()) { + return Status::OK(); + } + if (!core_options.RealtimeEnabled()) { + return Status::Invalid("real-time scan requires realtime.enabled=true"); + } + if (core_options.GetBucket() <= 0) { + return Status::Invalid("real-time union read requires fixed bucket mode"); + } + if (core_options.DataEvolutionEnabled()) { + return Status::Invalid("real-time union read does not support data evolution"); + } + if (!table_schema.PrimaryKeys().empty()) { + if (read_optimized) { + return Status::NotImplemented( + "PK real-time union read does not support read-optimized scans"); + } + PAIMON_RETURN_NOT_OK( + PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); + } + if (context.IsStreamingMode()) { + return Status::Invalid("real-time union read currently supports batch scans only"); + } + if (context.GetLimit()) { + return Status::Invalid("real-time union read does not support scan limit pushdown"); + } + if (context.GetGlobalIndexResult()) { + return Status::Invalid("real-time union read does not support global index splits"); + } + StartupMode startup_mode = core_options.GetStartupMode(); + const bool specified_snapshot = + startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); + if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || + specified_snapshot)) { + return Status::Invalid( + "real-time union read requires the latest snapshot or an explicit snapshot id"); + } + return Status::OK(); +} + RealtimeTableScan::RealtimeTableScan(std::unique_ptr&& disk_scan, bool pk_table, const std::shared_ptr& realtime_context, const std::shared_ptr& path_factory, diff --git a/src/paimon/core/table/source/realtime_table_scan.h b/src/paimon/core/table/source/realtime_table_scan.h index 821e67fd7..46227d327 100644 --- a/src/paimon/core/table/source/realtime_table_scan.h +++ b/src/paimon/core/table/source/realtime_table_scan.h @@ -30,14 +30,21 @@ namespace paimon { +class CoreOptions; class FileStorePathFactory; class FileSystem; +class ScanContext; class ScanFilter; class SnapshotManager; +class TableSchema; /// Adds process-local memory splits to a normal data-table batch scan. class RealtimeTableScan : public TableScan { public: + /// Validates table and scan options for real-time union reads; a no-op for ordinary scans. + static Status ValidateScan(const TableSchema& table_schema, const CoreOptions& core_options, + const ScanContext& context, bool read_optimized); + RealtimeTableScan(std::unique_ptr&& disk_scan, bool pk_table, const std::shared_ptr& realtime_context, const std::shared_ptr& path_factory, diff --git a/src/paimon/core/table/source/snapshot/static_from_tag_starting_scanner.h b/src/paimon/core/table/source/snapshot/static_from_tag_starting_scanner.h index a10b11b85..3229a76ef 100644 --- a/src/paimon/core/table/source/snapshot/static_from_tag_starting_scanner.h +++ b/src/paimon/core/table/source/snapshot/static_from_tag_starting_scanner.h @@ -36,7 +36,8 @@ class StaticFromTagStartingScanner : public StartingScanner { Result> Scan( const std::shared_ptr& snapshot_reader) override { - const TagManager tag_manager(snapshot_manager_->Fs(), snapshot_manager_->RootPath()); + const TagManager tag_manager(snapshot_manager_->Fs(), snapshot_manager_->RootPath(), + snapshot_manager_->Branch()); PAIMON_ASSIGN_OR_RAISE(const Tag tag, tag_manager.GetOrThrow(tag_name_)); PAIMON_ASSIGN_OR_RAISE(const Snapshot snapshot, tag.TrimToSnapshot()); PAIMON_ASSIGN_OR_RAISE( diff --git a/src/paimon/core/table/source/table_read.cpp b/src/paimon/core/table/source/table_read.cpp index e723c54bb..648713628 100644 --- a/src/paimon/core/table/source/table_read.cpp +++ b/src/paimon/core/table/source/table_read.cpp @@ -19,8 +19,20 @@ #include "paimon/table/source/table_read.h" +#include "paimon/catalog/table.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" + namespace paimon { +Result> TableRead::Create( + std::unique_ptr ctx, const std::map& options) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + FileStoreTableFactory::Create(*ctx, options)); + return table->NewRead(std::move(ctx)); +} + Result> TableRead::CreateCountReader( const std::vector>& splits) { (void)splits; diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index ead31b17a..8659f871a 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -36,6 +36,7 @@ #include "paimon/core/operation/split_read.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" @@ -60,7 +61,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0", "f1", "non-exist"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), {}), "Get field non-exist failed: not exist in table schema"); } { @@ -71,7 +72,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableRead::Create(std::move(read_context)), + TableRead::Create(std::move(read_context), {}), "field f3 has field type BIGINT in literal, mismatch field type DOUBLE in predicate"); } { @@ -81,7 +82,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { ReadContextBuilder context_builder(path); context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), {}), "schema type double mismatches predicate field type BIGINT"); } { @@ -93,7 +94,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableRead::Create(std::move(read_context)), + TableRead::Create(std::move(read_context), {}), "field f3 has field idx 0 in input schema, mismatch field idx 2 in predicate"); } { @@ -103,7 +104,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { ReadContextBuilder context_builder(path); context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), {}), "literal cannot be null in predicate, field name f3"); } { @@ -111,7 +112,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f3", "f1", "f3"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), {}), "validate schema failed: read schema has duplicate field f3"); } } @@ -121,7 +122,7 @@ TEST(TableReadTest, TestReadWithSpecifiedInvalidSchema) { ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"field_no_exist"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), {}), "Get field field_no_exist failed: not exist in table schema"); } @@ -134,20 +135,24 @@ TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) { auto fs = std::make_shared(); SchemaManager schema_manager(fs, table_path); ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); - Table table(fs, table_path, schema, Identifier("append_09.db", "append_09")); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"), + /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); for (bool table_entry : {false, true}) { ReadContextBuilder builder(table_path); builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - auto result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + auto result = table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context), {}); ASSERT_NOK_WITH_MSG(result, "FullTextMatch predicate execution is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); builder.SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(context, builder.Finish()); - result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + result = table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context), {}); ASSERT_NOK_WITH_MSG(result, "Score-producing FullTextMatch cannot appear under OR"); } } @@ -159,13 +164,17 @@ TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { auto fs = std::make_shared(); SchemaManager schema_manager(fs, path); ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); - Table table(fs, path, schema, Identifier("append_09.db", "append_09")); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"), + /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); for (bool table_entry : {false, true}) { ReadContextBuilder builder(path); builder.SetFullTextSearch(search).SetReadFieldNames({"f1", "_f0_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - auto result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + auto result = table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context), {}); ASSERT_NOK_WITH_MSG(result, "Per-field index score output is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); } @@ -174,12 +183,16 @@ TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { TEST(TableReadTest, TestCreateKeyValueTableRead) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["orc.read.enable-lazy-decoding"] = "true"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); auto key_value_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(key_value_table_read); } @@ -192,7 +205,7 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0"}).SetFullTextSearch(search); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader({}), "VectorSearch or FullTextSearch"); } { @@ -204,7 +217,7 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0"}).SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader({}), "VectorSearch or FullTextSearch"); } } @@ -218,7 +231,8 @@ TEST(TableReadTest, TestPrimaryKeyFileIndexSearchDoesNotFallBackToMergeReader) { ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0"}).SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + TableRead::Create(std::move(context), {})); BinaryRow partition = BinaryRowGenerator::GenerateRow({10}, GetDefaultPool().get()); DataSplitImpl::Builder builder(partition, /*bucket=*/0, /*bucket_path=*/"", /*data_files=*/{}); @@ -238,37 +252,64 @@ TEST(TableReadTest, TestPrimaryKeyFileIndexSearchDoesNotFallBackToMergeReader) { TEST(TableReadTest, TestCreateAppendOnlyTableRead) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["orc.read.enable-lazy-decoding"] = "true"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); auto append_only_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(append_only_table_read); } TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); - context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); - context_builder.AddOption("bucket", "10"); - context_builder.AddOption("bucket-key", "f3"); - ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - auto append_only_table_read = dynamic_cast(table_read.get()); - ASSERT_TRUE(append_only_table_read); - auto* typed_split_read = - dynamic_cast(append_only_table_read->split_reads_[0].get()); - ASSERT_TRUE(typed_split_read); - const auto& core_options = typed_split_read->options_; - std::map expected_options = { - {"read.batch-size", "2"}, {"orc.read.enable-lazy-decoding", "true"}, - {"bucket", "10"}, {"bucket-key", "f3"}, - {"manifest.format", "orc"}, {"file.format", "orc"}}; - ASSERT_EQ(expected_options, core_options.ToMap()); + { + std::map context_builder_options; + ReadContextBuilder context_builder(path); + context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["orc.read.enable-lazy-decoding"] = "true"; + + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); + auto append_only_table_read = dynamic_cast(table_read.get()); + ASSERT_TRUE(append_only_table_read); + auto* typed_split_read = + dynamic_cast(append_only_table_read->split_reads_[0].get()); + ASSERT_TRUE(typed_split_read); + const auto& core_options = typed_split_read->options_; + std::map expected_options = { + {"read.batch-size", "2"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"bucket", "2"}, + {"bucket-key", "f2"}, + {"manifest.format", "orc"}, + {"file.format", "orc"}}; + ASSERT_EQ(expected_options, core_options.ToMap()); + } + { + std::map context_builder_options; + ReadContextBuilder context_builder(path); + context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); + context_builder_options["bucket"] = "10"; + + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); + auto append_only_table_read = dynamic_cast(table_read.get()); + ASSERT_TRUE(append_only_table_read); + auto* typed_split_read = + dynamic_cast(append_only_table_read->split_reads_[0].get()); + ASSERT_TRUE(typed_split_read); + ASSERT_EQ(10, typed_split_read->options_.GetBucket()); + } } TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { @@ -282,15 +323,16 @@ TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { ASSERT_OK_AND_ASSIGN(std::string schema_json, table_schema->ToJsonString()); for (bool explicit_header : {false, true}) { + std::map builder_options; ReadContextBuilder builder("warehouse/db.db/table"); builder.SetTableSchema(schema_json); if (explicit_header) { - builder.AddOption(read_via_option, "read-view"); + builder_options[read_via_option] = "read-view"; } ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, builder.Finish()); // Exercise the actual table/read merge before creating the dependency context. ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), builder_options)); auto* append_only_table_read = dynamic_cast(table_read.get()); ASSERT_NE(append_only_table_read, nullptr); const auto& internal_context = append_only_table_read->context_; @@ -300,7 +342,7 @@ TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { expected_options[read_via_option] = explicit_header ? "read-view" : "table-view"; ASSERT_EQ(dependency_context->root_path, "warehouse"); ASSERT_EQ(dependency_context->options, expected_options); - ASSERT_EQ(internal_context->GetTableSchema()->Options(), table_options); + ASSERT_EQ(internal_context->GetTableSchema()->Options(), expected_options); } } diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index be5d5b955..9684d609f 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -19,10 +19,22 @@ #include "paimon/table/source/table_scan.h" +#include "paimon/catalog/table.h" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/scan_context.h" namespace paimon { +Result> TableScan::Create( + std::unique_ptr context, const std::map& options) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + FileStoreTableFactory::Create(*context, options)); + return table->NewScan(std::move(context)); +} + std::shared_ptr TableScan::GetMetrics() const { return std::make_shared(); } diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp index 435e2254f..59cae2906 100644 --- a/src/paimon/core/table/source/table_scan_test.cpp +++ b/src/paimon/core/table/source/table_scan_test.cpp @@ -19,12 +19,15 @@ #include "paimon/table/source/table_scan.h" +#include #include #include #include #include #include "gtest/gtest.h" +#include "paimon/core/core_options.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/defs.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/metrics.h" @@ -60,10 +63,12 @@ TEST(TableScanTest, TestDefaultMetricsSnapshot) { TEST(TableScanTest, TestNoSnapshot) { std::string path = paimon::test::GetDataDir() + "/orc/append_table_with_nested_type.db/append_table_with_nested_type/"; + std::map builder_options; ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + builder_options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_FALSE(plan->SnapshotId()); ASSERT_TRUE(plan->Splits().empty()); @@ -71,23 +76,27 @@ TEST(TableScanTest, TestNoSnapshot) { TEST(TableScanTest, TestNonExistTable) { std::string path = paimon::test::GetDataDir() + "/non-exist.db/non-exist/"; + std::map builder_options; ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + builder_options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), "not found latest schema"); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context), builder_options), + "not found latest schema"); } TEST(TableScanTest, TestStreamingSnapshotsRemoved) { auto dir = UniqueTestDirectory::Create("local"); ASSERT_TRUE(dir); ASSERT_TRUE(TestUtil::CopyDirectory(GetDataDir() + "/orc/append_09.db/append_09", dir->Str())); + std::map builder_options; ScanContextBuilder builder(dir->Str()); - builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_MODE, "latest") - .WithStreamingMode(true); + builder_options[Options::FILE_FORMAT] = "orc"; + builder_options[Options::SCAN_MODE] = "latest"; + builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + TableScan::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_plan, table_scan->CreatePlan()); ASSERT_TRUE(initial_plan->Splits().empty()); ASSERT_OK_AND_ASSIGN(std::shared_ptr waiting_plan, table_scan->CreatePlan()); @@ -103,14 +112,15 @@ TEST(TableScanTest, TestStreamingNextSnapshotBoundary) { std::string path = GetDataDir() + "/orc/append_09.db/append_09"; for (const std::string next_snapshot_id : {"6", "7"}) { SCOPED_TRACE(next_snapshot_id); + std::map builder_options; ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .AddOption(Options::SCAN_SNAPSHOT_ID, next_snapshot_id) - .WithStreamingMode(true); + builder_options[Options::FILE_FORMAT] = "orc"; + builder_options[Options::SCAN_MODE] = "from-snapshot"; + builder_options[Options::SCAN_SNAPSHOT_ID] = next_snapshot_id; + builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + TableScan::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_plan, table_scan->CreatePlan()); ASSERT_TRUE(initial_plan->Splits().empty()); @@ -131,10 +141,12 @@ TEST(TableScanTest, TestStreamingNextSnapshotBoundary) { TEST(TableScanTest, TestPkSchemaEvolutionScan) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table/"; + std::map builder_options; ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + builder_options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_TRUE(plan->SnapshotId()); ASSERT_FALSE(plan->Splits().empty()); @@ -164,12 +176,20 @@ TEST(TableScanTest, TestPkSchemaEvolutionScan) { TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table$ro"; + std::map options = {{Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), path, options)); + std::map builder_options; ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + builder_options[Options::FILE_FORMAT] = "orc"; + builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_1, table->Copy(builder_options)); + ASSERT_NOK_WITH_MSG(configured_table_1->NewScan(std::move(context)), "read-optimized system table does not support streaming scan for primary " "key table"); } diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 8de9742e1..9b4e1aaa7 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -42,8 +42,10 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -357,16 +359,21 @@ std::unique_ptr CreateChangelogBatchReader( AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, + std::string data_table_name, std::map options) - : fs_(std::move(fs)), + : AuditLogSystemTable(std::move(fs), std::move(table_path), std::move(table_schema), + std::move(options), ComposeName(data_table_name, kName)) {} + +AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, + std::shared_ptr table_schema, + std::map options, + std::string name) + : SystemTable(fs, table_path, std::move(name)), + fs_(std::move(fs)), table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} -std::string AuditLogSystemTable::Name() const { - return kName; -} - Result> AuditLogSystemTable::ArrowSchema() const { std::shared_ptr row_kind_field = DataField::ConvertDataFieldToArrowField(SpecialFields::RowKind()); @@ -383,17 +390,18 @@ Result> AuditLogSystemTable::ArrowSchema() const } Result> AuditLogSystemTable::NewScan( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { return Status::NotImplemented("audit_log system table predicate pushdown is not supported"); } std::shared_ptr scan_filter = context->GetScanFilters(); ScanContextBuilder builder(table_path_); - builder.SetOptions(options_) - .WithStreamingMode(context->IsStreamingMode()) + builder.WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + .WithFileSystem(GetFileSystem()) .WithCache(context->GetCache()); if (scan_filter) { if (scan_filter->GetBucketFilter()) { @@ -407,11 +415,16 @@ Result> AuditLogSystemTable::NewScan( builder.SetLimit(context->GetLimit().value()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, builder.Finish()); - return TableScan::Create(std::move(data_context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, options_)); + return data_table->NewScan(std::move(data_context)); } Result> AuditLogSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return NewChangelogRead(context, std::make_shared()); } @@ -430,8 +443,10 @@ Result> AuditLogSystemTable::NewChangelogRead( PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, CreateDataReadContext(context, read_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, read_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_read, - TableRead::Create(std::move(data_context))); + data_table->NewRead(std::move(data_context))); auto* key_value_read = dynamic_cast(data_read.get()); if (!key_value_read) { return Status::Invalid("audit_log system table requires key-value table read"); @@ -450,13 +465,10 @@ Result> AuditLogSystemTable::CreateDataReadContext( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr base_read_schema, BaseReadSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); ReadContextBuilder builder(table_path_); - builder.SetOptions(read_options) - .SetReadFieldNames(base_read_schema->field_names()) - .WithBranch(core_options.GetBranch()) + builder.SetReadFieldNames(base_read_schema->field_names()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) - .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) + .WithFileSystem(GetFileSystem()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) .SetPrefetchMaxParallelNum(context->GetPrefetchMaxParallelNum()) diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index 780f5fa72..bc15c385f 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -54,18 +54,21 @@ class AuditLogSystemTable : public SystemTable { public: static constexpr const char* kName = "audit_log"; + /// @param data_table_name Logical table name, including any branch suffix. AuditLogSystemTable(std::shared_ptr fs, std::string table_path, - std::shared_ptr table_schema, + std::shared_ptr table_schema, std::string data_table_name, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; - Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; protected: + /// Constructor for derived changelog views with a different system-table name. + AuditLogSystemTable(std::shared_ptr fs, std::string table_path, + std::shared_ptr table_schema, + std::map options, std::string name); + Result> NewChangelogRead( const std::shared_ptr& context, std::shared_ptr converter) const; diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index 88541194d..7b73ce1dc 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -31,6 +31,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/table/source/table_read.h" @@ -72,13 +73,10 @@ std::shared_ptr CreateBinlogBatchConverter() { BinlogSystemTable::BinlogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, + std::string data_table_name, std::map options) : AuditLogSystemTable(std::move(fs), std::move(table_path), std::move(table_schema), - std::move(options)) {} - -std::string BinlogSystemTable::Name() const { - return kName; -} + std::move(options), ComposeName(data_table_name, kName)) {} Result> BinlogSystemTable::ArrowSchema() const { std::shared_ptr rowkind_field = @@ -98,7 +96,10 @@ Result> BinlogSystemTable::ArrowSchema() const { } Result> BinlogSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return NewChangelogRead(context, CreateBinlogBatchConverter()); } diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index 3b4f45bd0..fcdcb84b6 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -36,14 +36,13 @@ class BinlogSystemTable : public AuditLogSystemTable { public: static constexpr const char* kName = "binlog"; + /// @param data_table_name Logical table name, including any branch suffix. BinlogSystemTable(std::shared_ptr fs, std::string table_path, - std::shared_ptr table_schema, + std::shared_ptr table_schema, std::string data_table_name, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewRead(std::unique_ptr context) const override; }; } // namespace paimon diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 647da76c3..c04900638 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -126,6 +126,16 @@ Result IsEnabled(const GlobalSystemTableRegistryEntry& entry, CatalogOptionsSystemTable::kEnabledOption, false); } +bool RequiresCatalog(const std::string& table_name) { + std::string normalized = StringUtils::ToLowerCase(table_name); + for (const auto& entry : GlobalSystemTableRegistry()) { + if (entry.name == normalized) { + return entry.requires_catalog; + } + } + return false; +} + struct CatalogTableInfo { std::string database_name; std::string table_name; @@ -134,11 +144,15 @@ struct CatalogTableInfo { // Match Java CatalogUtils::listAllTables: tolerate databases or tables removed concurrently, but // propagate all other catalog and schema errors instead of returning incomplete system-table rows. -Result> LoadAllDataTables(const Catalog& catalog) { +Result> LoadAllDataTables( + const std::shared_ptr& catalog, const std::string& table_name) { + if (catalog == nullptr) { + return Status::NotImplemented("global system table requires catalog context: ", table_name); + } std::vector result; - PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog.ListDatabases()); + PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog->ListDatabases()); for (const std::string& database : databases) { - Result> tables_result = catalog.ListTables(database); + Result> tables_result = catalog->ListTables(database); if (!tables_result.ok()) { if (tables_result.status().IsNotExist()) { continue; @@ -147,7 +161,7 @@ Result> LoadAllDataTables(const Catalog& catalog) } for (const std::string& table : tables_result.value()) { Identifier identifier(database, table); - Result> schema_result = catalog.LoadTableSchema(identifier); + Result> schema_result = catalog->LoadTableSchema(identifier); if (!schema_result.ok()) { if (schema_result.status().IsNotExist()) { continue; @@ -288,6 +302,20 @@ Result GlobalSystemTableLoader::IsSupported( } Result> GlobalSystemTableLoader::Load( + const std::string& table_name, const GlobalSystemTableContext& context) { + if (context.catalog == nullptr && RequiresCatalog(table_name)) { + return Status::NotImplemented("global system table requires catalog context: ", table_name); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + LoadForSchema(table_name, context)); + SystemTable::Origin origin; + origin.kind = StringUtils::ToLowerCase(table_name); + origin.context = std::make_shared(context); + system_table->origin_.emplace(std::move(origin)); + return system_table; +} + +Result> GlobalSystemTableLoader::LoadForSchema( const std::string& table_name, const GlobalSystemTableContext& context) { std::string normalized = StringUtils::ToLowerCase(table_name); for (const auto& entry : GlobalSystemTableRegistry()) { @@ -296,11 +324,10 @@ Result> GlobalSystemTableLoader::Load( if (!enabled) { return Status::NotExist("global system table is disabled: ", table_name); } - if (entry.requires_catalog && context.catalog == nullptr) { - return Status::NotImplemented("global system table requires catalog context: ", - table_name); - } - return entry.factory(context); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + entry.factory(context)); + PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); + return system_table; } } return Status::NotImplemented("unsupported global system table: ", table_name); @@ -324,11 +351,7 @@ Result> GlobalSystemTableLoader::GetSupportedTableNames // ============================================================================= CatalogOptionsSystemTable::CatalogOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/catalog_options"), context_(std::move(context)) {} - -std::string CatalogOptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/catalog_options", kName), context_(std::move(context)) {} Result> CatalogOptionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -360,11 +383,8 @@ Result> CatalogOptionsSystemTable::BuildRows() const { // ============================================================================= AllTableOptionsSystemTable::AllTableOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/all_table_options"), context_(std::move(context)) {} - -std::string AllTableOptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/all_table_options", kName), + context_(std::move(context)) {} Result> AllTableOptionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -380,7 +400,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { for (const auto& [key, value] : table.schema->Options()) { GenericRow row(schema->num_fields()); @@ -399,11 +419,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { // ============================================================================= TablesSystemTable::TablesSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/tables"), context_(std::move(context)) {} - -std::string TablesSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/tables", kName), context_(std::move(context)) {} Result> TablesSystemTable::ArrowSchema() const { return arrow::schema({ @@ -429,7 +445,7 @@ Result> TablesSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { const std::shared_ptr& data_schema = table.schema; @@ -471,11 +487,7 @@ Result> TablesSystemTable::BuildRows() const { // ============================================================================= PartitionsSystemTable::PartitionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/partitions"), context_(std::move(context)) {} - -std::string PartitionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/partitions", kName), context_(std::move(context)) {} Result> PartitionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -495,7 +507,7 @@ Result> PartitionsSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { const std::shared_ptr& data_schema = table.schema; Identifier id(table.database_name, table.table_name); diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h index 3d6936262..6ac4fbb30 100644 --- a/src/paimon/core/table/system/global_system_tables.h +++ b/src/paimon/core/table/system/global_system_tables.h @@ -33,7 +33,8 @@ class FileSystem; /// Context passed to global system table constructors, providing catalog-level /// access for enumerating databases, tables, and reading metadata. struct GlobalSystemTableContext { - const Catalog* catalog = nullptr; // non-owning pointer + /// Shared catalog for lazy row loading; may be empty for schema-only tables. + std::shared_ptr catalog; std::shared_ptr fs; std::string warehouse; std::map catalog_options; @@ -48,7 +49,6 @@ class CatalogOptionsSystemTable : public InMemorySystemTable { explicit CatalogOptionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -64,7 +64,6 @@ class AllTableOptionsSystemTable : public InMemorySystemTable { explicit AllTableOptionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -80,7 +79,6 @@ class TablesSystemTable : public InMemorySystemTable { explicit TablesSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -96,7 +94,6 @@ class PartitionsSystemTable : public InMemorySystemTable { explicit PartitionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -113,9 +110,14 @@ class GlobalSystemTableLoader { static Result IsSupported(const std::string& table_name, const std::map& catalog_options); + /// Loads a global system table. Views that enumerate tables require a catalog in context. static Result> Load(const std::string& table_name, const GlobalSystemTableContext& context); + /// Loads only the system table schema; no catalog is required. + static Result> LoadForSchema( + const std::string& table_name, const GlobalSystemTableContext& context); + static Result> GetSupportedTableNames( const std::map& catalog_options); }; diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index 8d6111d79..a2c69195a 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -29,8 +29,10 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/core/io/generic_row_to_arrow_array_converter.h" #include "paimon/core/table/system/system_table_scan.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/memory/memory_pool.h" #include "paimon/read_context.h" +#include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/source/table_read.h" @@ -104,16 +106,20 @@ class InMemorySystemTableRead : public TableRead { } // namespace -InMemorySystemTable::InMemorySystemTable(std::string table_path) - : table_path_(std::move(table_path)) {} +InMemorySystemTable::InMemorySystemTable(const std::shared_ptr& fs, + std::string table_path, std::string name) + : SystemTable(fs, table_path, std::move(name)), table_path_(std::move(table_path)) {} Result> InMemorySystemTable::NewScan( - const std::shared_ptr& /*context*/) const { + std::unique_ptr /*context*/) const { return std::make_unique(table_path_); } Result> InMemorySystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return std::make_unique( checked_pointer_cast(shared_from_this()), context->GetMemoryPool()); diff --git a/src/paimon/core/table/system/in_memory_system_table.h b/src/paimon/core/table/system/in_memory_system_table.h index f9fcebdb3..c72d7389e 100644 --- a/src/paimon/core/table/system/in_memory_system_table.h +++ b/src/paimon/core/table/system/in_memory_system_table.h @@ -28,18 +28,20 @@ namespace paimon { +class FileSystem; + /// Base class for system tables whose result can be materialized as a single in-memory /// RecordBatch. /// /// It provides the common singleton split scan and one-shot batch reader. class InMemorySystemTable : public SystemTable { public: - explicit InMemorySystemTable(std::string table_path); + InMemorySystemTable(const std::shared_ptr& fs, std::string table_path, + std::string name); Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + std::unique_ptr /*context*/) const override; + Result> NewRead(std::unique_ptr context) const override; virtual Result> BuildRows() const = 0; protected: diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index 5279e6871..3ccb488de 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -504,13 +504,11 @@ Result> WriteColsValue( } // namespace -OptionsSystemTable::OptionsSystemTable(std::string table_path, +OptionsSystemTable::OptionsSystemTable(const std::shared_ptr& fs, + std::string table_path, std::shared_ptr table_schema) - : InMemorySystemTable(std::move(table_path)), table_schema_(std::move(table_schema)) {} - -std::string OptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), + table_schema_(std::move(table_schema)) {} Result> OptionsSystemTable::ArrowSchema() const { return arrow::schema({arrow::field("key", arrow::utf8(), /*nullable=*/false), @@ -532,13 +530,9 @@ Result> OptionsSystemTable::BuildRows() const { SnapshotsSystemTable::SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string SnapshotsSystemTable::Name() const { - return kName; -} - Result> SnapshotsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("snapshot_id", arrow::int64(), /*nullable=*/false), @@ -594,13 +588,9 @@ Result> SnapshotsSystemTable::BuildRows() const { SchemasSystemTable::SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string SchemasSystemTable::Name() const { - return kName; -} - Result> SchemasSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("schema_id", arrow::int64(), /*nullable=*/false), @@ -650,13 +640,9 @@ Result> SchemasSystemTable::BuildRows() const { TagsSystemTable::TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string TagsSystemTable::Name() const { - return kName; -} - Result> TagsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("tag_name", arrow::utf8(), /*nullable=*/false), @@ -700,13 +686,9 @@ Result> TagsSystemTable::BuildRows() const { BranchesSystemTable::BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string BranchesSystemTable::Name() const { - return kName; -} - Result> BranchesSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("branch_name", arrow::utf8(), /*nullable=*/false), @@ -739,13 +721,9 @@ Result> BranchesSystemTable::BuildRows() const { ConsumersSystemTable::ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string ConsumersSystemTable::Name() const { - return kName; -} - Result> ConsumersSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("consumer_id", arrow::utf8(), /*nullable=*/false), @@ -774,14 +752,10 @@ ManifestsSystemTable::ManifestsSystemTable(std::shared_ptr fs, std:: std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} -std::string ManifestsSystemTable::Name() const { - return kName; -} - Result> ManifestsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("file_name", arrow::utf8(), /*nullable=*/false), @@ -843,14 +817,10 @@ Result> ManifestsSystemTable::BuildRows() const { FilesSystemTable::FilesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} -std::string FilesSystemTable::Name() const { - return kName; -} - Result> FilesSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("partition", arrow::utf8(), /*nullable=*/true), diff --git a/src/paimon/core/table/system/metadata_system_tables.h b/src/paimon/core/table/system/metadata_system_tables.h index e3961e476..1127e37ed 100644 --- a/src/paimon/core/table/system/metadata_system_tables.h +++ b/src/paimon/core/table/system/metadata_system_tables.h @@ -35,9 +35,9 @@ class OptionsSystemTable : public InMemorySystemTable { public: static constexpr const char* kName = "options"; - OptionsSystemTable(std::string table_path, std::shared_ptr table_schema); + OptionsSystemTable(const std::shared_ptr& fs, std::string table_path, + std::shared_ptr table_schema); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -62,7 +62,6 @@ class SnapshotsSystemTable : public InMemorySystemTable { SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -77,7 +76,6 @@ class SchemasSystemTable : public InMemorySystemTable { SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -92,7 +90,6 @@ class TagsSystemTable : public InMemorySystemTable { TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -107,7 +104,6 @@ class BranchesSystemTable : public InMemorySystemTable { BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -123,7 +119,6 @@ class ConsumersSystemTable : public InMemorySystemTable { ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -140,7 +135,6 @@ class ManifestsSystemTable : public InMemorySystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -157,7 +151,6 @@ class FilesSystemTable : public InMemorySystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 8d5696447..e7bd4a06a 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -26,7 +26,9 @@ #include "arrow/c/bridge.h" #include "paimon/common/types/data_field.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/read_optimized_scan_options.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -35,17 +37,16 @@ namespace paimon { -ReadOptimizedSystemTable::ReadOptimizedSystemTable(std::string table_path, +ReadOptimizedSystemTable::ReadOptimizedSystemTable(const std::shared_ptr& fs, + std::string table_path, std::shared_ptr table_schema, + std::string data_table_name, std::map options) - : table_path_(std::move(table_path)), + : SystemTable(fs, table_path, ComposeName(data_table_name, kName)), + table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} -std::string ReadOptimizedSystemTable::Name() const { - return kName; -} - Result> ReadOptimizedSystemTable::ArrowSchema() const { return DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields()); } @@ -57,18 +58,19 @@ std::map ReadOptimizedSystemTable::ReadOptimizedOption } Result> ReadOptimizedSystemTable::NewScan( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetRealtimeContext() && !table_schema_->PrimaryKeys().empty()) { return Status::NotImplemented( "PK real-time union read does not support read-optimized scans"); } auto options = ReadOptimizedOptions(); ScanContextBuilder builder(table_path_); - builder.SetOptions(options) - .WithStreamingMode(context->IsStreamingMode()) + builder.WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + .WithFileSystem(GetFileSystem()) .WithCache(context->GetCache()); if (context->GetLimit().has_value()) { builder.SetLimit(context->GetLimit().value()); @@ -83,26 +85,16 @@ Result> ReadOptimizedSystemTable::NewScan( if (context->GetGlobalIndexResult()) { builder.SetGlobalIndexResult(context->GetGlobalIndexResult()); } - if (context->GetSpecificTableSchema().has_value()) { - builder.SetTableSchema(context->GetSpecificTableSchema().value()); - } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, builder.Finish()); - return TableScan::Create(std::move(base_context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, options)); + return data_table->NewScan(std::move(base_context)); } Result> ReadOptimizedSystemTable::CreateDataReadContext( const std::shared_ptr& context) const { - auto options = options_; - std::string branch = context->GetBranch(); - // SystemTableLoader injects Options::BRANCH when parsing paths such as `T$branch_dev$ro`. - auto branch_iter = options.find(Options::BRANCH); - if (branch_iter != options.end()) { - branch = branch_iter->second; - } ReadContextBuilder builder(table_path_); - builder.SetOptions(options) - .WithBranch(branch) - .SetPredicate(context->GetPredicate()) + builder.SetPredicate(context->GetPredicate()) .EnablePredicateFilter(context->EnablePredicateFilter()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) @@ -111,8 +103,7 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) - .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) + .WithFileSystem(GetFileSystem()) .SetReadAheadCacheEnabled(context->ReadAheadCacheEnabled()) .WithCacheConfig(context->GetCacheConfig()) .SetWarmupLevel(context->GetWarmupLevel()) @@ -126,17 +117,19 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, c_read_schema.get())); builder.SetReadSchema(std::move(c_read_schema)); } - if (context->GetSpecificTableSchema().has_value()) { - builder.SetTableSchema(context->GetSpecificTableSchema().value()); - } return builder.Finish(); } Result> ReadOptimizedSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, CreateDataReadContext(context)); - return TableRead::Create(std::move(base_context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, options_)); + return data_table->NewRead(std::move(base_context)); } } // namespace paimon diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h index 3123fe03b..164b2c79c 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.h +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -26,6 +26,7 @@ #include "paimon/core/table/system/system_table.h" namespace paimon { +class FileSystem; class TableSchema; /// System table for `T$ro`, exposing read-optimized data. @@ -33,15 +34,14 @@ class ReadOptimizedSystemTable : public SystemTable { public: static constexpr const char* kName = "ro"; - ReadOptimizedSystemTable(std::string table_path, std::shared_ptr table_schema, + /// @param data_table_name Logical table name, including any branch suffix. + ReadOptimizedSystemTable(const std::shared_ptr& fs, std::string table_path, + std::shared_ptr table_schema, std::string data_table_name, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; - Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; private: /// Build the context for the data table underneath this read-optimized view. diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 5a1a1de00..06291ebea 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -26,26 +26,39 @@ #include #include +#include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" +#include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/metadata_system_tables.h" #include "paimon/core/table/system/read_optimized_system_table.h" +#include "paimon/core/table/system/system_table_schema.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/read_context.h" #include "paimon/status.h" +#include "paimon/write_context.h" namespace paimon { namespace { using SystemTableFactory = std::function>( const std::shared_ptr&, const std::string&, const std::shared_ptr&, - const std::map&)>; + const std::map&, const std::string&)>; struct SystemTableRegistryEntry { std::string name; @@ -70,81 +83,87 @@ std::string LoadBranch(const std::map& options) { const std::vector& SystemTableRegistry() { static const std::vector registry = { {OptionsSystemTable::kName, - [](const std::shared_ptr& /*fs*/, const std::string& table_path, + [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& /*dynamic_options*/) - -> Result> { - return std::make_shared(table_path, table_schema); + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged, + table_schema->MergeOptions(dynamic_options)); + return std::make_shared( + fs, table_path, std::shared_ptr(std::move(merged))); }}, {AuditLogSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& data_table_name) -> Result> { return std::make_shared( - fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, data_table_name, + MergeOptions(table_schema, dynamic_options)); }}, {BinlogSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& data_table_name) -> Result> { return std::make_shared( - fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, data_table_name, + MergeOptions(table_schema, dynamic_options)); }}, {ReadOptimizedSystemTable::kName, - [](const std::shared_ptr& /*fs*/, const std::string& table_path, + [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& data_table_name) -> Result> { return std::make_shared( - table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, data_table_name, + MergeOptions(table_schema, dynamic_options)); }}, {SnapshotsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {SchemasSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {TagsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {BranchesSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {ConsumersSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {ManifestsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); auto branch = LoadBranch(options); return std::make_shared(fs, table_path, std::move(branch), @@ -153,8 +172,8 @@ const std::vector& SystemTableRegistry() { {FilesSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); auto branch = LoadBranch(options); return std::make_shared(fs, table_path, std::move(branch), @@ -176,6 +195,93 @@ std::optional FindSystemTableFactory(const std::string& syst } // namespace +SystemTable::SystemTable(const std::shared_ptr& fs, const std::string& table_path, + std::string name) + : Table(fs, table_path, /*schema=*/nullptr, + Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/""), + name_(std::move(name)) {} + +std::string SystemTable::ComposeName(const std::string& data_table_name, const std::string& kind) { + return data_table_name + Catalog::SYSTEM_TABLE_SPLITTER + kind; +} + +Status SystemTable::CheckNoSearch(const std::shared_ptr& context) { + if (context->HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch does not support system-table reads yet"); + } + if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } + return Status::OK(); +} + +Status SystemTable::AdoptOwnSchema() { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, ArrowSchema()); + schema_ = std::make_shared(std::move(arrow_schema)); + return Status::OK(); +} + +Result> SystemTable::CreateDataTable( + const std::shared_ptr& table_schema, + const std::map& options) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, + table_schema->MergeOptions(options)); + const std::string branch = BranchManager::NormalizeBranch(LoadBranch(merged_schema->Options())); + return FileStoreTable::Create( + GetFileSystem(), GetRootPath(), std::shared_ptr(std::move(merged_schema)), + CatalogUtils::IdentifierFromPath(GetRootPath(), branch), "", branch, + origin_ ? origin_->fs_scheme_map : std::map{}, options); +} + +Result> SystemTable::Copy( + const std::map& dynamic_options) const { + if (!origin_) { + return Status::Invalid(Name(), + " was not loaded by a system table loader: copying it is " + "not supported"); + } + if (origin_->context) { + // Global system tables ignore dynamic options when copied. + return GlobalSystemTableLoader::Load(origin_->kind, *origin_->context); + } + if (origin_->kind == AuditLogSystemTable::kName || origin_->kind == BinlogSystemTable::kName) { + PAIMON_ASSIGN_OR_RAISE( + bool sequence_number, + OptionsUtils::GetValueFromMap( + dynamic_options, Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, false)); + if (sequence_number) { + return Status::Invalid("table-read.sequence-number.enabled is not supported by hint."); + } + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr current_schema, + origin_->table_schema->MergeOptions(origin_->dynamic_options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr copied_schema, + FileStoreTable::CopyTableSchema(GetFileSystem(), GetRootPath(), + std::shared_ptr(std::move(current_schema)), + origin_->dynamic_options, dynamic_options)); + auto accumulated = origin_->dynamic_options; + for (const auto& [key, value] : dynamic_options) { + if (key != Options::TYPE) { + accumulated.insert_or_assign(key, value); + } + } + return SystemTableLoader::Load(origin_->kind, GetFileSystem(), GetRootPath(), copied_schema, + accumulated, origin_->data_table_name, origin_->fs_scheme_map); +} + +Result> SystemTable::NewWrite( + std::unique_ptr /*context*/) const { + return Status::NotImplemented(Name(), " system table is read-only: writing is not supported"); +} + +Result> SystemTable::NewCommit( + std::unique_ptr /*context*/) const { + return Status::NotImplemented(Name(), + " system table is read-only: committing for it is not supported"); +} + bool SystemTableLoader::IsSupported(const std::string& system_table_name) { return FindSystemTableFactory(system_table_name).has_value(); } @@ -183,10 +289,24 @@ bool SystemTableLoader::IsSupported(const std::string& system_table_name) { Result> SystemTableLoader::Load( const std::string& system_table_name, const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) { + const std::map& dynamic_options, const std::string& data_table_name, + const std::map& fs_scheme_map) { std::optional factory = FindSystemTableFactory(system_table_name); if (factory) { - return factory.value()(fs, table_path, table_schema, dynamic_options); + const std::string effective_name = + data_table_name.empty() ? PathUtil::GetName(table_path) : data_table_name; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + factory.value()(fs, table_path, table_schema, dynamic_options, effective_name)); + SystemTable::Origin origin; + origin.kind = StringUtils::ToLowerCase(system_table_name); + origin.table_schema = table_schema; + origin.dynamic_options = dynamic_options; + origin.fs_scheme_map = fs_scheme_map; + origin.data_table_name = data_table_name; + system_table->origin_.emplace(std::move(origin)); + PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); + return system_table; } return Status::NotImplemented("unsupported system table: ", system_table_name); } @@ -237,7 +357,6 @@ Result> SystemTableLoader::LoadFromPath( // The warehouse is the grandparent of the sys/
path context.warehouse = PathUtil::GetParentDirPath(PathUtil::GetParentDirPath(path)); context.catalog_options = dynamic_options; - // Note: context.catalog is intentionally left as nullptr here. // Global tables loaded from path do not have a Catalog reference and // cannot enumerate databases/tables. Only tables that don't require // catalog enumeration (e.g. catalog_options) will work in this path. @@ -255,7 +374,12 @@ Result> SystemTableLoader::LoadFromPath( if (parsed.branch) { options[Options::BRANCH] = parsed.branch.value(); } - return Load(parsed.system_table_name, fs, parsed.table_path, latest_schema.value(), options); + // Preserve branch-qualified logical names when deriving identifiers from paths. + const Identifier data_identifier(Identifier::kUnknownDatabase, + PathUtil::GetName(parsed.table_path), + parsed.branch.value_or(std::string())); + return Load(parsed.system_table_name, fs, parsed.table_path, latest_schema.value(), options, + data_identifier.GetObjectName()); } } // namespace paimon diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 39e1da31b..e76c21e83 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -25,17 +25,20 @@ #include #include "arrow/api.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/result.h" #include "paimon/status.h" #include "paimon/type_fwd.h" namespace paimon { class FileSystem; +class FileStoreTable; class ReadContext; class ScanContext; -class TableScan; -class TableRead; class TableSchema; +class SystemTableSchema; +struct GlobalSystemTableContext; /// Parsed information for a table-scoped system table path. struct SystemTablePath { @@ -49,19 +52,71 @@ struct SystemTablePath { bool is_global = false; }; -/// Base interface for table-scoped system tables such as `T$options` and `T$snapshots`. -/// -/// Implementations expose a read-only schema and create their own scan/read objects. -class SystemTable : public std::enable_shared_from_this { +/// Read-only table-scoped or global system view. +/// Table-scoped names are $; global names are . FullName() equals Name(). +class SystemTable : public Table, public std::enable_shared_from_this { public: - virtual ~SystemTable() = default; + std::string Name() const override { + return name_; + } + + std::string FullName() const override { + return name_; + } - virtual std::string Name() const = 0; + /// Returns this view's schema, rather than its data table's schema. virtual Result> ArrowSchema() const = 0; - virtual Result> NewScan( - const std::shared_ptr& context) const = 0; - virtual Result> NewRead( - const std::shared_ptr& context) const = 0; + + /// Copies a table-scoped view with dynamic option overrides. Global views ignore overrides. + Result> Copy( + const std::map& dynamic_options) const override; + + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; + + protected: + /// @param fs Data table file system. + /// @param table_path Data table root path. + /// @param name System table name. + SystemTable(const std::shared_ptr& fs, const std::string& table_path, + std::string name); + + /// Composes $, preserving any branch suffix. + static std::string ComposeName(const std::string& data_table_name, const std::string& kind); + + /// Rejects TopK and full-text search requests, which system-table readers cannot evaluate. + static Status CheckNoSearch(const std::shared_ptr& context); + + /// Initializes the system schema after construction, when ArrowSchema() can be called. + Status AdoptOwnSchema(); + + /// Builds the data table using this view's schema, including one selected by time travel. + Result> CreateDataTable( + const std::shared_ptr& table_schema, + const std::map& options) const; + + friend class SystemTableLoader; + friend class GlobalSystemTableLoader; + + private: + /// Loader inputs retained for Copy. + struct Origin { + std::string kind; + /// Data table schema the view was loaded with. Null for a global system table. + std::shared_ptr table_schema; + std::map dynamic_options; + std::map fs_scheme_map; + /// Logical data table name, including its branch; empty uses the directory name. + std::string data_table_name; + /// Global table context; null for table-scoped views. + std::shared_ptr context; + }; + + std::string name_; + std::optional origin_; }; /// Loads system table implementations from parsed table identifiers or table paths. @@ -74,7 +129,9 @@ class SystemTableLoader { static Result> Load( const std::string& system_table_name, const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options); + const std::map& dynamic_options, + const std::string& data_table_name, + const std::map& fs_scheme_map = {}); static Result> TryParsePath(const std::string& path); diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 7591e53bf..9f4924c61 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -31,6 +31,7 @@ #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" #include "paimon/common/data/generic_row.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/core_options.h" @@ -38,20 +39,32 @@ #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" #include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/metadata_system_tables.h" #include "paimon/core/table/system/read_optimized_system_table.h" #include "paimon/core/table/system/system_table_scan.h" #include "paimon/defs.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" #include "paimon/fs/file_system_factory.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" +#include "paimon/predicate/full_text_match.h" +#include "paimon/predicate/predicate.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" #include "paimon/result.h" +#include "paimon/scan_context.h" #include "paimon/status.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_file_batch_reader.h" +#include "paimon/testing/mock/mock_file_system.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/write_context.h" namespace paimon::test { namespace { @@ -77,11 +90,11 @@ TEST(SystemTableTest, TestChangelogArrowSchemaReturnsInvalidOptions) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, CreateTableSchemaForTest(options)); - AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); ASSERT_NOK_WITH_MSG(audit_log.ArrowSchema(), "Invalid Config [table-read.sequence-number.enabled: invalid]"); - BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); ASSERT_NOK_WITH_MSG(binlog.ArrowSchema(), "Invalid Config [table-read.sequence-number.enabled: invalid]"); } @@ -92,7 +105,7 @@ TEST(SystemTableTest, TestBinlogArrowSchemaWithSequenceNumber) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, CreateTableSchemaForTest(options)); - BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, binlog.ArrowSchema()); ASSERT_EQ(schema->field_names(), @@ -218,8 +231,14 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table, SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, "/tmp/table", table_schema, - /*dynamic_options=*/{})); - ASSERT_EQ(system_table->Name(), ReadOptimizedSystemTable::kName); + /*dynamic_options=*/{}, /*data_table_name=*/"")); + ASSERT_EQ(system_table->Name(), "table$ro"); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr branch_view, + SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, + "/tmp/uuid-dir", table_schema, + /*dynamic_options=*/{}, "tbl$branch_dev")); + ASSERT_EQ(branch_view->Name(), "tbl$branch_dev$ro"); ASSERT_OK_AND_ASSIGN(std::shared_ptr arrow_schema, system_table->ArrowSchema()); ASSERT_EQ(arrow_schema->field_names(), (std::vector{"pk", "v"})); @@ -227,6 +246,60 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { ASSERT_EQ(arrow_schema->field(1)->type()->id(), arrow::Type::INT32); } +TEST(SystemTableTest, TestCopyRebuildsThroughTheLoader) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table, + SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, + "/tmp/table", table_schema, + /*dynamic_options=*/{}, /*data_table_name=*/"")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, system_table->Copy({})); + EXPECT_EQ(copy->Name(), system_table->Name()); + EXPECT_NE(copy, system_table); + EXPECT_TRUE(std::dynamic_pointer_cast(copy) != nullptr); + + ASSERT_NOK_WITH_MSG(system_table->Copy({{Options::BUCKET_KEY, "other"}}), + "Change 'bucket-key' is not supported yet."); + ASSERT_NOK_WITH_MSG(system_table->Copy({{Options::TYPE, "format-table"}}), + "Change 'type' is not supported yet."); +} + +TEST(SystemTableTest, TestCopyOptionsAndAuditSequenceNumber) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr schema, + CreateTableSchemaForTest({{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr options, + SystemTableLoader::Load(OptionsSystemTable::kName, dir->GetFileSystem(), + dir->Str(), schema, {}, "tbl")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, + options->Copy({{Options::READ_BATCH_SIZE, "2"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
again, copied->Copy({{"custom", "value"}})); + auto options_copy = std::dynamic_pointer_cast(again); + ASSERT_TRUE(options_copy); + ASSERT_OK_AND_ASSIGN(std::vector rows, options_copy->BuildRows()); + std::map values; + for (const auto& row : rows) { + values.emplace(std::string(row.GetStringView(0)), std::string(row.GetStringView(1))); + } + EXPECT_EQ(values.at(Options::READ_BATCH_SIZE), "2"); + EXPECT_EQ(values.at("custom"), "value"); + EXPECT_EQ(schema->Options().count(Options::READ_BATCH_SIZE), 0); + for (const std::string& kind : + {std::string(AuditLogSystemTable::kName), std::string(BinlogSystemTable::kName)}) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr view, + SystemTableLoader::Load(kind, dir->GetFileSystem(), dir->Str(), schema, {}, "tbl")); + ASSERT_NOK_WITH_MSG(view->Copy({{Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, "true"}}), + "is not supported by hint"); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
allowed, + view->Copy({{Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, "false"}})); + } +} + TEST(SystemTableTest, TestReadOptimizedSystemTablePathParsing) { ASSERT_OK_AND_ASSIGN(std::optional parsed, SystemTableLoader::TryParsePath("/tmp/db.db/t$branch_audit$ro")); @@ -241,19 +314,57 @@ TEST(SystemTableTest, TestReadOptimizedSystemTablePathParsing) { // defaults, so a setting that is not copied across silently reverts: a caller that asked for NONE // or DECODED would get RAW back. `$ro` has its own builder chain and `$audit_log` and `$binlog` // share one, so all three are pinned here. +TEST(SystemTableTest, TestDataReadContextsUseBoundStorage) { + const std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, CreateTableSchemaForTest(options)); + auto fs = std::make_shared(); + auto caller_fs = std::make_shared(); + ReadOptimizedSystemTable ro(fs, "/table", schema, "table", options); + AuditLogSystemTable audit(fs, "/table", schema, "table", options); + BinlogSystemTable binlog(fs, "/table", schema, "table", options); + for (bool use_scheme_map : {false, true}) { + ReadContextBuilder builder("another-table"); + builder.SetTableSchema("invalid schema").SetWarmupLevel(WarmupLevel::DECODED); + if (use_scheme_map) { + builder.WithFileSystemSchemeToIdentifierMap({{"file", "missing"}}); + } else { + builder.WithFileSystem(caller_fs); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_context, + ro.CreateDataReadContext(context)); + ASSERT_OK_AND_ASSIGN(auto audit_options, audit.ReadOptions()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr audit_context, + audit.CreateDataReadContext(context, audit_options)); + ASSERT_OK_AND_ASSIGN(auto binlog_options, binlog.ReadOptions()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr binlog_context, + binlog.CreateDataReadContext(context, binlog_options)); + for (const ReadContext* effective : + {ro_context.get(), audit_context.get(), binlog_context.get()}) { + ASSERT_EQ(effective->GetSpecificFileSystem(), fs); + ASSERT_TRUE(effective->GetFileSystemSchemeToIdentifierMap().empty()); + ASSERT_FALSE(effective->GetSpecificTableSchema()); + ASSERT_EQ(effective->GetPath(), "/table"); + ASSERT_EQ(effective->GetWarmupLevel(), WarmupLevel::DECODED); + } + } +} + TEST(SystemTableTest, TestNewReadPropagatesWarmupLevel) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, CreateTableSchemaForTest(options)); - AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); - BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); - ReadOptimizedSystemTable read_optimized("/tmp/table", table_schema, options); + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, "table", + options); for (WarmupLevel level : {WarmupLevel::NONE, WarmupLevel::RAW, WarmupLevel::DECODED}) { ReadContextBuilder builder("/tmp/table"); - builder.SetOptions(options).SetWarmupLevel(level); + builder.SetWarmupLevel(level); ASSERT_OK_AND_ASSIGN(std::unique_ptr caller_unique_context, builder.Finish()); std::shared_ptr caller_context(std::move(caller_unique_context)); ASSERT_EQ(level, caller_context->GetWarmupLevel()); @@ -287,6 +398,111 @@ TEST(SystemTableTest, TestGlobalSystemTableWithoutCatalogReturnsNotImplemented) "global system table requires catalog context: tables"); } +TEST(SystemTableTest, TestOperatorEntriesRefuseANullContext) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, "table", + options); + + ASSERT_NOK_WITH_MSG(audit_log.NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(binlog.NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(read_optimized.NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(audit_log.NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(binlog.NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(read_optimized.NewRead(nullptr), "read context is null pointer"); +} + +TEST(SystemTableTest, TestTheOperatorEntriesRefuseASystemTablePath) { + const std::string system_table_path = "/tmp/warehouse/db.db/tbl$options"; + + ScanContextBuilder scan_builder(system_table_path); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), {}), "names a system table"); + + ReadContextBuilder read_builder(system_table_path); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), {}), "names a system table"); + + WriteContextBuilder write_builder(system_table_path, "commit_user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context), {}), + "names a system table"); + + CommitContextBuilder commit_builder(system_table_path, "commit_user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context), {}), + "names a system table"); +} + +TEST(SystemTableTest, TestNewReadRefusesASearch) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, "table", + options); + + auto make_context = [] { + auto vector_search = std::make_shared( + "v", /*limit=*/3, std::vector{1.0f}, /*pre_filter=*/nullptr, + /*predicate=*/nullptr, VectorSearch::DistanceType::COSINE, + std::map{}); + ReadContextBuilder builder("/tmp/table"); + builder.SetVectorSearch(vector_search); + return builder.Finish(); + }; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr audit_log_context, make_context()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr binlog_context, make_context()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_optimized_context, make_context()); + + ASSERT_NOK_WITH_MSG(audit_log.NewRead(std::move(audit_log_context)), + "VectorSearch or FullTextSearch does not support system-table reads yet"); + ASSERT_NOK_WITH_MSG(binlog.NewRead(std::move(binlog_context)), + "VectorSearch or FullTextSearch does not support system-table reads yet"); + ASSERT_NOK_WITH_MSG(read_optimized.NewRead(std::move(read_optimized_context)), + "VectorSearch or FullTextSearch does not support system-table reads yet"); +} + +TEST(SystemTableTest, TestNewReadRefusesAFullTextMatchPredicate) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr in_memory, + SystemTableLoader::Load(OptionsSystemTable::kName, /*fs=*/nullptr, + "/tmp/table", table_schema, + /*dynamic_options=*/{}, /*data_table_name=*/"")); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, "table", + options); + + auto match = std::make_shared("pk", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::FullTextMatch(/*field_index=*/0, match)); + + auto make_context = [&predicate] { + ReadContextBuilder builder("/tmp/table"); + builder.SetPredicate(predicate); + return builder.Finish(); + }; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr in_memory_context, make_context()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_optimized_context, make_context()); + + ASSERT_NOK_WITH_MSG(in_memory->NewRead(std::move(in_memory_context)), + "FullTextMatch predicate execution is not implemented yet"); + ASSERT_NOK_WITH_MSG(read_optimized.NewRead(std::move(read_optimized_context)), + "FullTextMatch predicate execution is not implemented yet"); +} + namespace { /// A catalog that serves the files of every table through `GetTableFileSystem`, like one @@ -319,21 +535,23 @@ TEST(SystemTableTest, TestGlobalPartitionsUsesTheFileSystemOfEachTable) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - PerTableFileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); - ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + std::shared_ptr catalog = + std::make_shared(core_options.GetFileSystem(), dir->Str(), + options); + ASSERT_OK(catalog->CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::Schema typed_schema( {arrow::field("dt", arrow::utf8()), arrow::field("v", arrow::int32(), /*nullable=*/true)}); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(typed_schema, &c_schema).ok()); - Status created = catalog.CreateTable(Identifier("db1", "t1"), &c_schema, - /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, options, - /*ignore_if_exists=*/false); + Status created = catalog->CreateTable(Identifier("db1", "t1"), &c_schema, + /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false); ArrowSchemaRelease(&c_schema); ASSERT_OK(created); GlobalSystemTableContext context; - context.catalog = &catalog; + context.catalog = catalog; context.fs = core_options.GetFileSystem(); context.warehouse = dir->Str(); context.catalog_options = options; @@ -343,16 +561,16 @@ TEST(SystemTableTest, TestGlobalPartitionsUsesTheFileSystemOfEachTable) { // still the ones asked for before its files are listed. ASSERT_OK_AND_ASSIGN(std::vector rows, partitions.BuildRows()); ASSERT_TRUE(rows.empty()); - ASSERT_EQ((std::vector{"db1.t1"}), catalog.requested); + ASSERT_EQ((std::vector{"db1.t1"}), catalog->requested); // A table whose credentials cannot be issued fails the query, so that a partial result // is never mistaken for the whole one. - catalog.failure = Status::Invalid("no credentials for the table"); + catalog->failure = Status::Invalid("no credentials for the table"); ASSERT_NOK_WITH_MSG(partitions.BuildRows(), "no credentials for the table"); // A table that went away between being listed and being read is skipped instead, like // one whose location has already been removed. - catalog.failure = Status::NotExist("table dropped"); + catalog->failure = Status::NotExist("table dropped"); ASSERT_OK_AND_ASSIGN(std::vector skipped, partitions.BuildRows()); ASSERT_TRUE(skipped.empty()); } diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index dd7e18f20..3180a48fe 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -9,202 +9,17 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/catalog/table.h" -#include -#include -#include -#include -#include -#include -#include -#include - -#include "arrow/c/bridge.h" -#include "fmt/format.h" -#include "paimon/catalog/catalog.h" -#include "paimon/commit_context.h" -#include "paimon/common/global_index/btree/btree_defs.h" -#include "paimon/common/predicate/predicate_validator.h" -#include "paimon/common/types/data_field.h" -#include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/common/utils/binary_row_partition_computer.h" -#include "paimon/common/utils/fields_comparator.h" -#include "paimon/common/utils/options_utils.h" -#include "paimon/common/utils/string_utils.h" -#include "paimon/core/catalog/catalog_snapshot_commit.h" -#include "paimon/core/catalog/catalog_utils.h" -#include "paimon/core/catalog/commit_lock.h" -#include "paimon/core/catalog/renaming_snapshot_commit.h" -#include "paimon/core/catalog/snapshot_commit.h" -#include "paimon/core/catalog/version_managed_catalog.h" -#include "paimon/core/core_options.h" -#include "paimon/core/disk/io_manager.h" -#include "paimon/core/index/index_file_handler.h" -#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" -#include "paimon/core/index/pk/primary_key_index_definitions.h" -#include "paimon/core/manifest/index_manifest_entry.h" -#include "paimon/core/manifest/index_manifest_file.h" -#include "paimon/core/mergetree/compact/lookup_merge_function.h" -#include "paimon/core/mergetree/compact/merge_function.h" -#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" -#include "paimon/core/operation/append_only_file_store_scan.h" -#include "paimon/core/operation/append_only_file_store_write.h" -#include "paimon/core/operation/commit/realtime_commit_properties.h" -#include "paimon/core/operation/data_evolution_file_store_scan.h" -#include "paimon/core/operation/expire_snapshots.h" -#include "paimon/core/operation/file_store_commit_impl.h" -#include "paimon/core/operation/file_store_scan.h" -#include "paimon/core/operation/internal_read_context.h" -#include "paimon/core/operation/key_value_file_store_scan.h" -#include "paimon/core/operation/key_value_file_store_write.h" -#include "paimon/core/options/merge_engine.h" -#include "paimon/core/postpone/postpone_bucket_file_store_write.h" -#include "paimon/core/realtime/realtime_context_impl.h" -#include "paimon/core/realtime/realtime_schema_layout.h" -#include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/schema_validation.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/table/bucket_mode.h" -#include "paimon/core/table/format/format_table_file_store_commit.h" -#include "paimon/core/table/format/format_table_file_store_write.h" -#include "paimon/core/table/format/format_table_loader.h" -#include "paimon/core/table/format/format_table_read.h" -#include "paimon/core/table/format/format_table_scan.h" -#include "paimon/core/table/source/abstract_table_scan.h" -#include "paimon/core/table/source/append_only_split_generator.h" -#include "paimon/core/table/source/append_only_table_read.h" -#include "paimon/core/table/source/data_evolution_batch_scan.h" -#include "paimon/core/table/source/data_evolution_split_generator.h" -#include "paimon/core/table/source/data_table_batch_scan.h" -#include "paimon/core/table/source/data_table_stream_scan.h" -#include "paimon/core/table/source/fallback_table_read.h" -#include "paimon/core/table/source/key_value_table_read.h" -#include "paimon/core/table/source/merge_tree_split_generator.h" -#include "paimon/core/table/source/primary_key_index_batch_scan.h" -#include "paimon/core/table/source/read_optimized_scan_options.h" -#include "paimon/core/table/source/realtime_table_scan.h" -#include "paimon/core/table/source/snapshot/snapshot_reader.h" -#include "paimon/core/table/source/split_generator.h" -#include "paimon/core/table/system/system_table.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/core/utils/field_mapping.h" -#include "paimon/core/utils/file_store_path_factory.h" -#include "paimon/core/utils/index_file_path_factories.h" -#include "paimon/core/utils/primary_key_table_utils.h" -#include "paimon/core/utils/snapshot_manager.h" -#include "paimon/defs.h" -#include "paimon/file_store_commit.h" -#include "paimon/file_store_write.h" -#include "paimon/format/file_format.h" -#include "paimon/fs/file_system.h" -#include "paimon/predicate/predicate_utils.h" -#include "paimon/read_context.h" -#include "paimon/realtime/realtime_context.h" -#include "paimon/result.h" -#include "paimon/scan_context.h" -#include "paimon/schema/schema.h" -#include "paimon/status.h" -#include "paimon/table/format/format_table.h" -#include "paimon/table/source/table_read.h" -#include "paimon/table/source/table_scan.h" -#include "paimon/write_context.h" - -namespace arrow { -class Schema; -} // namespace arrow - namespace paimon { -class DataSplit; -class Executor; -struct KeyValue; -class MemoryPool; -template -class MergeFunctionWrapper; - -namespace { - -Status RestoreRealtimeCommittedProgress(const std::shared_ptr& realtime_context, - const std::shared_ptr& snapshot_manager, - const CoreOptions& options) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, - RealtimeContextImpl::Cast(realtime_context)); - PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_ASSIGN_OR_RAISE( - RealtimeOffsetMap realtime_committed_offsets, - RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); - PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( - latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); - } - return Status::OK(); -} - -/// Maps a `WriteContext` onto `FormatTableFileStoreWrite`, refusing by name what a format table -/// cannot honour rather than silently dropping it. -Result> NewFormatTableWrite( - const std::shared_ptr& table, const WriteContext& ctx) { - if (ctx.GetCatalog() != nullptr) { - return Status::Invalid( - "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " - "format table"); - } - if (ctx.IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing a streaming write could " - "commit against"); - } - if (ctx.GetRealtimeContext() != nullptr) { - return Status::NotImplemented("a format table has no real-time store to write into"); - } - if (!ctx.GetWriteSchema().empty()) { - return Status::NotImplemented( - "a format table write takes the table's own columns; a write schema naming a " - "subset of them is not supported yet"); - } - // A write id prefixes a postpone-bucket writer's files so that one compaction reader can - // put them back in order. A format table has no buckets, so it would identify nothing. - if (ctx.GetWriteId().has_value()) { - return Status::NotImplemented( - "a format table has no buckets, so a write id would name nothing"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, - FormatTableFileStoreWrite::Create(table, ctx.GetMemoryPool())); - return std::unique_ptr(std::move(format_write)); -} - -Result HasHistoricalPrimaryKeyBTreeDefinition( - const std::shared_ptr& schema_manager, int64_t current_schema_id) { - if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { - return false; - } - PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); - for (int64_t schema_id : schema_ids) { - if (schema_id >= current_schema_id) { - continue; - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, - schema_manager->ReadSchema(schema_id)); - const auto& historical_options = historical_schema->Options(); - auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); - if (option != historical_options.end() && - !StringUtils::IsNullOrWhitespaceOnly(option->second)) { - return true; - } - } - return false; -} - -} // namespace Table::Table(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, @@ -215,1227 +30,4 @@ Table::Table(const std::shared_ptr& file_system, const std::string& identifier_(identifier), uuid_(uuid) {} -Result> Table::NewScan(std::unique_ptr context) const { - return TableScan::Create(std::move(context)); -} - -Result> Table::NewRead(std::unique_ptr context) const { - return TableRead::Create(std::move(context)); -} - -Result> Table::NewWrite( - std::unique_ptr context) const { - return FileStoreWrite::Create(std::move(context)); -} - -Result> Table::NewCommit( - std::unique_ptr context) const { - return FileStoreCommit::Create(std::move(context)); -} - -Result> FileStoreWrite::Create(std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("write context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableWrite(given_table, *ctx); - } - - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && - ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); - } - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs, - ctx->GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx->GetBranch(); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } - // Only the main branch may have a current schema the catalog alone holds. A branch publishes - // its own under `branch/branch-`, which is where the load below and a read of the branch - // both read it, so a catalog answering for no branch schema does not fail a write to one. - std::optional catalog_table_schema; - if (ctx->GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog write requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); - } - } - // A format table writes plain data files into a directory, so it never reaches the manifest - // path below. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch, - ctx->GetOptions(), catalog_table_schema, schema_manager.get(), - &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableWrite(format_table, *ctx); - } - // The schema the dispatch above already read through `schema_manager`, rather than a second - // read of the same file. - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths above are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; - PAIMON_ASSIGN_OR_RAISE( - CoreOptions options, - CoreOptions::FromMap(opts, specific_fs, ctx->GetFileSystemSchemeToIdentifierMap())); - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); - if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) { - return Status::NotImplemented( - "Dedicated VECTOR storage does not yet support custom write schemas or partial writes"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); - - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr file_store_path_factory, - FileStorePathFactory::Create(ctx->GetRootPath(), arrow_schema, schema->PartitionKeys(), - options.GetPartitionDefaultName(), - options.GetWriteFileFormat(/*level=*/0)->Identifier(), - options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); - auto snapshot_manager = - std::make_shared(options.GetFileSystem(), ctx->GetRootPath(), branch); - if (VersionManagedCatalog* versioned = AsVersionManaged(ctx->GetCatalog())) { - std::shared_ptr catalog = ctx->GetCatalog(); - Identifier identifier = catalog_identifier.value(); - // Keep the catalog alive while the writer uses its snapshot loader. - snapshot_manager->SetSnapshotLoader( - [catalog, versioned, identifier]() -> Result> { - return versioned->LoadSnapshot(identifier); - }); - } - std::shared_ptr io_manager; - const auto& io_temp_dir = ctx->GetTempDirectory(); - if (!io_temp_dir.empty()) { - io_manager = std::make_shared(io_temp_dir, options.GetFileSystem()); - } - - bool ignore_previous_files = ctx->IgnorePreviousFiles(); - if (ctx->GetRealtimeContext() && !options.RealtimeEnabled()) { - return Status::Invalid("real-time write requires realtime.enabled=true"); - } - if (ctx->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, - RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); - PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); - } - if (schema->PrimaryKeys().empty()) { - // append table - const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); - bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); - if (options.GetBucket() == -1) { - need_dv_maintainer_factory = false; - ignore_previous_files = true; - } else if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in append table", options.GetBucket())); - } - if (ctx->GetRealtimeContext()) { - if (!ctx->GetWriteSchema().empty()) { - return Status::Invalid("real-time append write does not support a custom schema"); - } - if (!realtime_deduplicate) { - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - } else if (realtime_deduplicate) { - return Status::Invalid("real-time deduplicate write requires a real-time context"); - } - std::shared_ptr write_schema = arrow_schema; - const auto& write_field_names = ctx->GetWriteSchema(); - if (!write_field_names.empty()) { - arrow::FieldVector write_fields; - write_fields.reserve(write_field_names.size()); - for (const auto& field_name : write_field_names) { - auto field = arrow_schema->GetFieldByName(field_name); - if (!field) { - // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER - return Status::Invalid( - fmt::format("write field {} does not exist in table schema", field_name)); - } - write_fields.push_back(field); - } - write_schema = arrow::schema(write_fields); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - const RealtimeStoreMode mode = realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE - : RealtimeStoreMode::APPEND_ONLY; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, - RealtimeSchemaLayout::Create(mode, write_schema)); - realtime_schema_layout = std::move(layout); - } - - std::shared_ptr dv_maintainer_factory; - if (need_dv_maintainer_factory) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), file_store_path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); - auto index_file_handler = std::make_shared( - options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(file_store_path_factory), - options.DeletionVectorsBitmap64(), ctx->GetMemoryPool()); - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - - auto file_store_write = std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, write_schema, realtime_schema_layout, - partition_schema, dv_maintainer_factory, io_manager, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetRealtimeContext(), - ctx->GetExecutor(), ctx->GetMemoryPool()); - if (ctx->GetRealtimeContext() && realtime_deduplicate) { - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_RETURN_NOT_OK( - file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); - } - } - return std::unique_ptr(std::move(file_store_write)); - } else { - // pk table - if (ctx->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); - if (ignore_previous_files) { - return Status::NotImplemented( - "PK realtime requires restore from the latest snapshot"); - } - if (!ctx->GetWriteSchema().empty()) { - return Status::NotImplemented("PK realtime does not support a custom write schema"); - } - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { - return PostponeBucketFileStoreWrite::Create( - snapshot_manager, schema_manager, ctx->GetCommitUser(), ctx->GetRootPath(), schema, - arrow_schema, partition_schema, io_manager, options, ctx->IsStreamingMode(), - ctx->IgnoreNumBucketCheck(), ctx->GetWriteId(), - ctx->GetFileSystemSchemeToIdentifierMap(), ctx->GetExecutor(), ctx->GetMemoryPool(), - specific_fs); - } - if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in key value table", options.GetBucket())); - } - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, - schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_primary_key_fields, - options.SequenceFieldSortOrderIsAscending())); - auto primary_keys = schema->PrimaryKeys(); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, - PrimaryKeyTableUtils::CreateMergeFunction( - arrow_schema, primary_keys, options, ctx->GetMemoryPool())); - if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { - // don't wrap first row, it is already OK - merge_function = std::make_unique(std::move(merge_function)); - } - std::shared_ptr> merge_function_wrapper = - std::make_shared(std::move(merge_function)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr sequence_fields_comparator, - PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); - - std::shared_ptr dv_maintainer_factory; - PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, - PrimaryKeyIndexDefinitions::Create(*schema)); - bool has_btree_index = false; - for (const PrimaryKeyIndexDefinition& definition : - primary_key_index_definitions.Definitions()) { - if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { - has_btree_index = true; - break; - } - } - std::optional latest_snapshot; - bool has_existing_index_manifest = false; - if (!has_btree_index) { - PAIMON_ASSIGN_OR_RAISE( - bool has_historical_btree_index, - HasHistoricalPrimaryKeyBTreeDefinition(schema_manager, schema->Id())); - if (has_historical_btree_index) { - PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); - has_existing_index_manifest = - latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); - } - } - std::shared_ptr index_file_handler; - if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), file_store_path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); - index_file_handler = std::make_shared( - options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(file_store_path_factory), - options.DeletionVectorsBitmap64(), ctx->GetMemoryPool()); - } - bool has_restored_btree_index = false; - if (!has_btree_index && has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - std::vector restored_btree_entries, - index_file_handler->Scan( - latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { - return entry.index_file->IndexType() == BtreeDefs::kIdentifier && - IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); - })); - has_restored_btree_index = !restored_btree_entries.empty(); - } - if (options.DeletionVectorsEnabled()) { - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - std::shared_ptr - primary_key_index_maintainer_factory; - if (has_btree_index || has_restored_btree_index) { - primary_key_index_maintainer_factory = - std::make_shared( - ctx->GetRootPath(), branch, schema, primary_key_index_definitions.Definitions(), - file_store_path_factory, index_file_handler, options, io_manager, - ctx->EnableMultiThreadSpill(), ctx->GetExecutor(), ctx->GetMemoryPool()); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr layout, - RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); - realtime_schema_layout = std::move(layout); - } - - return std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, realtime_schema_layout, partition_schema, - dv_maintainer_factory, primary_key_index_maintainer_factory, io_manager, key_comparator, - sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->EnableMultiThreadSpill(), - ctx->GetRealtimeContext(), ctx->GetExecutor(), ctx->GetMemoryPool()); - } -} - -namespace { - -class TableScanImpl { - public: - static Result> CreateFileStoreScan( - const std::shared_ptr& path_factory, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const std::shared_ptr& executor, const std::shared_ptr& memory_pool, - const ScanContext* context) { - auto fs = core_options.GetFileSystem(); - auto manifest_file_format = core_options.GetManifestFormat(); - std::string branch = BranchManager::NormalizeBranch(core_options.GetBranch()); - auto snapshot_manager = std::make_shared(fs, context->GetPath(), branch); - // TODO(liancheng.lsz): support fallback branch in scan - auto schema_manager = std::make_shared(fs, context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_list, - ManifestList::Create(fs, manifest_file_format, core_options.GetManifestCompression(), - path_factory, core_options.GetCache(), memory_pool)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - ManifestFile::Create(fs, manifest_file_format, core_options.GetManifestCompression(), - path_factory, core_options.GetManifestTargetFileSize(), - memory_pool, core_options, partition_schema)); - std::unique_ptr scan; - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - PAIMON_ASSIGN_OR_RAISE( - scan, DataEvolutionFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, AppendOnlyFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, - context->GetScanFilters(), core_options, - executor, memory_pool)); - } - return WithTablePath(std::move(scan), context); - } - - static std::unique_ptr WithTablePath(std::unique_ptr&& scan, - const ScanContext* context) { - scan->WithTablePath(context->GetPath()); - return std::move(scan); - } - - static Result> CreateSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const ScanContext* context) { - auto source_split_target_size = core_options.GetSourceSplitTargetSize(); - auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.BlobSplitByFileSize()); - } - BucketMode bucket_mode = (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE - : BucketMode::HASH_FIXED); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, bucket_mode); - } else { - // TODO(liancheng.lsz): support evolution - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - table_schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, - table_schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), - key_comparator); - } - } - - static Result> CreateIndexFileHandler( - const CoreOptions& core_options, const std::shared_ptr& path_factory, - const std::shared_ptr& memory_pool) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_manifest_file, - IndexManifestFile::Create( - core_options.GetFileSystem(), core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetBucket(), memory_pool, core_options)); - return std::make_unique( - core_options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(path_factory), - core_options.DeletionVectorsBitmap64(), memory_pool); - } -}; - -/// @param loaded_schema The table's schema, when the caller already read it, or null when it has -/// to be read here. `TableScan::Create()` reads one to dispatch on its table type. -Result> NewDataTableScan( - const std::shared_ptr& context, const std::shared_ptr& loaded_schema); - -/// Maps a `ScanContext` onto `FormatTableScan`, which takes one partition filter and a limit and -/// nothing else. -/// -/// A format table has no buckets, so a bucket filter has nothing to act on. A predicate is -/// refused too, which is narrower than Java: `FormatReadBuilder.newScan()` hands the partition -/// half of one to the scan to prune directories with, which this does not do yet. -Result> NewFormatTableScan(const std::shared_ptr& table, - const std::shared_ptr& context) { - // Anything a `ScanContext` carries that a format table cannot honour is refused rather than - // silently dropped. - if (context->IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing for a streaming scan to follow"); - } - if (context->GetRealtimeContext() != nullptr) { - return Status::NotImplemented( - "a format table has no real-time store to union with what is on disk"); - } - if (context->GetGlobalIndexResult() != nullptr) { - return Status::NotImplemented("a format table carries no global index"); - } - std::map partition_filter; - std::shared_ptr filters = context->GetScanFilters(); - if (filters != nullptr) { - if (filters->GetPredicate() != nullptr) { - return Status::NotImplemented( - "a format table scan does not take a predicate: its files carry no statistics to " - "skip by, and pruning by the partition columns a predicate names is not " - "implemented yet; give the partition values as a partition filter instead"); - } - if (filters->GetBucketFilter()) { - return Status::NotImplemented("a format table has no buckets to filter by"); - } - const std::vector>& partition_filters = - filters->GetPartitionFilters(); - if (partition_filters.size() > 1) { - return Status::NotImplemented("a format table scan takes at most one partition filter"); - } - if (partition_filters.size() == 1) { - partition_filter = partition_filters.front(); - } - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - FormatTableScan::Create(table, partition_filter, context->GetLimit())); - return std::unique_ptr(std::move(scan)); -} - -} // namespace - -Result> TableScan::Create(std::unique_ptr context) { - if (context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - std::shared_ptr shared_context = std::move(context); - // A table the caller already loaded says what it is, so nothing is read to find out. - if (shared_context->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTable::Copy(shared_context->GetFormatTable(), shared_context->GetOptions())); - return NewFormatTableScan(format_table, shared_context); - } - // load schema - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(shared_context->GetOptions(), - shared_context->GetSpecificFileSystem(), {})); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(shared_context->GetPath())); - if (system_table_path) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), shared_context->GetPath(), - shared_context->GetOptions())); - return system_table->NewScan(shared_context); - } - // A format table is planned by listing directories, so it never reaches the manifest path - // below. One `TableScan` interface serves both, as Java Paimon serves both through one - // `ReadBuilder`. - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), shared_context->GetPath(), - BranchManager::NormalizeBranch(tmp_options.GetBranch()), - shared_context->GetOptions(), - shared_context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableScan(format_table, shared_context); - } - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableScan(shared_context, latest_schema); -} - -namespace { - -Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, - const ScanContext& context, bool read_optimized) { - if (!context.GetRealtimeContext()) { - return Status::OK(); - } - if (!core_options.RealtimeEnabled()) { - return Status::Invalid("real-time scan requires realtime.enabled=true"); - } - if (core_options.GetBucket() <= 0) { - return Status::Invalid("real-time union read requires fixed bucket mode"); - } - if (core_options.DataEvolutionEnabled()) { - return Status::Invalid("real-time union read does not support data evolution"); - } - if (!table_schema.PrimaryKeys().empty()) { - if (read_optimized) { - return Status::NotImplemented( - "PK real-time union read does not support read-optimized scans"); - } - PAIMON_RETURN_NOT_OK( - PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); - } - if (context.IsStreamingMode()) { - return Status::Invalid("real-time union read currently supports batch scans only"); - } - if (context.GetLimit()) { - return Status::Invalid("real-time union read does not support scan limit pushdown"); - } - if (context.GetGlobalIndexResult()) { - return Status::Invalid("real-time union read does not support global index splits"); - } - StartupMode startup_mode = core_options.GetStartupMode(); - const bool specified_snapshot = - startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); - if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || - specified_snapshot)) { - return Status::Invalid( - "real-time union read requires the latest snapshot or an explicit snapshot id"); - } - return Status::OK(); -} - -Result> NewDataTableScan( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); - std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema, - schema_manager.Latest()); - if (latest_table_schema == std::nullopt) { - return Status::Invalid("not found latest schema"); - } - table_schema = latest_table_schema.value(); - } - PAIMON_ASSIGN_OR_RAISE(bool read_optimized, - OptionsUtils::GetValueFromMap(context->GetOptions(), - kReadOptimizedScanOption, false)); - // merge options - auto options = table_schema->Options(); - for (const auto& [key, value] : context->GetOptions()) { - options[key] = value; - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, context->GetSpecificFileSystem(), {})); - core_options.WithCache(context->GetCache()); - - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); - - PAIMON_RETURN_NOT_OK( - ValidateRealtimeScan(*table_schema, core_options, *context, read_optimized)); - // validate options - if (core_options.GetBucket() == -1) { - if (!table_schema->PrimaryKeys().empty()) { - return Status::NotImplemented(fmt::format( - "do not support pk table bucket={} in scan process", core_options.GetBucket())); - } - } else if (core_options.GetBucket() < 1 && - !SchemaValidation::IsPostponeBucketTable(*table_schema, core_options.GetBucket())) { - return Status::Invalid( - fmt::format("do not support bucket={} in scan process", core_options.GetBucket())); - } - - // validate schema and scan filter - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { - PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( - *arrow_schema, context->GetScanFilters()->GetPredicate(), - /*validate_field_idx=*/false)); - } - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, - core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - FileStorePathFactory::Create( - context->GetPath(), arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), - core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - context->GetMemoryPool())); - - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_store_scan, - TableScanImpl::CreateFileStoreScan( - path_factory, arrow_schema, table_schema, core_options, - context->GetExecutor(), context->GetMemoryPool(), context.get())); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr split_generator, - TableScanImpl::CreateSplitGenerator(table_schema, core_options, context.get())); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_file_handler, - TableScanImpl::CreateIndexFileHandler(core_options, path_factory, - context->GetMemoryPool())); - auto snapshot_reader = std::make_shared( - file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler)); - const bool pk_table = !table_schema->PrimaryKeys().empty(); - if (read_optimized && pk_table && context->IsStreamingMode()) { - return Status::NotImplemented( - "read-optimized system table does not support streaming scan for primary key table"); - } - if (context->IsStreamingMode()) { - return std::make_unique(core_options, snapshot_reader); - } - auto batch_scan = std::make_unique( - /*pk_table=*/pk_table, core_options, snapshot_reader, read_optimized, context->GetLimit(), - /*realtime_pk_scan=*/pk_table && context->GetRealtimeContext() != nullptr); - if (context->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, - RealtimeContextImpl::Cast(context->GetRealtimeContext())); - return std::make_unique( - std::move(batch_scan), pk_table, realtime_context, path_factory, - snapshot_reader->GetSnapshotManager(), core_options.GetFileSystem(), - context->GetScanFilters(), core_options.GetRealtimeReadViewTtlMillis()); - } - if (core_options.DataEvolutionEnabled()) { - return std::make_unique( - context->GetPath(), snapshot_reader, std::move(batch_scan), table_schema, - context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), - context->GetExecutor()); - } - if (pk_table && !read_optimized && core_options.GlobalIndexEnabled()) { - PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, - PrimaryKeyIndexDefinitions::Create(*table_schema)); - if (!definitions.ScalarDefinitions().empty()) { - return PrimaryKeyIndexBatchScan::Create(snapshot_reader, std::move(batch_scan), - table_schema, path_factory, core_options, - context->GetMemoryPool()); - } - } - return batch_scan; -} - -} // namespace - -namespace { - -/// @param loaded_schema The schema of `branch`, when the caller already read it, or null when it -/// has to be read here: `TableRead::Create()` reads one to dispatch on the table type, and -/// reading it again would cost a second listing and read. -Result> CreateInternalReadContext( - const std::shared_ptr& context, const std::string& branch, - const std::shared_ptr& loaded_schema) { - std::map tmp_options = context->GetOptions(); - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, - CoreOptions::FromMap(tmp_options, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - SchemaManager schema_manager(tmp_core_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - schema_manager.Latest()); - if (!latest_schema) { - return Status::Invalid(fmt::format("schema file not found in path {}, branch {}", - context->GetPath(), branch)); - } - table_schema = latest_schema.value(); - } - assert(table_schema); - - // merge options - auto options = table_schema->Options(); - for (const auto& [key, value] : tmp_options) { - options[key] = value; - } - if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { - options[Options::BRANCH] = branch; - } - return InternalReadContext::Create(context, table_schema, options); -} - -Result> CreateTableRead( - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor) { - if (PredicateUtils::ContainsFullTextMatch(internal_context->GetPredicate())) { - return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); - } - if (!internal_context->GetIndexScoreBindings().empty()) { - return Status::NotImplemented("Per-field index score output is not implemented yet"); - } - const auto& core_options = internal_context->GetCoreOptions(); - const auto& table_schema = internal_context->GetTableSchema(); - if (internal_context->GetRealtimeContext() && !core_options.RealtimeEnabled()) { - return Status::Invalid("real-time read requires realtime.enabled=true"); - } - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, - core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - FileStorePathFactory::Create( - internal_context->GetPath(), arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), - core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - memory_pool)); - - if (internal_context->GetPrimaryKeys().empty()) { - return std::make_unique(path_factory, internal_context, memory_pool, - executor); - } - return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); -} - -Result> NewDataTableRead( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, - CreateInternalReadContext(context, context->GetBranch(), loaded_schema)); - - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr table_read, - CreateTableRead(internal_context, context->GetMemoryPool(), context->GetExecutor())); - - std::optional scan_fallback_branch = - internal_context->GetCoreOptions().GetScanFallbackBranch(); - if (!scan_fallback_branch || - StringUtils::IsNullOrWhitespaceOnly(scan_fallback_branch.value())) { - return std::move(table_read); - } - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr fallback_context, - CreateInternalReadContext(context, /*branch=*/scan_fallback_branch.value(), - /*loaded_schema=*/nullptr)); - - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr fallback_table_read, - CreateTableRead(fallback_context, context->GetMemoryPool(), context->GetExecutor())); - const std::shared_ptr& arrow_pool = internal_context->GetArrowMemoryPool(); - return std::make_unique(std::move(table_read), - std::move(fallback_table_read), arrow_pool); -} - -/// Maps a `ReadContext` onto `FormatTableRead`, which reads everything it needs out of the context -/// itself: the columns, the predicate, the pool and executor, and what a file is opened with. It -/// refuses by name what a format table cannot honour. -Result> NewFormatTableRead(const std::shared_ptr& table, - const std::shared_ptr& context) { - if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { - return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - FormatTableRead::Create(table, context)); - return std::unique_ptr(std::move(read)); -} - -} // namespace - -Result> TableRead::Create(std::unique_ptr ctx) { - std::shared_ptr context = std::move(ctx); - if (context == nullptr) { - return Status::Invalid("read context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - // A table the caller already loaded says what it is, so nothing is read to find out. - if (context->GetFormatTable() != nullptr) { - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for format tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); - return NewFormatTableRead(format_table, context); - } - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_core_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(context->GetPath())); - if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { - return Status::NotImplemented( - "FullTextMatch predicate execution is not implemented yet"); - } - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for system tables"); - } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetOptions())); - return system_table->NewRead(context); - } - - // A format table has no manifest and a file another engine wrote carries no field ids, so - // which files there are and how a row is put back together is its own. How a file is opened - // is not: both paths go through `DataFileReaderFactory`. - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetBranch(), context->GetOptions(), - context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for format tables"); - } - return NewFormatTableRead(format_table, context); - } - - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableRead(context, latest_schema); -} - -namespace { - -CommitScanner::ScanSupplier CreateAppendScanSupplier( - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - -CommitScanner::ScanSupplier CreatePkScanSupplier( - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - -bool IsCatalogCommit(const CommitContext& ctx) { - return AsVersionManaged(ctx.GetCatalog()) != nullptr; -} - -Result> NewSnapshotCommit( - const CommitContext& ctx, const std::optional& catalog_identifier, - const CoreOptions& options, const std::shared_ptr& snapshot_manager) { - if (IsCatalogCommit(ctx)) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr lock, - AsVersionManaged(ctx.GetCatalog()) - ->CreateCommitLock(catalog_identifier.value(), ctx.GetTableId(), - ctx.GetCommitUser(), options)); - // Reuse the caller's UUID so table recreation cannot redirect this commit. - return std::shared_ptr(std::make_shared( - ctx.GetCatalog(), catalog_identifier.value(), ctx.GetTableId(), std::move(lock))); - } - if (ctx.UseRESTCatalogCommit()) { - if (options.RestCommitAcquireLock()) { - return Status::Invalid(fmt::format( - "'{}' needs a commit through a REST catalog with WithCatalog(); a commit table " - "request built for the caller to send cannot hold a commit lease", - Options::REST_COMMIT_ACQUIRE_LOCK)); - } - return std::shared_ptr( - std::make_shared(ctx.GetTableId())); - } - // Catalogs without version management may still supply a table ID. - if (ctx.GetTableId() && ctx.GetCatalog() == nullptr) { - return Status::Invalid( - "a commit which publishes the snapshot by writing it to the table directory sends no " - "commit table request, so there is nowhere to put a table id; commit through a " - "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); - } - return std::shared_ptr( - std::make_shared(options.GetFileSystem(), snapshot_manager)); -} - -/// Creates a format-table commit, rejecting non-default options that require snapshots. -Result> NewFormatTableCommit( - const std::shared_ptr& table, const CommitContext& ctx) { - if (!ctx.IgnoreEmptyCommit()) { - return Status::NotImplemented( - "a format table cannot record an empty commit: keeping one means writing a " - "snapshot that adds no files, and there are no snapshots here"); - } - if (ctx.UseRESTCatalogCommit()) { - return Status::NotImplemented( - "a format table commits by renaming files into place, not by sending a snapshot " - "to a rest catalog"); - } - if (ctx.GetTableId()) { - return Status::NotImplemented( - "a format table sends no commit table request, so there is nowhere to put a table id"); - } - if (ctx.GetCatalog() != nullptr) { - return Status::NotImplemented( - "a format table keeps no snapshot for a catalog to take, so it commits by renaming " - "files into place rather than through the catalog it was loaded from"); - } - if (ctx.AppendCommitCheckConflict()) { - return Status::NotImplemented( - "a format table has no manifests to check a concurrent commit against"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, - FormatTableFileStoreCommit::Create(table)); - return std::unique_ptr(std::move(format_commit)); -} - -} // namespace - -Result> FileStoreCommit::Create( - std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("commit context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableCommit(given_table, *ctx); - } - - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - // Use catalog credentials unless the caller supplied a file system. - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - // Use the credentials of this table, which a catalog issuing per-table temporary - // ones only hands out through GetTableFileSystem. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); - const std::string& root_path = ctx->GetRootPath(); - // The branch the context was built for, already validated by `CommitContextBuilder`. - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - ctx->GetIdentifier(), ctx->GetOptions(), - /*explicit_branch=*/std::nullopt, "commit")); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } - // Catalog-managed schemas may be absent from the table directory, but only on the main branch: - // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail - // a commit to one. - std::optional catalog_table_schema; - FileStoreCommitImpl::SchemaIdLoader schema_id_loader; - if (ctx->GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); - catalog_table_schema = std::move(schema_json); - } - std::shared_ptr catalog = ctx->GetCatalog(); - Identifier identifier = catalog_identifier.value(); - schema_id_loader = [catalog, identifier]() -> Result { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - catalog->LoadTableSchema(identifier)); - std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); - if (data_schema == nullptr) { - return Status::Invalid( - "the catalog did not hand back a data table schema, which has the id a " - "snapshot records"); - } - return data_schema->Id(); - }; - } - // A format table commits by renaming files into place, so it never reaches the snapshot path - // below. The managed path here reads the commit's branch, so this reads the same one: the two - // must not dispatch on different schemas. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), root_path, branch); - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTableLoader::TryLoad( - tmp_options.GetFileSystem(), root_path, branch, ctx->GetOptions(), - catalog_table_schema, schema_manager.get(), &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableCommit(format_table, *ctx); - } - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths below are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; - std::shared_ptr arrow_schema = - DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, specific_fs)); - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); - assert(options.GetFileSystem()); - assert(options.GetFileFormat()); - PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); - PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path)); - // Catalog commits do not require atomic rename on the object store. - if (is_object_store && !ctx->UseRESTCatalogCommit() && !IsCatalogCommit(*ctx) && - opts.find("enable-object-store-commit-in-inte-test") == opts.end()) { - return Status::NotImplemented( - "commit operation does not support object store file system for now"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr partition_computer, - BinaryRowPartitionComputer::Create(schema->PartitionKeys(), arrow_schema, - options.GetPartitionDefaultName(), - options.LegacyPartitionNameEnabled(), - ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - FileStorePathFactory::Create( - root_path, arrow_schema, schema->PartitionKeys(), options.GetPartitionDefaultName(), - options.GetFileFormat()->Identifier(), options.DataFilePrefix(), - options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); - - auto snapshot_manager = - std::make_shared(options.GetFileSystem(), root_path, branch); - if (IsCatalogCommit(*ctx)) { - std::shared_ptr catalog = ctx->GetCatalog(); - VersionManagedCatalog* versioned = AsVersionManaged(catalog); - Identifier identifier = catalog_identifier.value(); - // Capture the catalog to keep the versioned interface alive. - snapshot_manager->SetSnapshotLoader( - [catalog, versioned, identifier]() -> Result> { - return versioned->LoadSnapshot(identifier); - }); - } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_list, - ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, options.GetCache(), - ctx->GetMemoryPool())); - - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - ManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, - options.GetManifestTargetFileSize(), ctx->GetMemoryPool(), options, - partition_schema)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); - - auto expire_snapshots = std::make_shared( - snapshot_manager, path_factory, manifest_list, manifest_file, index_manifest_file, - options.GetFileSystem(), options.GetExpireConfig(), options.RealtimeEnabled(), - ctx->GetExecutor()); - - CommitScanner::ScanSupplier scan_supplier; - if (schema->PrimaryKeys().empty()) { - scan_supplier = CreateAppendScanSupplier(snapshot_manager, schema_manager, manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); - } else { - scan_supplier = CreatePkScanSupplier(snapshot_manager, schema_manager, manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); - } - - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_commit, - NewSnapshotCommit(*ctx, catalog_identifier, options, snapshot_manager)); - - return std::make_unique( - ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path, ctx->GetCommitUser(), - options, path_factory, std::move(partition_computer), snapshot_manager, snapshot_commit, - ctx->IgnoreEmptyCommit(), ctx->AppendCommitCheckConflict(), schema, - std::move(schema_id_loader), manifest_file, manifest_list, index_manifest_file, - expire_snapshots, schema_manager, std::move(scan_supplier)); -} - } // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 4792a8461..492ddc5a3 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -18,6 +18,7 @@ #include "paimon/catalog/table.h" +#include #include #include #include @@ -25,20 +26,41 @@ #include #include "arrow/api.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/cache/cache.h" #include "paimon/commit_context.h" +#include "paimon/commit_message.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/table/source/fallback_data_split.h" +#include "paimon/core/table/system/metadata_system_tables.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/utils/tag_manager.h" +#include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/record_batch.h" #include "paimon/scan_context.h" #include "paimon/schema/schema.h" +#include "paimon/table/format/format_table.h" +#include "paimon/table/source/plan.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" +#include "paimon/testing/mock/mock_file_system.h" +#include "paimon/testing/utils/counting_cache_test_utils.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -46,24 +68,710 @@ namespace paimon::test { namespace { -// Creates a managed table directory and a `Table` over it, holding the schema written to -// `branch`'s directory under `identifier`. -Result> CreateManagedTable(const std::shared_ptr& file_system, - const std::string& path, - const Identifier& identifier, - const std::string& branch = "main") { +Result> CreateManagedTable( + const std::shared_ptr& file_system, const std::string& path, + const Identifier& identifier, const std::string& branch = "main", + const std::map& options = {}) { SchemaManager schema_manager(file_system, path, branch); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr created, schema_manager.CreateTable(schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, /*options=*/{})); - return std::make_shared
( - file_system, path, - checked_pointer_cast(std::shared_ptr(std::move(created))), identifier); + /*primary_keys=*/{}, options)); + return FileStoreTable::Create( + file_system, path, std::shared_ptr(std::move(created)), identifier, + /*uuid=*/"", branch, /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); +} + +void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::string& root_path) { + WriteContextBuilder write_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); +} + +void WriteAndCommitRow(const std::shared_ptr
& table, const std::string& root_path, + int32_t id, int32_t v) { + WriteContextBuilder write_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + arrow::Int32Builder vs_builder; + ASSERT_TRUE(ids_builder.AppendValues({id}).ok()); + ASSERT_TRUE(vs_builder.AppendValues({v}).ok()); + std::shared_ptr ids_array; + std::shared_ptr vs_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + ASSERT_TRUE(vs_builder.Finish(&vs_array).ok()); + auto struct_array = + arrow::StructArray::Make({ids_array, vs_array}, + {arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32())}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); } } // namespace +TEST(TableTest, TestTableCarriesManagersOfItsBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(fs, path, Identifier(Identifier::kUnknownDatabase, "tbl"), + /*branch=*/"dev")); + auto file_store_table = std::dynamic_pointer_cast(table); + ASSERT_TRUE(file_store_table != nullptr); + EXPECT_EQ(file_store_table->CurrentBranch(), "dev"); + ASSERT_TRUE(file_store_table->GetSchemaManager() != nullptr); + ASSERT_TRUE(file_store_table->GetSnapshotManager() != nullptr); + EXPECT_EQ(file_store_table->GetSnapshotManager()->Branch(), "dev"); + EXPECT_EQ(file_store_table->GetSnapshotManager()->RootPath(), path); + EXPECT_EQ(file_store_table->GetSnapshotManager()->Fs(), fs); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({})); + auto file_store_copy = std::dynamic_pointer_cast(copy); + ASSERT_TRUE(file_store_copy != nullptr); + EXPECT_EQ(file_store_copy->CurrentBranch(), "dev"); + ASSERT_TRUE(file_store_copy->GetSchemaManager() != nullptr); + ASSERT_TRUE(file_store_copy->GetSnapshotManager() != nullptr); + EXPECT_NE(file_store_copy->GetSchemaManager(), file_store_table->GetSchemaManager()); + EXPECT_NE(file_store_copy->GetSnapshotManager(), file_store_table->GetSnapshotManager()); + EXPECT_EQ(file_store_copy->GetSnapshotManager()->Branch(), "dev"); + EXPECT_EQ(file_store_copy->GetSnapshotManager()->RootPath(), path); + EXPECT_EQ(file_store_copy->GetSnapshotManager()->Fs(), fs); +} + +TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, + table->Copy({{Options::BRANCH, "main"}, {Options::BUCKET, "2"}})); + auto file_store_copy = std::dynamic_pointer_cast(copy); + ASSERT_TRUE(file_store_copy != nullptr); + EXPECT_EQ(file_store_copy->CurrentBranch(), "main"); + std::shared_ptr copied_schema = + std::dynamic_pointer_cast(copy->GetSchema()); + ASSERT_TRUE(copied_schema != nullptr); + EXPECT_EQ(copied_schema->Options().at(Options::BUCKET), "2"); + { + std::map scan_builder_options; + ScanContextBuilder scan_builder(dir->Str()); + scan_builder_options[Options::BRANCH] = "main"; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_1, + file_store_copy->Copy(scan_builder_options)); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + configured_table_1->NewScan(std::move(ctx))); + } + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, + CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", + identifier, /*branch=*/"dev", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_copy, + branch_table->Copy({{Options::BRANCH, "dev"}})); + auto branch_file_store_copy = std::dynamic_pointer_cast(branch_copy); + ASSERT_TRUE(branch_file_store_copy != nullptr); + EXPECT_EQ(branch_file_store_copy->CurrentBranch(), "dev"); + { + std::map scan_builder_options; + ScanContextBuilder scan_builder(dir->Str() + "/branch_tbl"); + scan_builder_options[Options::BRANCH] = "dev"; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_2, + branch_file_store_copy->Copy(scan_builder_options)); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + configured_table_2->NewScan(std::move(ctx))); + } +} + +TEST(TableTest, TestCopyBranchRetainsCurrentSchemaAndIdentifier) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, path, identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + SchemaManager dev_schema_manager(fs, path, "dev"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr dev_schema, + dev_schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{}, + {{Options::BUCKET, "4"}, {Options::BUCKET_KEY, "id"}})); + + // Branch overrides retain the current schema and identifier. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_copy, table->Copy({{Options::BRANCH, "dev"}})); + auto dev_file_store_copy = std::dynamic_pointer_cast(dev_copy); + ASSERT_TRUE(dev_file_store_copy != nullptr); + EXPECT_EQ("dev", dev_file_store_copy->CurrentBranch()); + EXPECT_EQ(table->Name(), dev_copy->Name()); + std::shared_ptr dev_copied_schema = + std::dynamic_pointer_cast(dev_copy->GetSchema()); + ASSERT_TRUE(dev_copied_schema != nullptr); + EXPECT_EQ(dev_copied_schema->Options().at(Options::BUCKET), "1"); + EXPECT_EQ(dev_copied_schema->Options().at(Options::BRANCH), "dev"); + EXPECT_EQ("dev", dev_file_store_copy->GetSnapshotManager()->Branch()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_copy_restyled, + table->Copy({{Options::BRANCH, "dev"}, {Options::BUCKET, "2"}})); + std::shared_ptr restyled_schema = + std::dynamic_pointer_cast(dev_copy_restyled->GetSchema()); + ASSERT_TRUE(restyled_schema != nullptr); + EXPECT_EQ(restyled_schema->Options().at(Options::BUCKET), "2"); + EXPECT_EQ(restyled_schema->Options().at(Options::BUCKET_KEY), "id"); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
missing, + table->Copy({{Options::BRANCH, "missing"}})); + EXPECT_EQ(table->GetSchema()->FieldNames(), missing->GetSchema()->FieldNames()); +} + +TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}})); + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr fallback_schema, + fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + FileStoreTable::Create( + fs, path, std::shared_ptr(std::move(created)), identifier, + /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, + table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "_SEQUENCE_NUMBER"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + copy->NewRead(std::move(read_context))); +} + +TEST(TableTest, TestAReadContextMatchingTheSchemaKeepsItsOptionsForTheFallbackReader) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + + // Persisted sequence-number options must also reach the fallback reader. + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}, + {Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr fallback_schema, + fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + std::map read_builder_options; + ReadContextBuilder read_builder(path); + read_builder_options[Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED] = "true"; + + read_builder.SetReadFieldNames({"id", "_SEQUENCE_NUMBER"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + TableRead::Create(std::move(read_context), read_builder_options)); +} + +TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}})); + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr fallback_schema, + fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + FileStoreTable::Create( + fs, path, std::shared_ptr(std::move(created)), identifier, + /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
off, + table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "false"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, + off->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "_SEQUENCE_NUMBER"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + copy->NewRead(std::move(read_context))); +} + +TEST(TableTest, TestACopySwitchingBranchesKeepsItsAccumulatedDynamicOptions) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + SchemaManager dev_schema_manager(fs, path, "dev"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr dev_schema, + dev_schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + FileStoreTable::Create( + fs, path, std::shared_ptr(std::move(created)), identifier, + /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + + // Verify accumulated options also reach the reader after changing branches. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
with_sequence, + table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_copy, + with_sequence->Copy({{Options::BRANCH, "dev"}})); + auto dev_file_store_copy = std::dynamic_pointer_cast(dev_copy); + ASSERT_TRUE(dev_file_store_copy != nullptr); + EXPECT_EQ("dev", dev_file_store_copy->CurrentBranch()); + std::shared_ptr dev_copied_schema = + std::dynamic_pointer_cast(dev_copy->GetSchema()); + ASSERT_TRUE(dev_copied_schema != nullptr); + EXPECT_EQ("true", dev_copied_schema->Options().at(Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED)); + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "_SEQUENCE_NUMBER"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + dev_copy->NewRead(std::move(read_context))); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
off, + with_sequence->Copy( + {{Options::BRANCH, "dev"}, {Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "false"}})); + std::shared_ptr off_schema = + std::dynamic_pointer_cast(off->GetSchema()); + ASSERT_TRUE(off_schema != nullptr); + EXPECT_EQ("false", off_schema->Options().at(Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED)); +} + +TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + SchemaManager main_schema_manager(fs, path, "main"); + auto row_schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32())}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_created, + main_schema_manager.CreateTable(row_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
main, + FileStoreTable::Create(fs, path, std::shared_ptr(std::move(main_created)), + identifier, + /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr fallback_created, + fallback_schema_manager.CreateTable(row_schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
fallback, + FileStoreTable::Create(fs, path, std::shared_ptr(std::move(fallback_created)), + identifier, + /*uuid=*/"", /*branch=*/"fb", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/10); + WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); + + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "v"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, main->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(path).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fallback_scan, + fallback->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fallback_plan, fallback_scan->CreatePlan()); + ASSERT_FALSE(fallback_plan->Splits().empty()); + std::vector> fallback_splits; + for (const std::shared_ptr& split : fallback_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split != nullptr); + fallback_splits.push_back( + std::make_shared(data_split, /*is_fallback=*/true)); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(fallback_splits)); + + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_EQ(array->length(), 1); + auto struct_array = std::dynamic_pointer_cast(array); + ASSERT_TRUE(struct_array != nullptr); + EXPECT_EQ(struct_array->num_fields(), 3); + auto value_kind_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto id_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto v_array = std::dynamic_pointer_cast(struct_array->field(2)); + ASSERT_TRUE(value_kind_array != nullptr); + ASSERT_TRUE(id_array != nullptr); + ASSERT_TRUE(v_array != nullptr); + EXPECT_EQ(value_kind_array->Value(0), 0); + EXPECT_EQ(id_array->Value(0), 1); + EXPECT_EQ(v_array->Value(0), 20); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof_batch, batch_reader->NextBatch()); + EXPECT_TRUE(BatchReader::IsEofBatch(eof_batch)); +} + +TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnPartitionKeys) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + SchemaManager main_schema_manager(fs, path, "main"); + auto partitioned_schema = + arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32()), arrow::field("dt", arrow::utf8())}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_created, + main_schema_manager.CreateTable(partitioned_schema, + /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, + {{Options::SCAN_FALLBACK_BRANCH, "fb"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
main, + FileStoreTable::Create(fs, path, std::shared_ptr(std::move(main_created)), + identifier, + /*uuid=*/"", /*branch=*/"main", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + + SchemaManager fallback_schema_manager(fs, path, "fb"); + auto row_schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32())}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr fallback_created, + fallback_schema_manager.CreateTable(row_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
fallback, + FileStoreTable::Create(fs, path, std::shared_ptr(std::move(fallback_created)), + identifier, + /*uuid=*/"", /*branch=*/"fb", /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); + + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "v"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, main->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(path).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fallback_scan, + fallback->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fallback_plan, fallback_scan->CreatePlan()); + ASSERT_FALSE(fallback_plan->Splits().empty()); + std::vector> fallback_splits; + for (const std::shared_ptr& split : fallback_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split != nullptr); + fallback_splits.push_back( + std::make_shared(data_split, /*is_fallback=*/true)); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(fallback_splits)); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_EQ(array->length(), 1); + auto struct_array = std::dynamic_pointer_cast(array); + ASSERT_TRUE(struct_array != nullptr); + // Every batch reader leads with `_VALUE_KIND`, ahead of the fields the read named. + EXPECT_EQ(struct_array->num_fields(), 3); + auto value_kind_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto id_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto v_array = std::dynamic_pointer_cast(struct_array->field(2)); + ASSERT_TRUE(value_kind_array != nullptr); + ASSERT_TRUE(id_array != nullptr); + ASSERT_TRUE(v_array != nullptr); + EXPECT_EQ(value_kind_array->Value(0), 0); + EXPECT_EQ(id_array->Value(0), 1); + EXPECT_EQ(v_array->Value(0), 20); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof_batch, batch_reader->NextBatch()); + EXPECT_TRUE(BatchReader::IsEofBatch(eof_batch)); +} + +TEST(TableTest, TestACopyServesItsS3OptionThroughManagersOfItsOwn) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable( + fs, path, identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}, {"s3.endpoint", "http://old"}})); + auto file_store_table = std::dynamic_pointer_cast(table); + ASSERT_TRUE(file_store_table != nullptr); + + // Copies rebuild managers even when the resolved file system is unchanged. + for (const std::string& endpoint : {std::string("http://new"), std::string("http://old")}) { + SCOPED_TRACE(endpoint); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{"s3.endpoint", endpoint}})); + auto copy_file_store_table = std::dynamic_pointer_cast(copy); + ASSERT_TRUE(copy_file_store_table != nullptr); + EXPECT_NE(copy_file_store_table->GetSchemaManager(), file_store_table->GetSchemaManager()); + EXPECT_NE(copy_file_store_table->GetSnapshotManager(), + file_store_table->GetSnapshotManager()); + EXPECT_EQ("main", copy_file_store_table->CurrentBranch()); + auto copy_schema = std::dynamic_pointer_cast(copy->GetSchema()); + ASSERT_TRUE(copy_schema != nullptr); + EXPECT_EQ(endpoint, copy_schema->Options().at("s3.endpoint")); + } +} + +TEST(TableTest, TestSwitchToBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier(Identifier::kUnknownDatabase, "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, CreateManagedTable(fs, path, identifier)); + SchemaManager dev_schema_manager(fs, path, "dev"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr dev_schema, + dev_schema_manager.CreateTable(arrow::schema({arrow::field("c0", arrow::int32())}), + /*partition_keys=*/{}, /*primary_keys=*/{}, {})); + + auto main_file_store_table = std::dynamic_pointer_cast(table); + ASSERT_TRUE(main_file_store_table != nullptr); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_table, + main_file_store_table->SwitchToBranch("dev")); + auto dev_file_store_table = std::dynamic_pointer_cast(dev_table); + ASSERT_TRUE(dev_file_store_table != nullptr); + EXPECT_EQ("dev", dev_file_store_table->CurrentBranch()); + EXPECT_EQ("tbl$branch_dev", dev_table->Name()); + EXPECT_EQ("tbl$branch_dev", dev_table->FullName()); + ASSERT_EQ((std::vector{"c0"}), dev_table->GetSchema()->FieldNames()); + auto dev_table_schema = std::dynamic_pointer_cast(dev_table->GetSchema()); + ASSERT_TRUE(dev_table_schema != nullptr); + ASSERT_EQ("dev", dev_table_schema->Options().at(Options::BRANCH)); + ASSERT_TRUE(dev_file_store_table->GetSchemaManager() != nullptr); + ASSERT_TRUE(dev_file_store_table->GetSnapshotManager() != nullptr); + EXPECT_EQ("dev", dev_file_store_table->GetSnapshotManager()->Branch()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
same_table, + dev_file_store_table->SwitchToBranch("dev")); + auto same_file_store_table = std::dynamic_pointer_cast(same_table); + ASSERT_TRUE(same_file_store_table != nullptr); + // Switching to the same branch still rebuilds its managers. + EXPECT_EQ(same_table->GetSchema(), dev_table->GetSchema()); + EXPECT_NE(same_file_store_table->GetSchemaManager(), dev_file_store_table->GetSchemaManager()); + EXPECT_NE(same_file_store_table->GetSnapshotManager(), + dev_file_store_table->GetSnapshotManager()); + EXPECT_EQ("dev", same_file_store_table->CurrentBranch()); + EXPECT_EQ(dev_table->GetRootPath(), same_table->GetRootPath()); + EXPECT_EQ(path, same_file_store_table->GetSnapshotManager()->RootPath()); + EXPECT_EQ("dev", same_file_store_table->GetSnapshotManager()->Branch()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_again, + dev_file_store_table->SwitchToBranch("main")); + auto main_again_file_store_table = std::dynamic_pointer_cast(main_again); + ASSERT_TRUE(main_again_file_store_table != nullptr); + EXPECT_EQ("main", main_again_file_store_table->CurrentBranch()); + EXPECT_EQ("tbl", main_again->Name()); + EXPECT_EQ("tbl", main_again->FullName()); + ASSERT_EQ((std::vector{"id"}), main_again->GetSchema()->FieldNames()); + auto main_again_schema = std::dynamic_pointer_cast(main_again->GetSchema()); + ASSERT_TRUE(main_again_schema != nullptr); + EXPECT_EQ(0, main_again_schema->Options().count(Options::BRANCH)); + + ASSERT_NOK_WITH_MSG(main_file_store_table->SwitchToBranch("missing"), "branch"); +} + +TEST(TableTest, TestCreateRefusesATableWithoutASchema) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::string path = dir->Str() + "/tbl"; + SchemaManager dev_schema_manager(dir->GetFileSystem(), path, "dev"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr dev_schema, + dev_schema_manager.CreateTable(arrow::schema({arrow::field("c0", arrow::int32())}), + /*partition_keys=*/{}, /*primary_keys=*/{}, {})); + + ASSERT_NOK_WITH_MSG(FileStoreTable::Create(/*file_system=*/nullptr, path, /*schema=*/nullptr, + Identifier("db", "tbl"), /*uuid=*/"", + /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{}), + "not found latest schema of db.tbl"); +} + +TEST(TableTest, TestNewWriteServesTheBranchTheTableWasLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"dev", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::vector> messages, + write->PrepareCommit()); + ASSERT_OK(write->Close()); + + std::vector branch_entries; + ASSERT_OK(fs->ListDir(BranchManager::BranchPath(dir->Str(), "dev"), &branch_entries)); + EXPECT_FALSE(branch_entries.empty()); +} + +TEST(TableTest, TestOperatorsServeTheBranchTheTableWasLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"dev", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + ASSERT_OK(write->Close()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + EXPECT_FALSE(plan->Splits().empty()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_copy, + table->Copy({{Options::BRANCH, "main"}})); + EXPECT_EQ(table->GetSchema()->FieldNames(), main_copy->GetSchema()->FieldNames()); +} + TEST(TableTest, TestGetTableWithUnknownDatabase) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -94,7 +802,7 @@ TEST(TableTest, TestGetTableWithUnknownDatabase) { EXPECT_EQ(table->Name(), "tbl1"); EXPECT_EQ(table->FullName(), "tbl1"); - std::shared_ptr latest_schema = table->LatestSchema(); + std::shared_ptr latest_schema = table->GetSchema(); ASSERT_NE(latest_schema, nullptr); auto data_schema = std::dynamic_pointer_cast(latest_schema); ASSERT_TRUE(data_schema != nullptr); @@ -123,11 +831,11 @@ TEST(TableTest, TestGetTableWithBranch) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "tbl1$branch_b1"))); ASSERT_EQ("tbl1$branch_b1", table->Name()); - ASSERT_EQ((std::vector{"b1_c0"}), table->LatestSchema()->FieldNames()); + ASSERT_EQ((std::vector{"b1_c0"}), table->GetSchema()->FieldNames()); for (const auto& main_identifier : {Identifier("db1", "tbl1$branch_main"), Identifier("db1", "tbl1$branch_MAIN")}) { ASSERT_OK_AND_ASSIGN(table, catalog.GetTable(main_identifier)); - ASSERT_EQ((std::vector{"main_c0"}), table->LatestSchema()->FieldNames()); + ASSERT_EQ((std::vector{"main_c0"}), table->GetSchema()->FieldNames()); } Status missing_branch = catalog.GetTable(Identifier("db1", "tbl1$branch_missing")).status(); @@ -156,17 +864,18 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { ASSERT_OK_AND_ASSIGN(std::unique_ptr created, schema_manager.CreateTable( arrow::schema({arrow::field("c0", arrow::int32())}), {}, {}, {})); - auto schema = checked_pointer_cast(std::shared_ptr(std::move(created))); + auto schema = std::shared_ptr(std::move(created)); for (const auto& main_identifier : {Identifier("db", "tbl"), Identifier("db", "tbl$branch_main"), Identifier("db", "tbl$branch_MAIN")}) { SCOPED_TRACE(main_identifier.ToString()); - // A catalog that loaded the schema itself calls the constructor directly, which a - // rest catalog is shaped like. The alias resolves to the main branch as well. - Table table(fs, dir->Str(), schema, main_identifier); - ASSERT_OK_AND_ASSIGN(std::string branch, table.CurrentBranch()); - EXPECT_EQ("main", branch); + ASSERT_OK_AND_ASSIGN(std::string branch, main_identifier.GetBranchNameOrDefault()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + FileStoreTable::Create(fs, dir->Str(), schema, main_identifier, + /*uuid=*/"", branch, /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); + EXPECT_EQ("main", table->CurrentBranch()); } } @@ -178,15 +887,16 @@ TEST(TableTest, TestCurrentBranch) { ASSERT_OK_AND_ASSIGN( std::shared_ptr
main_table, CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); - ASSERT_OK_AND_ASSIGN(std::string branch, main_table->CurrentBranch()); - EXPECT_EQ(branch, "main"); + auto main_file_store_table = std::dynamic_pointer_cast(main_table); + ASSERT_TRUE(main_file_store_table != nullptr); + EXPECT_EQ(main_file_store_table->CurrentBranch(), "main"); - // The identifier a table was opened with names the branch it is for. ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", Identifier("db", "tbl", "dev"), /*branch=*/"dev")); - ASSERT_OK_AND_ASSIGN(branch, branch_table->CurrentBranch()); - EXPECT_EQ(branch, "dev"); + auto branch_file_store_table = std::dynamic_pointer_cast(branch_table); + ASSERT_TRUE(branch_file_store_table != nullptr); + EXPECT_EQ(branch_file_store_table->CurrentBranch(), "dev"); } TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { @@ -197,12 +907,10 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); auto catalog = std::make_shared(); - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); - // A catalog that hands out no per-table file system opens the table without one, and one - // that does opens it with the file system issued for that table. ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog->GetTable(identifier)); - EXPECT_EQ(opened->GetFileSystem(), nullptr); + EXPECT_NE(opened->GetFileSystem(), nullptr); catalog->SetTableFileSystem(dir->GetFileSystem()); ASSERT_OK_AND_ASSIGN(opened, catalog->GetTable(identifier)); @@ -211,10 +919,7 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { EXPECT_EQ(opened->GetRootPath(), ""); } -TEST(TableTest, TestGetTableRejectsATableTypeThatIsNotAManagedTable) { - // A `Table` describes a managed paimon table. Handing back another type as one would promise - // snapshots and manifests it never had, so the failure belongs at the call that opens the - // table rather than at the first read. +TEST(TableTest, TestGetTableServesEachKindOfTableAsWhatItIs) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); @@ -230,7 +935,9 @@ TEST(TableTest, TestGetTableRejectsATableTypeThatIsNotAManagedTable) { format_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {{"type", "format-table"}, {"file.format", "parquet"}})); - ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db", "format")), "Cannot open format table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, + catalog.GetTable(Identifier("db", "format"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(opened)); // A managed table with no `type` option at all is what a `Table` is for, and still opens. const std::string managed_dir = dir->Str() + "/db.db/managed"; @@ -287,6 +994,183 @@ TEST(TableTest, TestNewScanCreatesATableScan) { table->NewScan(std::move(ctx))); } +TEST(TableTest, TestNewScanServesTheSchemaTheTableWasLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier)); + + SchemaManager schema_manager(fs, dir->Str()); + ASSERT_OK(fs->Delete(schema_manager.ToSchemaPath(0), /*recursive=*/false)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr held_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + table->NewScan(std::move(held_ctx))); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ScanContextBuilder(dir->Str()).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx), {}), "not found latest schema"); +} + +TEST(TableTest, TestStaticFactoriesResolveCatalogFileSystemWhenLoading) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier)); + auto catalog = std::make_shared(); + catalog->SetTableFileSystem(fs); + catalog->SetTableSchema(table->GetSchema()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_context, + WriteContextBuilder(dir->Str(), "user").WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_context, + CommitContextBuilder(dir->Str(), "user").WithCatalog(catalog, identifier).Finish()); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
scan_table, + FileStoreTableFactory::Create(*scan_context, {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
read_table, + FileStoreTableFactory::Create(*read_context, {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
write_table, + FileStoreTableFactory::Create(*write_context, {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
commit_table, + FileStoreTableFactory::Create(*commit_context, {})); + for (const auto& loaded : {scan_table, read_table, write_table, commit_table}) { + ASSERT_EQ(loaded->GetFileSystem(), fs); + } + ASSERT_EQ(catalog->TableFileSystemRequests().size(), 4U); +} + +TEST(TableTest, TestSystemScansUseBoundFileSystem) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"), "main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + WriteAndCommitTwoRows(table, dir->Str()); + auto schema = std::dynamic_pointer_cast(table->GetSchema()); + ASSERT_TRUE(schema); + for (const std::string& kind : {"ro", "audit_log", "binlog"}) { + SCOPED_TRACE(kind); + ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table, + SystemTableLoader::Load(kind, fs, dir->Str(), schema, {}, "tbl")); + for (bool use_scheme_map : {false, true}) { + ScanContextBuilder builder("another-table"); + builder.SetTableSchema("invalid schema"); + if (use_scheme_map) { + builder.WithFileSystemSchemeToIdentifierMap({{"file", "missing"}}); + } else { + builder.WithFileSystem(std::make_shared()); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + system_table->NewScan(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + } + } +} + +TEST(TableTest, TestNewScanServesTheFileSystemTheTableWasOpenedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr held_ctx, + ScanContextBuilder(dir->Str()).WithFileSystem(std::make_shared()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(held_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr ctx, + ScanContextBuilder(dir->Str()).WithFileSystem(std::make_shared()).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx), {}), "not found latest schema"); +} + +TEST(TableTest, TestNewScanServesTheRootPathTheTableWasOpenedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr held_ctx, + ScanContextBuilder(dir->Str() + "/not-the-table").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(held_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + ScanContextBuilder(dir->Str() + "/not-the-table").Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx), {}), "not found latest schema"); +} + TEST(TableTest, TestNewReadCreatesATableRead) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -313,7 +1197,7 @@ TEST(TableTest, TestNewWriteCreatesAFileStoreWrite) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); // A catalog write reads the current schema from the catalog on the main branch. - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); ASSERT_OK_AND_ASSIGN( std::unique_ptr ctx, @@ -332,7 +1216,7 @@ TEST(TableTest, TestNewCommitCreatesAFileStoreCommit) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); // Like a write, a catalog commit reads the current schema from the catalog. - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); ASSERT_OK_AND_ASSIGN( std::unique_ptr ctx, @@ -341,6 +1225,155 @@ TEST(TableTest, TestNewCommitCreatesAFileStoreCommit) { table->NewCommit(std::move(ctx))); } +TEST(TableTest, TestNewWriteAndCommitRefuseAContextThatNamesAnotherTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto other_dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(other_dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier)); + + { + WriteContextBuilder write_builder(other_dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(ctx)), "names root path"); + } + { + CommitContextBuilder commit_builder(other_dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "names root path"); + } + + auto catalog = std::make_shared(); + catalog->SetFileSystem(fs); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); + { + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + write_builder.WithCatalog(catalog, Identifier("db", "other")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(ctx)), "names table"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.WithCatalog(catalog, Identifier("db", "other")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "names table"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.WithCatalog(catalog, Identifier("db", "tbl$branch_dev")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "names branch"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.UseRESTCatalogCommit(true).WithTableId("another-table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "table id"); + } + + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, + table->NewCommit(std::move(ctx))); +} + +TEST(TableTest, TestFactoriesApplyOptionsWithoutContextConfiguration) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::map schema_options = {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}}; + for (const std::string& branch : {std::string("main"), std::string("dev")}) { + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl", branch), + branch, schema_options)); + } + for (const std::string& branch : + {std::string("main"), std::string(" "), std::string(""), std::string("dev")}) { + SCOPED_TRACE(branch); + const std::map options = {{Options::BRANCH, branch}, + {Options::READ_BATCH_SIZE, "17"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + FileStoreTableFactory::Create(*scan_context, options)); + auto managed = std::dynamic_pointer_cast(loaded); + ASSERT_TRUE(managed); + ASSERT_EQ(managed->CurrentBranch(), BranchManager::NormalizeBranch(branch)); + auto effective_schema = std::dynamic_pointer_cast(loaded->GetSchema()); + ASSERT_TRUE(effective_schema); + ASSERT_EQ(effective_schema->Options().at(Options::READ_BATCH_SIZE), "17"); + ASSERT_OK(TableScan::Create(std::move(scan_context), options).status()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(dir->Str()).Finish()); + ASSERT_OK(TableRead::Create(std::move(read_context), options).status()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + WriteContextBuilder(dir->Str(), "commit-user").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + FileStoreWrite::Create(std::move(write_context), options)); + ASSERT_OK(write->Close()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + CommitContextBuilder(dir->Str(), "commit-user").Finish()); + ASSERT_OK(FileStoreCommit::Create(std::move(commit_context), options).status()); + } +} + +TEST(TableTest, TestFactoryOptionsCannotContradictTheCatalogBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto catalog = std::make_shared(); + catalog->SetFileSystem(dir->GetFileSystem()); + const Identifier identifier("db", "tbl$branch_dev"); + const std::map options = {{Options::BRANCH, "other"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), options), "aimed at one branch"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), options), "aimed at one branch"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_context, + WriteContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context), options), + "aimed at one branch"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_context, + CommitContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context), options), + "aimed at one branch"); +} + +TEST(TableTest, TestNewCommitRefusesABranchACatalogCannotAddress) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"dev$x")); + + CommitContextBuilder fs_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fs_ctx, fs_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr fs_commit, + table->NewCommit(std::move(fs_ctx))); + + CommitContextBuilder rest_builder(dir->Str(), "commit-user"); + rest_builder.UseRESTCatalogCommit(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_ctx, rest_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(rest_ctx)), "cannot contain"); + + auto catalog = std::make_shared(); + catalog->SetFileSystem(fs); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + write_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_ctx)), "cannot contain"); +} + TEST(TableTest, TestNewScanReadWriteCommitRejectANullContext) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -356,6 +1389,641 @@ TEST(TableTest, TestNewScanReadWriteCommitRejectANullContext) { ASSERT_NOK_WITH_MSG(table->NewCommit(nullptr), "commit context is null pointer"); } +TEST(TableTest, TestCreateFailsWhenTheSchemaOptionsDoNotParse) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + TableSchema::Create( + /*schema_id=*/0, + arrow::schema({arrow::field("id", arrow::int32(), + /*nullable=*/false)}), + /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::TARGET_FILE_ROW_NUM, "0"}})); + auto schema = std::shared_ptr(std::move(created)); + ASSERT_NOK_WITH_MSG( + FileStoreTable::Create(dir->GetFileSystem(), dir->Str(), schema, Identifier("db", "tbl"), + /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{}), + "target-file-row-num should be at least 1"); +} + +TEST(TableTest, TestTheCacheAScanContextCarriesReachesTheScan) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, + /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + WriteAndCommitTwoRows(table, dir->Str()); + + auto cache = std::make_shared( + std::map{{CacheKind::MANIFEST, 64 * 1024 * 1024}, + {CacheKind::SNAPSHOT_LIVE_MANIFEST, 64 * 1024 * 1024}}); + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.WithCache(cache); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + ASSERT_GT(cache->GetCount(CacheKind::MANIFEST), 0) + << "the cache the scan context carries never reached the scan"; +} + +TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + // A matching type is accepted case-insensitively and omitted from overrides. + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET, "2"}, {Options::TYPE, "format-table"}}), + "Change 'type' is not supported yet."); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, + table->Copy({{Options::BUCKET, "2"}, {Options::TYPE, "TABLE"}})); + std::shared_ptr copied_schema = + std::dynamic_pointer_cast(copied->GetSchema()); + ASSERT_TRUE(copied_schema != nullptr); + EXPECT_EQ(copied_schema->Options().at(Options::BUCKET), "2"); + EXPECT_EQ(copied_schema->Options().count(Options::TYPE), 0); + EXPECT_EQ(copied_schema->Options().at(Options::BUCKET_KEY), "id"); + EXPECT_EQ(copied_schema->Id(), std::dynamic_pointer_cast(table->GetSchema())->Id()); + EXPECT_EQ(copied->FullName(), table->FullName()); + EXPECT_EQ(copied->GetRootPath(), table->GetRootPath()); + EXPECT_EQ(copied->GetFileSystem(), table->GetFileSystem()); + EXPECT_EQ( + std::dynamic_pointer_cast(table->GetSchema())->Options().at(Options::BUCKET), + "1"); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, copied->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); +} + +TEST(TableTest, TestSchemaRemainsBoundUntilTableIsReloaded) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/db.db/tbl"; + const Identifier identifier("db", "tbl"); + FileSystemCatalog catalog(fs, dir->Str(), {}); + ASSERT_OK(catalog.CreateDatabase("db", {}, false)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
original, CreateManagedTable(fs, path, identifier)); + const std::shared_ptr bound_schema = original->GetSchema(); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr updated, + TableSchema::Create(1, + arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("added", arrow::utf8())}), + {}, {}, {})); + ASSERT_OK_AND_ASSIGN(std::string schema_json, updated->ToJsonString()); + SchemaManager schemas(fs, path); + ASSERT_OK(fs->AtomicStore(schemas.SchemaDirectory() + "/schema-1", schema_json)); + + EXPECT_EQ(original->GetSchema(), bound_schema); + EXPECT_EQ(original->GetSchema()->FieldNames(), (std::vector{"id"})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, original->Copy({})); + EXPECT_EQ(copied->GetSchema()->FieldNames(), (std::vector{"id"})); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
reloaded, catalog.GetTable(identifier)); + EXPECT_EQ(reloaded->GetSchema()->FieldNames(), (std::vector{"id", "added"})); + EXPECT_EQ(std::dynamic_pointer_cast(reloaded->GetSchema())->Id(), 1); + EXPECT_EQ(original->GetSchema(), bound_schema); +} + +TEST(TableTest, TestCopyTimeTravelUsesTargetBranchSchema) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + const std::map options = {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main, + CreateManagedTable(fs, dir->Str(), identifier, "main", options)); + WriteAndCommitTwoRows(main, dir->Str()); + SchemaManager dev_schemas(fs, dir->Str(), "dev"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr dev_schema, + dev_schemas.CreateTable(arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("v", arrow::int32())}), + {}, {}, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr dev, + FileStoreTable::Create(fs, dir->Str(), std::shared_ptr(std::move(dev_schema)), + identifier, "", "dev", {}, {})); + WriteAndCommitRow(dev, dir->Str(), 7, 70); + + std::map> tables; + int64_t timestamp = 0; + for (const auto& original : {main, std::shared_ptr
(dev)}) { + auto managed = std::dynamic_pointer_cast(original); + const std::string branch = managed->CurrentBranch(); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, + managed->GetSnapshotManager()->LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_EQ(snapshot->Id(), 1); + timestamp = std::max(timestamp, snapshot->TimeMillis()); + ASSERT_OK_AND_ASSIGN(std::string snapshot_json, snapshot->ToJsonString()); + TagManager tags(fs, dir->Str(), branch); + ASSERT_OK(fs->AtomicStore(tags.TagPath("old"), snapshot_json)); + + auto schema = std::dynamic_pointer_cast(original->GetSchema()); + auto fields = schema->Fields(); + fields.emplace_back(static_cast(fields.size()), + arrow::field("added", arrow::utf8())); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr latest_schema, + TableSchema::Create(1, DataField::ConvertDataFieldsToArrowSchema(fields), {}, {}, + schema->Options())); + ASSERT_OK_AND_ASSIGN(std::string schema_json, latest_schema->ToJsonString()); + SchemaManager schemas(fs, dir->Str(), branch); + ASSERT_OK(fs->AtomicStore(schemas.ToSchemaPath(1), schema_json)); + ASSERT_OK_AND_ASSIGN( + tables[branch], + FileStoreTable::Create(fs, dir->Str(), + std::shared_ptr(std::move(latest_schema)), + identifier, "", branch, {}, {})); + } + + for (const auto& [source_branch, table] : tables) { + for (const std::string& kind : {std::string("managed"), std::string("ro")}) { + std::shared_ptr
source = table; + if (kind == "ro") { + ASSERT_OK_AND_ASSIGN( + source, + SystemTableLoader::Load( + kind, fs, dir->Str(), + std::dynamic_pointer_cast(table->GetSchema()), {}, "tbl")); + } + for (const std::string& named_branch : + {std::string("dev"), std::string("main"), std::string(""), std::string(" \t ")}) { + const std::string target_branch = BranchManager::NormalizeBranch(named_branch); + const bool target_dev = target_branch == "dev"; + const std::vector expected_fields = + target_dev ? std::vector{"id", "v"} + : std::vector{"id"}; + for (const auto& selector : std::vector>{ + {Options::SCAN_SNAPSHOT_ID, "1"}, + {Options::SCAN_TAG_NAME, "old"}, + {Options::SCAN_TIMESTAMP_MILLIS, std::to_string(timestamp)}}) { + SCOPED_TRACE(source_branch + "/" + kind + "/" + named_branch + "/" + + selector.first); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
historical, + source->Copy({{Options::BRANCH, named_branch}, selector})); + ASSERT_EQ(historical->GetSchema()->FieldNames(), expected_fields); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
again, historical->Copy({})); + ASSERT_EQ(again->GetSchema()->FieldNames(), expected_fields); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + historical->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + for (bool explicit_projection : {false, true}) { + ReadContextBuilder builder(dir->Str()); + if (explicit_projection) { + builder.SetReadFieldNames(expected_fields); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + historical->NewRead(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + read->CreateReader(plan->Splits())); + int64_t rows = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + auto values = std::dynamic_pointer_cast(array); + ASSERT_TRUE(values); + ASSERT_TRUE(values->GetFieldByName("id")); + ASSERT_FALSE(values->GetFieldByName("added")); + ASSERT_EQ(values->GetFieldByName("v") != nullptr, target_dev); + if (target_dev) { + auto v = std::dynamic_pointer_cast( + values->GetFieldByName("v")); + ASSERT_TRUE(v); + for (int64_t i = 0; i < v->length(); ++i) { + ASSERT_FALSE(v->IsNull(i)); + ASSERT_EQ(v->Value(i), 70); + } + } + rows += values->length(); + } + ASSERT_EQ(rows, target_dev ? 1 : 2); + } + } + } + } + } +} + +TEST(TableTest, TestCopyTimeTravelUsesHistoricalSchema) { + for (const std::string& branch : {std::string("main"), std::string("dev")}) { + SCOPED_TRACE(branch); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
original, + CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"), branch, + {{Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}})); + WriteAndCommitTwoRows(original, dir->Str()); + auto managed = std::dynamic_pointer_cast(original); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, + managed->GetSnapshotManager()->LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_OK_AND_ASSIGN(std::string snapshot_json, snapshot->ToJsonString()); + TagManager tags(fs, dir->Str(), branch); + ASSERT_OK(fs->AtomicStore(tags.TagPath("old"), snapshot_json)); + + // Model a newly added column and its storage declarations in schema 1. + auto latest_options = + std::dynamic_pointer_cast(original->GetSchema())->Options(); + latest_options[Options::BLOB_FIELD] = "payload"; + latest_options[Options::FALLBACK_BLOB_DESCRIPTOR_FIELD] = "payload"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr latest_schema, + TableSchema::Create(1, + arrow::schema({arrow::field("id", arrow::int32(), false), + BlobUtils::ToArrowField("payload", true)}), + {}, {}, latest_options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
latest, + FileStoreTable::Create(fs, dir->Str(), + std::shared_ptr(std::move(latest_schema)), + Identifier("db", "tbl"), "", branch, {}, {})); + for (const auto& selector : std::vector>{ + {Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot->Id())}, + {Options::SCAN_TAG_NAME, "old"}, + {Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snapshot->TimeMillis())}}) { + SCOPED_TRACE(selector.first); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
historical, + latest->Copy({selector, {Options::READ_BATCH_SIZE, "1"}})); + auto schema = std::dynamic_pointer_cast(historical->GetSchema()); + ASSERT_TRUE(schema); + EXPECT_EQ(schema->Id(), 0); + EXPECT_EQ(schema->FieldNames(), (std::vector{"id"})); + EXPECT_EQ(schema->Options().count(Options::BLOB_FIELD), 0); + EXPECT_EQ(schema->Options().count(Options::FALLBACK_BLOB_DESCRIPTOR_FIELD), 0); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
again, historical->Copy({})); + EXPECT_EQ(std::dynamic_pointer_cast(again->GetSchema())->Id(), 0); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + historical->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + historical->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + read->CreateReader(plan->Splits())); + int64_t rows = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + rows += array->length(); + } + EXPECT_EQ(rows, 2); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr
unavailable, + original->Copy({{Options::SCAN_SNAPSHOT_ID, "999"}})); + EXPECT_EQ(std::dynamic_pointer_cast(unavailable->GetSchema())->Id(), 0); + ASSERT_NOK_WITH_MSG( + latest->Copy({{Options::SCAN_TAG_NAME, "old"}, {Options::BLOB_FIELD, "payload"}}), + "payload"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
explicitly_declared, + latest->Copy({{Options::BLOB_FIELD, "payload"}})); + ASSERT_NOK_WITH_MSG(explicitly_declared->Copy({{Options::SCAN_TAG_NAME, "old"}}), + "payload"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr options_view, + SystemTableLoader::Load(OptionsSystemTable::kName, fs, dir->Str(), + std::dynamic_pointer_cast(latest->GetSchema()), {}, + "tbl")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
old_options, + options_view->Copy({{Options::SCAN_TAG_NAME, "old"}})); + auto old_view = std::dynamic_pointer_cast(old_options); + ASSERT_TRUE(old_view); + ASSERT_OK_AND_ASSIGN(std::vector option_rows, old_view->BuildRows()); + for (const auto& row : option_rows) { + EXPECT_NE(row.GetStringView(0), Options::BLOB_FIELD); + EXPECT_NE(row.GetStringView(0), Options::FALLBACK_BLOB_DESCRIPTOR_FIELD); + } + + // Finding a snapshot but failing to load its schema must not silently use the current one. + ASSERT_OK(fs->Delete(managed->GetSchemaManager()->SchemaDirectory() + "/schema-0", false)); + ASSERT_NOK_WITH_MSG(original->Copy({{Options::SCAN_TAG_NAME, "old"}}), "schema-0"); + } +} + +TEST(TableTest, TestCopyRejectsConflictingTimeTravelOptions) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "t"))); + const std::vector> invalid = { + {{Options::SCAN_SNAPSHOT_ID, "1"}, {Options::SCAN_TAG_NAME, "tag"}}, + {{Options::SCAN_SNAPSHOT_ID, "1"}, {Options::SCAN_TIMESTAMP_MILLIS, "1"}}, + {{Options::SCAN_TAG_NAME, "tag"}, {Options::SCAN_TIMESTAMP_MILLIS, "1"}}, + {{Options::SCAN_MODE, "latest-full"}, {Options::SCAN_SNAPSHOT_ID, "1"}}, + {{Options::SCAN_MODE, "latest"}, {Options::SCAN_TIMESTAMP_MILLIS, "1"}}, + {{Options::SCAN_MODE, "from-snapshot"}, {Options::SCAN_TIMESTAMP_MILLIS, "1"}}, + {{Options::SCAN_MODE, "from-snapshot-full"}, {Options::SCAN_TAG_NAME, "tag"}}, + {{Options::SCAN_MODE, "from-timestamp"}, {Options::SCAN_SNAPSHOT_ID, "1"}}, + {{Options::SCAN_MODE, "from-snapshot"}}, + {{Options::SCAN_MODE, "from-snapshot-full"}}, + {{Options::SCAN_MODE, "from-timestamp"}}, + }; + for (const auto& options : invalid) { + ASSERT_NOK_WITH_MSG(table->Copy(options), "Invalid time travel options"); + } + // Unavailable snapshots defer their error to the scan. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
historical, + table->Copy({{Options::SCAN_SNAPSHOT_ID, "1"}})); + ASSERT_NOK_WITH_MSG(historical->Copy({{Options::SCAN_TAG_NAME, "tag"}}), + "Invalid time travel options"); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
another_snapshot, + historical->Copy({{Options::SCAN_SNAPSHOT_ID, "2"}})); +} + +TEST(TableTest, TestCopyProtectsDeletionVectorsMode) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + for (bool enabled : {false, true}) { + SCOPED_TRACE(enabled); + const std::string value = enabled ? "true" : "false"; + const std::string changed = enabled ? "false" : "true"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr schema, + TableSchema::Create( + 0, arrow::schema({arrow::field("id", arrow::int32(), false)}), {}, {"id"}, + {{Options::BUCKET, "1"}, {Options::DELETION_VECTORS_ENABLED, value}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + FileStoreTable::Create(fs, dir->Str(), std::shared_ptr(std::move(schema)), + Identifier("db", "t"), "", "main", {}, {})); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
unchanged, + table->Copy({{Options::DELETION_VECTORS_ENABLED, value}})); + ASSERT_NOK_WITH_MSG(table->Copy({{Options::DELETION_VECTORS_ENABLED, changed}}), + "Cannot change deletion vectors mode"); + // Check the original configuration, not the modifiable flag in this hint. + ASSERT_NOK_WITH_MSG(table->Copy({{Options::DELETION_VECTORS_ENABLED, changed}, + {"deletion-vectors.modifiable", "true"}}), + "Cannot change deletion vectors mode"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
modifiable, + table->Copy({{"deletion-vectors.modifiable", "true"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
toggled, + modifiable->Copy({{Options::DELETION_VECTORS_ENABLED, changed}})); + EXPECT_EQ(std::dynamic_pointer_cast(toggled->GetSchema()) + ->Options() + .at(Options::DELETION_VECTORS_ENABLED), + changed); + } +} + +TEST(TableTest, TestSystemTableCopyUsesHistoricalSchemaForOperators) { + for (const std::string& branch : {std::string("main"), std::string("dev")}) { + for (const std::string& kind : + {std::string("ro"), std::string("audit_log"), std::string("binlog")}) { + SCOPED_TRACE(branch + "/" + kind); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + std::map options = {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}}; + const std::vector primary_keys = + kind == "ro" ? std::vector{} : std::vector{"id"}; + SchemaManager schemas(fs, dir->Str(), branch); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr schema, + schemas.CreateTable(arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("v", arrow::int32())}), + {}, primary_keys, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
original, + FileStoreTable::Create(fs, dir->Str(), + std::shared_ptr(std::move(schema)), + Identifier("db", "tbl"), "", branch, {}, {})); + WriteAndCommitRow(original, dir->Str(), 1, 10); + options = std::dynamic_pointer_cast(original->GetSchema())->Options(); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr latest, + TableSchema::Create(1, + arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("new", arrow::utf8())}), + {}, primary_keys, options)); + ASSERT_OK_AND_ASSIGN(std::string latest_json, latest->ToJsonString()); + ASSERT_OK(fs->AtomicStore(schemas.SchemaDirectory() + "/schema-1", latest_json)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr view, + SystemTableLoader::Load( + kind, fs, dir->Str(), + std::shared_ptr(std::move(latest)), {}, "tbl")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
historical, + view->Copy({{Options::SCAN_SNAPSHOT_ID, "1"}})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + historical->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + ReadContextBuilder read_builder(dir->Str()); + read_builder.SetReadFieldNames({"id", "v"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + historical->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + ASSERT_FALSE(BatchReader::IsEofBatch(batch)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + auto rows = std::dynamic_pointer_cast(array); + ASSERT_TRUE(rows); + ASSERT_EQ(rows->length(), 1); + auto values = rows->GetFieldByName("v"); + ASSERT_TRUE(values); + if (kind == "binlog") { + auto list = std::dynamic_pointer_cast(values); + ASSERT_TRUE(list); + values = list->values(); + } + auto ints = std::dynamic_pointer_cast(values); + ASSERT_TRUE(ints); + EXPECT_EQ(ints->Value(0), 10); + } + } +} + +TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, + /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_KEY, "v"}}), + "Change 'bucket-key' is not supported yet."); + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_FUNCTION_TYPE, "hash"}}), + "Change 'bucket-function.type' is not supported yet."); + for (const std::string& key : + {std::string(Options::MERGE_ENGINE), std::string(Options::ROW_TRACKING_ENABLED), + std::string(Options::DATA_EVOLUTION_ENABLED)}) { + ASSERT_NOK_WITH_MSG(table->Copy({{key, "true"}}), "is not supported yet"); + } + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET, "-1"}}), "Cannot change bucket to -1"); + { + std::map scan_builder_options; + ScanContextBuilder scan_builder(dir->Str()); + scan_builder_options[Options::BUCKET_KEY] = "v"; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx), scan_builder_options), + "Change 'bucket-key' is not supported yet."); + } + + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
unchanged, + table->Copy({{Options::BUCKET_KEY, "id"}, {Options::BUCKET, "1"}})); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
rescaled, + table->Copy({{Options::BUCKET, "2"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
normalized, + table->Copy({{"primary-key", " , "}, {"partition", " "}})); + auto normalized_schema = std::dynamic_pointer_cast(normalized->GetSchema()); + ASSERT_TRUE(normalized_schema); + EXPECT_EQ(normalized_schema->Options().count("primary-key"), 0); + EXPECT_EQ(normalized_schema->Options().count("partition"), 0); + ASSERT_NOK_WITH_MSG(table->Copy({{"primary-key", "id"}}), "is not supported yet"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
bucket_unaware, + CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/bucket_unaware", + Identifier("db", "bucket_unaware"))); + ASSERT_NOK_WITH_MSG(bucket_unaware->Copy({{Options::BUCKET, "1"}}), + "Cannot change bucket when it is -1"); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
ignoring_deletes, + table->Copy({{Options::IGNORE_DELETE, "true"}})); + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr
ignore_delete_table, + CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/ignore_delete", + Identifier("db", "igd"), /*branch=*/"main", + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::IGNORE_DELETE, "true"}})); + ASSERT_NOK_WITH_MSG(ignore_delete_table->Copy({{Options::IGNORE_DELETE, "false"}}), + "Cannot change ignore-delete from true to false."); + } +} + +TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("dt", arrow::utf8())}); + + const std::string table_dir = dir->Str() + "/my_db.db/my_table"; + ASSERT_OK(fs->Mkdirs(table_dir)); + SchemaManager schema_manager(fs, table_dir); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr managed_schema, + schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, {{"file.format", "orc"}})); + { + ScanContextBuilder builder(table_dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx, {})); + ASSERT_EQ(table->FullName(), "my_db.my_table"); + } + + const std::string loose_table_dir = dir->Str() + "/loose_table"; + ASSERT_OK(fs->Mkdirs(loose_table_dir)); + SchemaManager loose_schema_manager(fs, loose_table_dir); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr loose_schema, + loose_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, + {{"file.format", "orc"}})); + { + ScanContextBuilder builder(loose_table_dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx, {})); + ASSERT_EQ(table->FullName(), "loose_table"); + } +} + +TEST(TableTest, TestALoadResolvesTheContextFormatOverTheSchemaOne) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + // Apply dynamic overrides before resolving the stored file format. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr stored_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, {{"file.format", "unavailable-format"}})); + ASSERT_OK_AND_ASSIGN(std::string schema_json, stored_schema->GetJsonSchema()); + { + ScanContextBuilder builder(dir->Str()); + builder.SetTableSchema(schema_json); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_NOK(FileStoreTableFactory::Create(*ctx, {})); + } + { + std::map builder_options; + ScanContextBuilder builder(dir->Str()); + builder_options[Options::FILE_FORMAT] = "parquet"; + builder.SetTableSchema(schema_json); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + FileStoreTableFactory::Create(*ctx, builder_options)); + ASSERT_NE(table, nullptr); + ScanContextBuilder scan_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_NE(scan, nullptr); + } +} + TEST(TableTest, TestCreateBuildsTheOperator) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -366,31 +2034,91 @@ TEST(TableTest, TestCreateBuildsTheOperator) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); // A catalog write and commit read the current schema from the catalog on the main branch. - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); // `TableScan::Create()` and friends build the operator straight from the context, which // carries the catalog and identifier it resolves the table through. ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, ScanContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, - TableScan::Create(std::move(scan_ctx))); + TableScan::Create(std::move(scan_ctx), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, ReadContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, - TableRead::Create(std::move(read_ctx))); + TableRead::Create(std::move(read_ctx), {})); ASSERT_OK_AND_ASSIGN( std::unique_ptr write_ctx, WriteContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr write, - FileStoreWrite::Create(std::move(write_ctx))); + FileStoreWrite::Create(std::move(write_ctx), {})); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit_ctx, CommitContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_ctx))); + FileStoreCommit::Create(std::move(commit_ctx), {})); +} + +TEST(TableTest, TestCreateWithSpecificTableSchemaSkipsTheCatalogSchema) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); + + auto catalog = std::make_shared(); + catalog->SetFileSystem(dir->GetFileSystem()); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); + + // An explicit schema bypasses catalog schema loading on main. + ASSERT_OK_AND_ASSIGN(std::string schema_json, table->GetSchema()->GetJsonSchema()); + + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_ctx, + write_builder.WithCatalog(catalog, identifier).SetTableSchema(schema_json).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr write, + FileStoreWrite::Create(std::move(write_ctx), {})); + + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_ctx, + commit_builder.WithCatalog(catalog, identifier).SetTableSchema(schema_json).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, + FileStoreCommit::Create(std::move(commit_ctx), {})); + + ASSERT_EQ(catalog->LoadTableSchemaCalls(), 0U); +} + +TEST(TableTest, TestCreateWithCatalogReadsTheMainBranchSchemaFromIt) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); + + auto catalog = std::make_shared(); + catalog->SetFileSystem(dir->GetFileSystem()); + catalog->SetTableSchema(checked_pointer_cast(table->GetSchema())); + + // Without an explicit schema, each operator loads the catalog schema once. + ScanContextBuilder scan_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, + scan_builder.WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
scanned, + FileStoreTableFactory::Create(*scan_ctx, {})); + ASSERT_EQ(scanned->FullName(), "db.tbl"); + + ReadContextBuilder read_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, + read_builder.WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
read_table, + FileStoreTableFactory::Create(*read_ctx, {})); + ASSERT_EQ(read_table->FullName(), "db.tbl"); + + ASSERT_EQ(catalog->LoadTableSchemaCalls(), 2U); } } // namespace paimon::test diff --git a/src/paimon/core/utils/blob_view_lookup.cpp b/src/paimon/core/utils/blob_view_lookup.cpp index 30999fe62..7dec800b8 100644 --- a/src/paimon/core/utils/blob_view_lookup.cpp +++ b/src/paimon/core/utils/blob_view_lookup.cpp @@ -161,20 +161,21 @@ Result BlobViewLookup::LoadTableDescriptorChu .WithCatalog(catalog, identifier); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), {})); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + std::map read_builder_options; ReadContextBuilder read_builder(table_path); std::vector read_field_ids = field_ids; read_field_ids.push_back(SpecialFieldIds::ROW_ID); + read_builder_options[Options::BLOB_AS_DESCRIPTOR] = "true"; read_builder.SetReadFieldIds(read_field_ids) - .AddOption(Options::BLOB_AS_DESCRIPTOR, "true") .EnablePrefetch(true) .WithMemoryPool(pool) .WithCatalog(catalog, identifier); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); diff --git a/src/paimon/core/utils/branch_manager.cpp b/src/paimon/core/utils/branch_manager.cpp index 6a9a46b3d..2592fb979 100644 --- a/src/paimon/core/utils/branch_manager.cpp +++ b/src/paimon/core/utils/branch_manager.cpp @@ -36,7 +36,6 @@ namespace paimon { Result BranchManager::ResolveBranch(const std::optional& identifier, const std::map& options, - const std::optional& explicit_branch, const std::string& operation) { std::optional identifier_branch; if (identifier) { @@ -48,8 +47,7 @@ Result BranchManager::ResolveBranch(const std::optional : std::optional(option_iter->second); std::optional resolved; - for (const std::optional& named_branch : - {identifier_branch, option_branch, explicit_branch}) { + for (const std::optional& named_branch : {identifier_branch, option_branch}) { if (!named_branch) { continue; } diff --git a/src/paimon/core/utils/branch_manager.h b/src/paimon/core/utils/branch_manager.h index eebb45647..ee87017e9 100644 --- a/src/paimon/core/utils/branch_manager.h +++ b/src/paimon/core/utils/branch_manager.h @@ -74,17 +74,10 @@ class BranchManager { return branch == DEFAULT_MAIN_BRANCH; } - /// Returns the branch an operation is aimed at, which the table identifier, the `branch` - /// option and `explicit_branch` may each name. - /// - /// An absent name and a name which `NormalizeBranch` maps to `main` both mean the main - /// branch. Names which disagree are refused rather than silently chosen between, with - /// `operation` naming the operation in that error. Every name is put through - /// `CheckValidBranch` before it is compared, so neither the branch this returns nor a - /// branch it names in an error is one that would leave the table root. + /// Resolves and validates the identifier's branch and the branch option. + /// Rejects conflicting names and invalid branch paths; operation labels errors. static Result ResolveBranch(const std::optional& identifier, const std::map& options, - const std::optional& explicit_branch, const std::string& operation); /// Fails when the branch object name `Identifier` builds, as `tbl$branch_dev`, does not read diff --git a/src/paimon/core/utils/branch_manager_test.cpp b/src/paimon/core/utils/branch_manager_test.cpp index c73459241..caa8d96a7 100644 --- a/src/paimon/core/utils/branch_manager_test.cpp +++ b/src/paimon/core/utils/branch_manager_test.cpp @@ -55,49 +55,49 @@ TEST(BranchManagerTest, TestBranchPath) { TEST(BranchManagerTest, TestResolveBranch) { const std::map no_options; - ASSERT_OK_AND_ASSIGN(std::string unnamed, - BranchManager::ResolveBranch(/*identifier=*/std::nullopt, no_options, - /*explicit_branch=*/std::nullopt, "commit")); + ASSERT_OK_AND_ASSIGN( + std::string unnamed, + BranchManager::ResolveBranch(/*identifier=*/std::nullopt, no_options, "commit")); ASSERT_EQ(BranchManager::DEFAULT_MAIN_BRANCH, unnamed); - ASSERT_OK_AND_ASSIGN(std::string from_identifier, - BranchManager::ResolveBranch(Identifier("db", "tbl$branch_data"), - no_options, std::nullopt, "commit")); + ASSERT_OK_AND_ASSIGN( + std::string from_identifier, + BranchManager::ResolveBranch(Identifier("db", "tbl$branch_data"), no_options, "commit")); ASSERT_EQ("data", from_identifier); - ASSERT_OK_AND_ASSIGN(std::string from_option, - BranchManager::ResolveBranch(std::nullopt, {{Options::BRANCH, "data"}}, - std::nullopt, "commit")); + ASSERT_OK_AND_ASSIGN( + std::string from_option, + BranchManager::ResolveBranch(std::nullopt, {{Options::BRANCH, "data"}}, "commit")); ASSERT_EQ("data", from_option); - ASSERT_OK_AND_ASSIGN(std::string from_given, - BranchManager::ResolveBranch(std::nullopt, no_options, "data", "write")); + ASSERT_OK_AND_ASSIGN( + std::string from_given, + BranchManager::ResolveBranch(std::nullopt, {{Options::BRANCH, "data"}}, "write")); ASSERT_EQ("data", from_given); - ASSERT_OK_AND_ASSIGN(std::string empty, - BranchManager::ResolveBranch(std::nullopt, no_options, "", "write")); + ASSERT_OK_AND_ASSIGN(std::string empty, BranchManager::ResolveBranch( + std::nullopt, {{Options::BRANCH, ""}}, "write")); ASSERT_EQ(BranchManager::DEFAULT_MAIN_BRANCH, empty); - ASSERT_OK_AND_ASSIGN(std::string agreed, BranchManager::ResolveBranch( - Identifier("db", "tbl$branch_data"), - {{Options::BRANCH, "data"}}, "data", "write")); + ASSERT_OK_AND_ASSIGN(std::string agreed, + BranchManager::ResolveBranch(Identifier("db", "tbl$branch_data"), + {{Options::BRANCH, "data"}}, "write")); ASSERT_EQ("data", agreed); - ASSERT_NOK_WITH_MSG( - BranchManager::ResolveBranch(Identifier("db", "tbl$branch_data"), {{Options::BRANCH, "rt"}}, - std::nullopt, "commit"), - "a commit is aimed at one branch, but both 'data' and 'rt' were named"); - ASSERT_NOK_WITH_MSG( - BranchManager::ResolveBranch(std::nullopt, {{Options::BRANCH, "main"}}, "rt", "write"), - "a write is aimed at one branch, but both 'main' and 'rt' were named"); + ASSERT_NOK_WITH_MSG(BranchManager::ResolveBranch(Identifier("db", "tbl$branch_data"), + {{Options::BRANCH, "rt"}}, "commit"), + "a commit is aimed at one branch, but both 'data' and 'rt' were named"); + ASSERT_NOK_WITH_MSG(BranchManager::ResolveBranch(Identifier("db", "tbl$branch_main"), + {{Options::BRANCH, "rt"}}, "write"), + "a write is aimed at one branch, but both 'main' and 'rt' were named"); - ASSERT_NOK_WITH_MSG(BranchManager::ResolveBranch(Identifier("db", "tbl$a$b$c"), no_options, - std::nullopt, "commit"), - "Invalid object name"); + ASSERT_NOK_WITH_MSG( + BranchManager::ResolveBranch(Identifier("db", "tbl$a$b$c"), no_options, "commit"), + "Invalid object name"); ASSERT_NOK_WITH_MSG( BranchManager::ResolveBranch(Identifier("db", "tbl$branch_data"), - {{Options::BRANCH, "rt/../../outside"}}, std::nullopt, "read"), + {{Options::BRANCH, "rt/../../outside"}}, "read"), "branch name cannot contain path separators"); ASSERT_NOK_WITH_MSG( - BranchManager::ResolveBranch(std::nullopt, no_options, "line\nfeed", "write"), + BranchManager::ResolveBranch(std::nullopt, {{Options::BRANCH, "line\nfeed"}}, "write"), "branch name cannot contain control characters"); } diff --git a/src/paimon/core/utils/context_utils.h b/src/paimon/core/utils/context_utils.h new file mode 100644 index 000000000..f3a3cb85e --- /dev/null +++ b/src/paimon/core/utils/context_utils.h @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// Shared operator-context checks. +class ContextUtils { + public: + ContextUtils() = delete; + ~ContextUtils() = delete; + + template + static Result> ResolveFileSystemForLoad(const Context& context) { + if (context.GetSpecificFileSystem() != nullptr || context.GetCatalog() == nullptr || + !context.GetFileSystemSchemeToIdentifierMap().empty()) { + return context.GetSpecificFileSystem(); + } + if (!context.GetIdentifier()) { + return Status::Invalid("a catalog requires a table identifier"); + } + return context.GetCatalog()->GetTableFileSystem(context.GetIdentifier().value()); + } + + /// Requires a context, memory pool and executor; kind labels errors. + template + static Status CheckOperatorContext(const ContextPointer& context, const char* kind) { + if (context == nullptr) { + return Status::Invalid(fmt::format("{} context is null pointer", kind)); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + return Status::OK(); + } +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 273804d0d..a4481df3d 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -32,8 +32,10 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" @@ -383,84 +385,112 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } GlobalSystemTableContext context; - context.catalog = this; context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = api_->GetMergedOptions(); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + GlobalSystemTableLoader::LoadForSchema(identifier.GetObjectName(), context)); + return system_table->GetSchema(); } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); branch = NormalizeBranch(std::move(branch)); PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, - identifier.GetSystemTableName()); - if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { - return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); - } - std::string data_table_path; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, - LoadDataTableSchema(load_identifier, branch, &data_table_path, - /*table_id=*/nullptr)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + LoadSystemTable(identifier, load_identifier, branch)); if (table_path != nullptr) { - *table_path = data_table_path; - } - std::map dynamic_options; - if (branch) { - dynamic_options[Options::BRANCH] = branch.value(); + *table_path = system_table->GetRootPath(); } - // The file system is only used to build the system table, whose schema does not - // read any file, so the catalog wide one is enough here. The credentials of the - // table are applied when its rows are read, through the file system of the read or - // scan context. - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::Load(system_table_name.value(), fs_, data_table_path, latest_schema, - dynamic_options)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + return system_table->GetSchema(); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadDataTableSchema(load_identifier, branch, table_path, table_id)); return checked_pointer_cast(schema); } -Result> RestCatalog::LoadFormatTable( - const Identifier& identifier) const { - // A system table is already refused by `Catalog::GetFormatTable()`, which every caller goes - // through, so this only has to load. - PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); - branch = NormalizeBranch(std::move(branch)); - PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); - std::string location; +Result> RestCatalog::LoadSystemTable( + const Identifier& identifier, const Identifier& load_identifier, + const std::optional& branch) const { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::string data_table_path; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, + LoadDataTableSchema(load_identifier, branch, &data_table_path, + /*table_id=*/nullptr)); + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, + GetTableFileSystem(identifier)); + // Use the catalog name, including any branch, even when data lives under a UUID directory. PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr schema, - LoadDataTableSchema(load_identifier, branch, &location, /*table_id=*/nullptr)); - // A rest catalog holds the schema itself, so everything below the location is data. - return FormatTable::Create(fs_, location, identifier, checked_pointer_cast(schema), - /*location_carries_paimon_metadata=*/false, - /*dynamic_options=*/{}); + Identifier data_identifier, + CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); + return SystemTableLoader::Load(system_table_name.value(), table_file_system, data_table_path, + latest_schema, dynamic_options, data_identifier.GetObjectName(), + fs_scheme_to_identifier_map_); } Result> RestCatalog::GetTable(const Identifier& identifier) const { - // Read UUID, path and schema together to avoid mixing table generations after recreation. + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, + GlobalSystemTableLoader::IsSupported(identifier.GetObjectName(), + api_->GetMergedOptions())); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::shared_ptr self = weak_from_this().lock(); + if (self == nullptr) { + return Status::Invalid( + "a global system table retains its catalog, so the catalog must be owned by " + "a std::shared_ptr before it is asked for one"); + } + GlobalSystemTableContext context; + context.catalog = std::move(self); + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = api_->GetMergedOptions(); + return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + if (is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + return LoadSystemTable(identifier, load_identifier, NormalizeBranch(std::move(branch))); + } + // Load UUID, location and schema together to avoid mixing table generations. + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + branch = NormalizeBranch(std::move(branch)); + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); std::string uuid; std::string table_path; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - LoadTableSchema(identifier, &uuid, &table_path)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + LoadDataTableSchema(load_identifier, branch, &table_path, &uuid)); + std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); + if (data_schema != nullptr) { + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(data_schema->Options())); + if (is_format_table) { + return FormatTable::Create(fs_, table_path, identifier, data_schema, + /*location_carries_paimon_metadata=*/false, + /*dynamic_options=*/{}); + } + } PAIMON_RETURN_NOT_OK( CatalogUtils::CheckManagedTableType(identifier, schema, "Catalog::GetTable")); // The table holds the file system its own data is served by, which a server issuing data // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - return std::make_shared
(table_file_system, table_path, schema, identifier, uuid); + return FileStoreTable::Create(table_file_system, table_path, schema, identifier, uuid, + branch.value_or(Identifier::kDefaultMainBranch), + fs_scheme_to_identifier_map_, /*dynamic_options=*/{}); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 2f57e912d..a489d2fdf 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -36,12 +36,15 @@ struct ArrowSchema; namespace paimon { class FileSystem; +class SystemTable; class TableSchema; /// A catalog backed by a REST catalog server. Metadata operations are delegated to /// `RestApi`; table data is accessed through the file system configured by the /// server-merged options. -class RestCatalog : public Catalog, public VersionManagedCatalog { +class RestCatalog : public Catalog, + public VersionManagedCatalog, + public std::enable_shared_from_this { public: /// Creates the catalog: fetches and merges "/v1/config" from the server configured /// by `CatalogOptions::URI`, then builds the file system from the merged options. @@ -117,12 +120,6 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { /// Options merged with the server side config. const std::map& GetOptions() const override; - protected: - /// Loads the location and the schema from one `GetTable` response, so the table cannot be - /// built from a location and a schema that two requests disagreed on. - Result> LoadFormatTable( - const Identifier& identifier) const override; - private: RestCatalog(std::shared_ptr api, const std::shared_ptr& fs, const std::string& warehouse, bool data_token_enabled, bool fs_explicitly_supplied, @@ -140,6 +137,11 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { const Identifier& data_identifier, const std::optional& branch, std::string* table_path, std::string* table_id) const; + /// Loads a T$system view from the same response as its data table's schema and path. + Result> LoadSystemTable( + const Identifier& identifier, const Identifier& load_identifier, + const std::optional& branch) const; + /// Converts a `GetTableResponse` into a `TableSchema`, applying the table path, audit /// and branch options. static Result> ToTableSchema( diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 147a908ce..6e9f420f6 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -49,15 +49,24 @@ #include "paimon/core/core_options.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" #include "paimon/rest/mock_rest_server.h" #include "paimon/rest/rest_api.h" +#include "paimon/scan_context.h" #include "paimon/schema/schema.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/plan.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/snapshot_test_helper.h" #include "paimon/testing/utils/testharness.h" @@ -801,7 +810,7 @@ TEST_F(RestCatalogTest, RecreatedTableLoadsNewCredentialsBeforeOldTokenExpires) } TEST_F(RestCatalogTest, CatalogFactoryMetastoreDispatch) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(kWarehouse, options_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(kWarehouse, options_)); ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); ASSERT_TRUE(databases.empty()); options_[CatalogOptions::METASTORE] = "something-else"; @@ -918,8 +927,7 @@ TEST_F(RestCatalogTest, TableOperations) { std::lock_guard lock(state_->mutex); ASSERT_EQ(requests_before_get_table + 1, state_->get_table_requests); } - std::shared_ptr schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, schema); ASSERT_EQ((std::vector{"f0", "f1"}), schema->FieldNames()); ASSERT_EQ((std::vector{"f0"}), schema->PrimaryKeys()); @@ -937,7 +945,7 @@ TEST_F(RestCatalogTest, TableOperations) { // timeMillis is backed by the server's audit "updatedAt" instead of the current // time, keeping the conversion deterministic std::shared_ptr table_schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, table_schema); ASSERT_EQ(123, table_schema->TimeMillis()); @@ -959,7 +967,7 @@ TEST_F(RestCatalogTest, TableOperations) { } ASSERT_OK_AND_ASSIGN(std::shared_ptr
commented_table, catalog->GetTable(Identifier("db1", "commented"))); - ASSERT_EQ("a table comment", commented_table->LatestSchema()->Comment().value_or("")); + ASSERT_EQ("a table comment", commented_table->GetSchema()->Comment().value_or("")); ASSERT_OK(catalog->RenameTable(identifier, Identifier("db1", "t2"), /*ignore_if_not_exists=*/false)); @@ -1061,9 +1069,11 @@ TEST_F(RestCatalogTest, SystemTableSchema) { requests_before = state_->get_table_requests; } ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); - ASSERT_EQ("t1$options", table->Name()); - ASSERT_EQ((std::vector{"key", "value"}), table->LatestSchema()->FieldNames()); - ASSERT_EQ("db1.t1$options", table->Uuid()); + std::shared_ptr system_table = std::dynamic_pointer_cast(table); + ASSERT_NE(nullptr, system_table); + ASSERT_EQ("t1$options", system_table->Name()); + ASSERT_EQ((std::vector{"key", "value"}), table->GetSchema()->FieldNames()); + ASSERT_EQ("t1$options", table->Uuid()); { std::lock_guard lock(state_->mutex); ASSERT_EQ(requests_before + 1, state_->get_table_requests); @@ -1105,8 +1115,7 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(branch_identifier)); ASSERT_EQ("t1$branch_b1", table->Name()); - std::shared_ptr schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, schema); // the branch's own schema and schema id, not the main table's ASSERT_EQ((std::vector{"b0"}), schema->FieldNames()); @@ -1118,7 +1127,7 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_table, catalog->GetTable(Identifier("db1", "t1$branch_main"))); std::shared_ptr main_schema = - std::dynamic_pointer_cast(main_table->LatestSchema()); + std::dynamic_pointer_cast(main_table->GetSchema()); ASSERT_NE(nullptr, main_schema); ASSERT_EQ((std::vector{"f0", "f1"}), main_schema->FieldNames()); ASSERT_EQ(0, main_schema->Options().count(Options::BRANCH)); @@ -1128,14 +1137,15 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_case_table, catalog->GetTable(Identifier("db1", "t1$branch_MAIN"))); std::shared_ptr main_case_schema = - std::dynamic_pointer_cast(main_case_table->LatestSchema()); + std::dynamic_pointer_cast(main_case_table->GetSchema()); ASSERT_NE(nullptr, main_case_schema); ASSERT_EQ((std::vector{"f0", "f1"}), main_case_schema->FieldNames()); ASSERT_EQ(0, main_case_schema->Options().count(Options::BRANCH)); // the branch the alias spells resolves to the main branch, not to a branch // directory named after the alias - ASSERT_OK_AND_ASSIGN(std::string main_case_branch, main_case_table->CurrentBranch()); - ASSERT_EQ("main", main_case_branch); + auto main_case_file_store_table = std::dynamic_pointer_cast(main_case_table); + ASSERT_NE(nullptr, main_case_file_store_table); + ASSERT_EQ("main", main_case_file_store_table->CurrentBranch()); // a system table on a branch resolves against the branch's data table: the system // suffix is stripped while the branch stays in the identifier sent to the server @@ -1213,8 +1223,10 @@ TEST_F(RestCatalogTest, CreateRefusesAFormatTableThisClientCouldNotOpen) { // And a schema this library can open is created and opens. ASSERT_OK(create("fine", good, {"dt"}, format_table)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr table, - catalog->GetFormatTable(Identifier("db1", "fine"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + catalog->GetTable(Identifier("db1", "fine"))); + std::shared_ptr table = std::dynamic_pointer_cast(loaded); + ASSERT_NE(nullptr, table); ASSERT_EQ(FormatTable::Format::PARQUET, table->GetFormat()); } @@ -1236,7 +1248,7 @@ TEST_F(RestCatalogTest, TableResponseWithoutAPathIsRejected) { // would be checked against an empty one has no boundary at all, so the response is refused // rather than turned into a table. What the response said is not repeated back: a body may // carry credentials, so a failure to read one names the request and nothing else. - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "no_path")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "no_path")), "failed to deserialize the response"); ASSERT_NOK_WITH_MSG(catalog->GetTableLocation(Identifier("db1", "no_path")), "failed to deserialize the response"); @@ -1259,8 +1271,10 @@ TEST_F(RestCatalogTest, FormatTableIsLoadedInOneRequest) { state_->get_table_requests = 0; } - ASSERT_OK_AND_ASSIGN(std::shared_ptr table, - catalog->GetFormatTable(Identifier("db1", "fmt"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + catalog->GetTable(Identifier("db1", "fmt"))); + std::shared_ptr table = std::dynamic_pointer_cast(loaded); + ASSERT_NE(nullptr, table); ASSERT_EQ("wh1/db1.db/fmt", table->Location()); ASSERT_EQ(FormatTable::Format::PARQUET, table->GetFormat()); ASSERT_EQ((std::vector{"dt"}), table->PartitionKeys()); @@ -1275,18 +1289,10 @@ TEST_F(RestCatalogTest, FormatTableIsLoadedInOneRequest) { // directory named "schema" there is a partition value, not metadata to skip. ASSERT_FALSE(table->LocationCarriesPaimonMetadata()); - ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "fmt")), "Cannot open format table"); - - // A managed table stays out of this path, and so does a system table. ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "t1")), "is not a format table"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "t1$snapshots")), - "for system table"); - // The database is checked before the table name is parsed, so a malformed name under "sys" - // is refused as a system table rather than surfacing a parse error. - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("sys", "all_table_options")), - "for system table"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("sys", "a$b$c$d")), "for system table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
managed, + catalog->GetTable(Identifier("db1", "t1"))); + ASSERT_EQ(nullptr, std::dynamic_pointer_cast(managed)); } TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { @@ -1312,24 +1318,23 @@ TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { // A format table format with no reader here. seed("csv_table", R"({"type": "format-table", "file.format": "csv"})", R"(["dt"])", default_fields); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "csv_table")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "csv_table")), "not supported by paimon-cpp yet"); // A row count no file could ever reach. seed("bad_rows", R"({"type": "format-table", "file.format": "parquet", "target-file-row-num": "0"})", R"(["dt"])", default_fields); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "bad_rows")), - "should be at least 1"); + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "bad_rows")), "should be at least 1"); // The failure names the table, which is all a caller holding several has to go on. - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "bad_rows")), "db1.bad_rows"); + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "bad_rows")), "db1.bad_rows"); // The structural rules run here too, not only the format-table ones: a partition key that is // not a field of the schema is a table nothing could ever read. seed("bad_partition_key", R"({"type": "format-table", "file.format": "parquet"})", R"(["nosuchfield"])", default_fields); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "bad_partition_key")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "bad_partition_key")), "should include all partition fields"); // Two schemas this library cannot open, refused when the table is opened rather than when a @@ -1339,12 +1344,12 @@ TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { R"([{"id": 0, "name": "f0", "type": "INT NOT NULL"},)" // Not a raw string: `TIMESTAMP(6)"` holds the sequence that would end one. " {\"id\": 1, \"name\": \"dt\", \"type\": \"TIMESTAMP(6)\"}]"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "timestamp_partition")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "timestamp_partition")), "cannot be TIMESTAMP/DECIMAL/BLOB"); seed("only_partitions", R"({"type": "format-table", "file.format": "parquet"})", R"(["dt"])", R"([{"id": 0, "name": "dt", "type": "STRING"}])"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "only_partitions")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "only_partitions")), "every one of its columns"); // An option this library does not honour keeps its own status code, so a caller can still @@ -1353,8 +1358,7 @@ TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { R"({"type": "format-table", "file.format": "parquet",)" R"( "metastore.partitioned-table": "true"})", R"(["dt"])", default_fields); - Status not_implemented = - catalog->GetFormatTable(Identifier("db1", "catalog_partitions")).status(); + Status not_implemented = catalog->GetTable(Identifier("db1", "catalog_partitions")).status(); ASSERT_TRUE(not_implemented.IsNotImplemented()) << not_implemented.ToString(); } @@ -1389,8 +1393,7 @@ TEST_F(RestCatalogTest, NestedSchemaHighestFieldId) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("db1", "nested"))); - std::shared_ptr schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, schema); ASSERT_EQ(5, schema->Id()); // 9 lives inside the value row of the map: ROW, ARRAY element and MAP key/value @@ -1484,8 +1487,7 @@ TEST_F(RestCatalogTest, PartitionKeysRoundTrip) { /*ignore_if_exists=*/false)); // the partition keys survive both the create request and the load response conversion ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("db1", "pt"))); - std::shared_ptr loaded = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr loaded = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, loaded); ASSERT_EQ((std::vector{"f1"}), loaded->PartitionKeys()); } @@ -1535,6 +1537,59 @@ TEST_F(RestCatalogTest, SystemDatabase) { ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); } +TEST_F(RestCatalogTest, GlobalSystemTableRetainsItsCatalog) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr created, CreateRestCatalog()); + std::shared_ptr catalog = std::move(created); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("sys", "tables"))); + std::weak_ptr weak = catalog; + catalog.reset(); + ASSERT_FALSE(weak.expired()) << "the global system table does not retain its catalog"; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_GT(array->length(), 0); +} + +TEST_F(RestCatalogTest, GlobalSystemTableWorksThroughThePublicCatalogFactory) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(kWarehouse, options_)); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("sys", "tables"))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_GT(array->length(), 0); +} + TEST_F(RestCatalogTest, ListSnapshots) { ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); @@ -1658,7 +1713,7 @@ TEST_F(RestCatalogTest, FileStoreCommitIsBuiltFromTheCatalog) { CommitContextBuilder builder(dir->Str(), "commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.WithCatalog(shared_catalog, Identifier("db1", "t1")).Finish()); - ASSERT_OK(FileStoreCommit::Create(std::move(context))); + ASSERT_OK(FileStoreCommit::Create(std::move(context), {})); { std::lock_guard lock(state_->mutex); ASSERT_GT(state_->get_table_requests, requests_before); @@ -1891,7 +1946,7 @@ TEST_P(RestCatalogCommitRecoveryTest, CommitAcceptedThenLostRecoversAfterRestart CommitContextBuilder builder(dir->Str(), "commit-user"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.WithCatalog(catalog, identifier).Finish()); - return FileStoreCommit::Create(std::move(context)); + return FileStoreCommit::Create(std::move(context), {}); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, create_commit()); @@ -2063,16 +2118,17 @@ class RestCatalogCommitLeaseTest : public RestCatalogTest { Result> CreateCommit( const std::shared_ptr& catalog, const std::optional& table_id, const std::map& options) { + std::map builder_options; CommitContextBuilder builder(dir_->Str(), "commit-user"); builder.WithCatalog(catalog, Identifier("db1", "t1")); if (table_id) { builder.WithTableId(table_id.value()); } for (const auto& [key, value] : options) { - builder.AddOption(key, value); + builder_options[key] = value; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreCommit::Create(std::move(context)); + return FileStoreCommit::Create(std::move(context), builder_options); } std::vector TableEvents() { diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 1a2f0b7d7..4db1b0ccb 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -34,7 +34,9 @@ #include "paimon/core/catalog/commit_lock.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/partition/partition_statistics.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/result.h" #include "paimon/schema/schema.h" #include "paimon/status.h" @@ -112,12 +114,19 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { Result> GetTable(const Identifier& identifier) const override { ++get_table_calls_; + std::shared_ptr managed_schema = + std::dynamic_pointer_cast(table_schema_); + if (table_schema_ != nullptr && managed_schema == nullptr) { + return Status::Invalid(identifier.ToString() + " is not a managed table schema"); + } // Serves the per-table file system when one was set, matching `GetTableFileSystem` // without recording the request, so a test can still tell the two apart. No location: // this mock's `GetTableLocation` is the unsupported stub. - return std::make_shared
( + return FileStoreTable::Create( table_file_system_ != nullptr ? table_file_system_ : file_system_, - /*root_path=*/"", table_schema_, identifier, table_uuid_); + /*root_path=*/"", managed_schema, identifier, table_uuid_, + /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, /*fs_scheme_map=*/{}, + /*dynamic_options=*/{}); } void SetTableSchema(const std::shared_ptr& table_schema) { diff --git a/src/paimon/testing/utils/test_helper.h b/src/paimon/testing/utils/test_helper.h index 2bc1e9bbc..ad996cd6a 100644 --- a/src/paimon/testing/utils/test_helper.h +++ b/src/paimon/testing/utils/test_helper.h @@ -94,19 +94,16 @@ class TestHelper { if (!temp_directory.empty()) { context_builder.WithTempDirectory(temp_directory); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, - context_builder.SetOptions(options) - .WithStreamingMode(is_streaming_mode) - .WithWriteId(12345) - .Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr write_context, + context_builder.WithStreamingMode(is_streaming_mode).WithWriteId(12345).Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), options)); CommitContextBuilder commit_context_builder(table_path, commit_user); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).IgnoreEmptyCommit(false).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), options)); return std::unique_ptr(new TestHelper(std::move(file_system), std::move(write), std::move(commit), commit_user, table_path, options)); @@ -157,15 +154,14 @@ class TestHelper { std::optional snapshot_id, bool is_streaming = true) { ScanContextBuilder scan_context_builder(table_path_); - scan_context_builder.WithStreamingMode(is_streaming) - .SetOptions(options_) - .AddOption(Options::SCAN_MODE, startup_mode.ToString()); + scan_context_builder.WithStreamingMode(is_streaming); + std::map options = options_; + options[Options::SCAN_MODE] = startup_mode.ToString(); if (snapshot_id) { - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, - std::to_string(snapshot_id.value())); + options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id.value()); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(scan_, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(scan_, TableScan::Create(std::move(scan_context), options)); return Scan(); } @@ -277,13 +273,13 @@ class TestHelper { const std::vector>& splits, std::unique_ptr<::ArrowSchema> read_schema) { ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options_); if (read_schema != nullptr) { read_context_builder.SetReadSchema(std::move(read_schema)); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, + TableRead::Create(std::move(read_context), options_)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr collected, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -303,10 +299,10 @@ class TestHelper { const std::vector>& splits, const std::string& expected_result) { ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, + TableRead::Create(std::move(read_context), options_)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -335,7 +331,7 @@ class TestHelper { return commit_impl->snapshot_manager_->LatestSnapshotOfUser(commit_user_); } - Result>> LatestSchema() const { + Result>> GetSchema() const { auto commit_impl = dynamic_cast(commit_.get()); return commit_impl->schema_manager_->Latest(); } diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index 1424533de..dd6cff70a 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -224,7 +224,7 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompaction) { auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, /*is_streaming_mode=*/true)); ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); auto gen = std::make_shared(table_schema.value(), pool_); int64_t commit_identifier = 0; @@ -412,10 +412,13 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithMap auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar")); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -548,7 +551,7 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithDv) auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, /*is_streaming_mode=*/true)); ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); auto gen = std::make_shared(table_schema.value(), pool_); int64_t commit_identifier = 0; @@ -634,7 +637,7 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteBestEffortCompaction) auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, /*is_streaming_mode=*/true)); ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); auto gen = std::make_shared(table_schema.value(), pool_); int64_t commit_identifier = 0; @@ -725,7 +728,7 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteCompactionWithExterna auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, /*is_streaming_mode=*/true)); ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); auto gen = std::make_shared(table_schema.value(), pool_); int64_t commit_identifier = 0; @@ -813,7 +816,7 @@ TEST_F(AppendCompactionInteTest, TestAppendTableCompactionWithIOException) { TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, /*is_streaming_mode=*/true)); ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); auto gen = std::make_shared(table_schema.value(), pool_); @@ -1035,10 +1038,13 @@ TEST_P(AppendCompactionInteTest, TestAppendTableCompactionDictionaryPassthrough) auto predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"s", FieldType::STRING, Literal(FieldType::STRING, indexed_value.data(), indexed_value.size())); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar")); - read_context_builder.SetOptions(options).SetPredicate(predicate); + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto filtered, ReadResultCollector::CollectResult(batch_reader.get())); auto expected = std::make_shared( diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 4cbfda4df..0850740d4 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -145,7 +145,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter write_builder.WithWriteSchema(write_cols); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); for (const auto& write_array : write_arrays) { ArrowArray c_array; @@ -180,7 +180,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); return file_store_commit->Commit(commit_msgs); } @@ -216,6 +216,12 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter return fields; } + /// The identifier `WriteDanglingBlobViewRows` stamps on row 0: no table exists where the + /// catalog issues it, so the catalog the blob view lookup resolves the reference through + /// reports the identifier not exist, and every read that reaches the reference fails on it. + static constexpr const char* kDanglingUpstreamNotFound = + "Identifier{database='nonexistent_db', object='nonexistent_table'} not exist"; + /// Writes and commits the two rows the blob-view skip tests share: row 0 holds a blob view /// reference to an upstream table that does not exist, row 1 holds no blob view at all. /// Resolving row 0 fails the whole read, so any read the pre-read still reaches it from must @@ -274,7 +280,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter scan_context_builder.SetGlobalIndexResult(global_index_result); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), {})); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); return result_plan; } @@ -285,15 +291,17 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter const std::shared_ptr& plan, const std::shared_ptr& predicate = nullptr, const std::map& options = {}) const { auto splits = plan->Splits(); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); read_context_builder.SetReadFieldNames(read_schema).SetPredicate(predicate); read_context_builder.EnableLateMaterializing(false); if (!options.empty()) { - read_context_builder.SetOptions(options); + read_context_builder_options = options; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4589,10 +4597,11 @@ TEST_P(BlobTableInteTest, TestBlobViewFailsWhenUpstreamTableAbsent) { ASSERT_OK(WriteDanglingBlobViewRows(table_path)); - // reading fails because the upstream table does not exist at the catalog-issued location + // reading fails because the upstream table does not exist at the catalog-issued location: + // the catalog the lookup resolves the reference through reports its identifier not exist ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr), - "not found latest schema"); + kDanglingUpstreamNotFound); } TEST_P(BlobTableInteTest, TestBlobViewSkipsDanglingReferenceOfDeletedRow) { @@ -4611,7 +4620,7 @@ TEST_P(BlobTableInteTest, TestBlobViewSkipsDanglingReferenceOfDeletedRow) { // without the deletion vector the dangling reference fails the read ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr), - "not found latest schema"); + kDanglingUpstreamNotFound); ASSERT_OK_AND_ASSIGN(std::string anchor_file_name, DeletionVectorTestHelper::RetrieveAnchorFileName(commit_msgs)); @@ -4655,7 +4664,7 @@ TEST_P(BlobTableInteTest, TestBlobViewSkipsDanglingReferenceInEveryRowRangeGroup // without deletion vectors both groups' dangling references fail the read ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr), - "not found latest schema"); + kDanglingUpstreamNotFound); // each group's anchor gets a deletion vector dropping its row 0, the dangling one ASSERT_OK_AND_ASSIGN(std::string anchor_file_name0, @@ -4692,7 +4701,7 @@ TEST_P(BlobTableInteTest, TestBlobViewPreReadHonorsRowRangesNotPredicate) { // reading the whole split fails on the dangling reference of row 0 ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr), - "not found latest schema"); + kDanglingUpstreamNotFound); // a predicate keeping only row 1 does not rescue the query: it is evaluated above the split // read, while the pre-read takes none, so row 0's reference is still collected and resolved @@ -4700,7 +4709,7 @@ TEST_P(BlobTableInteTest, TestBlobViewPreReadHonorsRowRangesNotPredicate) { FieldType::INT, Literal(101)); ASSERT_OK_AND_ASSIGN(auto filtered_plan, ScanTable(table_path, keep_row_1)); ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), filtered_plan, keep_row_1), - "not found latest schema"); + kDanglingUpstreamNotFound); // dropping the same row through the row ranges does: the pre-read honors them the same way // the main read does, so row 0 is never resolved @@ -4758,10 +4767,13 @@ TEST_P(BlobTableInteTest, TestReadArrayBlobTableFromJava) { const std::vector read_fields = {"id", "array_payloads"}; for (int64_t snapshot_id : {1, 2, 3}) { + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); size_t array_layer_count = 0; @@ -4819,10 +4831,13 @@ TEST_P(BlobTableInteTest, TestReadMapBlobTableFromJava) { "large_decimal_payloads"}; for (int64_t snapshot_id : {1, 3}) { + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); size_t string_layer_count = 0; diff --git a/test/inte/clean_inte_test.cpp b/test/inte/clean_inte_test.cpp index ec418d2c0..24dcd51a7 100644 --- a/test/inte/clean_inte_test.cpp +++ b/test/inte/clean_inte_test.cpp @@ -204,11 +204,14 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { auto dir = UniqueTestDirectory::Create(); std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 4); } @@ -219,11 +222,14 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath( table_path, "manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1"))); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 4); } @@ -234,11 +240,14 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath( table_path, "f1=10/bucket-1/data-7a912f84-04b7-4bbb-8dc6-53f4a292ea25-0.parquet"))); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 4); } @@ -248,11 +257,14 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath(table_path, "snapshot/snapshot-3"))); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 1); } @@ -282,12 +294,15 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { CreateTestTable(dir->Str(), /*db_name=*/"foo", /*table_name=*/"bar", &arrow_schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); @@ -303,18 +318,20 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "2") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .AddOption(Options::MANIFEST_MERGE_MIN_COUNT, "2") - .AddOption(Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true") - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "2"; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + commit_context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; + commit_context_builder_options[Options::MANIFEST_MERGE_MIN_COUNT] = "2"; + commit_context_builder_options[Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES] = "true"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_context, + + commit_context_builder.WithFileSystem(file_system_).IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0, 10)); ASSERT_OK_AND_ASSIGN( @@ -430,12 +447,16 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { /*table_name=*/"bar", &arrow_schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + context_builder.WithStreamingMode(true).Finish()); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_write, + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); @@ -451,19 +472,20 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "2"; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + commit_context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; + commit_context_builder_options[Options::MANIFEST_MERGE_MIN_COUNT] = "2"; + commit_context_builder_options[Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES] = "true"; ASSERT_OK_AND_ASSIGN( std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "2") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .AddOption(Options::MANIFEST_MERGE_MIN_COUNT, "2") - .AddOption(Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true") - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + commit_context_builder.WithFileSystem(file_system_).IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( @@ -588,12 +610,15 @@ TEST_F(CleanInteTest, TestOrphanFilesClean) { /*partition_keys=*/{"f1", "f2"}, /*primary_keys=*/{}, options)); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, commit_user); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}, {"f2", "1"}}, 0)); @@ -609,13 +634,15 @@ TEST_F(CleanInteTest, TestOrphanFilesClean) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, commit_user); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_context, + + commit_context_builder.WithFileSystem(file_system_).IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( @@ -702,12 +729,15 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithFileRetainCondition) { /*partition_keys=*/{"f1", "f2"}, /*primary_keys=*/{}, options)); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, commit_user); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}, {"f2", "1"}}, 0)); @@ -723,13 +753,15 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithFileRetainCondition) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, commit_user); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_context, + + commit_context_builder.WithFileSystem(file_system_).IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( @@ -812,12 +844,15 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithIOException) { /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, commit_user); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); std::string root_path = write_context->GetRootPath(); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); @@ -833,14 +868,16 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithIOException) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(root_path, commit_user); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .AddOption(Options::BUCKET, "2") - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_options[Options::BUCKET] = "2"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_context, + + commit_context_builder.WithFileSystem(file_system_).IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp index 395de88d0..585ee2621 100644 --- a/test/inte/data_evolution_table_test.cpp +++ b/test/inte/data_evolution_table_test.cpp @@ -113,7 +113,7 @@ class DataEvolutionTableTest : public ::testing::Test, write_builder.WithWriteSchema(write_cols); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); for (const auto& write_array : write_arrays) { ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); @@ -163,7 +163,7 @@ class DataEvolutionTableTest : public ::testing::Test, PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); return file_store_commit->Commit(commit_msgs); } @@ -300,6 +300,7 @@ class DataEvolutionTableTest : public ::testing::Test, const std::string& table_path, const std::vector& read_schema, int32_t limit, const std::shared_ptr& predicate = nullptr, const std::vector& row_ranges = {}, bool enable_predicate_filter = false) const { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetLimit(limit); scan_context_builder.SetPredicate(predicate); @@ -307,10 +308,12 @@ class DataEvolutionTableTest : public ::testing::Test, scan_context_builder.SetGlobalIndexResult( BitmapGlobalIndexResult::FromRanges(row_ranges)); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - FinishScanContext(scan_context_builder)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result_plan, table_scan->CreatePlan()); LimitScanResult result; @@ -322,7 +325,7 @@ class DataEvolutionTableTest : public ::testing::Test, PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(result.splits)); PAIMON_ASSIGN_OR_RAISE(result.rows, @@ -335,11 +338,14 @@ class DataEvolutionTableTest : public ::testing::Test, /// the row count derived from them. Reading the splits back is ScanAndReadWithLimit's job, /// which verifies the returned rows after their reader has been destroyed. Result>> PlanSplits(const std::string& table_path) const { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - FinishScanContext(scan_context_builder)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); return plan->Splits(); } @@ -412,7 +418,7 @@ class DataEvolutionTableTest : public ::testing::Test, PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); file_store_commit->RowIdCheckConflict(row_id_check_from_snapshot); return file_store_commit->Commit(commit_msgs); } @@ -470,11 +476,14 @@ class DataEvolutionTableTest : public ::testing::Test, ASSERT_OK_AND_ASSIGN(std::string anchor, PlannedAnchorFileName(table_path)); ASSERT_OK(CommitDeletionVectors(table_path, base_msgs[0], {{anchor, deleted}})); } + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path); scan_builder.SetGlobalIndexResult(std::make_shared( RoaringBitmap64::From(candidates), std::vector(scores))); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_builder, scan_builder_options)); + ASSERT_OK_AND_ASSIGN(auto scan, + TableScan::Create(std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); auto scored_split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -510,7 +519,7 @@ class DataEvolutionTableTest : public ::testing::Test, .SetPredicate(bound_predicate) .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context), {})); // Reuse the same TableRead with and without scores: the internal row-id // projection must not affect a subsequent split's output schema. for (bool with_scores : {true, false}) { @@ -560,14 +569,17 @@ class DataEvolutionTableTest : public ::testing::Test, bool check_scan_plan_when_empty_result = true, bool apply_predicate_to_scan = true) const { // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetPredicate(apply_predicate_to_scan ? predicate : nullptr); if (!row_ranges.empty()) { auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); scan_context_builder.SetGlobalIndexResult(global_index_result); } - PAIMON_ASSIGN_OR_RAISE(auto scan_context, FinishScanContext(scan_context_builder)); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, FinishScanContext(scan_context_builder, + scan_context_builder_options)); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); if (!expected_array && check_scan_plan_when_empty_result) { if (!result_plan->Splits().empty()) { @@ -581,7 +593,7 @@ class DataEvolutionTableTest : public ::testing::Test, read_context_builder.SetReadFieldNames(read_schema).SetPredicate(predicate); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), {})); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -626,14 +638,17 @@ class DataEvolutionTableTest : public ::testing::Test, const std::vector>& expected_first_row_ids, const std::vector& expected_row_counts) { ASSERT_EQ(expected_first_row_ids.size(), expected_row_counts.size()); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetPredicate(predicate); if (!row_ranges.empty()) { auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); scan_context_builder.SetGlobalIndexResult(global_index_result); } - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); const auto& result_splits = result_plan->Splits(); if (expected_first_row_ids.empty()) { @@ -658,14 +673,15 @@ class DataEvolutionTableTest : public ::testing::Test, ASSERT_EQ(result_row_counts, expected_row_counts); } - Result> FinishScanContext(ScanContextBuilder& builder) const { + Result> FinishScanContext( + ScanContextBuilder& builder, std::map& options) const { if (EnableSnapshotLiveManifestCache()) { if (!snapshot_live_manifest_cache_) { snapshot_live_manifest_cache_ = std::make_shared(/*max_weight=*/64 * 1024 * 1024); } - builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .WithCache(snapshot_live_manifest_cache_); + options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder.WithCache(snapshot_live_manifest_cache_); } return builder.Finish(); } @@ -1090,16 +1106,19 @@ TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite) auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ReadContextBuilder read_context_builder(table_path); read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2015,7 +2034,7 @@ TEST_P(DataEvolutionTableTest, TestReadJavaIndependentlyRolledVectorStore) { read_builder.SetReadFieldNames({"embedding"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); const std::vector, std::shared_ptr>> gap_cases = {{gap_split, gap_expected}, {std::make_shared(gap_split, selected_ranges), @@ -2081,7 +2100,7 @@ TEST_P(DataEvolutionTableTest, TestReadJavaIndependentlyRolledVectorStore) { ASSERT_OK_AND_ASSIGN(std::unique_ptr sequence_context, sequence_read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr sequence_read, - TableRead::Create(std::move(sequence_context))); + TableRead::Create(std::move(sequence_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr sequence_reader, sequence_read->CreateReader(split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr sequence_result, @@ -2192,14 +2211,17 @@ TEST_P(DedicatedVectorTableTest, TestDedicatedVectorStore) { WriteContextBuilder partial_builder(table_path, "partial"); partial_builder.WithWriteSchema({"embedding"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr partial_context, partial_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(partial_context)).status(), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(partial_context), {}).status(), "partial writes"); + std::map invalid_builder_options; WriteContextBuilder invalid_builder(table_path, "invalid"); - invalid_builder.AddOption(Options::DATA_EVOLUTION_ENABLED, "false"); + invalid_builder_options[Options::DATA_EVOLUTION_ENABLED] = "false"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_context, invalid_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(invalid_context)).status(), - "requires row-tracking.enabled and data-evolution.enabled"); + ASSERT_NOK_WITH_MSG( + FileStoreWrite::Create(std::move(invalid_context), invalid_builder_options).status(), + "requires row-tracking.enabled and data-evolution.enabled"); } TEST_P(DedicatedVectorTableTest, TestIndependentVectorRolling) { @@ -3826,7 +3848,7 @@ TEST_P(DataEvolutionDvTableTest, TestReadWithDeletionVectorsOnEveryRowRangeGroup ASSERT_OK_AND_ASSIGN(std::unique_ptr count_context, count_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr count_table_read, - TableRead::Create(std::move(count_context))); + TableRead::Create(std::move(count_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr count_reader, count_table_read->CreateCountReader(planned_splits)); ASSERT_OK_AND_ASSIGN(int64_t counted_rows, count_reader->CountRows()); diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index c5b421629..0de3ccb01 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -129,7 +129,7 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter write_builder.WithFileSystem(fs_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); ArrowArray c_array; EXPECT_TRUE(arrow::ExportArray(*write_array, &c_array).ok()); auto record_batch = std::make_unique( @@ -157,7 +157,7 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); return file_store_commit->Commit(commit_msgs); } @@ -168,7 +168,7 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter scan_context_builder.SetPartitionFilter(partition_filters); scan_context_builder.WithFileSystem(fs_); PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), {})); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); EXPECT_EQ(result_plan->Splits().size(), 1); return std::dynamic_pointer_cast(result_plan->Splits()[0]); @@ -396,16 +396,18 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter const std::map& options = {}, const std::shared_ptr& index_result = nullptr, const std::shared_ptr& cache = nullptr) const { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); + scan_context_builder_options = options; scan_context_builder.SetPredicate(predicate) - .SetOptions(options) .SetGlobalIndexResult(index_result) .WithFileSystem(fs_); if (cache) { scan_context_builder.WithCache(cache); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); return result_plan; } @@ -444,7 +446,7 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter .EnableLateMaterializing(false); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), {})); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); diff --git a/test/inte/nested_column_pruning_inte_test.cpp b/test/inte/nested_column_pruning_inte_test.cpp index b9b1b99ca..b20aab5f1 100644 --- a/test/inte/nested_column_pruning_inte_test.cpp +++ b/test/inte/nested_column_pruning_inte_test.cpp @@ -92,13 +92,16 @@ class NestedColumnPruningInteTest : public ::testing::Test, const std::shared_ptr& expected_schema, const std::string& expected_json, const std::shared_ptr& predicate = nullptr) const { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + if (predicate) { scan_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); @@ -110,7 +113,7 @@ class NestedColumnPruningInteTest : public ::testing::Test, read_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -350,10 +353,12 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsAllNonExistent) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), read_context_builder_options), "does not support schema evolution inside struct"); } @@ -412,10 +417,12 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithNonExistentField) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), read_context_builder_options), "does not support schema evolution inside struct"); } @@ -491,10 +498,13 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithTypeMismatchShouldFa helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); ASSERT_FALSE(data_splits.empty()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options); + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateReader(data_splits), "PruneDataType nested field type mismatch for 'a': read string vs " @@ -581,10 +591,13 @@ TEST_P(NestedColumnPruningInteTest, auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); ASSERT_FALSE(data_splits.empty()); @@ -766,10 +779,13 @@ TEST_P(NestedColumnPruningInteTest, PruneNestedStructWithSpecialFields) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1178,10 +1194,15 @@ TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) { ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + std::map write_context_builder_options; WriteContextBuilder write_context_builder(table_path_, "commit_user_1"); + write_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + + write_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(std::move(write_context), write_context_builder_options)); auto write_one_row = [&](const std::string& data) -> Status { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, @@ -1201,10 +1222,14 @@ TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) { /*commit_identifier=*/0)); ASSERT_OK(file_store_write->Close()); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path_, "commit_user_1"); + commit_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + commit_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(commit_msgs, /*commit_identifier=*/0)); // Read selected MAP keys together with nested STRUCT sub-fields. @@ -1309,10 +1334,13 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysWithOrcDictionaryEncodedMap) auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1464,10 +1492,13 @@ TEST_P(NestedColumnPruningInteTest, PruneListStructSubFields) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); auto create_reader_result = table_read->CreateReader(data_splits); ASSERT_NOK_WITH_MSG(create_reader_result, "partial projection inside list"); } diff --git a/test/inte/paimon_read_compat_inte_test.cpp b/test/inte/paimon_read_compat_inte_test.cpp index 5761720d4..23c1ecca9 100644 --- a/test/inte/paimon_read_compat_inte_test.cpp +++ b/test/inte/paimon_read_compat_inte_test.cpp @@ -71,20 +71,26 @@ Result> ReadTable( std::string table_path = GetDataDir() + "/" + file_format + "/" + kDatabaseName + ".db/" + table_name; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options); + scan_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames(field_names); + read_context_builder_options = options; + read_context_builder.SetReadFieldNames(field_names); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); return ReadResultCollector::CollectResult(std::move(batch_reader)); diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp index 153a7926c..c3f96b258 100644 --- a/test/inte/pk_compaction_inte_test.cpp +++ b/test/inte/pk_compaction_inte_test.cpp @@ -351,15 +351,17 @@ class PkCompactionInteTest : public ::testing::Test { int32_t bucket, const std::shared_ptr& write_array, int64_t commit_identifier, bool is_streaming = true, const std::vector& row_kinds = {}, bool write_only = true) const { + std::map write_builder_options; WriteContextBuilder write_builder(table_path, "commit_user_1"); write_builder.WithStreamingMode(is_streaming) .WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp")); if (write_only) { - write_builder.AddOption(Options::WRITE_ONLY, "true"); + write_builder_options[Options::WRITE_ONLY] = "true"; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(std::move(write_context), write_builder_options)); ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); auto record_batch = std::make_unique(partition, bucket, row_kinds, &c_array); @@ -376,7 +378,7 @@ class PkCompactionInteTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); return file_store_commit->Commit(commit_msgs); } @@ -397,13 +399,16 @@ class PkCompactionInteTest : public ::testing::Test { const std::string& table_path, const std::map& partition, int32_t bucket, bool full_compaction, int64_t commit_identifier, const std::map& options = {}) { + std::map write_builder_options; WriteContextBuilder write_builder(table_path, "commit_user_1"); - write_builder.SetOptions(options); + write_builder_options = options; + write_builder.WithStreamingMode(true).WithTempDirectory( PathUtil::JoinPath(dir_->Str(), "tmp")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(std::move(write_context), write_builder_options)); PAIMON_RETURN_NOT_OK(file_store_write->Compact(partition, bucket, full_compaction)); PAIMON_ASSIGN_OR_RAISE( std::vector> commit_messages, @@ -422,16 +427,19 @@ class PkCompactionInteTest : public ::testing::Test { std::map options = {{Options::FILE_SYSTEM, "local"}}; ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(table_path, options, /*is_streaming_mode=*/false)); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false).SetOptions(options).AddOption( - Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); if (predicate) { scan_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_plan, table_scan->CreatePlan()); std::vector> data_splits = result_plan->Splits(); @@ -456,15 +464,18 @@ class PkCompactionInteTest : public ::testing::Test { auto iter = expected_data_per_partition_bucket.find(key); ASSERT_TRUE(iter != expected_data_per_partition_bucket.end()) << "Unexpected partition=" << key.first << " bucket=" << key.second; + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options); + read_context_builder_options = options; + if (predicate) { read_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_result, @@ -490,20 +501,26 @@ class PkCompactionInteTest : public ::testing::Test { template void ScanAllRows(const std::string& table_path, Fn consume) { std::map options = {{Options::FILE_SYSTEM, "local"}}; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false).SetOptions(options).AddOption( - Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_plan, table_scan->CreatePlan()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options); + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -617,13 +634,14 @@ TEST_F(PkCompactionInteTest, TestMetadataOnlyLevelUpgradeKeepsValueStats) { ASSERT_FALSE(before->value_stats == SimpleStats::EmptyStats()); ASSERT_EQ(before->value_stats, after->value_stats); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false) - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + scan_context_builder_options[Options::FILE_SYSTEM] = "local"; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_EQ(1u, plan->Splits().size()); auto split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -999,7 +1017,7 @@ TEST_F(PkCompactionInteTest, DeduplicateWithBitmap64HighPositions) { // Reload committed metadata and DV bytes; no test-created DV or lookup result is injected. ScanContextBuilder scan_builder(table_path); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); int32_t dv_count = 0; for (const auto& split : plan->Splits()) { @@ -1746,10 +1764,13 @@ TEST_F(PkCompactionInteTest, CompactWithSchemaEvolution) { // Step 5: Read and verify using DataSplits from CompactAfter in write CommitMessages. { + std::map context_builder_options; ReadContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "parquet"); + context_builder_options[Options::FILE_FORMAT] = "parquet"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); // Use the latest snapshot id for both splits. // The original table has 6 snapshots. Each write with force-compact produces @@ -2016,7 +2037,7 @@ TEST_F(PkCompactionInteTest, ContinuousWriteWithBackgroundCompact) { PathUtil::JoinPath(dir_->Str(), "tmp")); ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); auto write_batch = [&](const std::string& json_data) -> Status { auto array = @@ -2220,12 +2241,13 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { auto data_type = arrow::struct_(fields); // Step 2: Write data to the branch-rt and compact. + std::map write_builder_options; WriteContextBuilder write_builder(table_path, "commit_user_1"); - write_builder.WithStreamingMode(true) - .WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp")) - .WithBranch("rt"); + write_builder_options[Options::BRANCH] = "rt"; + write_builder.WithStreamingMode(true).WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp")); ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), write_builder_options)); // dt=20240726, bucket-0 only has ["20240726", "cherry", 3] one row auto write_array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ @@ -2276,11 +2298,15 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { ASSERT_EQ(compact_after[0]->row_count, 2); // Step 4: Commit to branch-rt. + std::map commit_builder_options; CommitContextBuilder commit_builder(table_path, "commit_user_1"); - commit_builder.AddOption(Options::FILE_SYSTEM, "local").AddOption(Options::BRANCH, "rt"); + commit_builder_options[Options::FILE_SYSTEM] = "local"; + commit_builder_options[Options::BRANCH] = "rt"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(std::move(commit_context), commit_builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); auto fs = std::make_shared(); @@ -2299,13 +2325,14 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { // Step 5: Scan and read branch-rt. std::map branch_options = {{Options::FILE_SYSTEM, "local"}, {Options::BRANCH, "rt"}}; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false) - .SetOptions(branch_options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + scan_context_builder_options = branch_options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); std::vector> splits = plan->Splits(); @@ -2316,11 +2343,13 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { ASSERT_EQ(split_impl->SnapshotId(), 3); } + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(branch_options); + read_context_builder_options = branch_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -3174,7 +3203,7 @@ TEST_F(PkCompactionInteTest, TestKeyValueTableCompactionWithIOException) { TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options, /*is_streaming_mode=*/true)); ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); auto gen = std::make_shared(table_schema.value(), pool_); @@ -3236,7 +3265,7 @@ TEST_P(PkCompactionFormatInteTest, TestKeyValueTableStreamWriteFullCompaction) { auto helper, TestHelper::Create(dir_->Str(), schema, partition_keys, primary_keys, options, /*is_streaming_mode=*/true)); ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); auto gen = std::make_shared(table_schema.value(), pool_); int64_t commit_identifier = 0; diff --git a/test/inte/primary_key_sorted_index_inte_test.cpp b/test/inte/primary_key_sorted_index_inte_test.cpp index f2811a699..bd5729ce8 100644 --- a/test/inte/primary_key_sorted_index_inte_test.cpp +++ b/test/inte/primary_key_sorted_index_inte_test.cpp @@ -199,24 +199,25 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, const std::optional& snapshot_id, bool index_enabled, const std::vector>& partition_filters, const std::string& branch, const std::shared_ptr& file_system) const { + std::map builder_options; ScanContextBuilder builder(TablePath(partitioned)); - builder.SetPredicate(predicate).AddOption(Options::GLOBAL_INDEX_ENABLED, - index_enabled ? "true" : "false"); + builder_options[Options::GLOBAL_INDEX_ENABLED] = index_enabled ? "true" : "false"; + builder.SetPredicate(predicate); if (snapshot_id != std::nullopt) { - builder.AddOption(Options::SCAN_SNAPSHOT_ID, fmt::format("{}", snapshot_id.value())); + builder_options[Options::SCAN_SNAPSHOT_ID] = fmt::format("{}", snapshot_id.value()); } if (!partition_filters.empty()) { builder.SetPartitionFilter(partition_filters); } if (!branch.empty()) { - builder.AddOption(Options::BRANCH, branch); + builder_options[Options::BRANCH] = branch; } if (file_system != nullptr) { builder.WithFileSystem(file_system); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + TableScan::Create(std::move(context), builder_options)); return table_scan->CreatePlan(); } @@ -224,6 +225,7 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, const std::vector>& splits, const std::map& options, const std::shared_ptr& file_system) const { + std::map builder_options; ReadContextBuilder builder(TablePath(partitioned)); if (partitioned) { builder.SetReadFieldNames({"pt", "id", "score", "tag"}); @@ -232,14 +234,14 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, } builder.SetPredicate(predicate).EnablePredicateFilter(true); for (const auto& [key, value] : options) { - builder.AddOption(key, value); + builder_options[key] = value; } if (file_system != nullptr) { builder.WithFileSystem(file_system); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(context))); + TableRead::Create(std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -547,7 +549,7 @@ TEST_P(PrimaryKeySortedIndexInteTest, ScoredSplitFailsBeforeForceKeepDeleteFallb .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(context))); + TableRead::Create(std::move(context), {})); auto* key_value_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(key_value_table_read != nullptr); key_value_table_read->ForceKeepDelete(true); diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 284cd362f..9adc162b1 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -49,6 +49,7 @@ #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/read_ahead_cache.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/snapshot.h" @@ -57,6 +58,7 @@ #include "paimon/core/table/source/deletion_file.h" #include "paimon/core/table/source/fallback_data_split.h" #include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/tag/tag.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/data/decimal.h" @@ -274,22 +276,32 @@ Result> ReadSystemTable( const std::string& system_table_path, const std::map& options, bool streaming_mode = false, const std::shared_ptr& predicate = nullptr, const std::vector& read_field_names = {}, bool read_next_plan = false) { + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), system_table_path, options)); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(system_table_path); - scan_context_builder.SetOptions(options).WithStreamingMode(streaming_mode); + scan_context_builder_options = options; + scan_context_builder.WithStreamingMode(streaming_mode); if (predicate) { scan_context_builder.SetPredicate(predicate); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_1, + table->Copy(scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + configured_table_1->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); if (read_next_plan) { PAIMON_ASSIGN_OR_RAISE(plan, table_scan->CreatePlan()); } + std::map read_context_builder_options; ReadContextBuilder read_context_builder(system_table_path); - read_context_builder.SetOptions(options); + read_context_builder_options = options; + if (predicate) { read_context_builder.SetPredicate(predicate); } @@ -298,8 +310,10 @@ Result> ReadSystemTable( } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_2, + table->Copy(read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + configured_table_2->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -371,18 +385,21 @@ TEST_P(ReadInteTest, TestAppendSimple) { DataField(2, arrow::field("f2", arrow::int32())), DataField(3, arrow::field("f3", arrow::float64()))}; + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption("orc.read.enable-metrics", "true"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder.EnablePrefetch(param.enable_prefetch); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); if (specific_table_schema) { context_builder.SetTableSchema(specific_table_schema.value()); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -454,18 +471,21 @@ TEST_P(ReadInteTest, TestReadWithLimits) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption(Options::READ_BATCH_SIZE, "1"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options[Options::READ_BATCH_SIZE] = "1"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options["orc.read.enable-metrics"] = "true"; context_builder.EnablePrefetch(param.enable_prefetch) .SetReadAheadCacheEnabled(param.read_ahead_cache_enabled) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .AddOption("orc.read.enable-metrics", "true") .SetPrefetchBatchCount(10); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -517,14 +537,17 @@ TEST_P(ReadInteTest, TestReadAheadCacheMetrics) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", "false") .SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -591,16 +614,19 @@ TEST_P(ReadInteTest, TestReadOnlyPartitionField) { DataField(0, arrow::field("dt", arrow::utf8())), }; + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder.SetReadFieldNames({"dt"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector> data_splits; data_splits.reserve(3); @@ -650,17 +676,26 @@ TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) { ArrowSchemaRelease(&schema); std::string system_table_path = PathUtil::JoinPath(dir->Str(), "warehouse/db1.db/tbl1$options"); + ASSERT_OK_AND_ASSIGN(auto table, catalog->GetTable(Identifier("db1", "tbl1$options"))); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(system_table_path); - scan_context_builder.SetOptions(options); + scan_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_3, + table->Copy(scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, configured_table_3->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(system_table_path); - read_context_builder.SetOptions(options); + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_4, + table->Copy(read_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_read, configured_table_4->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); ASSERT_TRUE(result); @@ -682,22 +717,36 @@ TEST(SystemTableReadInteTest, TestReadBranchOptionsSystemTable) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}}; std::string system_table_path = table_path + "$branch_rt$options"; + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN(auto table, SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), + system_table_path, options)); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(system_table_path); - scan_context_builder.SetOptions(options); + scan_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_5, + table->Copy(scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, configured_table_5->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(system_table_path); - read_context_builder.SetOptions(options); + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_6, + table->Copy(read_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_read, configured_table_6->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); - std::map expected = { - {"bucket", "2"}, {"file.format", "parquet"}, {"manifest.format", "avro"}}; + std::map expected = {{"branch", "rt"}, + {"bucket", "2"}, + {"file-system", "local"}, + {"file.format", "parquet"}, + {"manifest.format", "avro"}}; ASSERT_EQ(CollectStringMap(result), expected) << result->ToString(); } @@ -1002,23 +1051,35 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP std::shared_ptr predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k", FieldType::INT, Literal(2)); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), table_path + "$ro", options)); + std::map ro_scan_context_builder_options; ScanContextBuilder ro_scan_context_builder(table_path + "$ro"); - ro_scan_context_builder.SetOptions(options).SetPredicate(predicate); + ro_scan_context_builder_options = options; + ro_scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_scan_context, ro_scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_7, + table->Copy(ro_scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_table_scan, - TableScan::Create(std::move(ro_scan_context))); + configured_table_7->NewScan(std::move(ro_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr ro_plan, ro_table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(int64_t ro_file_count, CountDataFiles(ro_plan->Splits())); ASSERT_EQ(ro_file_count, 1); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path + "$ro"); // Do not enable row-level predicate filtering: the result must come from the file reader's // row-group/page predicate pushdown. - read_context_builder.SetOptions(options).SetPredicate(predicate).SetReadFieldNames({"k", "v"}); + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate).SetReadFieldNames({"k", "v"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_8, + table->Copy(read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + configured_table_8->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(ro_plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -1062,11 +1123,19 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(WriteAndFullCompact(std::move(batch), /*commit_identifier=*/0, helper.get())); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), table_path + "$ro", options)); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path + "$ro"); - scan_context_builder.SetOptions(options); + scan_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_9, + table->Copy(scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + configured_table_9->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); auto projected_schema = arrow::schema({ @@ -1075,11 +1144,15 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { }); auto c_projected_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_projected_schema.get()).ok()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path + "$ro"); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_projected_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_projected_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_10, + table->Copy(read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + configured_table_10->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -1276,7 +1349,7 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); ASSERT_OK_AND_ASSIGN( std::unique_ptr batch, TestHelper::MakeRecordBatch(arrow::struct_({fields[2]}), R"([[10], [20]])", @@ -1290,7 +1363,7 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); ASSERT_OK(commit->Commit(commit_messages)); ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); @@ -1600,7 +1673,7 @@ TEST(SystemTableReadInteTest, TestReadAuditLogSystemTableWithSequenceNumber) { std::map options = { {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, - + {Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, "true"}, {Options::BUCKET, "1"}, }; auto dir = UniqueTestDirectory::Create(); @@ -1617,7 +1690,8 @@ TEST(SystemTableReadInteTest, TestReadAuditLogSystemTableWithSequenceNumber) { /*expected_commit_messages=*/std::nullopt)); auto read_options = options; - read_options[Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED] = "true"; + // Java allows the persisted table property, but rejects enabling it through a Copy hint. + read_options.erase(Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED); ASSERT_OK_AND_ASSIGN( auto result, ReadSystemTable(PathUtil::JoinPath(dir->Str(), "foo.db/bar$audit_log"), read_options)); @@ -1910,17 +1984,19 @@ TEST_P(ReadInteTest, TestAppendReadWithMultipleBuckets) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .EnablePrefetch(param.enable_prefetch); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -1991,19 +2067,20 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicate) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .SetPredicate(predicate) - .EnablePredicateFilter(true) - .EnablePrefetch(param.enable_prefetch) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .AddOption("orc.read.enable-metrics", "true"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder.SetPredicate(predicate).EnablePredicateFilter(true).EnablePrefetch( + param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2093,18 +2170,21 @@ TEST_P(ReadInteTest, TestAppendReadWithComplexTypePredicate) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_complex_data.db/append_complex_data"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f6", "f2", "f4", "f3", "f5"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2167,19 +2247,20 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateOnlyPushdown) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .SetPredicate(predicate) - .EnableLateMaterializing(false) - .EnablePrefetch(param.enable_prefetch); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.SetPredicate(predicate).EnableLateMaterializing(false).EnablePrefetch( + param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2251,18 +2332,19 @@ TEST_P(ReadInteTest, TestAppendReadWithLateMaterializing) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .SetPredicate(predicate) - .EnableLateMaterializing(true) - .EnablePrefetch(param.enable_prefetch); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.SetPredicate(predicate).EnableLateMaterializing(true).EnablePrefetch( + param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2329,19 +2411,20 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateAllFiltered) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .SetPredicate(predicate) - .EnablePredicateFilter(true) - .EnablePrefetch(param.enable_prefetch); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.SetPredicate(predicate).EnablePredicateFilter(true).EnablePrefetch( + param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2415,18 +2498,20 @@ TEST_P(ReadInteTest, TestAppendReadIOException) { for (size_t i = 0; i < 200; i++) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + std::map context_builder_options; ReadContextBuilder context_builder(paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09/"); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = + TableRead::Create(std::move(read_context), context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(data_splits); @@ -2461,16 +2546,19 @@ TEST_P(ReadInteTest, TestPkTableWithDeletionVectorSimple) { data_file = "data-ed5d7184-cf42-4f4a-bf83-1e3080d9012d-0.parquet"; dv_file = "index/index-62372b7b-d1cf-4dde-a162-c9d9d35006b4-1"; } + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); DataSplitsDv input_data_splits = { {path + "f1=10/bucket-1", @@ -2510,16 +2598,18 @@ TEST_P(ReadInteTest, TestPkTableWithDeletionVector) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list; std::string deletion_file; @@ -2577,19 +2667,22 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot6) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(15.0)); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2663,17 +2756,20 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot8) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f0", "f3", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2742,20 +2838,23 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolution) { DataField(5, arrow::field("a", arrow::int32())), DataField(8, arrow::field("e", arrow::int32()))}; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { context_builder.SetTableSchema(specific_table_schema.value()); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2849,15 +2948,18 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithReaderBuildParallelism /*schema ids*/ {1, 0}}}; auto data_splits = CreateDataSplits(input_data_splits, /*snapshot_id=*/2); + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2914,20 +3016,23 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateFilter) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_with_alter_table.db/append_table_with_alter_table/"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"a", "k", "key1", "d", "key0", "c"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2994,18 +3099,21 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateOnlyPushDown) std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_with_alter_table.db/" "append_table_with_alter_table/"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"a", "k", "key1", "d", "key0", "c"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3068,17 +3176,20 @@ TEST_P(ReadInteTest, TestPkReadSnapshot5WithSchemaEvolution) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3154,17 +3265,20 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolution) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3240,18 +3354,20 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateOnlyPush FieldType::INT, Literal(510)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetPredicate(predicate); - context_builder.EnablePrefetch(param.enable_prefetch) - .EnableLateMaterializing(false) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch).EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3326,18 +3442,20 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithLateMaterializing FieldType::INT, Literal(510)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetPredicate(predicate); - context_builder.EnableLateMaterializing(true) - .EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnableLateMaterializing(true).EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3410,19 +3528,22 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateFilter) FieldType::INT, Literal(510)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3503,17 +3624,20 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithBuildInFieldId) { "data-11b12094-192f-4ad8-92a8-ae8cba5e25ef-0.parquet"}; } + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key0", "key1", "k", "c", "d", "a", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); DataSplitsSchema input_data_splits = { {path + "key0=0/key1=1/bucket-0", BinaryRowGenerator::GenerateRow({0, 1}, pool_.get()), @@ -3567,16 +3691,19 @@ TEST_P(ReadInteTest, TestAppendReadNestedType) { DataField(7, arrow::field("f6", arrow::decimal128(2, 2)))}; std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -3623,17 +3750,20 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCast) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_alter_table_with_cast.db/" "append_table_alter_table_with_cast/"; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3709,15 +3839,15 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCastWithPredicatePushD ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + const std::map read_options = { + {Options::FILE_FORMAT, param.file_format}, + {"read.batch-size", "2"}, + {"test.enable-adaptive-prefetch-strategy", param.enable_adaptive_prefetch_strategy}}; context_builder.SetPredicate(predicate); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), read_options)); std::vector file_list_0; std::vector file_list_1; @@ -3788,15 +3918,16 @@ TEST_P(ReadInteTest, TestReadWithPKFallBackBranch) { ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder.EnablePrefetch(param.enable_prefetch); + const std::map read_options = { + {Options::FILE_FORMAT, param.file_format}, + {"test.enable-adaptive-prefetch-strategy", param.enable_adaptive_prefetch_strategy}}; if (specific_table_schema) { context_builder.SetTableSchema(specific_table_schema.value()); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3846,7 +3977,7 @@ TEST_P(ReadInteTest, TestReadWithAppendFallBackBranch) { context_builder.EnablePrefetch(param.enable_prefetch); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3886,13 +4017,15 @@ TEST_P(ReadInteTest, TestFallBackBranchStreamRead) { std::vector read_fields = {DataField(0, arrow::field("dt", arrow::utf8())), DataField(1, arrow::field("name", arrow::utf8())), DataField(2, arrow::field("amount", arrow::int32()))}; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_split)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3932,17 +4065,19 @@ TEST_P(ReadInteTest, TestReadWithPKRtBranch) { DataField(2, arrow::field("amount", arrow::int32())), }; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .WithBranch("rt"); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options[Options::BRANCH] = "rt"; + context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { context_builder.SetTableSchema(specific_table_schema.value()); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3989,17 +4124,19 @@ TEST_P(ReadInteTest, TestReadWithAppendPtBranch) { DataField(2, arrow::field("value2", arrow::int32())), }; + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .WithBranch("test"); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options[Options::BRANCH] = "test"; + context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { context_builder.SetTableSchema(specific_table_schema.value()); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4120,15 +4257,17 @@ TEST_P(ReadInteTest, TestSpecificFs) { auto countable_fs = std::make_shared(std::make_shared(), &io_count); + std::map context_builder_options; ReadContextBuilder context_builder(path); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption("orc.read.enable-metrics", "true") - .WithFileSystem(countable_fs); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder.EnablePrefetch(param.enable_prefetch).WithFileSystem(countable_fs); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -4173,8 +4312,9 @@ TEST_P(ReadInteTest, TestSpecificFs) { namespace { Result> ReadGlobalSystemTable( - const std::string& table_name, Catalog* catalog, const std::shared_ptr& fs, - const std::string& warehouse, const std::map& options) { + const std::string& table_name, const std::shared_ptr& catalog, + const std::shared_ptr& fs, const std::string& warehouse, + const std::map& options) { GlobalSystemTableContext ctx; ctx.catalog = catalog; ctx.fs = fs; @@ -4188,20 +4328,28 @@ Result> ReadGlobalSystemTable( std::string sys_path = PathUtil::JoinPath(PathUtil::JoinPath(warehouse, "sys"), table_name); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(sys_path); - scan_context_builder.SetOptions(options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr scan_context, + scan_context_builder_options = options; + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_11, + system_table->Copy(scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - system_table->NewScan(scan_context)); + configured_table_11->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(sys_path); - read_context_builder.SetOptions(options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, + read_context_builder_options = options; + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_12, + system_table->Copy(read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - system_table->NewRead(read_context)); + configured_table_12->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -4229,8 +4377,8 @@ TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) { std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); ASSERT_OK_AND_ASSIGN(auto result, - ReadGlobalSystemTable("catalog_options", catalog.get(), - catalog->GetFileSystem(), warehouse, options)); + ReadGlobalSystemTable("catalog_options", catalog, catalog->GetFileSystem(), + warehouse, options)); auto struct_array = SingleStructChunk(result); ASSERT_TRUE(struct_array); auto key_array = std::dynamic_pointer_cast(struct_array->field(0)); @@ -4301,7 +4449,7 @@ TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { // Directly test BuildRows { GlobalSystemTableContext ctx; - ctx.catalog = catalog.get(); + ctx.catalog = catalog; ctx.fs = catalog->GetFileSystem(); ctx.warehouse = warehouse; ctx.catalog_options = options; @@ -4310,9 +4458,9 @@ TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { ASSERT_GT(rows.size(), 0) << "BuildRows returned empty, expected at least 1 row"; } - ASSERT_OK_AND_ASSIGN(auto result, - ReadGlobalSystemTable("all_table_options", catalog.get(), - catalog->GetFileSystem(), warehouse, options)); + ASSERT_OK_AND_ASSIGN( + auto result, ReadGlobalSystemTable("all_table_options", catalog, catalog->GetFileSystem(), + warehouse, options)); auto struct_array = SingleStructChunk(result); ASSERT_TRUE(struct_array); ASSERT_GE(struct_array->length(), 1) << "result has " << struct_array->length() << " rows"; @@ -4392,7 +4540,7 @@ TEST(SystemTableReadInteTest, TestReadGlobalTables) { ArrowSchemaRelease(&no_pk_schema); ASSERT_OK_AND_ASSIGN(auto result, - ReadGlobalSystemTable("tables", catalog.get(), fs, warehouse, options)); + ReadGlobalSystemTable("tables", catalog, fs, warehouse, options)); auto struct_array = SingleStructChunk(result); ASSERT_TRUE(struct_array); ASSERT_GE(struct_array->length(), 1); @@ -4490,8 +4638,8 @@ TEST(SystemTableReadInteTest, TestReadGlobalPartitions) { ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); auto fs = catalog->GetFileSystem(); - ASSERT_OK_AND_ASSIGN(auto part_result, ReadGlobalSystemTable("partitions", catalog.get(), fs, - warehouse, options)); + ASSERT_OK_AND_ASSIGN(auto part_result, + ReadGlobalSystemTable("partitions", catalog, fs, warehouse, options)); auto struct_array = SingleStructChunk(part_result); ASSERT_TRUE(struct_array); ASSERT_EQ(StructFieldNames(struct_array), @@ -4542,7 +4690,7 @@ TEST(SystemTableReadInteTest, TestGlobalSystemTablesPropagateCorruptSchema) { /*overwrite=*/true)); for (const std::string system_table : {"all_table_options", "tables", "partitions"}) { - ASSERT_NOK(ReadGlobalSystemTable(system_table, catalog.get(), fs, warehouse, options)); + ASSERT_NOK(ReadGlobalSystemTable(system_table, catalog, fs, warehouse, options)); } } @@ -4578,7 +4726,7 @@ TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptSnapshot ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(table_path, "snapshot/snapshot-1"), "{invalid-json", /*overwrite=*/true)); - ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); + ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog, fs, warehouse, options)); } TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptManifest) { @@ -4617,7 +4765,7 @@ TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptManifest PathUtil::JoinPath(table_path, "manifest/" + snapshot->BaseManifestList()), "corrupt", /*overwrite=*/true)); - ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); + ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog, fs, warehouse, options)); } } // namespace paimon::test diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 5de6c9fa7..7aad2410f 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -61,6 +61,7 @@ #include "paimon/core/table/sink/commit_message_impl.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/realtime_split.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/data/shredding/map_shared_shredding_schema_utils.h" #include "paimon/data/variant.h" @@ -504,7 +505,7 @@ class RealtimeWriteInteTest : public ::testing::Test { void CreateTable(const std::vector& partition_keys) const { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(dir_->Str(), options_)); ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), c_schema.get(), partition_keys, @@ -516,7 +517,7 @@ class RealtimeWriteInteTest : public ::testing::Test { const std::vector& primary_keys = {"id"}) const { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(dir_->Str(), options_)); ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); std::vector table_primary_keys = partition_keys; @@ -550,13 +551,15 @@ class RealtimeWriteInteTest : public ::testing::Test { Result> CreateRealtimeWriter( const std::shared_ptr& realtime_context, const std::string& temp_directory) const { + std::map builder_options; WriteContextBuilder builder(table_path_, commit_user_); - builder.SetOptions(options_).WithStreamingMode(true).WithRealtimeContext(realtime_context); + builder_options = options_; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); if (!temp_directory.empty()) { builder.WithTempDirectory(temp_directory); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreWrite::Create(std::move(context)); + return FileStoreWrite::Create(std::move(context), builder_options); } Result> CreateRealtimeWriter() const { @@ -730,22 +733,28 @@ class RealtimeWriteInteTest : public ::testing::Test { Result Commit(const std::vector& realtime_commits, int64_t commit_identifier) const { + std::map builder_options; CommitContextBuilder builder(table_path_, commit_user_); + builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); + + builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); return commit->CommitWithProgress(realtime_commits, commit_identifier, /*watermark=*/std::nullopt); } Result DropPartition(const std::map& partition, int64_t commit_identifier) const { + std::map builder_options; CommitContextBuilder builder(table_path_, kDropPartitionCommitUser); + builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); + + builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); PAIMON_RETURN_NOT_OK(commit->DropPartition({partition}, commit_identifier)); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_)); SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_); @@ -758,11 +767,14 @@ class RealtimeWriteInteTest : public ::testing::Test { } Result TruncateTable(int64_t commit_identifier) const { + std::map builder_options; CommitContextBuilder builder(table_path_, kTruncateCommitUser); + builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); + + builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); PAIMON_RETURN_NOT_OK(commit->TruncateTable(commit_identifier)); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_)); SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_); @@ -775,11 +787,14 @@ class RealtimeWriteInteTest : public ::testing::Test { } Result RollbackToAsLatest(int64_t target_snapshot_id) const { + std::map builder_options; CommitContextBuilder builder(table_path_, kRollbackCommitUser); + builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); + + builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(bool rolled_back, commit->RollbackToAsLatest(target_snapshot_id)); if (!rolled_back) { return Status::Invalid("failed to commit rollback snapshot"); @@ -797,12 +812,15 @@ class RealtimeWriteInteTest : public ::testing::Test { Result CompactAndCommit(const std::map& partition, int32_t bucket, int64_t commit_identifier, bool full_compaction = true) const { + std::map write_builder_options; WriteContextBuilder write_builder(table_path_, commit_user_); - write_builder.SetOptions(options_).WithStreamingMode(true).WithTempDirectory( + write_builder_options = options_; + write_builder.WithStreamingMode(true).WithTempDirectory( PathUtil::JoinPath(dir_->Str(), "compact-tmp")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr compaction_writer, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr compaction_writer, + FileStoreWrite::Create(std::move(write_context), write_builder_options)); PAIMON_RETURN_NOT_OK(compaction_writer->Compact(partition, bucket, full_compaction)); PAIMON_ASSIGN_OR_RAISE( std::vector> compaction_messages, @@ -811,11 +829,15 @@ class RealtimeWriteInteTest : public ::testing::Test { return Status::Invalid("compaction did not produce a commit message"); } + std::map commit_builder_options; CommitContextBuilder commit_builder(table_path_, commit_user_); + commit_builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr compaction_commit, - FileStoreCommit::Create(std::move(commit_context))); + + commit_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr compaction_commit, + FileStoreCommit::Create(std::move(commit_context), commit_builder_options)); PAIMON_RETURN_NOT_OK(compaction_commit->Commit(compaction_messages, commit_identifier)); PAIMON_RETURN_NOT_OK(compaction_writer->Close()); @@ -830,40 +852,47 @@ class RealtimeWriteInteTest : public ::testing::Test { } Result ExpireSnapshots() const { + std::map builder_options; CommitContextBuilder builder(table_path_, commit_user_); + builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); + + builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); return commit->Expire(); } Result> CreatePlan( const std::shared_ptr& realtime_context, const std::shared_ptr& predicate) const { + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path_); if (realtime_context) { scan_builder.WithRealtimeContext(realtime_context); } scan_builder.SetPredicate(predicate).WithMemoryPool(pool_); + scan_builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - scan_builder.SetOptions(options_).Finish()); + + scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), scan_builder_options)); return scan->CreatePlan(); } Result> CreateQueryReader( const std::shared_ptr& plan, const std::shared_ptr& realtime_context) const { + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); return table_read->CreateReader(plan->Splits()); } @@ -878,16 +907,17 @@ class RealtimeWriteInteTest : public ::testing::Test { const std::shared_ptr& plan, const std::shared_ptr& realtime_context, const std::vector& read_fields, const std::shared_ptr& predicate, bool enable_predicate_filter) const { + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames(read_fields) + read_builder_options = options_; + read_builder.SetReadFieldNames(read_fields) .SetPredicate(predicate) .EnablePredicateFilter(enable_predicate_filter) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -901,14 +931,15 @@ class RealtimeWriteInteTest : public ::testing::Test { const std::string& expected_json) const { std::unique_ptr c_read_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_read_schema.get()).ok()); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadSchema(std::move(c_read_schema)) + read_builder_options = options_; + read_builder.SetReadSchema(std::move(c_read_schema)) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -987,13 +1018,13 @@ class RealtimeWriteInteTest : public ::testing::Test { Result CountRows(const std::shared_ptr& plan, const std::shared_ptr& realtime_context) const { + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .WithRealtimeContext(realtime_context) - .WithMemoryPool(pool_); + read_builder_options = options_; + read_builder.WithRealtimeContext(realtime_context).WithMemoryPool(pool_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr count_reader, table_read->CreateCountReader(plan->Splits())); return count_reader->CountRows(); @@ -1250,31 +1281,39 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeOperationsRequireEnabledOption) { ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); + std::map write_builder_options; WriteContextBuilder write_builder(table_path_, commit_user_); - write_builder.SetOptions(disabled_options) - .WithStreamingMode(true) - .WithRealtimeContext(realtime_context); + write_builder_options = disabled_options; + write_builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context), write_builder_options), "real-time write requires realtime.enabled=true"); + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path_); - scan_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); + scan_builder_options = disabled_options; + scan_builder.WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), scan_builder_options), "real-time scan requires realtime.enabled=true"); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); + read_builder_options = disabled_options; + read_builder.WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context), read_builder_options), "real-time read requires realtime.enabled=true"); + std::map commit_builder_options; CommitContextBuilder commit_builder(table_path_, commit_user_); + commit_builder_options = disabled_options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_builder.SetOptions(disabled_options).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + + commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(std::move(commit_context), commit_builder_options)); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(/*realtime_commits=*/{}, /*commit_identifier=*/0, /*watermark=*/std::nullopt), @@ -1495,11 +1534,15 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryMergesLatestDv) auto concurrent_message = std::make_shared( initial_message->Partition(), /*bucket=*/0, /*total_buckets=*/1, DataIncrement({}, {}, {}, {concurrent_dv}, {}), CompactIncrement({}, {}, {}, {}, {})); + std::map concurrent_builder_options; CommitContextBuilder concurrent_builder(table_path_, "concurrent-dv-writer"); + concurrent_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_context, - concurrent_builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_commit, - FileStoreCommit::Create(std::move(concurrent_context))); + + concurrent_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr concurrent_commit, + FileStoreCommit::Create(std::move(concurrent_context), concurrent_builder_options)); bool interleaved = false; auto fs = std::make_shared( PathUtil::JoinPath(table_path_, "snapshot/snapshot-2"), [&]() -> Status { @@ -1507,11 +1550,14 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryMergesLatestDv) return concurrent_commit->Commit({concurrent_message}, /*commit_identifier=*/0, /*watermark=*/std::nullopt); }); + std::map builder_options; CommitContextBuilder builder(table_path_, commit_user_); + builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).WithFileSystem(fs).Finish()); + + builder.WithFileSystem(fs).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN( int64_t snapshot_id, commit->CommitWithProgress(progress, /*commit_identifier=*/1, /*watermark=*/std::nullopt)); @@ -1573,11 +1619,14 @@ TEST_F(RealtimeWriteInteTest, DISABLED_TestRealtimeDeduplicateCommitRetryRemapsC interleaved = true; return Status::OK(); }); + std::map builder_options; CommitContextBuilder builder(table_path_, commit_user_); + builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).WithFileSystem(fs).Finish()); + + builder.WithFileSystem(fs).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN( int64_t snapshot_id, commit->CommitWithProgress(progress, /*commit_identifier=*/2, /*watermark=*/std::nullopt)); @@ -2459,18 +2508,24 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesSnapshotConflict) { const std::string first_snapshot_path = PathUtil::JoinPath(table_path_, "snapshot/snapshot-1"); auto conflict_file_system = std::make_shared(first_snapshot_path); + std::map bucket0_builder_options; CommitContextBuilder bucket0_builder(table_path_, "bucket0_realtime_commit_user"); + bucket0_builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket0_context, + + bucket0_builder.WithFileSystem(conflict_file_system).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr bucket0_context, - bucket0_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket0_commit, - FileStoreCommit::Create(std::move(bucket0_context))); + std::unique_ptr bucket0_commit, + FileStoreCommit::Create(std::move(bucket0_context), bucket0_builder_options)); + std::map bucket1_builder_options; CommitContextBuilder bucket1_builder(table_path_, "bucket1_realtime_commit_user"); + bucket1_builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket1_context, + + bucket1_builder.WithFileSystem(conflict_file_system).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr bucket1_context, - bucket1_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket1_commit, - FileStoreCommit::Create(std::move(bucket1_context))); + std::unique_ptr bucket1_commit, + FileStoreCommit::Create(std::move(bucket1_context), bucket1_builder_options)); std::optional> bucket0_result; std::optional> bucket1_result; @@ -2538,13 +2593,16 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesConflictWithCompaction) { ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id)); } + std::map compact_write_builder_options; WriteContextBuilder compact_write_builder(table_path_, "compaction_write_user"); - compact_write_builder.SetOptions(options_).WithStreamingMode(true).WithTempDirectory( + compact_write_builder_options = options_; + compact_write_builder.WithStreamingMode(true).WithTempDirectory( PathUtil::JoinPath(dir_->Str(), "conflict-compact-tmp")); ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_write_context, compact_write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_writer, - FileStoreWrite::Create(std::move(compact_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr compact_writer, + FileStoreWrite::Create(std::move(compact_write_context), compact_write_builder_options)); ASSERT_OK(compact_writer->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN( @@ -2568,18 +2626,24 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesConflictWithCompaction) { PathUtil::JoinPath(table_path_, "snapshot/snapshot-3"); auto conflict_file_system = std::make_shared(conflict_snapshot_path); + std::map append_commit_builder_options; CommitContextBuilder append_commit_builder(table_path_, commit_user_); + append_commit_builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr append_commit_context, + + append_commit_builder.WithFileSystem(conflict_file_system).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr append_commit_context, - append_commit_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr append_commit, - FileStoreCommit::Create(std::move(append_commit_context))); + std::unique_ptr append_commit, + FileStoreCommit::Create(std::move(append_commit_context), append_commit_builder_options)); + std::map compact_commit_builder_options; CommitContextBuilder compact_commit_builder(table_path_, "compaction_commit_user"); + compact_commit_builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_commit_context, + + compact_commit_builder.WithFileSystem(conflict_file_system).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr compact_commit_context, - compact_commit_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_commit, - FileStoreCommit::Create(std::move(compact_commit_context))); + std::unique_ptr compact_commit, + FileStoreCommit::Create(std::move(compact_commit_context), compact_commit_builder_options)); std::optional> append_result; std::optional compact_status; @@ -2843,14 +2907,15 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) { ASSERT_OK_AND_ASSIGN(std::shared_ptr lifetime_plan, CreatePlan(realtime_context, /*predicate=*/nullptr)); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(lifetime_plan->Splits())); ASSERT_FALSE(query_view->expired()); @@ -2918,10 +2983,18 @@ TEST_F(RealtimeWriteInteTest, TestPkRealtimeReadOptimizedScanUnsupported) { ASSERT_OK_AND_ASSIGN(std::vector disk_rows, ReadRows()); ASSERT_EQ(rows, disk_rows); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), + table_path_ + "$ro", options_)); + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path_ + "$ro"); - scan_builder.SetOptions(options_).WithRealtimeContext(realtime_context).WithMemoryPool(pool_); + scan_builder_options = options_; + scan_builder.WithRealtimeContext(realtime_context).WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - Result> scan = TableScan::Create(std::move(scan_context)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_1, + table->Copy(scan_builder_options)); + Result> scan = configured_table_1->NewScan(std::move(scan_context)); ASSERT_TRUE(scan.status().IsNotImplemented()) << scan.status().ToString(); ASSERT_NE(std::string::npos, scan.status().ToString().find( "PK real-time union read does not support read-optimized")); @@ -2951,14 +3024,15 @@ TEST_F(RealtimeWriteInteTest, TestPkDeleteInsertAndPinnedReadsAcrossRefresh) { CreatePlan(realtime_context, /*predicate=*/nullptr)); ASSERT_OK_AND_ASSIGN(std::shared_ptr reader_plan, CreatePlan(realtime_context, /*predicate=*/nullptr)); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr pinned_reader, table_read->CreateReader(reader_plan->Splits())); @@ -3149,14 +3223,15 @@ TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) { }); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadSchema(std::move(c_schema)) + read_builder_options = options_; + read_builder.SetReadSchema(std::move(c_schema)) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3444,22 +3519,28 @@ TEST_F(RealtimeWriteInteTest, TestPkPartitionBucketRecovery) { TEST_F(RealtimeWriteInteTest, TestPkRecovery) { CreatePkTable(); + std::map seed_builder_options; WriteContextBuilder seed_builder(table_path_, commit_user_); - seed_builder.SetOptions(options_).WithStreamingMode(true); + seed_builder_options = options_; + seed_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_context, seed_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_writer, - FileStoreWrite::Create(std::move(seed_context))); + FileStoreWrite::Create(std::move(seed_context), seed_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_batch, MakeLogicalBatch({Row{99, "seed", "p0"}}, /*bucket=*/0)); ASSERT_OK(seed_writer->Write(std::move(seed_batch))); ASSERT_OK_AND_ASSIGN(std::vector> seed_messages, seed_writer->PrepareCommit(/*wait_compaction=*/false, /*commit_identifier=*/0)); + std::map seed_commit_builder_options; CommitContextBuilder seed_commit_builder(table_path_, commit_user_); + seed_commit_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_commit_context, - seed_commit_builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_commit, - FileStoreCommit::Create(std::move(seed_commit_context))); + + seed_commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr seed_commit, + FileStoreCommit::Create(std::move(seed_commit_context), seed_commit_builder_options)); ASSERT_OK(seed_commit->Commit(seed_messages, /*commit_identifier=*/0)); ASSERT_OK(seed_writer->Close()); const std::vector mutations = { @@ -3546,11 +3627,14 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) { ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context)); ASSERT_EQ(0, memory_usage); } + std::map compact_builder_options; WriteContextBuilder compact_builder(table_path_, commit_user_); - compact_builder.SetOptions(options_).WithStreamingMode(true); + compact_builder_options = options_; + compact_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_context, compact_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_writer, - FileStoreWrite::Create(std::move(compact_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr compact_writer, + FileStoreWrite::Create(std::move(compact_context), compact_builder_options)); ASSERT_OK(compact_writer->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN( @@ -3578,11 +3662,15 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) { std::max(compacted_live_max_sequence_number, file->max_sequence_number); } ASSERT_LT(compacted_live_max_sequence_number, kHistoricalMaxSequenceNumber); + std::map commit_builder_options; CommitContextBuilder commit_builder(table_path_, commit_user_); + commit_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + + commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(std::move(commit_context), commit_builder_options)); ASSERT_OK(commit->Commit(compact_messages, /*commit_identifier=*/4)); ASSERT_OK(compact_writer->Close()); @@ -3902,11 +3990,14 @@ TEST_F(RealtimeWriteInteTest, TestCommitWithProgressRejectsCoveredRangesFromAnot writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); ASSERT_OK(Commit(commits, /*commit_identifier=*/0)); + std::map builder_options; CommitContextBuilder builder(table_path_, "another_realtime_commit_user"); + builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).Finish()); + + builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + FileStoreCommit::Create(std::move(context), builder_options)); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(commits, /*commit_identifier=*/0, /*watermark=*/std::nullopt), "another commit user or identifier"); @@ -4722,14 +4813,15 @@ TEST_F(RealtimeWriteInteTest, TestSuccessfulReaderCreationConsumesRealtimeSplitT ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, CreatePlan(realtime_context, /*predicate=*/nullptr)); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr first_reader, table_read->CreateReader(plan->Splits())); first_reader->Close(); @@ -4754,16 +4846,17 @@ TEST_F(RealtimeWriteInteTest, TestFailedReaderCreationPreservesRealtimeSplitTick CreatePlan(realtime_context, /*predicate=*/nullptr)); ASSERT_EQ(1, plan->Splits().size()); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .SetPredicate(std::make_shared()) .EnablePredicateFilter(true) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateReader(plan->Splits()), "cannot cast predicate unsupported"); @@ -5617,14 +5710,15 @@ TEST_F(RealtimeWriteInteTest, TestReaderPinsMemoryAcrossRefresh) { std::dynamic_pointer_cast(plan->Splits()[0]); ASSERT_NE(nullptr, realtime_split); + std::map read_builder_options; ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context_impl, diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index b7cc73b96..2e23606e3 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -144,21 +144,23 @@ class ScanAndReadInteTest : public testing::Test, } } - void AddReadOptionsForPrefetch(ReadContextBuilder* read_context_builder) { - read_context_builder->AddOption("test.enable-adaptive-prefetch-strategy", "false"); + void AddReadOptionsForPrefetch(ReadContextBuilder* read_context_builder, + std::map& options) { + options["test.enable-adaptive-prefetch-strategy"] = "false"; if (EnablePrefetch()) { read_context_builder->EnablePrefetch(true).SetPrefetchBatchCount(3); } } - Result> FinishScanContext(ScanContextBuilder& builder) { + Result> FinishScanContext( + ScanContextBuilder& builder, std::map& options) { if (EnableSnapshotLiveManifestCache()) { if (!snapshot_live_manifest_cache_) { snapshot_live_manifest_cache_ = std::make_shared(/*max_weight=*/64 * 1024 * 1024); } - builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .WithCache(snapshot_live_manifest_cache_); + options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder.WithCache(snapshot_live_manifest_cache_); } return builder.Finish(); } @@ -240,12 +242,15 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - Result> scan_context = FinishScanContext(scan_context_builder); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + Result> scan_context = + FinishScanContext(scan_context_builder, scan_context_builder_options); CHECK_HOOK_STATUS(scan_context.status(), i); Result> table_scan = - TableScan::Create(std::move(scan_context).value()); + TableScan::Create(std::move(scan_context).value(), scan_context_builder_options); CHECK_HOOK_STATUS(table_scan.status(), i); Result> result_plan = table_scan.value()->CreatePlan(); CHECK_HOOK_STATUS(result_plan.status(), i); @@ -254,12 +259,14 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { auto splits = result_plan.value()->Splits(); ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = + TableRead::Create(std::move(read_context), read_context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(splits); @@ -297,12 +304,15 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - Result> scan_context = FinishScanContext(scan_context_builder); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + + Result> scan_context = + FinishScanContext(scan_context_builder, scan_context_builder_options); CHECK_HOOK_STATUS(scan_context.status(), i); Result> table_scan = - TableScan::Create(std::move(scan_context).value()); + TableScan::Create(std::move(scan_context).value(), scan_context_builder_options); CHECK_HOOK_STATUS(table_scan.status(), i); Result> result_plan = table_scan.value()->CreatePlan(); CHECK_HOOK_STATUS(result_plan.status(), i); @@ -310,12 +320,14 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { auto splits = result_plan.value()->Splits(); ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = + TableRead::Create(std::move(read_context), read_context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(splits); @@ -350,21 +362,27 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot1) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -392,10 +410,14 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -403,10 +425,12 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -505,21 +529,20 @@ TEST(SelectiveManifestDecodeInteTest, TestPartitionedPointLookup) { bool entry_cache) -> Result { std::shared_ptr pool = GetMemoryPool(); scan_pools.push_back(pool); + std::map builder_options; ScanContextBuilder builder(table_path); - builder.WithMemoryPool(pool) - .WithExecutor(executor) - .WithCache(cache) - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, entry_cache ? "3" : "0") - .AddOption(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED, - lazy_decode ? "true" : "false") - .AddOption(Options::READ_BATCH_SIZE, "128"); + builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = entry_cache ? "3" : "0"; + builder_options[Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED] = + lazy_decode ? "true" : "false"; + builder_options[Options::READ_BATCH_SIZE] = "128"; + builder.WithMemoryPool(pool).WithExecutor(executor).WithCache(cache); if (entry_cache) { // No explicit bucket or partition filter: the predicate must select the bucket. builder.SetPredicate(predicate); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + TableScan::Create(std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); PAIMON_ASSIGN_OR_RAISE(uint64_t enabled, table_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED)); @@ -541,7 +564,7 @@ TEST(SelectiveManifestDecodeInteTest, TestPartitionedPointLookup) { builder.SetPredicate(predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(context))); + TableRead::Create(std::move(context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(result.plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, @@ -690,15 +713,15 @@ TEST(SelectiveManifestDecodeInteTest, TestSchemaEvolutionPreservesBucketPruning) } auto cache = std::make_shared(capacities); for (int32_t attempt = 0; attempt < 2; ++attempt) { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.SetPredicate(predicate) - .WithCache(cache) - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .AddOption(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED, - lazy_decode ? "true" : "false"); + builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder_options[Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED] = + lazy_decode ? "true" : "false"; + builder.SetPredicate(predicate).WithCache(cache); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(context))); + TableScan::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); auto split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -712,7 +735,7 @@ TEST(SelectiveManifestDecodeInteTest, TestSchemaEvolutionPreservesBucketPruning) ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, @@ -779,10 +802,11 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { // All buckets have overlapping stats for this key. An explicit filter bypasses // inference, proving that ordinary statistics cannot account for the pruning. for (int32_t bucket = 0; bucket < kNumBuckets; ++bucket) { + std::map builder_options; ScanContextBuilder builder(table_path); builder.SetPredicate(predicate).SetBucketFilter(bucket); - ASSERT_OK_AND_ASSIGN(auto context, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto context, FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_FALSE(plan->Splits().empty()); @@ -804,23 +828,29 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { auto lookup_predicate = PredicateBuilder::Equal( 0, "rowkey", field_type, Literal(field_type, lookup_key.data(), lookup_key.size())); + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path); - scan_builder.SetPredicate(lookup_predicate) - .AddOption(Options::BUCKET, current_bucket_count); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + scan_builder_options[Options::BUCKET] = current_bucket_count; + scan_builder.SetPredicate(lookup_predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_builder, scan_builder_options)); + ASSERT_OK_AND_ASSIGN( + auto scan, TableScan::Create(std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); if (EnableSnapshotLiveManifestCache()) { ASSERT_OK_AND_ASSIGN( uint64_t enabled, scan->GetMetrics()->GetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED)); ASSERT_EQ(enabled, 1); + std::map cached_builder_options; ScanContextBuilder cached_builder(table_path); - cached_builder.SetPredicate(lookup_predicate) - .AddOption(Options::BUCKET, current_bucket_count); - ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); - ASSERT_OK_AND_ASSIGN(auto cached_scan, - TableScan::Create(std::move(cached_context))); + cached_builder_options[Options::BUCKET] = current_bucket_count; + cached_builder.SetPredicate(lookup_predicate); + ASSERT_OK_AND_ASSIGN(auto cached_context, + FinishScanContext(cached_builder, cached_builder_options)); + ASSERT_OK_AND_ASSIGN( + auto cached_scan, + TableScan::Create(std::move(cached_context), cached_builder_options)); ASSERT_OK_AND_ASSIGN(plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); @@ -833,11 +863,13 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { ASSERT_TRUE(data_split); ASSERT_EQ(data_split->Bucket(), bucket_ids[lookup_index]); } + std::map read_builder_options; ReadContextBuilder read_builder(table_path); - AddReadOptionsForPrefetch(&read_builder); + AddReadOptionsForPrefetch(&read_builder, read_builder_options); read_builder.SetPredicate(lookup_predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + auto read, TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(reader))); @@ -861,10 +893,14 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -872,10 +908,12 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -911,18 +949,24 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5WithReaderBuildParallelism) { auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -953,15 +997,21 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotWithStreamWithDefaultMode) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 1, 2, 3, 4}; auto expected_snapshot1 = std::make_shared( @@ -1002,10 +1052,14 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshot1) { std::string table_path = GetDataDir() + "/" + file_format + "/append_10.db/append_10"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1013,10 +1067,12 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshot1) { ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1041,10 +1097,14 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshotOfNestedType) { "/append_complex_build_in_fieldid.db/" "append_complex_build_in_fieldid/"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1069,10 +1129,12 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshotOfNestedType) { std::shared_ptr arrow_data_type = DataField::ConvertDataFieldsToArrowStructType(read_fields); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1096,14 +1158,18 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6) { "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; // normal batch scan case for pk+dv, all data in level 0 is filtered out + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ReadContextBuilder read_context_builder(table_path); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1137,15 +1203,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot1) { auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1160,16 +1232,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPartitionAndBu std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1196,8 +1274,9 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; // predicate: f0 != "Alice" (key predicate) and f3 > 18 (value predicate) and all data in level // 0 + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; std::string literal_str = "Alice"; auto not_equal = PredicateBuilder::NotEqual( @@ -1207,14 +1286,18 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { FieldType::DOUBLE, Literal(18.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, greater_than})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1237,8 +1320,9 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLateMaterializ auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; std::string literal_str = "Alice"; auto not_equal = PredicateBuilder::NotEqual( @@ -1248,14 +1332,18 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLateMaterializ FieldType::DOUBLE, Literal(18.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, greater_than})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1278,15 +1366,19 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot4WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + scan_context_builder.SetBucketFilter(0).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will be applied, as in kv mode value filter is enabled auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -1300,15 +1392,20 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLimit) { // normal batch scan case for pk+dv, all data in level 0 is filtered out, set row limits to 6, // data in partition 20 is truncated + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetLimit(6); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1336,17 +1433,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot4) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // from an compact snapshot std::vector> expected_snapshot_ids = {4, 5, 6}; @@ -1388,17 +1490,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot5) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // from an append snapshot, first plan is snapshot5 with merge, second plan is snapshot6 // (isStreaming=true) @@ -1434,15 +1541,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot6) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, "latest-full").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_MODE] = "latest-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // from an append snapshot, first plan is snapshot6 with merge, level 0 data in base manifest // list is contained @@ -1471,17 +1584,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot1) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // from the first snapshot std::vector> expected_snapshot_ids = {1, 3, 5, 6}; @@ -1530,17 +1648,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot2) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // use from-snapshot mode, first plan is empty, second plan is snapshot 3 with delta files // (snapshot 2 is compact snapshot, so skipped) @@ -1575,18 +1698,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithNestedType) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_nested_type.db/pk_table_nested_type/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN( auto read_context, read_context_builder.SetReadFieldNames({"shopId", "dt", "hr", "col0", "col1", "col2"}) .Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -1629,14 +1756,19 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanLatestSnapshot) { "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // normal batch scan case for pk+mor, use latest snapshot if not specified + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1676,15 +1808,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { // normal batch scan case for pk+mor, read snapshot 2 (append snapshot), all data is in level 0 // with merge read + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -1720,16 +1858,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPartitionAndB "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // all data not in partition 10, bucket 1 is filtered out + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1758,8 +1902,9 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; // predicate: f0 != "Alice" and f0 < "Lucy" and f3 <= 30.0 // as f3 <= 30.0 is a value filter, it does not take effect @@ -1780,14 +1925,18 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { PredicateBuilder::And({not_equal, less_than, less_or_equal})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1821,8 +1970,9 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLateMateriali std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; std::string literal_str = "Alice"; auto not_equal = PredicateBuilder::NotEqual( @@ -1837,14 +1987,18 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLateMateriali ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, less_than, less_or_equal})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1869,16 +2023,20 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot3WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will be applied as all data files in bucket 1 in compact snapshot 3 are // all filtered auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -1890,20 +2048,27 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvWithInvalidAggregateBatchScanSnapsho std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + scan_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + read_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -1930,20 +2095,27 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorWithInvalidAggregateBatchScanSnapsh std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + scan_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + read_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -1957,24 +2129,31 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithAggregateBatchScanSnapshot3WithPredica std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "sum"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + scan_context_builder_options["fields.f3.aggregate-function"] = "sum"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will not be applied, as merge engine is aggregation auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "sum"); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + read_context_builder_options["fields.f3.aggregate-function"] = "sum"; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -2001,24 +2180,31 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithPartialUpdateBatchScanSnapshot3WithPre std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "partial-update") - .AddOption(Options::IGNORE_DELETE, "true"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "partial-update"; + scan_context_builder_options[Options::IGNORE_DELETE] = "true"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will not be applied, as merge engine is partial-update auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "partial-update") - .AddOption(Options::IGNORE_DELETE, "true"); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "partial-update"; + read_context_builder_options[Options::IGNORE_DELETE] = "true"; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -2047,15 +2233,20 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLimit) { // in pk mor mode, limit does not take effect, as we do not know the number of records after // merging + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetLimit(6); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -2087,17 +2278,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot4) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // first plan is snapshot4 (isStreaming=false), second plan is snapshot5 std::vector> expected_snapshot_ids = {4, 5}; @@ -2132,17 +2328,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot1) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {1, 2, 4, 5}; auto expected_snapshot1_batch = std::make_shared( @@ -2193,17 +2394,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot2) { // use from-snapshot mode, first plan is empty, snapshot 2 in second plan only includes delta // files + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 2, 4, 5}; auto expected_snapshot2_stream = std::make_shared( @@ -2238,21 +2444,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot5WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(50.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // first plan is snapshot 5 with empty plan, second plan is snapshot 5 with delta files // (predicates take no effects in delta mode) @@ -2274,24 +2485,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamPredicateFilter) { std::string table_path = paimon::test::GetDataDir() + FileFormat() + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); std::shared_ptr predicate = PredicateBuilder::GreaterThan( /*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(31.05)); scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - FinishScanContext(scan_context_builder)); + FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 5}; auto expected_snapshot5_stream = std::make_shared( @@ -2313,15 +2526,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowBatchScanSnapshot5) { "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; // normal batch scan case for pk+first row, all data in level 0 is filtered out + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -2353,17 +2572,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot3) { paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // from a compact snapshot std::vector> expected_snapshot_ids = {3, 4, 5}; @@ -2406,15 +2630,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot5) { paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, "latest-full").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_MODE] = "latest-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // from an append snapshot, first plan is snapshot5 with merge, all level 0 data is merged std::vector> expected_snapshot_ids = {5}; @@ -2445,15 +2675,20 @@ TEST_P(ScanAndReadInteTest, TestWithPKWith09VersionDvBatchScanLatestSnapshot) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_09.db/pk_09/"; // normal batch scan case for pk+dv (09 version) + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetLimit(2); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 8); @@ -2486,15 +2721,21 @@ TEST_P(ScanAndReadInteTest, TestWithEmptyPartitionValue) { [&](const std::string& table_path, const std::vector>& partition_filters, const std::shared_ptr& expected) { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetPartitionFilter(partition_filters); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -2555,15 +2796,21 @@ TEST_P(ScanAndReadInteTest, TestWithMultipleEmptyPartitionValue) { auto check_result = [&](const std::vector>& partition_filters, const std::shared_ptr& expected) { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetPartitionFilter(partition_filters); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -2604,10 +2851,14 @@ TEST_P(ScanAndReadInteTest, TestMemoryUse) { std::string table_path = paimon::test::GetDataDir() + file_format + "/append_09.db/append_09/"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -2616,12 +2867,14 @@ TEST_P(ScanAndReadInteTest, TestMemoryUse) { // read std::shared_ptr read_pool = GetMemoryPool(); { + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.WithMemoryPool(read_pool); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2705,26 +2958,35 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { { // batch scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.WithStreamingMode(false); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_TRUE(result_plan->Splits().empty()); } { // stream scan: from snapshot 1 + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 1, 2}; auto expected_snapshot1 = std::make_shared( @@ -2749,16 +3011,22 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { } { // stream scan: from snapshot 2 + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 2}; auto expected_snapshot2 = std::make_shared( @@ -2775,17 +3043,20 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { } { // stream scan: from snapshot full + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); ReadContextBuilder read_context_builder(table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); std::vector> expected_snapshot_ids = {1, 2}; auto expected_snapshot1 = std::make_shared( @@ -2817,22 +3088,27 @@ TEST_P(ScanAndReadInteTest, TestScanWithPredicateAndReadWithUnorderedFieldForPar } std::string table_path = paimon::test::GetDataDir() + "parquet/parquet_append_table.db/parquet_append_table"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); auto predicate = PredicateBuilder::LessThan( /*field_index=*/3, /*field_name=*/"f4", FieldType::INT, Literal(300006)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_EQ(result_plan->Splits().size(), 1); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetReadFieldNames({"f10", "f8", "f4", "f13"}); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2868,11 +3144,15 @@ TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { auto predicate = PredicateBuilder::GreaterThan( /*field_index=*/2, /*field_name=*/"key_2", FieldType::INT, Literal(500)); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); ASSERT_EQ(result_plan->Splits().size(), 1); @@ -2885,11 +3165,13 @@ TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { } ASSERT_EQ(data_file_count, 1); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2921,20 +3203,26 @@ TEST_P(ScanAndReadInteTest, TestAppendTableWithMultipleFileFormat) { paimon::test::GetDataDir() + "/append_table_with_multiple_file_format.db/append_table_with_multiple_file_format"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); auto splits = result_plan->Splits(); ASSERT_EQ(1, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2960,19 +3248,24 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndNoExternalPath) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_no_external.db/pk_dv_index_in_data_no_external"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3000,19 +3293,24 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndNoExternalPath) { paimon::test::GetDataDir() + file_format + "/pk_dv_index_not_in_data_no_external.db/pk_dv_index_not_in_data_no_external"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3042,9 +3340,12 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { std::string external_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_not_in_data_with_external.db/external"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -3053,10 +3354,12 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { AdjustSplitWithExternalPath("FILE:/tmp/external", external_path, /*adjust_index=*/false, &splits); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3087,22 +3390,28 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { Literal(FieldType::STRING, "Alice0", 6)); // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN( - auto scan_context, - scan_context_builder.AddOption("file-index.read.enabled", "false").Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + scan_context_builder_options["file-index.read.enabled"] = "false"; + ASSERT_OK_AND_ASSIGN(auto scan_context, + + scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); auto splits = result_plan->Splits(); ASSERT_EQ(1, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption("file-index.read.enabled", "false"); - AddReadOptionsForPrefetch(&read_context_builder); + read_context_builder_options["file-index.read.enabled"] = "false"; + + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3162,24 +3471,29 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { // file. Disabling file-index reads verifies that baseline explicitly. const std::shared_ptr absent_predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"indexed_value", FieldType::INT, Literal(15)); + std::map stats_only_scan_builder_options; ScanContextBuilder stats_only_scan_builder(table_path); - stats_only_scan_builder.SetOptions(options) - .AddOption(Options::FILE_INDEX_READ_ENABLED, "false") - .SetPredicate(absent_predicate); + stats_only_scan_builder_options = options; + stats_only_scan_builder_options[Options::FILE_INDEX_READ_ENABLED] = "false"; + stats_only_scan_builder.SetPredicate(absent_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - TableScan::Create(std::move(stats_only_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr stats_only_scan, + TableScan::Create(std::move(stats_only_scan_context), stats_only_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); // The embedded bitmap knows that 15 is absent and eliminates the file during scan planning. + std::map indexed_scan_builder_options; ScanContextBuilder indexed_scan_builder(table_path); - indexed_scan_builder.SetOptions(options).SetPredicate(absent_predicate); + indexed_scan_builder_options = options; + indexed_scan_builder.SetPredicate(absent_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan_context, indexed_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan, - TableScan::Create(std::move(indexed_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr indexed_scan, + TableScan::Create(std::move(indexed_scan_context), indexed_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr indexed_plan, indexed_scan->CreatePlan()); ASSERT_TRUE(indexed_plan->Splits().empty()); @@ -3188,22 +3502,25 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { // the other rows. const std::shared_ptr present_predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"indexed_value", FieldType::INT, Literal(20)); + std::map retained_scan_builder_options; ScanContextBuilder retained_scan_builder(table_path); - retained_scan_builder.SetOptions(options).SetPredicate(present_predicate); + retained_scan_builder_options = options; + retained_scan_builder.SetPredicate(present_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr retained_scan_context, retained_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr retained_scan, - TableScan::Create(std::move(retained_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr retained_scan, + TableScan::Create(std::move(retained_scan_context), retained_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr retained_plan, retained_scan->CreatePlan()); ASSERT_FALSE(retained_plan->Splits().empty()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options) - .SetPredicate(present_predicate) - .EnablePredicateFilter(false); + read_context_builder_options = options; + read_context_builder.SetPredicate(present_predicate).EnablePredicateFilter(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(retained_plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3263,24 +3580,29 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc /*field_index=*/1, /*field_name=*/"renamed_value", FieldType::INT, Literal(15)); // The evolved min/max range is still [10, 30], so statistics alone retain the old file. + std::map stats_only_scan_builder_options; ScanContextBuilder stats_only_scan_builder(table_path); - stats_only_scan_builder.SetOptions(options) - .AddOption(Options::FILE_INDEX_READ_ENABLED, "false") - .SetPredicate(predicate); + stats_only_scan_builder_options = options; + stats_only_scan_builder_options[Options::FILE_INDEX_READ_ENABLED] = "false"; + stats_only_scan_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - TableScan::Create(std::move(stats_only_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr stats_only_scan, + TableScan::Create(std::move(stats_only_scan_context), stats_only_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); // Reconstructing the predicate to schema-0 lets its embedded bitmap prove that 15 is absent. + std::map indexed_scan_builder_options; ScanContextBuilder indexed_scan_builder(table_path); - indexed_scan_builder.SetOptions(options).SetPredicate(predicate); + indexed_scan_builder_options = options; + indexed_scan_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan_context, indexed_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan, - TableScan::Create(std::move(indexed_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr indexed_scan, + TableScan::Create(std::move(indexed_scan_context), indexed_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr indexed_plan, indexed_scan->CreatePlan()); ASSERT_TRUE(indexed_plan->Splits().empty()); } @@ -3293,9 +3615,12 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { std::string external_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_with_external.db/external"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -3304,10 +3629,12 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { AdjustSplitWithExternalPath("FILE:/tmp/external", external_path, /*adjust_index=*/true, &splits); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3334,9 +3661,12 @@ TEST_P(ScanAndReadInteTest, TestTimestampType) { std::string table_path = paimon::test::GetDataDir() + file_format + "/append_with_multiple_ts_precision_and_timezone.db" "/append_with_multiple_ts_precision_and_timezone/"; + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3344,10 +3674,12 @@ TEST_P(ScanAndReadInteTest, TestTimestampType) { ASSERT_EQ(1, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3382,9 +3714,12 @@ TEST_P(ScanAndReadInteTest, TestCastTimestampType) { "/append_with_cast_timestamp.db" "/append_with_cast_timestamp/"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3392,10 +3727,12 @@ TEST_P(ScanAndReadInteTest, TestCastTimestampType) { ASSERT_EQ(1, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3433,7 +3770,7 @@ TEST_F(ScanAndReadInteTest, TestParquetDictionaryBinaryStoredArrowSchema) { ScanContextBuilder scan_context_builder(table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_EQ(1, plan->SnapshotId().value()); ASSERT_EQ(1, plan->Splits().size()); @@ -3441,7 +3778,7 @@ TEST_F(ScanAndReadInteTest, TestParquetDictionaryBinaryStoredArrowSchema) { ReadContextBuilder read_context_builder(table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3509,6 +3846,7 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { const std::shared_ptr& expected_result, bool exact_filter = false) { ScanContextBuilder scan_builder(table_path); + std::map read_builder_options; ReadContextBuilder read_builder(table_path); if (predicate) { scan_builder.SetPredicate(predicate); @@ -3521,14 +3859,16 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { read_builder.SetReadFieldNames(projection); } read_builder.EnablePredicateFilter(exact_filter); - read_builder.AddOption("read.batch-size", "1"); + read_builder_options["read.batch-size"] = "1"; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId(), std::optional(2)); ASSERT_FALSE(plan->Splits().empty()); ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto read, + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); ASSERT_TRUE(actual); @@ -3597,21 +3937,27 @@ TEST_F(ScanAndReadInteTest, TestLancePythonCompatibility) { auto check = [&](int64_t snapshot_id, const std::vector& projection, const std::shared_ptr& expected_result) { SCOPED_TRACE("snapshot " + std::to_string(snapshot_id)); + std::map scan_builder_options; ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, + TableScan::Create(std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId(), std::optional(snapshot_id)); ASSERT_FALSE(plan->Splits().empty()); + std::map read_builder_options; ReadContextBuilder read_builder(table_path); - read_builder.AddOption("read.batch-size", "1"); + read_builder_options["read.batch-size"] = "1"; + if (!projection.empty()) { read_builder.SetReadFieldNames(projection); } ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto read, + TableRead::Create(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); ASSERT_TRUE(actual); @@ -3695,13 +4041,13 @@ TEST_F(ScanAndReadInteTest, TestMosaicJavaAndPythonCompatibility) { ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3754,11 +4100,11 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibility) { ReadContextBuilder read_context_builder(table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3783,12 +4129,12 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibilityLtzAcrossTimezones) { ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3857,11 +4203,11 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibilityAllTypes) { ReadContextBuilder read_context_builder(table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3876,22 +4222,28 @@ TEST_F(ScanAndReadInteTest, TestAvroWithAppendTable) { auto read_data = [](int64_t snapshot_id, const std::string& result_json) { std::string table_path = GetDataDir() + "/avro/append_multiple.db/append_multiple"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), snapshot_id); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption("test.enable-adaptive-prefetch-strategy", "false"); + read_context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + read_context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3951,10 +4303,13 @@ TEST_F(ScanAndReadInteTest, TestAvroWithPkTable) { std::string table_path = GetDataDir() + "/avro/pk_with_multiple_type.db/pk_with_multiple_type"; // scan + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), snapshot_id); @@ -3965,7 +4320,7 @@ TEST_F(ScanAndReadInteTest, TestAvroWithPkTable) { ReadContextBuilder read_context_builder(table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4024,32 +4379,41 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { // Historical files use two buckets even when the current option has changed. for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - scan_context_builder.AddOption(Options::BUCKET, current_bucket_count); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + + scan_context_builder_options[Options::BUCKET] = current_bucket_count; + scan_context_builder.SetPartitionFilter({{{"f1", "10"}}}); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); if (EnableSnapshotLiveManifestCache()) { ASSERT_OK_AND_ASSIGN(uint64_t enabled, table_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED)); ASSERT_EQ(enabled, 1); + std::map cached_builder_options; ScanContextBuilder cached_builder(table_path); - cached_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6") - .AddOption(Options::BUCKET, current_bucket_count) - .SetPartitionFilter({{{"f1", "10"}}}) - .SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); - ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(std::move(cached_context))); + cached_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + cached_builder_options[Options::BUCKET] = current_bucket_count; + cached_builder.SetPartitionFilter({{{"f1", "10"}}}).SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto cached_context, + FinishScanContext(cached_builder, cached_builder_options)); + ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(std::move(cached_context), + cached_builder_options)); ASSERT_OK_AND_ASSIGN(result_plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); @@ -4099,16 +4463,19 @@ TEST_P(ScanAndReadInteTest, TestReadNullableMapKey) { ScanContextBuilder scan_context_builder(table_path); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->SnapshotId()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); ASSERT_EQ(result_plan->Splits().size(), 1); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::READ_BATCH_SIZE, "1"); + read_context_builder_options[Options::READ_BATCH_SIZE] = "1"; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); const std::vector expected_rows = {R"([[0, 1, [["one", 10]]]])", @@ -4140,10 +4507,12 @@ TEST_P(ScanAndReadInteTest, TestCountRowsEmptySplits) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); // CreateCountReader with empty splits should return 0 rows. std::vector> empty_splits; @@ -4158,25 +4527,32 @@ TEST_P(ScanAndReadInteTest, TestCountRowsConsistencyWithCreateReader) { "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Scan latest snapshot + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // Method 1: CreateCountReader + iterate batches + std::map count_context_builder_options; ReadContextBuilder count_context_builder(table_path); - AddReadOptionsForPrefetch(&count_context_builder); + AddReadOptionsForPrefetch(&count_context_builder, count_context_builder_options); ASSERT_OK_AND_ASSIGN(auto count_read_context, count_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto count_table_read, TableRead::Create(std::move(count_read_context))); + ASSERT_OK_AND_ASSIGN(auto count_table_read, TableRead::Create(std::move(count_read_context), + count_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto count_reader, count_table_read->CreateCountReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(int64_t count_result, count_reader->CountRows()); // Method 2: CreateReader + iterate batches + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4192,19 +4568,24 @@ TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithPredicateNotSupported) { "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Create splits from latest snapshot. + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // Set predicate in read context. CountReader currently does not support predicate pushdown. auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT, Literal(static_cast(0))); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader(result_plan->Splits()), "predicate pushdown is not supported"); @@ -4216,15 +4597,20 @@ TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithForceKeepDeleteNotSupported "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Create splits from latest snapshot. + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); auto* key_value_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(key_value_table_read != nullptr); diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 3f949484d..814512b12 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -64,15 +64,16 @@ enum class ManifestCacheMode { NoCache, Cache }; class ScanInteTest : public testing::TestWithParam { public: - Result> FinishScanContext(ScanContextBuilder& builder) { + Result> FinishScanContext( + ScanContextBuilder& builder, std::map& options) { if (GetParam() == ManifestCacheMode::Cache) { if (!cache_) { cache_ = std::make_shared(std::map{ {CacheKind::MANIFEST, 64 * 1024 * 1024}, {CacheKind::SNAPSHOT_LIVE_MANIFEST, 64 * 1024 * 1024}}); } - builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .WithCache(cache_); + options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder.WithCache(cache_); } return builder.Finish(); } @@ -269,10 +270,13 @@ TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; auto run_scan = [&]() -> Result> { + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithCache(manifest_cache); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.WithCache(manifest_cache); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); return table_scan->CreatePlan(); }; @@ -293,19 +297,21 @@ TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) { Result>> RunBucketSnapshotScan( const std::string& table_path, int64_t snapshot_id, int32_t bucket, const std::shared_ptr& cache, int32_t max_snapshot_live_manifest_versions) { + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)) - .SetBucketFilter(bucket); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + context_builder.SetBucketFilter(bucket); if (max_snapshot_live_manifest_versions > 0) { - context_builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, - std::to_string(max_snapshot_live_manifest_versions)); + context_builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = + std::to_string(max_snapshot_live_manifest_versions); } if (cache) { context_builder.WithCache(cache); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); std::vector> data_splits; @@ -396,11 +402,14 @@ TEST(ScanInteManifestCacheTest, TestSnapshotLiveManifestCacheRetainsSnapshotsPer TEST_P(ScanInteTest, TestScanAppendWithSnapshot1) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -453,10 +462,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -514,13 +527,17 @@ TEST_P(ScanInteTest, TestScanAppendWithSpecificTableSchema) { std::string table_path = data_dir + "orc/append_09.db/append_09"; auto check_result = [&](const std::optional& specific_table_schema) { + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + if (specific_table_schema) { context_builder.SetTableSchema(specific_table_schema.value()); } - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -578,10 +595,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSpecificTableSchema) { TEST_P(ScanInteTest, TestScanInvalidSnapshot) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "100"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "100"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_NOK_WITH_MSG( table_scan->CreatePlan(), "The specified scan snapshotId 100 is out of available snapshotId range [1, 5]."); @@ -589,10 +610,14 @@ TEST_P(ScanInteTest, TestScanInvalidSnapshot) { TEST_P(ScanInteTest, TestBatchScanMultipleTimes) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // batch scan multiple ASSERT_NOK_WITH_MSG(table_scan->CreatePlan(), "end of scan"); @@ -600,13 +625,17 @@ TEST_P(ScanInteTest, TestBatchScanMultipleTimes) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::SOURCE_SPLIT_OPEN_FILE_COST, "1024") - .AddOption(Options::SOURCE_SPLIT_TARGET_SIZE, "2048"); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + context_builder_options[Options::SOURCE_SPLIT_OPEN_FILE_COST] = "1024"; + context_builder_options[Options::SOURCE_SPLIT_TARGET_SIZE] = "2048"; + // open cost = 1024, and split target size is 2048, indicates at most 2 files in a split - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -670,10 +699,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3").SetLimit(3); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + context_builder.SetLimit(3); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -716,10 +749,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetBucketFilter(0).AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + context_builder.SetBucketFilter(0); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -759,10 +796,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { TEST_P(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { // from snapshot is specified std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); DataSplitImpl::Builder builder1_1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -871,11 +912,14 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); DataSplitImpl::Builder builder1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -925,10 +969,14 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { TEST_P(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::Latest().ToString()); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_MODE] = StartupMode::Latest().ToString(); + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -981,11 +1029,14 @@ TEST_P(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::Latest().ToString()) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_MODE] = StartupMode::Latest().ToString(); + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); std::vector> expected_snapshot_ids = {}; CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, {}); @@ -994,12 +1045,15 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshot().ToString()) - .AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_MODE] = StartupMode::FromSnapshot().ToString(); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); DataSplitImpl::Builder builder2_1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -1066,13 +1120,16 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshotFull().ToString()) - .AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); + context_builder_options[Options::SCAN_MODE] = StartupMode::FromSnapshotFull().ToString(); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); ASSERT_TRUE(scan_context); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); DataSplitImpl::Builder builder1_1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -1151,20 +1208,26 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { TEST_P(ScanInteTest, TestScanAppendWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; { + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshot().ToString()) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_MODE] = StartupMode::FromSnapshot().ToString(); + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_NOK_WITH_MSG( table_scan->CreatePlan(), "scan.snapshot-id or scan.tag-name must be set when startup mode is FROM_SNAPSHOT"); } { + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::BUCKET, "-2").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + context_builder_options[Options::BUCKET] = "-2"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), context_builder_options), "do not support bucket=-2 in scan process"); } } @@ -1177,10 +1240,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEqualPredicate) { PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, val.data(), val.size())); + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1223,12 +1290,14 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { PredicateBuilder::And({greater_than_predicate, less_than_predicate, not_in_predicate, not_equal})); + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate) - .AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate).WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); DataSplitImpl::Builder builder1_2( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -1284,10 +1353,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilter) { std::map partition_keys; partition_keys["f1"] = "10"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1330,10 +1403,13 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithInvalidPartitionFilter) { std::map partition_keys; partition_keys["invalid_partition_key"] = "10"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), context_builder_options), "field invalid_partition_key does not exist in partition keys"); } @@ -1347,12 +1423,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilterAndPredicateF std::map partition_keys; partition_keys["f1"] = "10"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate) - .SetPartitionFilter({partition_keys}) - .AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate).SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1399,10 +1477,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) { partition_keys["f1"] = "10"; partition_keys["f2"] = "0"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1455,10 +1537,14 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter) Literal(paimon::Decimal(23, 5, 0))); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "4"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan); // check snapshot id @@ -1516,10 +1602,14 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) Literal(FieldType::BINARY, "zoo", 3)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "4"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -1571,12 +1661,16 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { std::string table_path = paimon::test::GetDataDir() + "orc/append_10_stats_dense_store.db/append_10_stats_dense_store"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(13.0)); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1668,6 +1762,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_10_stats_dense_store.db/append_10_stats_dense_store"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto greater_than = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(13.0)); @@ -1675,9 +1770,12 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { Literal(FieldType::STRING, "Emily", 5)); auto predicate = PredicateBuilder::And({greater_than, equal}).value(); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1714,10 +1812,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").SetBucketFilter(2); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetBucketFilter(2); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); ASSERT_TRUE(result_plan->Splits().empty()); @@ -1726,19 +1828,25 @@ TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::BUCKET, "-1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + context_builder_options[Options::BUCKET] = "-1"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context), context_builder_options), "do not support pk table bucket=-1 in scan process"); } TEST_P(ScanInteTest, TestReadWithNoSnapshot) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_nested_type.db/append_table_with_nested_type"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->SnapshotId()); ASSERT_TRUE(result_plan->Splits().empty()); @@ -1748,6 +1856,7 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_alter_table_with_cast.db/append_table_alter_table_with_cast"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto child1 = @@ -1770,9 +1879,12 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { FieldType::INT, Literal(80)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2, child3})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1813,6 +1925,7 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_alter_table.db/append_table_with_alter_table"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto child1 = PredicateBuilder::GreaterThan(/*field_index=*/2, /*field_name=*/"k", @@ -1821,9 +1934,12 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { Literal(96)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1891,6 +2007,7 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_alter_table_with_dense_field.db/" "append_table_with_alter_table_with_dense_field"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto child1 = PredicateBuilder::GreaterThan(/*field_index=*/2, /*field_name=*/"f1", @@ -1899,9 +2016,12 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { FieldType::DOUBLE, Literal(0.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child3})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1939,6 +2059,7 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap.db/append_with_bitmap/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto child1 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, @@ -1947,9 +2068,12 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { Literal(10)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -2016,6 +2140,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap.db/append_with_bitmap/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto child1 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, @@ -2024,9 +2149,12 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { Literal(10)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->Splits().empty()); } @@ -2035,6 +2163,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto child1 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, @@ -2043,9 +2172,12 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { Literal(10)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -2083,14 +2215,18 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); // file0 will be removed as f5 not exists auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f5", FieldType::INT, Literal(100)); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2152,15 +2288,19 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); auto child1 = PredicateBuilder::IsNull(/*field_index=*/3, /*field_name=*/"f5", FieldType::INT); auto child2 = PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2227,6 +2367,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); // in stats filter: predicate is trimmed as type for f1 is not consist: int->bigint @@ -2235,9 +2376,12 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, Literal(100l)); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2304,6 +2448,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); // child1 will remove file1 for schema1 @@ -2314,9 +2459,12 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { Literal(FieldType::STRING, "David", 5)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->Splits().empty()); } @@ -2324,11 +2472,14 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { TEST_P(ScanInteTest, TestScanAppendWithTag1) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_tag.db/append_table_with_tag"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_TAG_NAME, "1"); + context_builder_options[Options::SCAN_TAG_NAME] = "1"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(1, result_plan->SnapshotId().value()); @@ -2337,10 +2488,14 @@ TEST_P(ScanInteTest, TestScanAppendWithTag1) { TEST_P(ScanInteTest, TestScanInvalidTag) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_tag.db/append_table_with_tag"; + std::map context_builder_options; ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_TAG_NAME, "unknown"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_TAG_NAME] = "unknown"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), context_builder_options)); ASSERT_NOK_WITH_MSG(table_scan->CreatePlan(), "Tag 'unknown' doesn't exist."); } @@ -2353,38 +2508,54 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { // EarlierOrEqual(snap3.time) → snap-3 { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis())); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap3.TimeMillis()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 3); } // EarlierOrEqual(snap3.time - 1) → snap-2 { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis() - 1)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap3.TimeMillis() - 1); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 2); } // EarlierOrEqual(INT64_MAX) → snap-5 (latest) { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, - std::to_string(std::numeric_limits::max())); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = + std::to_string(std::numeric_limits::max()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 5); } // EarlierOrEqual(0) → no snapshot found, expect error { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, "0"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = "0"; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_NOK_WITH_MSG(scan->CreatePlan(), "There is currently no snapshot earlier than or equal to timestamp"); } @@ -2400,10 +2571,14 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { // T=0: no snapshot earlier than T=0, stream starts from snapshot 1. { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, "0").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = "0"; + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan1, scan->CreatePlan()); @@ -2411,11 +2586,14 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } // T=snap2.time+1: EarlierThan(snap2.time+1)=snap2, stream starts from snapshot 3. { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap2.TimeMillis() + 1)) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap2.TimeMillis() + 1); + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan1, scan->CreatePlan()); @@ -2423,11 +2601,14 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } // T=snap3.time: EarlierThan(snap3.time)=snap2, stream starts from snapshot 3. { + std::map builder_options; ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis())) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap3.TimeMillis()); + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan1, scan->CreatePlan()); @@ -2435,13 +2616,15 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } // T=INT64_MAX: EarlierThan(INT64_MAX)=snap5, stream starts from snapshot 6 (beyond latest). { + std::map builder_options; ScanContextBuilder builder(table_path); - builder - .AddOption(Options::SCAN_TIMESTAMP_MILLIS, - std::to_string(std::numeric_limits::max())) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = + std::to_string(std::numeric_limits::max()); + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_TRUE(plan0->Splits().empty()); diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 441c6d81f..bd3d9fd1b 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -154,13 +154,16 @@ class WriteAndReadInteTest Status CompactAndCommit(const std::string& table_path, const std::map& options, int64_t commit_identifier) const { + std::map write_context_builder_options; WriteContextBuilder write_context_builder(table_path, "commit_user"); write_context_builder.WithTempDirectory(LookupTempDirectory()); + write_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, + + write_context_builder.WithStreamingMode(true).Finish()); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr write_context, - write_context_builder.SetOptions(options).WithStreamingMode(true).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + std::unique_ptr file_store_write, + FileStoreWrite::Create(std::move(write_context), write_context_builder_options)); PAIMON_RETURN_NOT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); PAIMON_ASSIGN_OR_RAISE( @@ -170,11 +173,15 @@ class WriteAndReadInteTest if (compact_messages.empty()) { return Status::Invalid("expected compaction commit messages"); } + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user"); + commit_context_builder_options = options; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + + commit_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(std::move(commit_context), commit_context_builder_options)); return file_store_commit->Commit(compact_messages, commit_identifier); } @@ -185,10 +192,13 @@ class WriteAndReadInteTest std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames(read_fields); + read_context_builder_options = options; + read_context_builder.SetReadFieldNames(read_fields); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -208,10 +218,13 @@ class WriteAndReadInteTest auto c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, c_schema.get())); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -240,11 +253,14 @@ class WriteAndReadInteTest Result> InnerScan( const std::map& options) const { std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options).AddOption(Options::SCAN_MODE, - StartupMode::LatestFull().ToString()); + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); return table_scan->CreatePlan(); } @@ -442,16 +458,18 @@ TEST_P(WriteAndReadInteTest, TestAppendReadWithNestedPredicateAcrossBatches) { SCOPED_TRACE(batch_size); for (bool enable_filter : {false, true}) { SCOPED_TRACE(enable_filter); + std::map builder_options; ReadContextBuilder builder(table_path); - builder.SetOptions(options) - .SetReadFieldNames({"payload", "value", "id", "key"}) + builder_options = options; + builder_options[Options::READ_BATCH_SIZE] = std::to_string(batch_size); + builder.SetReadFieldNames({"payload", "value", "id", "key"}) .SetPredicate(predicate) .EnablePredicateFilter(enable_filter) .EnablePrefetch(false) - .EnableLateMaterializing(false) - .AddOption(Options::READ_BATCH_SIZE, std::to_string(batch_size)); + .EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); ASSERT_TRUE(actual); @@ -617,19 +635,24 @@ TEST_P(WriteAndReadInteTest, TestAppendVectorWithPredicate) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(2)); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetPredicate(predicate); + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -699,21 +722,26 @@ TEST_P(WriteAndReadInteTest, TestAppendWithExternalBitmapAndRangeBitmapIndexes) ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({name_predicate, score_predicate})); std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(1, plan->Splits().size()); // Keep precise post-read filtering disabled. The exact result therefore verifies that the // bitmap and range-bitmap indexes produced by the write path are consumed by the read path. + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetPredicate(predicate); + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -811,12 +839,12 @@ TEST_P(WriteAndReadInteTest, TestLuminaSearchAfterRenameAcrossSchemaVersions) { auto search = std::make_shared("vec", 3, std::vector{0.0f, 0.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); + std::map builder_options; ReadContextBuilder builder(path); - builder.SetOptions(current_options) - .SetReadFieldNames({"id", "_INDEX_SCORE"}) - .SetVectorSearch(search); + builder_options = current_options; + builder.SetReadFieldNames({"id", "_INDEX_SCORE"}).SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), @@ -908,13 +936,13 @@ TEST_P(WriteAndReadInteTest, TestLuminaVectorIndexAfterCompaction) { auto search = std::make_shared( "embedding", limit, std::vector{0.0f, 0.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); + std::map builder_options; ReadContextBuilder builder(path); - builder.SetOptions(options) - .SetReadFieldNames({"id", "_INDEX_SCORE"}) - .SetVectorSearch(search); + builder_options = options; + builder.SetReadFieldNames({"id", "_INDEX_SCORE"}).SetVectorSearch(search); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - TableRead::Create(std::move(context))); + TableRead::Create(std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr actual, @@ -1033,7 +1061,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithLuminaVectorIndex) { builder.SetReadFieldNames({"id", "_INDEX_SCORE"}).SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(context))); + TableRead::Create(std::move(context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -3290,11 +3318,13 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, literal_str.data(), literal_str.size())); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_FALSE(result_plan->Splits().empty()); @@ -3306,7 +3336,7 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { ReadContextBuilder read_context_builder(table_path); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3400,11 +3430,13 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, literal_str.data(), literal_str.size())); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_FALSE(result_plan->Splits().empty()); @@ -3415,7 +3447,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { ReadContextBuilder read_context_builder(table_path); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3486,12 +3518,14 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { {Literal(7), Literal(11), Literal(15)}); ASSERT_TRUE(predicate); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); @@ -3500,15 +3534,17 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { // therefore finds no data for it and skips ahead into the next row group it owns, whose first // selected row sits in the middle of that row group. Row level filtering stays off: the // expected rows below are exactly what page-index filtering selects. + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options) - .SetPredicate(predicate) + read_context_builder_options = options; + read_context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + read_context_builder.SetPredicate(predicate) .EnablePrefetch(true) .SetPrefetchMaxParallelNum(2) - .SetPrefetchBatchCount(3) - .AddOption("test.enable-adaptive-prefetch-strategy", "false"); + .SetPrefetchBatchCount(3); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3575,10 +3611,13 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { auto cache = std::make_shared(CacheKind::DATA_FILE_FOOTER, 128 * 1024 * 1024); auto read_once = [&]() -> Result { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); if (result_plan->SnapshotId() != std::optional(1)) { return Status::Invalid("unexpected snapshot id"); @@ -3590,7 +3629,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { ReadContextBuilder read_context_builder(table_path); read_context_builder.WithCache(cache); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), {})); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(result_plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3897,10 +3936,15 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { (void)helper; std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + std::map write_context_builder_options; WriteContextBuilder write_context_builder(table_path, "commit_user_1"); + write_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + + write_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(std::move(write_context), write_context_builder_options)); auto write_one_row = [&](const std::string& data) -> Status { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, @@ -3917,27 +3961,36 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { /*commit_identifier=*/0)); ASSERT_OK(file_store_write->Close()); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + commit_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + commit_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(commit_msgs, /*commit_identifier=*/0)); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(10)); + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetPredicate(predicate); + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4994,10 +5047,13 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingStructValueSchemaEvolutionRea auto read_fields = [&](const std::vector& field_names) -> Status { PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames(field_names); + read_context_builder_options = options; + read_context_builder.SetReadFieldNames(field_names); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp index efda3de03..aa0abb0d7 100644 --- a/test/inte/write_and_read_inte_with_index_test.cpp +++ b/test/inte/write_and_read_inte_with_index_test.cpp @@ -133,16 +133,17 @@ class WriteAndReadInteWithIndexTest const std::vector> splits, const std::shared_ptr& predicate, const std::shared_ptr& expected_array) const { + std::map context_builder_options; ReadContextBuilder context_builder(table_path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .SetPredicate(predicate) - .EnableLateMaterializing(false); + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder.SetPredicate(predicate).EnableLateMaterializing(false); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto result_array, @@ -1176,16 +1177,18 @@ TEST_P(WriteAndReadInteWithIndexTest, TestSimple) { &expected_array); ASSERT_TRUE(array_status.ok()); + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption("orc.read.enable-metrics", "true") - .SetPredicate(predicate); + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder.SetPredicate(predicate); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(split)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(batch_reader.get())); std::shared_ptr read_metrics = batch_reader->GetReaderMetrics(); @@ -1249,17 +1252,19 @@ TEST_P(WriteAndReadInteWithIndexTest, TestReadWithLimits) { auto predicate = PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption(Options::READ_BATCH_SIZE, "1") - .AddOption("orc.read.enable-metrics", "true") - .SetPredicate(predicate); + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options[Options::READ_BATCH_SIZE] = "1"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder.SetPredicate(predicate); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(split)); // simulate read limits, only read 3 batches std::vector batches; @@ -1533,17 +1538,18 @@ TEST_P(WriteAndReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, literal_str.data(), literal_str.size())); + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .SetPredicate(predicate) - .EnablePredicateFilter(true) - .EnableLateMaterializing(true); + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder.SetPredicate(predicate).EnablePredicateFilter(true).EnableLateMaterializing( + true); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(std::vector>{split})); ASSERT_OK_AND_ASSIGN(auto result_array, @@ -2934,15 +2940,17 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithIOException) { for (size_t i = 0; i < 200; i++) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + std::map context_builder_options; ReadContextBuilder context_builder(path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .SetPredicate(predicate); + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder.SetPredicate(predicate); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = + TableRead::Create(std::move(read_context), context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(split); CHECK_HOOK_STATUS(batch_reader.status(), i); diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index 31a325ec1..7e915e708 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -289,11 +289,14 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface const std::vector>& data_splits, const std::string& blob_field, const std::vector>& expected_blobs) const { + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames({blob_field}); + read_context_builder_options = options; + read_context_builder.SetReadFieldNames({blob_field}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(data_splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -353,7 +356,7 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface commit_builder.IgnoreEmptyCommit(ignore_empty_commit); PAIMON_ASSIGN_OR_RAISE(auto commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); return file_store_commit->Commit(commit_messages, commit_identifier); } @@ -684,7 +687,7 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithPartitionAndMultiBuckets) { /*is_streaming_mode=*/true)); int64_t commit_identifier = 0; ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); DataGenerator gen(table_schema.value(), pool_); @@ -994,7 +997,7 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { /*is_streaming_mode=*/true)); int64_t commit_identifier = 0; ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); // round 1 write @@ -1272,7 +1275,7 @@ TEST_P(WriteInteTest, TestPkTableBatchWrite) { /*is_streaming_mode=*/false)); int64_t commit_identifier = 0; ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); DataGenerator gen(table_schema.value(), pool_); @@ -1435,7 +1438,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) { /*is_streaming_mode=*/true)); int64_t commit_identifier = 0; ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); DataGenerator gen(table_schema.value(), pool_); @@ -1850,7 +1853,7 @@ TEST_P(WriteInteTest, TestPkTableForceLookup) { /*is_streaming_mode=*/true)); int64_t commit_identifier = 0; ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); DataGenerator gen(table_schema.value(), pool_); @@ -1914,7 +1917,7 @@ TEST_P(WriteInteTest, TestPkTableEnableDeletionVector) { /*is_streaming_mode=*/true)); int64_t commit_identifier = 0; ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); DataGenerator gen(table_schema.value(), pool_); @@ -2015,11 +2018,14 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { ASSERT_EQ(3, batches_2.size()); // write data + std::map context_builder_options; WriteContextBuilder context_builder(root_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); Result> write = - FileStoreWrite::Create(std::move(write_context)); + FileStoreWrite::Create(std::move(write_context), context_builder_options); CHECK_HOOK_STATUS(write.status(), i); auto& file_store_write = write.value(); // round 1 @@ -2451,12 +2457,15 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) { for (size_t i = 0; i < 200; i++) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); Result> write = - FileStoreWrite::Create(std::move(write_context)); + FileStoreWrite::Create(std::move(write_context), context_builder_options); CHECK_HOOK_STATUS(write.status(), i); auto& file_store_write = write.value(); @@ -2476,14 +2485,15 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) { for (size_t i = 0; i < 400; i += paimon::test::RandomNumber(5, 15)) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(false) - .Finish()); - auto commit = FileStoreCommit::Create(std::move(commit_context)); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + auto commit = + FileStoreCommit::Create(std::move(commit_context), commit_context_builder_options); CHECK_HOOK_STATUS(commit.status(), i); Result committed = commit.value()->FilterAndCommit({{0, commit_messages}}); CHECK_HOOK_STATUS(committed.status(), i); @@ -2545,12 +2555,15 @@ TEST_P(WriteInteTest, TestWriteWithFieldId) { /*partition_keys=*/{}, /*primary_keys=*/{}, options)); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + + context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ::ArrowArray c_array; ASSERT_TRUE(arrow::ExportArray(*src_array, &c_array).ok()); RecordBatchBuilder batch_builder(&c_array); @@ -2615,12 +2628,15 @@ TEST_P(WriteInteTest, TestAppendTableWriteAndReadWithExternalPath) { /*table_name=*/"append_table_with_external_path", &schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); // write + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + + context_builder.Finish()); std::string root_path = write_context->GetRootPath(); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ @@ -2652,15 +2668,21 @@ TEST_P(WriteInteTest, TestAppendTableWriteAndReadWithExternalPath) { ASSERT_TRUE(file_exist); // check read result + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); + read_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -2712,12 +2734,15 @@ TEST_P(WriteInteTest, TestPKTableWriteAndReadWithExternalPath) { /*table_name=*/"pk_table_with_external_path", &schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{"f0", "f1"}, options)); // write + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + + context_builder.Finish()); std::string root_path = write_context->GetRootPath(); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ @@ -2932,12 +2957,15 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { /*primary_keys=*/{}, options)); // write + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + + context_builder.Finish()); std::string root_path = write_context->GetRootPath(); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); auto write = [&](const std::string& json_array, const std::map& partition) -> void { std::shared_ptr array = @@ -2973,15 +3001,21 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { arrow::field("_VALUE_KIND", arrow::int8())); auto data_type = arrow::struct_(fields_with_row_kind); { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); + read_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3020,18 +3054,24 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + std::vector> partition_filters = { {{"dt", " "}, {"hr", "a=b?"}}}; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.SetPartitionFilter(partition_filters).Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); + read_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3047,18 +3087,24 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + std::vector> partition_filters = { {{"dt", ""}, {"hr", "a=b?"}}}; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.SetPartitionFilter(partition_filters).Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); + read_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3074,18 +3120,24 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } { + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + std::vector> partition_filters = { {{"dt", "__DEFAULT_PARTITION__"}, {"hr", "a=b?"}}}; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.SetPartitionFilter(partition_filters).Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context), + scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); + read_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3124,12 +3176,15 @@ TEST_P(WriteInteTest, TestWriteWithNestedSchema) { /*table_name=*/"bar", &schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options)); // write + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + + context_builder.Finish()); std::string root_path = write_context->GetRootPath(); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ @@ -3155,15 +3210,21 @@ TEST_P(WriteInteTest, TestWriteWithNestedSchema) { /*commit_identifier=*/1)); // check read result + std::map scan_context_builder_options; ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(std::move(scan_context), scan_context_builder_options)); + std::map read_context_builder_options; ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); + read_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3220,12 +3281,15 @@ TEST_P(WriteInteTest, TestWriteWithIOException) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); Result> write = - FileStoreWrite::Create(std::move(write_context)); + FileStoreWrite::Create(std::move(write_context), context_builder_options); CHECK_HOOK_STATUS(write.status(), i); auto& file_store_write = write.value(); @@ -3273,11 +3337,14 @@ TEST_P(WriteInteTest, TestCommitWithIOException) { /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); // write data + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); ASSERT_OK(file_store_write->Write(std::move(batch_0))); @@ -3293,14 +3360,15 @@ TEST_P(WriteInteTest, TestCommitWithIOException) { std::string tmp_table_path = tmp_dir->Str(); ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(tmp_table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(false) - .Finish()); - auto commit = FileStoreCommit::Create(std::move(commit_context)); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + auto commit = + FileStoreCommit::Create(std::move(commit_context), commit_context_builder_options); CHECK_HOOK_STATUS(commit.status(), i); Result committed = commit.value()->FilterAndCommit({{0, commit_messages}}); CHECK_HOOK_STATUS(committed.status(), i); @@ -3348,14 +3416,16 @@ TEST_P(WriteInteTest, TestWriteMemoryUse) { /*primary_keys=*/{}, options)); std::shared_ptr write_pool = GetMemoryPool(); { + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options) - .WithMemoryPool(write_pool) - .WithStreamingMode(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_context, + + context_builder.WithMemoryPool(write_pool).WithStreamingMode(true).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_write, + FileStoreWrite::Create(std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); ASSERT_OK(file_store_write->Write(std::move(batch_0))); @@ -3445,7 +3515,7 @@ TEST_P(WriteInteTest, TestPkTablePostponeBucket) { /*is_streaming_mode=*/false)); int64_t commit_identifier = 0; ASSERT_OK_AND_ASSIGN(std::optional> table_schema, - helper->LatestSchema()); + helper->GetSchema()); ASSERT_TRUE(table_schema); std::string data = R"([ @@ -3532,11 +3602,15 @@ TEST_F(WriteInteTest, TestBranchWrite) { std::map options = {{Options::FILE_FORMAT, "orc"}, {Options::FILE_SYSTEM, "local"}}; + std::map context_builder_options; WriteContextBuilder context_builder(table_path, "commit_user_1"); + context_builder_options = options; + context_builder_options[Options::BRANCH] = "rt"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithBranch("rt").Finish()); + + context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), context_builder_options)); // write to rt branch, pk table std::string data_0 = @@ -3577,11 +3651,15 @@ TEST_F(WriteInteTest, TestBranchWrite) { ASSERT_TRUE(expected_commit_message_0->TEST_Equal(*result_commit_msgs)); // write to main branch, append table + std::map context_builder1_options; WriteContextBuilder context_builder1(table_path, "commit_user_1"); + context_builder1_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context1, - context_builder1.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write1, - FileStoreWrite::Create(std::move(write_context1))); + + context_builder1.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_write1, + FileStoreWrite::Create(std::move(write_context1), context_builder1_options)); std::string data_1 = R"([["20240725", "watermelon", 3], ["20240725", "orange", 2] @@ -3719,7 +3797,7 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { builder.WithWriteSchema(write_cols); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(std::move(write_context), {})); ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); auto record_batch = std::make_unique( @@ -3752,7 +3830,7 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { CommitContextBuilder builder(table_path, commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context), {})); ASSERT_OK(file_store_commit->Commit(commit_msgs)); auto commit_impl = dynamic_cast(file_store_commit.get()); @@ -3768,7 +3846,7 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { ScanContextBuilder builder(table_path); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); auto data_split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -4144,13 +4222,14 @@ TEST_P(WriteInteTest, TestPkSpillableMapSharedShreddingReadWrite) { /*primary_keys=*/{"id"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); + std::map write_builder_options; WriteContextBuilder write_builder(table_path, "commit_user_1"); + write_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_builder.SetOptions(options) - .WithStreamingMode(true) - .WithTempDirectory(tmp_dir) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + + write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir).Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), write_builder_options)); ASSERT_OK_AND_ASSIGN( auto batch_0, TestHelper::MakeRecordBatch( @@ -4213,7 +4292,8 @@ TEST_P(WriteInteTest, TestPkSpillableDiskQuotaExhaustedFallsBackToFlush) { WriteContextBuilder write_builder(table_path, "commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); auto write_array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ ["Alice", 10, 1, 1.0], @@ -4269,7 +4349,8 @@ TEST_P(WriteInteTest, TestPkSpillableGlobalMemoryPreemptionDataCorrectness) { WriteContextBuilder write_builder(table_path, "commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); // Write long strings to different partitions to trigger global memory preemption. std::string long_str_a(48, 'a'); @@ -4367,7 +4448,8 @@ TEST_P(WriteInteTest, TestPkSpillableTempFilesCleanedAfterPrepareCommit) { WriteContextBuilder write_builder(table_path, "commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); auto batch1 = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([["Alice", 10, 1]])").ValueOrDie(); @@ -4431,7 +4513,8 @@ TEST_P(WriteInteTest, TestPkSpillableIntermediateMergeWithTempFileTracking) { WriteContextBuilder write_builder(table_path, "commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); auto write_array_fn = [](FileStoreWrite* writer, const std::map& partition, int32_t bucket, @@ -4514,7 +4597,8 @@ TEST_P(WriteInteTest, TestPkSpillableVector) { WriteContextBuilder write_builder(table_path, "commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); auto write_array = [](FileStoreWrite* writer, const std::shared_ptr& array) { ArrowArray c_array; @@ -4599,7 +4683,8 @@ TEST_P(WriteInteTest, TestPkSpillableMultiBucketMultiRoundDataCorrectness) { .WithTempDirectory(tmp_dir) .SetWriteBufferSpillThreadNumber(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context), {})); // Round 1: Bucket 0 writes Alice(dup) + Bob, Bucket 1 writes Dave(dup) + Eve auto r1_b0_batch1 = @@ -4787,14 +4872,16 @@ TEST_P(WriteInteTest, TestPkSpillableWithIOException) { // Write with spill enabled (WithTempDirectory) std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); + std::map context_builder_options; WriteContextBuilder context_builder(root_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options) - .WithStreamingMode(true) - .WithTempDirectory(tmp_dir) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_context, + + context_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir).Finish()); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(std::move(write_context), context_builder_options)); // Start IOHook only for spill write/read operations. io_hook->Reset(i, IOHook::Mode::RETURN_ERROR);