From 0d2ddf520957bf78fbd33ba9be4ede14b614af32 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 6 Nov 2026 11:19:00 +0800 Subject: [PATCH 1/3] refactor(table): introduce a common table hierarchy --- docs/source/user_guide/catalog.rst | 27 +- docs/source/user_guide/format_table.rst | 6 +- include/paimon/catalog/catalog.h | 11 +- include/paimon/catalog/table.h | 103 +- include/paimon/table/format/format_table.h | 28 +- src/paimon/CMakeLists.txt | 4 + src/paimon/core/catalog/catalog_utils.cpp | 12 + src/paimon/core/catalog/catalog_utils.h | 2 + .../core/catalog/file_system_catalog.cpp | 76 +- src/paimon/core/catalog/file_system_catalog.h | 2 + .../core/catalog/file_system_catalog_test.cpp | 11 +- .../core/operation/internal_read_context.cpp | 48 +- .../core/operation/internal_read_context.h | 18 +- src/paimon/core/operation/read_context.cpp | 11 +- .../core/operation/read_context_test.cpp | 26 +- src/paimon/core/operation/scan_context.cpp | 16 +- src/paimon/core/operation/write_context.cpp | 12 +- .../core/operation/write_context_test.cpp | 17 +- src/paimon/core/schema/table_schema.cpp | 43 +- src/paimon/core/schema/table_schema.h | 7 + .../table/append_only_file_store_table.cpp | 205 +++ .../core/table/append_only_file_store_table.h | 71 + src/paimon/core/table/file_store.cpp | 12 + src/paimon/core/table/file_store.h | 16 +- src/paimon/core/table/file_store_table.cpp | 679 +++++++++ src/paimon/core/table/file_store_table.h | 159 ++ src/paimon/core/table/format/format_table.cpp | 170 ++- .../core/table/format/format_table_test.cpp | 47 +- .../table/primary_key_file_store_table.cpp | 280 ++++ .../core/table/primary_key_file_store_table.h | 71 + .../core/table/source/table_read_test.cpp | 57 +- .../table/system/audit_log_system_table.cpp | 53 +- .../table/system/audit_log_system_table.h | 15 +- .../core/table/system/binlog_system_table.cpp | 13 +- .../core/table/system/binlog_system_table.h | 7 +- .../table/system/global_system_tables.cpp | 47 +- .../core/table/system/global_system_tables.h | 10 +- .../table/system/in_memory_system_table.cpp | 27 +- .../table/system/in_memory_system_table.h | 20 +- .../table/system/metadata_system_tables.cpp | 90 +- .../table/system/metadata_system_tables.h | 17 +- .../system/read_optimized_system_table.cpp | 58 +- .../system/read_optimized_system_table.h | 12 +- src/paimon/core/table/system/system_table.cpp | 149 +- src/paimon/core/table/system/system_table.h | 70 +- .../core/table/system/system_table_test.cpp | 15 +- src/paimon/core/table/table.cpp | 1327 ----------------- src/paimon/core/table/table_factory.cpp | 378 +++++ src/paimon/core/table/table_factory.h | 37 + src/paimon/core/table/table_test.cpp | 502 ++++++- src/paimon/core/utils/context_utils.h | 51 + src/paimon/rest/rest_catalog.cpp | 79 +- src/paimon/rest/rest_catalog.h | 4 + src/paimon/rest/rest_catalog_test.cpp | 63 +- src/paimon/testing/mock/mock_catalog.h | 18 +- test/inte/read_inte_test.cpp | 104 +- 56 files changed, 3653 insertions(+), 1760 deletions(-) create mode 100644 src/paimon/core/table/append_only_file_store_table.cpp create mode 100644 src/paimon/core/table/append_only_file_store_table.h create mode 100644 src/paimon/core/table/file_store_table.cpp create mode 100644 src/paimon/core/table/file_store_table.h create mode 100644 src/paimon/core/table/primary_key_file_store_table.cpp create mode 100644 src/paimon/core/table/primary_key_file_store_table.h create mode 100644 src/paimon/core/table/table_factory.cpp create mode 100644 src/paimon/core/table/table_factory.h create mode 100644 src/paimon/core/utils/context_utils.h diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 0592a7b74..684a990e7 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -158,12 +158,27 @@ offsets when a real-time writer is created: PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, paimon::FileStoreWrite::Create(std::move(write_context))); -The ``Table`` handle is an equivalent entry point: ``table->NewCommit()``, -``table->NewWrite()``, ``table->NewRead()`` and ``table->NewScan()`` each take the -same finished context and return the same operator as ``FileStoreCommit::Create()``, -``FileStoreWrite::Create()``, ``TableRead::Create()`` and ``TableScan::Create()``. -Pick whichever reads better; the context still carries the catalog when one is -needed. See the :ref:`Table API `. +``Catalog::GetTable()`` returns a common ``Table`` interface for managed, format and +system tables. The managed implementation selects append-only or primary-key behavior +from the loaded schema; system tables reject writes and commits. + +``table->NewCommit()``, ``NewWrite()``, ``NewRead()`` and ``NewScan()`` use the table's +bound type, path, schema and file system. ``GetSchema()`` returns that schema without +metadata I/O; reload through the catalog to refresh it. Contexts still provide execution +settings, dynamic options and optional catalog services. Branch or type options that +contradict a loaded managed table are rejected. The legacy static ``Create()`` methods +load the table from their context before delegating to these methods. +See the :ref:`Table API `. + +An explicit ``WithBranch()`` or branch-qualified identifier participates in the same +branch check. Omitting a branch allows the loaded table to supply it. The static +scan and read factories use an explicit context branch first, then the branch in +the schema options, and finally main. + +Global system tables under ``sys`` retain their catalog for deferred row loading. +Before calling ``GetTable()`` for one, place the catalog in a ``std::shared_ptr``, +as in the example above. ``LoadTableSchema()`` can inspect the schema without +retaining the catalog. For real-time writes, also set ``WithStreamingMode(true)`` and ``WithRealtimeContext(...)`` with ``realtime.enabled=true``. After diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 492b5f0d0..5bda38fcf 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -40,8 +40,10 @@ format table is an insert. Reading and writing ------------------- -A format table is not served through :cpp:func:`Catalog::GetTable`, which describes a managed -table; use :cpp:func:`Catalog::GetFormatTable` instead. +Use :cpp:func:`Catalog::GetTable` to load a format table through the common ``Table`` +interface. Its ``NewScan``, ``NewRead``, ``NewWrite`` and ``NewCommit`` methods use the +loaded schema, location and file system. :cpp:func:`Catalog::GetFormatTable` also returns +the concrete ``FormatTable`` interface for callers using the legacy context constructors. Reading and writing go through the entry points every other table uses: ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create`` and ``FileStoreCommit::Create``, each from its diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 69eb6187c..cd83bd219 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -48,7 +48,7 @@ class Tag; class Identifier; /// This interface is responsible for reading and writing metadata such as database/table from a /// paimon catalog. -class PAIMON_EXPORT Catalog { +class PAIMON_EXPORT Catalog : public std::enable_shared_from_this { public: static const char SYSTEM_DATABASE_NAME[]; static const char SYSTEM_TABLE_SPLITTER[]; @@ -146,7 +146,7 @@ class PAIMON_EXPORT Catalog { virtual Status RenameTable(const Identifier& from_table, const Identifier& to_table, bool ignore_if_not_exists) = 0; - /// Gets a table. + /// Gets a managed, format or read-only system table. /// /// @param identifier Identifier of the table to get. /// @return A result containing the table, or an error status. @@ -249,11 +249,8 @@ class PAIMON_EXPORT Catalog { /// Gets a format table: a directory of data files laid out like a standard Hive table. /// - /// A format table carries no snapshots and no manifests, so it is loaded through its own - /// method rather than `GetTable()`. Reading and writing it go through the same `TableScan`, - /// `TableRead`, `FileStoreWrite` and `FileStoreCommit` entry points every other table uses: - /// hand the table back to `ScanContextBuilder`, `ReadContextBuilder`, `WriteContextBuilder` or - /// `CommitContextBuilder`, each of which takes one. + /// Returns the concrete format-table interface. `GetTable()` also loads format tables + /// through the common Table interface. Legacy context builders accept this typed handle. /// /// @param identifier Identifier of the table to get. /// @return A result containing the format table, or an error status if the table does not diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index e542c9aee..aed1e34cc 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -33,50 +33,37 @@ namespace paimon { -class TableScan; +class CommitContext; +class FileStoreCommit; +class FileStoreWrite; +class ReadContext; +class ScanContext; class TableRead; +class TableScan; +class WriteContext; -/// A table provides basic abstraction for table type. +/// Common interface for managed, format and system tables. +/// Create operators with NewScan/NewRead/NewWrite/NewCommit; unsupported operations return errors. +/// Operators use this table's type, path, schema and file system, with context execution settings. class PAIMON_EXPORT Table { public: - /// @param file_system File system the table's data is accessed through. - /// @param root_path Root path of the table in the file system. - /// @param uuid Metastore UUID, or empty if unavailable. - Table(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = ""); - - ~Table() = default; + virtual ~Table() = default; - /// Returns the file system the table was opened with, which serves the table's - /// files. A catalog that hands out per-table file systems opens a table with the - /// one for that table; it is null when the opener had none. + /// Returns the table file system, or null if none was supplied. std::shared_ptr GetFileSystem() const { return file_system_; } - /// Returns the root path of the table in its file system. It is empty when the - /// opener had no location for the table, which a mock in a test is one case of. + /// Returns the table root path, or an empty string if unavailable. const std::string& GetRootPath() const { return root_path_; } - /// A name to identify this table. - std::string Name() const { - return identifier_.GetObjectName(); - } - - /// Full name of the table, default is database.tableName. - std::string FullName() const { - return identifier_.GetFullName(); - } + /// Returns the object name for data tables, or the system table name. + virtual std::string Name() const = 0; - /// Returns the branch the table was opened for, which its identifier carries; `main` - /// when it names none or a spelling of it, which its managers are bound to. A - /// malformed object name that hides the branch is reported. - Result CurrentBranch() const { - return identifier_.GetBranchNameOrDefault(); - } + /// Returns database.tableName for data tables, or Name() for system tables. + virtual std::string FullName() const = 0; /// Returns the metastore UUID, falling back to `FullName()`. /// @note Use `CatalogUuid()` for commit requests to avoid sending a table name as an ID. @@ -90,42 +77,42 @@ class PAIMON_EXPORT Table { return uuid_.empty() ? std::nullopt : std::optional(uuid_); } - /// Loads the latest schema of table. - std::shared_ptr LatestSchema() const { + /// Returns the schema bound to this table instance without metadata I/O. + /// Reload through the catalog to obtain the latest schema; this accessor does not refresh it. + std::shared_ptr GetSchema() const { return schema_; } - /// Creates a scan of this table from a finished scan context, the table-first entry point - /// that `TableScan::Create()` also builds through. The scan is derived from the context. - /// @param context The finished scan context. - /// @return The scan, or the error building it reports. - Result> NewScan(std::unique_ptr context) const; - - /// Creates a read of this table from a finished read context, the table-first entry point - /// that `TableRead::Create()` also builds through. The read is derived from the context. - /// @param context The finished read context. - /// @return The read, or the error building it reports. - Result> NewRead(std::unique_ptr context) const; - - /// Creates a write of this table from a finished write context, the table-first entry point - /// that `FileStoreWrite::Create()` also builds through. The write is derived from the context. - /// @param context The finished write context. - /// @return The write, or the error building it reports. - Result> NewWrite(std::unique_ptr context) const; - - /// Creates a commit of this table from a finished commit context, the table-first entry point - /// that `FileStoreCommit::Create()` also builds through. The commit is derived from the - /// context. - /// @param context The finished commit context. - /// @return The commit, or the error building it reports. - Result> NewCommit( - std::unique_ptr context) const; + /// Creates a scan using this table's schema and options. + virtual Result> NewScan( + std::unique_ptr context) const = 0; + + /// Creates a reader using this table's schema and options. + virtual Result> NewRead( + std::unique_ptr context) const = 0; + + /// Creates a writer using this table's schema and options. + virtual Result> NewWrite( + std::unique_ptr context) const = 0; + + /// Creates a commit using this table's schema and options. + virtual Result> NewCommit( + std::unique_ptr context) const = 0; + + protected: + /// @param file_system File system the table's data is accessed through. + /// @param root_path Root path of the table in the file system. + /// @param uuid Metastore UUID, or empty if unavailable. + Table(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid); - private: std::shared_ptr file_system_; std::string root_path_; std::shared_ptr schema_; Identifier identifier_; + + private: std::string uuid_; }; diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index a4c310254..917fcb46c 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -24,6 +24,7 @@ #include #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/result.h" #include "paimon/schema/schema.h" #include "paimon/status.h" @@ -53,7 +54,7 @@ class FileSystem; /// every context builder has, which is how a table whose schema lives in a metastore is reached. /// /// See `docs/source/user_guide/format_table.rst` for what is not supported yet. -class PAIMON_EXPORT FormatTable { +class PAIMON_EXPORT FormatTable : public Table { public: /// Formats a format table's files can be in. enum class Format { @@ -115,11 +116,18 @@ class PAIMON_EXPORT FormatTable { const std::shared_ptr& table, const std::map& dynamic_options); - ~FormatTable(); + ~FormatTable() override; + + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; + Result> NewWrite( + std::unique_ptr context) const override; + Result> NewCommit( + std::unique_ptr context) const override; /// Directory the data files live in. const std::string& Location() const { - return location_; + return root_path_; } /// Format of every data file in the directory. @@ -156,16 +164,16 @@ class PAIMON_EXPORT FormatTable { } /// A name to identify this table. - std::string Name() const { + std::string Name() const override { return identifier_.GetObjectName(); } /// Full name of the table, database.tableName. - std::string FullName() const; + std::string FullName() const override; /// Schema of the table, including its partition fields. - std::shared_ptr LatestSchema() const { - return schema_; + std::shared_ptr GetSchema() const { + return std::static_pointer_cast(schema_); } /// Schema of the table as an arrow schema, including its partition fields. @@ -186,16 +194,14 @@ class PAIMON_EXPORT FormatTable { } private: + Result> ForContext( + const std::map& options) const; FormatTable(const std::shared_ptr& file_system, const std::string& location, const Identifier& identifier, const std::shared_ptr& schema, const std::map& options, Format format, const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata); - std::shared_ptr file_system_; - std::string location_; - Identifier identifier_; - std::shared_ptr schema_; std::map options_; Format format_; std::string file_compression_; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 4a5077c31..f39b80698 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -450,6 +450,10 @@ set(PAIMON_CORE_SRCS core/stats/simple_stats_converter.cpp core/stats/simple_stats.cpp core/stats/simple_stats_evolution.cpp + core/table/append_only_file_store_table.cpp + core/table/primary_key_file_store_table.cpp + core/table/file_store_table.cpp + core/table/table_factory.cpp core/table/file_store.cpp core/table/table.cpp core/table/bucket_mode.cpp diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp index 1f2a3800a..587648943 100644 --- a/src/paimon/core/catalog/catalog_utils.cpp +++ b/src/paimon/core/catalog/catalog_utils.cpp @@ -26,6 +26,7 @@ #include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/options/table_type.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/defs.h" @@ -149,4 +150,15 @@ Result> CatalogUtils::LoadFormatTableInTwoRequests( location_carries_paimon_metadata, /*dynamic_options=*/{}); } +Identifier CatalogUtils::IdentifierFromPath(const std::string& table_path, + const std::string& branch) { + const std::string name = PathUtil::GetName(table_path); + const std::string parent = PathUtil::GetName(PathUtil::GetParentDirPath(table_path)); + const std::string suffix = Catalog::DB_SUFFIX; + if (StringUtils::EndsWith(parent, suffix)) { + return Identifier(parent.substr(0, parent.size() - suffix.size()), name, branch); + } + return Identifier(Identifier::kUnknownDatabase, name, branch); +} + } // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h index 565a6aef5..d222300e2 100644 --- a/src/paimon/core/catalog/catalog_utils.h +++ b/src/paimon/core/catalog/catalog_utils.h @@ -38,6 +38,8 @@ class CatalogUtils { CatalogUtils() = delete; ~CatalogUtils() = delete; + /// Derives an identifier for a table loaded directly from a path. + static Identifier IdentifierFromPath(const std::string& path, const std::string& branch); /// Returns whether `db_name` is the reserved system database "sys". static bool IsSystemDatabase(const std::string& db_name); diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 87c89de8c..4aeac4d53 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -50,6 +50,10 @@ class Schema; } // namespace arrow struct ArrowSchema; +#include "paimon/core/options/table_type.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/table/format/format_table.h" + namespace paimon { FileSystemCatalog::FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, @@ -334,16 +338,56 @@ Result> FileSystemCatalog::LoadTableSchema( return checked_pointer_cast(*latest_schema); } +Result> FileSystemCatalog::LoadSystemTable( + const Identifier& identifier) const { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + PAIMON_ASSIGN_OR_RAISE( + Identifier data_identifier, + CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + TableSchemaExists(data_identifier)); + if (!latest_schema) { + return Status::NotExist(fmt::format("{} not exist", data_identifier.ToString())); + } + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); + return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, + latest_schema.value(), dynamic_options, + data_identifier.GetObjectName(), {}); +} + Result> FileSystemCatalog::GetTable(const Identifier& identifier) const { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported( + identifier.GetObjectName(), catalog_options_)); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::shared_ptr self = weak_from_this().lock(); + if (self == nullptr) { + return Status::Invalid( + "a global system table retains its catalog, so the catalog must be owned by " + "a std::shared_ptr before it is asked for one"); + } + GlobalSystemTableContext context; + context.catalog = self.get(); + context.catalog_owner = std::move(self); + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = catalog_options_; + return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); + } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier)); - // A system table reads the files of the data table it belongs to, so its root path - // is that table's. - PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); - Identifier data_identifier(identifier.GetDatabaseName(), data_table_name); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); - return std::make_shared(fs_, table_path, schema, identifier); + return LoadSystemTable(identifier); } // The schema is read on the table's own branch: `root/branch/branch-` below the // table path, which `main` resolves to itself. `GetBranchNameOrDefault` folds any @@ -353,14 +397,20 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide if (!latest_schema) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - // Only a managed table is a `Table`; describing another type as one would hand back - // snapshots and manifests it never had, so the failure belongs to the call that opens the - // table rather than at the first read. + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); + std::shared_ptr data_schema = checked_pointer_cast(*latest_schema); + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(data_schema->Options())); + if (is_format_table) { + return FormatTable::Create(fs_, table_path, identifier, data_schema, + /*location_carries_paimon_metadata=*/true, + /*dynamic_options=*/{}); + } PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); - return std::make_shared
(fs_, table_path, checked_pointer_cast(*latest_schema), - identifier); + PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); + return FileStoreTable::Create(fs_, table_path, *latest_schema, identifier, /*uuid=*/"", branch, + {}); } Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index ccc45f666..37bb23ea6 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -33,6 +33,7 @@ struct ArrowSchema; namespace paimon { +class SystemTable; class TableSchema; class FileSystem; class Identifier; @@ -79,6 +80,7 @@ class FileSystemCatalog : public Catalog { const Identifier& identifier) const override; private: + Result> LoadSystemTable(const Identifier& identifier) const; /// Fails when `db_name` cannot be used as a single path component, so that the returned /// path always stays under `warehouse`. static Result NewDatabasePath(const std::string& warehouse, diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index f02af7be3..2409942fc 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -349,7 +349,7 @@ TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { ASSERT_TRUE(exists); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(branch_identifier)); - data_schema = std::dynamic_pointer_cast(table->LatestSchema()); + data_schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, data_schema); ASSERT_EQ((std::vector{"b1_c0"}), data_schema->FieldNames()); ASSERT_EQ(3, data_schema->Id()); @@ -679,7 +679,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { ASSERT_TRUE(typed_schema.Equals(loaded_schema)); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "tbl1"))); - ASSERT_OK_AND_ASSIGN(auto arrow_schema_from_get_table, table->LatestSchema()->GetArrowSchema()); + ASSERT_OK_AND_ASSIGN(auto arrow_schema_from_get_table, table->GetSchema()->GetArrowSchema()); auto schema_from_get_table = arrow::ImportSchema(arrow_schema_from_get_table.get()).ValueOrDie(); ASSERT_TRUE(typed_schema.Equals(schema_from_get_table)); @@ -693,7 +693,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { ArrowSchemaRelease(&schema); } -TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { +TEST(FileSystemCatalogTest, TestGetTableLoadsFormatTable) { std::map options; options[Options::FILE_SYSTEM] = "local"; options[Options::FILE_FORMAT] = "orc"; @@ -714,9 +714,8 @@ TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - // A format table has no snapshots and no manifests, so handing it back as a Table would - // describe it as something it is not. - ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db1", "fmt")), "Cannot open format table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog.GetTable(Identifier("db1", "fmt"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(opened)); ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, catalog.GetFormatTable(Identifier("db1", "fmt"))); ASSERT_EQ(format_table->GetFormat(), FormatTable::Format::PARQUET); diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 09661843e..89f2cd490 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -44,7 +44,9 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/common/utils/url_utils.h" #include "paimon/core/options/map_storage_layout.h" +#include "paimon/core/realtime/realtime_utils.h" #include "paimon/core/schema/arrow_schema_validator.h" +#include "paimon/core/schema/schema_manager.h" #include "paimon/core/utils/nested_projection_utils.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/full_text_search.h" @@ -279,6 +281,12 @@ Result> InternalReadContext::Create( PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, context->GetSpecificFileSystem(), context->GetFileSystemSchemeToIdentifierMap())); + return CreateFromCoreOptions(context, table_schema, std::move(core_options)); +} + +Result> InternalReadContext::CreateFromCoreOptions( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + CoreOptions core_options) { core_options.WithCache(context->GetCache()); if (context->EnablePreciseFileIndexEvaluate()) { if (!core_options.FileIndexReadEnabled()) { @@ -482,7 +490,8 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea table_schema_(table_schema), read_schema_(read_schema), arrow_pool_(arrow_pool), - options_(options) {} + options_(options), + fs_scheme_map_(read_context->GetFileSystemSchemeToIdentifierMap()) {} Result> InternalReadContext::CreateWithSchema( const std::shared_ptr& original, @@ -493,9 +502,44 @@ Result> InternalReadContext::CreateWithSche } // Create a new InternalReadContext sharing all properties except read_schema. // The new read_schema is the minimal column set for COUNT(*). - return std::shared_ptr( + auto result = std::shared_ptr( new InternalReadContext(original->read_context_, original->table_schema_, new_read_schema, original->options_, original->arrow_pool_)); + result->path_ = original->path_; + result->fs_scheme_map_ = original->fs_scheme_map_; + return result; +} + +Result> InternalReadContext::CreateForTable( + const std::shared_ptr& context, const std::shared_ptr& schema, + CoreOptions options, const std::string& root_path, + const std::map& fs_scheme_map) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr result, + CreateFromCoreOptions(context, schema, std::move(options))); + result->path_ = root_path; + result->fs_scheme_map_ = fs_scheme_map; + return result; +} +Result> InternalReadContext::CreateForFallbackBranch( + const std::shared_ptr& context, const std::string& branch, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& fs_scheme_map) { + if (context->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); + } + SchemaManager manager(file_system, root_path, branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> schema, manager.Latest()); + if (!schema) { + return Status::Invalid("fallback schema not found"); + } + auto options = schema.value()->Options(); + for (const auto& [key, value] : context->GetOptions()) { + options[key] = value; + } + options[Options::BRANCH] = branch; + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system)); + return CreateForTable(context, schema.value(), std::move(core_options), root_path, + fs_scheme_map); } } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 0f1c0f12b..dd13e0f13 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -61,6 +61,15 @@ class InternalReadContext { const std::shared_ptr& table_schema, const std::map& options); + static Result> CreateForTable( + const std::shared_ptr& context, const std::shared_ptr& schema, + CoreOptions options, const std::string& root_path, + const std::map& fs_scheme_map); + static Result> CreateForFallbackBranch( + const std::shared_ptr& context, const std::string& branch, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& fs_scheme_map); + /// Maps score output positions in the read schema to their logical search sources. const std::map& GetIndexScoreBindings() const { return index_score_bindings_; @@ -79,7 +88,7 @@ class InternalReadContext { return table_schema_; } const std::string& GetPath() const { - return read_context_->GetPath(); + return path_ ? *path_ : read_context_->GetPath(); } std::shared_ptr CreateDependencyCatalogContext() const; @@ -141,7 +150,7 @@ class InternalReadContext { /// The scheme map the file system of this read was resolved with, which a catalog reading a /// dependency table on its behalf needs to resolve that table's location the same way. const std::map& GetFileSystemSchemeToIdentifierMap() const { - return read_context_->GetFileSystemSchemeToIdentifierMap(); + return fs_scheme_map_; } bool ReadAheadCacheEnabled() const { @@ -163,6 +172,9 @@ class InternalReadContext { const std::shared_ptr& new_read_schema); private: + static Result> CreateFromCoreOptions( + const std::shared_ptr& context, const std::shared_ptr& schema, + CoreOptions options); InternalReadContext(const std::shared_ptr& read_context, const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, @@ -185,6 +197,8 @@ class InternalReadContext { std::shared_ptr read_schema_; std::shared_ptr arrow_pool_; CoreOptions options_; + std::optional path_; + std::map fs_scheme_map_; std::map index_score_bindings_; }; diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index f2bf754cf..d51b2eef7 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -27,6 +27,7 @@ #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/full_text_search.h" @@ -420,6 +421,14 @@ Result> ReadContextBuilder::Finish() { PAIMON_ASSIGN_OR_RAISE( std::string branch, BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, impl_->branch_, "read")); + std::map options = impl_->options_; + std::optional named_branch; + if (impl_->identifier_) { + PAIMON_ASSIGN_OR_RAISE(named_branch, impl_->identifier_->GetBranchName()); + } + if (impl_->branch_ || named_branch || options.count(Options::BRANCH) != 0) { + options[Options::BRANCH] = branch; + } if (impl_->enable_prefetch_ && impl_->prefetch_max_parallel_num_ == 0) { return Status::Invalid("prefetch max parallel num should be greater than 0"); } @@ -452,7 +461,7 @@ Result> ReadContextBuilder::Finish() { impl_->prefetch_batch_count_, impl_->prefetch_max_parallel_num_, impl_->enable_multi_thread_row_to_batch_, impl_->row_to_batch_thread_number_, impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_, - impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, impl_->options_, + impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, options, impl_->read_ahead_cache_enabled_, impl_->cache_config_, impl_->cache_, impl_->format_table_, impl_->warmup_level_, impl_->catalog_, impl_->identifier_); if (impl_->read_schema_ && impl_->read_schema_->release) { diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index 3d5040a8a..4bf1ce9da 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -40,6 +40,29 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { +TEST(ReadContextTest, TestFailedFinishDoesNotRetainImplicitBranchOption) { + for (bool explicit_option : {false, true}) { + SCOPED_TRACE(explicit_option); + ReadContextBuilder builder("table_root_path"); + builder.WithBranch("dev").EnablePrefetch(true).SetPrefetchMaxParallelNum(0).AddOption( + "custom.option", "value"); + if (explicit_option) { + builder.AddOption(Options::BRANCH, "dev"); + } + ASSERT_NOK_WITH_MSG(builder.Finish(), "prefetch max parallel num should be greater than 0"); + builder.WithBranch("main").SetPrefetchMaxParallelNum(1); + auto retried = builder.Finish(); + if (explicit_option) { + ASSERT_NOK_WITH_MSG(retried, "but both 'dev' and 'main' were named"); + } else { + ASSERT_OK(retried); + ASSERT_EQ("main", retried.value()->GetBranch()); + ASSERT_EQ("main", retried.value()->GetOptions().at(Options::BRANCH)); + ASSERT_EQ("value", retried.value()->GetOptions().at("custom.option")); + } + } +} + TEST(ReadContextTest, TestWithCatalogResolvesTableFileSystem) { // WithCatalog is a shorthand for WithFileSystem(catalog->GetTableFileSystem(identifier)); it // resolves the per-table file system when Finish() builds the context, which also keeps the @@ -197,7 +220,8 @@ TEST(ReadContextTest, TestSetContent) { ASSERT_EQ(128U, ctx->GetCacheConfig().GetHoleSizeLimit()); ASSERT_EQ(2048U, ctx->GetCacheConfig().GetPreBufferLimit()); ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); - std::map expected_options = {{"key", "value"}}; + std::map expected_options = {{"key", "value"}, + {Options::BRANCH, "rt"}}; ASSERT_EQ(expected_options, ctx->GetOptions()); ASSERT_EQ(ctx->GetSpecificFileSystem(), fs); ASSERT_TRUE(ctx->GetCache()); diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 89cccaa69..339538bb7 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -18,10 +18,13 @@ #include "paimon/scan_context.h" +#include #include #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" @@ -240,6 +243,17 @@ Result> ScanContextBuilder::Finish() { PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); } + PAIMON_ASSIGN_OR_RAISE( + std::string branch, + BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, std::nullopt, "scan")); + std::map options = impl_->options_; + std::optional named_branch; + if (impl_->identifier_) { + PAIMON_ASSIGN_OR_RAISE(named_branch, impl_->identifier_->GetBranchName()); + } + if (named_branch || options.count(Options::BRANCH) != 0) { + options[Options::BRANCH] = branch; + } std::shared_ptr executor = impl_->executor_ ? impl_->executor_ : CreateDefaultExecutor(); auto ctx = std::make_unique( @@ -247,7 +261,7 @@ Result> ScanContextBuilder::Finish() { std::make_shared(impl_->predicates_, impl_->partition_filters_, impl_->bucket_filter_), impl_->global_index_result_, impl_->realtime_context_, impl_->memory_pool_, executor, - impl_->specific_file_system_, impl_->table_schema_, impl_->options_, impl_->cache_, + impl_->specific_file_system_, impl_->table_schema_, options, impl_->cache_, impl_->format_table_, impl_->catalog_, impl_->identifier_); impl_->Reset(); return ctx; diff --git a/src/paimon/core/operation/write_context.cpp b/src/paimon/core/operation/write_context.cpp index e31d64ac4..2fdf0dab2 100644 --- a/src/paimon/core/operation/write_context.cpp +++ b/src/paimon/core/operation/write_context.cpp @@ -18,6 +18,7 @@ #include "paimon/write_context.h" +#include #include #include "arrow/util/thread_pool.h" @@ -26,6 +27,7 @@ #include "paimon/common/utils/path_util.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" @@ -263,6 +265,14 @@ Result> WriteContextBuilder::Finish() { PAIMON_ASSIGN_OR_RAISE( std::string branch, BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, impl_->branch_, "write")); + std::map options = impl_->options_; + std::optional named_branch; + if (impl_->identifier_) { + PAIMON_ASSIGN_OR_RAISE(named_branch, impl_->identifier_->GetBranchName()); + } + if (impl_->branch_ || named_branch || options.count(Options::BRANCH) != 0) { + options[Options::BRANCH] = branch; + } if (impl_->catalog_ != nullptr && !BranchManager::IsMainBranch(branch)) { // Refused here rather than at the commit of what is written to the branch. PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); @@ -286,7 +296,7 @@ Result> WriteContextBuilder::Finish() { impl_->ignore_num_bucket_check_, impl_->ignore_previous_files_, enable_multi_thread_spill, impl_->write_id_, branch, impl_->write_schema_, impl_->memory_pool_, impl_->executor_, impl_->temp_directory_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, - impl_->realtime_context_, impl_->options_, impl_->format_table_, impl_->catalog_, + impl_->realtime_context_, options, impl_->format_table_, impl_->catalog_, impl_->identifier_); impl_->Reset(); return ctx; diff --git a/src/paimon/core/operation/write_context_test.cpp b/src/paimon/core/operation/write_context_test.cpp index 5e777c351..337254482 100644 --- a/src/paimon/core/operation/write_context_test.cpp +++ b/src/paimon/core/operation/write_context_test.cpp @@ -33,6 +33,20 @@ namespace paimon::test { +TEST(WriteContextTest, TestFailedFinishDoesNotRetainImplicitBranchOption) { + auto catalog = std::make_shared(); + WriteContextBuilder builder("table_root_path", "user"); + builder.WithCatalog(catalog, Identifier("db", "tbl")) + .WithBranch("dev") + .AddOption("custom.option", "value"); + ASSERT_NOK_WITH_MSG(builder.Finish(), "addresses a branch by the table identifier"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + builder.WithBranch("main").Finish()); + ASSERT_EQ("main", context->GetBranch()); + ASSERT_EQ("main", context->GetOptions().at(Options::BRANCH)); + ASSERT_EQ("value", context->GetOptions().at("custom.option")); +} + TEST(WriteContextTest, TestDefaultValue) { WriteContextBuilder builder("table_root_path", "commit_user_1"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); @@ -187,7 +201,8 @@ TEST(WriteContextTest, TestSetContent) { ASSERT_EQ(ctx->GetWriteSchema(), write_schema); ASSERT_EQ(ctx->GetFileSystemSchemeToIdentifierMap(), fs_scheme_to_identifier_map); ASSERT_EQ(ctx->GetSpecificFileSystem(), file_system); - std::map expected_options = {{"key", "value"}}; + std::map expected_options = {{"key", "value"}, + {Options::BRANCH, "test_branch"}}; ASSERT_EQ(expected_options, ctx->GetOptions()); } diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index 1d69c033a..1c0cfd333 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -194,6 +194,36 @@ TableSchema::TableSchema(int32_t version, int64_t id, const std::vector> TableSchema::MergeOptions( + const std::map& dynamic_options) const { + std::map options = options_; + for (const auto& [key, value] : dynamic_options) { + options[key] = value; + } + return Copy(options); +} + +Result> TableSchema::Copy( + const std::map& options) const { + auto merged = std::unique_ptr( + new TableSchema(version_, id_, fields_, highest_field_id_, partition_keys_, primary_keys_, + options, comment_, time_millis_)); + PAIMON_RETURN_NOT_OK(merged->DeriveBucketInfo()); + return merged; +} + +Status TableSchema::DeriveBucketInfo() { + PAIMON_ASSIGN_OR_RAISE(std::vector keys, TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, OriginalBucketKeys()); + if (bucket_keys.empty()) { + bucket_keys = keys; + } + bucket_keys_ = bucket_keys; + PAIMON_ASSIGN_OR_RAISE(num_bucket_, + OptionsUtils::GetValueFromMap(Options(), Options::BUCKET, -1)); + return Status::OK(); +} + bool TableSchema::operator==(const TableSchema& other) const { return version_ == other.version_ && fields_ == other.fields_ && partition_keys_ == other.partition_keys_ && primary_keys_ == other.primary_keys_ && @@ -332,18 +362,7 @@ Result> TableSchema::InitSchema( auto table_schema = std::unique_ptr( new TableSchema(TableSchema::CURRENT_VERSION, schema_id, fields, highest_field_id, partition_keys, primary_keys, options, comment, time_millis)); - PAIMON_ASSIGN_OR_RAISE(std::vector keys, table_schema->TrimmedPrimaryKeys()); - - // Try to validate bucket keys - PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, - table_schema->OriginalBucketKeys()); - if (bucket_keys.empty()) { - bucket_keys = keys; - } - table_schema->bucket_keys_ = bucket_keys; - PAIMON_ASSIGN_OR_RAISE( - table_schema->num_bucket_, - OptionsUtils::GetValueFromMap(table_schema->Options(), Options::BUCKET, -1)); + PAIMON_RETURN_NOT_OK(table_schema->DeriveBucketInfo()); return table_schema; } diff --git a/src/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h index eab109cf1..9b6717446 100644 --- a/src/paimon/core/schema/table_schema.h +++ b/src/paimon/core/schema/table_schema.h @@ -52,6 +52,11 @@ class TableSchema : public DataSchema, public Jsonizable { const std::vector& primary_keys, const std::map& options); + Result> Copy( + const std::map& options) const; + Result> MergeOptions( + const std::map& options) const; + static Result> CreateFromJson(const std::string& json_str); /// Computes the highest non-system field id across all nesting levels of a "fields" JSON @@ -149,6 +154,8 @@ class TableSchema : public DataSchema, public Jsonizable { const std::shared_ptr& field, int32_t field_id); private: + Status DeriveBucketInfo(); + // version of schema for paimon int32_t version_ = -1; int64_t id_ = -1; diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp new file mode 100644 index 000000000..2a2110ef7 --- /dev/null +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/append_only_file_store_table.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/operation/append_only_file_store_scan.h" +#include "paimon/core/operation/append_only_file_store_write.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/data_evolution_file_store_scan.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/source/append_only_split_generator.h" +#include "paimon/core/table/source/data_evolution_split_generator.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { + +AppendOnlyFileStoreTable::AppendOnlyFileStoreTable(std::unique_ptr store, + const std::shared_ptr& schema, + const Identifier& identifier, + const std::string& uuid) + : FileStoreTable(std::move(store), schema, identifier, uuid) {} + +Result> AppendOnlyFileStoreTable::CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const { + const std::shared_ptr& file_store_path_factory = path_factory; + auto schema = std::static_pointer_cast(schema_); + auto schema_manager = GetSchemaManager(); + const std::string& branch = CurrentBranch(); + FileStore file_store(root_path_, options); + + // append table + const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); + bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); + if (options.GetBucket() == -1) { + need_dv_maintainer_factory = false; + ignore_previous_files = true; + } else if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in append table", options.GetBucket())); + } + if (ctx.GetRealtimeContext()) { + if (!ctx.GetWriteSchema().empty()) { + return Status::Invalid("real-time append write does not support a custom schema"); + } + if (!realtime_deduplicate) { + PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx.GetRealtimeContext(), + snapshot_manager, options)); + } + } else if (realtime_deduplicate) { + if (!ctx.GetWriteSchema().empty() || ignore_previous_files || options.WriteOnly()) { + return Status::Invalid( + "deduplicate compaction requires the full schema, restored files and " + "write-only=false"); + } + } + std::shared_ptr write_schema = arrow_schema; + const auto& write_field_names = ctx.GetWriteSchema(); + if (!write_field_names.empty()) { + arrow::FieldVector write_fields; + write_fields.reserve(write_field_names.size()); + for (const auto& field_name : write_field_names) { + auto field = arrow_schema->GetFieldByName(field_name); + if (!field) { + // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER + return Status::Invalid( + fmt::format("write field {} does not exist in table schema", field_name)); + } + write_fields.push_back(field); + } + write_schema = arrow::schema(write_fields); + } + + std::shared_ptr realtime_schema_layout; + if (ctx.GetRealtimeContext()) { + const RealtimeStoreMode mode = + realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE : RealtimeStoreMode::APPEND_ONLY; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, + RealtimeSchemaLayout::Create(mode, write_schema)); + realtime_schema_layout = std::move(layout); + } + + std::shared_ptr dv_maintainer_factory; + if (need_dv_maintainer_factory) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr index_file_handler, + file_store.NewIndexFileHandler(ResolveBucketMode(options.GetBucket(), schema), + file_store_path_factory, ctx.GetMemoryPool())); + dv_maintainer_factory = std::make_shared(index_file_handler); + } + + auto file_store_write = std::make_unique( + file_store_path_factory, snapshot_manager, schema_manager, ctx.GetCommitUser(), root_path_, + schema, arrow_schema, write_schema, realtime_schema_layout, partition_schema, + dv_maintainer_factory, io_manager, options, ignore_previous_files, ctx.IsStreamingMode(), + ctx.IgnoreNumBucketCheck(), ctx.GetRealtimeContext(), ctx.GetExecutor(), + ctx.GetMemoryPool()); + if (ctx.GetRealtimeContext() && realtime_deduplicate) { + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_RETURN_NOT_OK(file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); + } + } + return std::unique_ptr(std::move(file_store_write)); +} + +Result> AppendOnlyFileStoreTable::NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const { + // TODO(liancheng.lsz): support fallback branch in scan + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + std::unique_ptr scan; + if (core_options.DataEvolutionEnabled()) { + PAIMON_ASSIGN_OR_RAISE( + scan, DataEvolutionFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), + manifest_list, manifest_file, table_schema, + arrow_schema, context->GetScanFilters(), + core_options, executor, memory_pool)); + } else { + PAIMON_ASSIGN_OR_RAISE( + scan, AppendOnlyFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), + manifest_list, manifest_file, table_schema, + arrow_schema, context->GetScanFilters(), + core_options, executor, memory_pool)); + } + return WithTablePath(std::move(scan), root_path_); +} + +Result> AppendOnlyFileStoreTable::NewSplitGenerator( + const CoreOptions& core_options) const { + auto source_split_target_size = core_options.GetSourceSplitTargetSize(); + auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); + if (core_options.DataEvolutionEnabled()) { + return std::make_unique(source_split_target_size, + source_split_open_file_cost, + core_options.BlobSplitByFileSize()); + } + BucketMode bucket_mode = + (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE : BucketMode::HASH_FIXED); + return std::make_unique(source_split_target_size, + source_split_open_file_cost, bucket_mode); +} + +CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) const { + const std::shared_ptr& schema_manager = GetSchemaManager(); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h new file mode 100644 index 000000000..a83632dfd --- /dev/null +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class FileStoreScan; +class IOManager; +class SplitGenerator; +class TableSchema; + +/// Managed table without primary keys; uses the shared FileStoreTable operator flows. +class AppendOnlyFileStoreTable : public FileStoreTable { + public: + AppendOnlyFileStoreTable(std::unique_ptr store, + const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid); + + private: + Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const override; + + Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const override; + + Result> NewSplitGenerator( + const CoreOptions& core_options) const override; + + CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index 8f1c6d7a8..68bf717de 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -34,6 +34,7 @@ #include "paimon/core/manifest/index_manifest_file.h" #include "paimon/core/manifest/manifest_file.h" #include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/schema/schema_manager.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/index_file_path_factories.h" @@ -46,6 +47,17 @@ namespace paimon { FileStore::FileStore(std::string root_path, CoreOptions options) : root_path_(std::move(root_path)), options_(std::move(options)) {} +std::string FileStore::CurrentBranch() const { + return BranchManager::NormalizeBranch(options_.GetBranch()); +} + +std::shared_ptr FileStore::GetSchemaManager() const { + return std::make_shared(options_.GetFileSystem(), root_path_, CurrentBranch()); +} +std::shared_ptr FileStore::GetSnapshotManager() const { + return std::make_shared(options_.GetFileSystem(), root_path_, CurrentBranch()); +} + Result> FileStore::NewPathFactory( const std::shared_ptr& arrow_schema, const std::vector& partition_keys, const std::string& file_format_identifier, diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 521a7bc39..0c536132d 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -43,12 +43,26 @@ class IndexManifestFile; class ManifestFile; class ManifestList; class SnapshotManager; +class SchemaManager; -/// Component factories for one managed-table operation using already resolved options. +/// Managed-table storage configuration and component factories using resolved options. class FileStore { public: FileStore(std::string root_path, CoreOptions options); + const CoreOptions& GetOptions() const { + return options_; + } + const std::string& GetRootPath() const { + return root_path_; + } + std::string CurrentBranch() const; + std::shared_ptr GetFileSystem() const { + return options_.GetFileSystem(); + } + std::shared_ptr GetSchemaManager() const; + std::shared_ptr GetSnapshotManager() const; + Result> NewPathFactory( const std::shared_ptr& arrow_schema, const std::vector& partition_keys, const std::string& file_format_identifier, diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp new file mode 100644 index 000000000..d4255fd71 --- /dev/null +++ b/src/paimon/core/table/file_store_table.cpp @@ -0,0 +1,679 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/file_store_table.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" +#include "paimon/common/predicate/predicate_validator.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/binary_row_partition_computer.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/sensitive_config_utils.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_snapshot_commit.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/commit_lock.h" +#include "paimon/core/catalog/renaming_snapshot_commit.h" +#include "paimon/core/catalog/snapshot_commit.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/disk/io_manager.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/operation/commit/realtime_commit_properties.h" +#include "paimon/core/operation/expire_snapshots.h" +#include "paimon/core/operation/file_store_commit_impl.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/internal_read_context.h" +#include "paimon/core/options/table_type.h" +#include "paimon/core/realtime/realtime_context_impl.h" +#include "paimon/core/realtime/realtime_utils.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/append_only_file_store_table.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/primary_key_file_store_table.h" +#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/source/data_evolution_batch_scan.h" +#include "paimon/core/table/source/data_table_batch_scan.h" +#include "paimon/core/table/source/data_table_stream_scan.h" +#include "paimon/core/table/source/fallback_table_read.h" +#include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/table/source/primary_key_index_batch_scan.h" +#include "paimon/core/table/source/read_optimized_scan_options.h" +#include "paimon/core/table/source/realtime_table_scan.h" +#include "paimon/core/table/source/snapshot/snapshot_reader.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/primary_key_table_utils.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/format/file_format.h" +#include "paimon/fs/file_system.h" +#include "paimon/read_context.h" +#include "paimon/realtime/realtime_context.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +namespace { + +/// Allows object-store commits in integration tests. +constexpr const char kObjectStoreCommitInteTestOption[] = "enable-object-store-commit-in-inte-test"; + +Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, + const ScanContext& context, bool read_optimized) { + if (!context.GetRealtimeContext()) { + return Status::OK(); + } + if (!core_options.RealtimeEnabled()) { + return Status::Invalid("real-time scan requires realtime.enabled=true"); + } + if (core_options.GetBucket() <= 0) { + return Status::Invalid("real-time union read requires fixed bucket mode"); + } + if (core_options.DataEvolutionEnabled()) { + return Status::Invalid("real-time union read does not support data evolution"); + } + if (!table_schema.PrimaryKeys().empty()) { + if (read_optimized) { + return Status::NotImplemented( + "PK real-time union read does not support read-optimized scans"); + } + PAIMON_RETURN_NOT_OK( + PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); + } + if (context.IsStreamingMode()) { + return Status::Invalid("real-time union read currently supports batch scans only"); + } + if (context.GetLimit()) { + return Status::Invalid("real-time union read does not support scan limit pushdown"); + } + if (context.GetGlobalIndexResult()) { + return Status::Invalid("real-time union read does not support global index splits"); + } + StartupMode startup_mode = core_options.GetStartupMode(); + const bool specified_snapshot = + startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); + if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || + specified_snapshot)) { + return Status::Invalid( + "real-time union read requires the latest snapshot or an explicit snapshot id"); + } + return Status::OK(); +} + +bool IsCatalogCommit(const CommitContext& ctx) { + return AsVersionManaged(ctx.GetCatalog()) != nullptr; +} + +Result> NewSnapshotCommit( + const CommitContext& ctx, const std::optional& catalog_identifier, + const CoreOptions& options, const std::shared_ptr& snapshot_manager) { + if (IsCatalogCommit(ctx)) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr lock, + AsVersionManaged(ctx.GetCatalog()) + ->CreateCommitLock(catalog_identifier.value(), ctx.GetTableId(), + ctx.GetCommitUser(), options)); + // Reuse the caller's UUID so table recreation cannot redirect this commit. + return std::shared_ptr(std::make_shared( + ctx.GetCatalog(), catalog_identifier.value(), ctx.GetTableId(), std::move(lock))); + } + if (ctx.UseRESTCatalogCommit()) { + if (options.RestCommitAcquireLock()) { + return Status::Invalid(fmt::format( + "'{}' needs a commit through a REST catalog with WithCatalog(); a commit table " + "request built for the caller to send cannot hold a commit lease", + Options::REST_COMMIT_ACQUIRE_LOCK)); + } + return std::shared_ptr( + std::make_shared(ctx.GetTableId())); + } + // Catalogs without version management may still supply a table ID. + if (ctx.GetTableId() && ctx.GetCatalog() == nullptr) { + return Status::Invalid( + "a commit which publishes the snapshot by writing it to the table directory sends no " + "commit table request, so there is nowhere to put a table id; commit through a " + "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); + } + return std::shared_ptr( + std::make_shared(options.GetFileSystem(), snapshot_manager)); +} + +} // namespace + +class Executor; +class MemoryPool; + +FileStoreTable::FileStoreTable(std::unique_ptr store, + const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid) + : Table(store->GetFileSystem(), store->GetRootPath(), schema, identifier, uuid), + store_(std::move(store)) {} + +Result> FileStoreTable::Create( + const std::shared_ptr& specified_file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid, const std::string& branch, + const std::map& fs_scheme_map) { + if (schema == nullptr) { + return Status::Invalid( + fmt::format("not found latest schema of {}", identifier.GetFullName())); + } + PAIMON_RETURN_NOT_OK( + CatalogUtils::CheckManagedTableType(identifier, schema, "FileStoreTable::Create")); + PAIMON_RETURN_NOT_OK(BranchManager::CheckValidBranch(branch)); + std::map options = schema->Options(); + const std::string normalized_branch = BranchManager::NormalizeBranch(branch); + if (BranchManager::IsMainBranch(normalized_branch)) { + options.erase(Options::BRANCH); + } else { + options[Options::BRANCH] = normalized_branch; + } + std::shared_ptr effective_schema = schema; + if (options != schema->Options()) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr normalized_schema, + schema->Copy(options)); + effective_schema = std::shared_ptr(std::move(normalized_schema)); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(options, specified_file_system, fs_scheme_map)); + std::unique_ptr store = + std::make_unique(root_path, std::move(core_options)); + return Create(std::move(store), effective_schema, identifier, uuid, fs_scheme_map); +} + +Result> FileStoreTable::Create( + std::unique_ptr store, const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid, + const std::map& fs_scheme_map) { + if (!store || !schema) { + return Status::Invalid("managed table requires a store and schema"); + } + PAIMON_RETURN_NOT_OK( + CatalogUtils::CheckManagedTableType(identifier, schema, "FileStoreTable::Create")); + PAIMON_RETURN_NOT_OK(BranchManager::CheckValidBranch(store->CurrentBranch())); + std::shared_ptr table; + if (!schema->PrimaryKeys().empty()) { + table = + std::make_shared(std::move(store), schema, identifier, uuid); + } else { + table = + std::make_shared(std::move(store), schema, identifier, uuid); + } + table->fs_scheme_map_ = fs_scheme_map; + return table; +} + +Result FileStoreTable::ResolveOptions( + const std::map& overrides) const { + if (overrides.empty()) { + return store_->GetOptions(); + } + auto options = store_->GetOptions().ToMap(); + for (const auto& [key, value] : overrides) { + options[key] = value; + } + auto branch = overrides.find(Options::BRANCH); + if (branch != overrides.end() && + BranchManager::NormalizeBranch(branch->second) != CurrentBranch()) { + return Status::Invalid("context branch contradicts the loaded table"); + } + PAIMON_ASSIGN_OR_RAISE(TableType type, TableTypeDefine::FromOptions(options)); + PAIMON_ASSIGN_OR_RAISE(TableType bound_type, + TableTypeDefine::FromOptions(store_->GetOptions().ToMap())); + if (type != bound_type) { + return Status::Invalid("context type contradicts the loaded table"); + } + return CoreOptions::FromMap(options, file_system_, fs_scheme_map_); +} + +Status FileStoreTable::RestoreRealtimeCommittedProgress( + const std::shared_ptr& realtime_context, + const std::shared_ptr& snapshot_manager, const CoreOptions& options) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, + RealtimeContextImpl::Cast(realtime_context)); + PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_ASSIGN_OR_RAISE( + RealtimeOffsetMap realtime_committed_offsets, + RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); + PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( + latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); + } + return Status::OK(); +} + +Status FileStoreTable::CheckContextNamesThisTable( + const std::string& kind, const std::string& context_root_path, + const std::optional& context_identifier, + const std::optional& context_table_id) const { + if (!root_path_.empty()) { + PAIMON_ASSIGN_OR_RAISE(std::string context_path, + PathUtil::NormalizePath(context_root_path)); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, PathUtil::NormalizePath(root_path_)); + if (context_path != table_path) { + return Status::Invalid(fmt::format( + "the {} context names root path '{}' but this table serves '{}': the store below " + "reads and writes this table's files, so a context naming another path would " + "leave this table's metadata there and publish it as the other's", + kind, context_root_path, root_path_)); + } + } + if (context_identifier) { + PAIMON_ASSIGN_OR_RAISE(std::optional context_branch, + context_identifier.value().GetBranchName()); + if (context_branch && context_branch.value() != CurrentBranch()) { + return Status::Invalid(fmt::format( + "the {} context names branch '{}' of '{}' but this table serves branch '{}', " + "which is the one a {} of this table writes and publishes", + kind, context_branch.value(), context_identifier.value().ToString(), + CurrentBranch(), kind)); + } + PAIMON_ASSIGN_OR_RAISE( + Identifier context_table, + CatalogUtils::BranchIdentifier(context_identifier.value(), CurrentBranch())); + PAIMON_ASSIGN_OR_RAISE(Identifier served_table, + CatalogUtils::BranchIdentifier(identifier_, CurrentBranch())); + const bool names_this_table = + (served_table.GetDatabaseName() == Identifier::kUnknownDatabase || + context_table.GetDatabaseName() == served_table.GetDatabaseName()) && + context_table.GetObjectName() == served_table.GetObjectName(); + if (!names_this_table) { + return Status::Invalid(fmt::format( + "the {} context names table '{}' but this table is '{}': the store below serves " + "this table's files, so a context naming another table would write this table's " + "metadata and publish it as the other's", + kind, context_identifier.value().ToString(), identifier_.ToString())); + } + } + if (context_table_id && context_table_id != CatalogUuid()) { + return Status::Invalid(fmt::format( + "the {} context carries table id '{}' but this table's is '{}': the id tells a " + "recreated table from the one the changes were prepared for, so one this table does " + "not carry would publish this table's metadata as the recreation's", + kind, context_table_id.value(), CatalogUuid().value_or("none"))); + } + return Status::OK(); +} + +std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr&& scan, + const std::string& root_path) { + scan->WithTablePath(root_path); + return std::move(scan); +} + +Result> FileStoreTable::NewWrite( + std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("write", ctx->GetRootPath(), + ctx->GetIdentifier(), + /*context_table_id=*/std::nullopt)); + if (AsVersionManaged(ctx->GetCatalog()) != nullptr) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && + ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + std::optional catalog_identifier; + if (ctx->GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE(Identifier identifier, CatalogUtils::BranchIdentifier( + *ctx->GetIdentifier(), CurrentBranch())); + catalog_identifier.emplace(std::move(identifier)); + } + if (ctx->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(CurrentBranch())); + } + std::shared_ptr schema = std::static_pointer_cast(schema_); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(ctx->GetOptions())); + FileStore file_store(root_path_, options); + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); + if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) { + return Status::NotImplemented( + "Dedicated VECTOR storage does not yet support custom write schemas or partial writes"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr file_store_path_factory, + file_store.NewPathFactory(arrow_schema, schema->PartitionKeys(), + options.GetWriteFileFormat(/*level=*/0)->Identifier(), + ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_manager, + file_store.NewSnapshotManager(ctx->GetCatalog(), catalog_identifier)); + std::shared_ptr io_manager; + const auto& io_temp_dir = ctx->GetTempDirectory(); + if (!io_temp_dir.empty()) { + io_manager = std::make_shared(io_temp_dir, options.GetFileSystem()); + } + + bool ignore_previous_files = ctx->IgnorePreviousFiles(); + if (ctx->GetRealtimeContext() && !options.RealtimeEnabled()) { + return Status::Invalid("real-time write requires realtime.enabled=true"); + } + if (ctx->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, + RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); + PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); + } + return CreateWrite(*ctx, arrow_schema, partition_schema, options, file_store_path_factory, + snapshot_manager, io_manager, ignore_previous_files); +} + +Result> FileStoreTable::NewScan( + std::unique_ptr context) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + if (context->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(CurrentBranch())); + } + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, ResolveOptions(context->GetOptions())); + PAIMON_ASSIGN_OR_RAISE( + bool read_optimized, + OptionsUtils::GetValueFromMap(core_options.ToMap(), kReadOptimizedScanOption, false)); + core_options.WithCache(context->GetCache()); + FileStore file_store(root_path_, core_options); + + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); + + PAIMON_RETURN_NOT_OK( + ValidateRealtimeScan(*table_schema, core_options, *context, read_optimized)); + if (core_options.GetBucket() == -1) { + if (!table_schema->PrimaryKeys().empty()) { + return Status::NotImplemented(fmt::format( + "do not support pk table bucket={} in scan process", core_options.GetBucket())); + } + } else if (core_options.GetBucket() < 1 && + !SchemaValidation::IsPostponeBucketTable(*table_schema, core_options.GetBucket())) { + return Status::Invalid( + fmt::format("do not support bucket={} in scan process", core_options.GetBucket())); + } + + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { + PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( + *arrow_schema, context->GetScanFilters()->GetPredicate(), + /*validate_field_idx=*/false)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, + file_store.NewPathFactory(arrow_schema, table_schema->PartitionKeys(), + core_options.GetFileFormat()->Identifier(), + context->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, + file_store.NewManifestList(path_factory, context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_file, + file_store.NewManifestFile(path_factory, partition_schema, context->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr file_store_scan, + NewFileStoreScan(manifest_list, manifest_file, arrow_schema, core_options, + context->GetExecutor(), context->GetMemoryPool(), context.get())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr split_generator, + NewSplitGenerator(core_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_file_handler, + file_store.NewIndexFileHandler(ResolveBucketMode(core_options.GetBucket(), table_schema), + path_factory, context->GetMemoryPool())); + auto snapshot_reader = std::make_shared( + file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler)); + const bool pk_table = !table_schema->PrimaryKeys().empty(); + if (read_optimized && pk_table && context->IsStreamingMode()) { + return Status::NotImplemented( + "read-optimized system table does not support streaming scan for primary key table"); + } + if (context->IsStreamingMode()) { + return std::make_unique(core_options, snapshot_reader); + } + auto batch_scan = std::make_unique( + /*pk_table=*/pk_table, core_options, snapshot_reader, read_optimized, context->GetLimit(), + /*realtime_pk_scan=*/pk_table && context->GetRealtimeContext() != nullptr); + if (context->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, + RealtimeContextImpl::Cast(context->GetRealtimeContext())); + return std::make_unique( + std::move(batch_scan), pk_table, realtime_context, path_factory, + snapshot_reader->GetSnapshotManager(), core_options.GetFileSystem(), + context->GetScanFilters(), core_options.GetRealtimeReadViewTtlMillis()); + } + if (core_options.DataEvolutionEnabled()) { + return std::make_unique( + root_path_, snapshot_reader, std::move(batch_scan), table_schema, + context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), + context->GetExecutor()); + } + if (pk_table && !read_optimized && core_options.GlobalIndexEnabled()) { + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*table_schema)); + if (!definitions.ScalarDefinitions().empty()) { + return PrimaryKeyIndexBatchScan::Create(snapshot_reader, std::move(batch_scan), + table_schema, path_factory, core_options, + context->GetMemoryPool()); + } + } + return batch_scan; +} + +Result> FileStoreTable::CreateTableRead( + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const { + const auto& core_options = internal_context->GetCoreOptions(); + FileStore file_store(root_path_, core_options); + const auto& table_schema = internal_context->GetTableSchema(); + if (internal_context->GetRealtimeContext() && !core_options.RealtimeEnabled()) { + return Status::Invalid("real-time read requires realtime.enabled=true"); + } + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + file_store.NewPathFactory(arrow_schema, table_schema->PartitionKeys(), + core_options.GetFileFormat()->Identifier(), memory_pool)); + if (internal_context->GetPrimaryKeys().empty()) { + return std::make_unique(path_factory, internal_context, memory_pool, + executor); + } + return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); +} + +Result> FileStoreTable::NewRead( + std::unique_ptr context) const { + std::shared_ptr read_context = std::move(context); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); + if (read_context->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(CurrentBranch())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(read_context->GetOptions())); + auto fs_scheme_map = fs_scheme_map_; + for (const auto& [scheme, identifier] : read_context->GetFileSystemSchemeToIdentifierMap()) { + fs_scheme_map[scheme] = identifier; + } + std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr internal_context, + InternalReadContext::CreateForTable(read_context, loaded_schema, std::move(options), + root_path_, fs_scheme_map)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + CreateTableRead(internal_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); + + std::optional scan_fallback_branch = + internal_context->GetCoreOptions().GetScanFallbackBranch(); + if (!scan_fallback_branch || + StringUtils::IsNullOrWhitespaceOnly(scan_fallback_branch.value())) { + return std::move(table_read); + } + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fallback_context, + InternalReadContext::CreateForFallbackBranch( + read_context, /*branch=*/scan_fallback_branch.value(), file_system_, + root_path_, fs_scheme_map)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, + CreateTableRead(fallback_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); + const std::shared_ptr& arrow_pool = internal_context->GetArrowMemoryPool(); + return std::make_unique(std::move(table_read), + std::move(fallback_table_read), arrow_pool); +} + +Result> FileStoreTable::NewCommit( + std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("commit", ctx->GetRootPath(), + ctx->GetIdentifier(), ctx->GetTableId())); + if (ctx->GetCatalog() != nullptr || ctx->UseRESTCatalogCommit()) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + std::optional catalog_identifier; + if (ctx->GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE(Identifier identifier, CatalogUtils::BranchIdentifier( + *ctx->GetIdentifier(), CurrentBranch())); + catalog_identifier.emplace(std::move(identifier)); + } + FileStoreCommitImpl::SchemaIdLoader schema_id_loader; + if (ctx->GetCatalog() != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + std::shared_ptr catalog = ctx->GetCatalog(); + Identifier identifier = catalog_identifier.value(); + schema_id_loader = [catalog, identifier]() -> Result { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + catalog->LoadTableSchema(identifier)); + std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); + if (data_schema == nullptr) { + return Status::Invalid( + "the catalog did not hand back a data table schema, which has the id a " + "snapshot records"); + } + return data_schema->Id(); + }; + } + std::shared_ptr schema = std::static_pointer_cast(schema_); + std::shared_ptr arrow_schema = + DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(ctx->GetOptions())); + FileStore file_store(root_path_, options); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); + assert(options.GetFileSystem()); + assert(options.GetFileFormat()); + PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); + PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); + if (is_object_store && !ctx->UseRESTCatalogCommit() && + AsVersionManaged(ctx->GetCatalog()) == nullptr && + options.ToMap().count(kObjectStoreCommitInteTestOption) == 0) { + return Status::NotImplemented( + "commit operation does not support object store file system for now"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr partition_computer, + file_store.NewPartitionComputer(arrow_schema, schema->PartitionKeys(), + ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + file_store.NewPathFactory(arrow_schema, schema->PartitionKeys(), + options.GetFileFormat()->Identifier(), ctx->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_manager, + file_store.NewSnapshotManager(ctx->GetCatalog(), catalog_identifier)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, + file_store.NewManifestList(path_factory, ctx->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_file, + file_store.NewManifestFile(path_factory, partition_schema, ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr index_manifest_file, + file_store.NewIndexManifestFile(ResolveBucketMode(options.GetBucket(), schema), + path_factory, ctx->GetMemoryPool())); + + auto expire_snapshots = std::make_shared( + snapshot_manager, path_factory, manifest_list, manifest_file, index_manifest_file, + options.GetFileSystem(), options.GetExpireConfig(), options.RealtimeEnabled(), + ctx->GetExecutor()); + + CommitScanner::ScanSupplier scan_supplier = + NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, arrow_schema, options, + ctx->GetExecutor(), ctx->GetMemoryPool()); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_commit, + NewSnapshotCommit(*ctx, catalog_identifier, options, snapshot_manager)); + + return std::make_unique( + ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path_, ctx->GetCommitUser(), + options, path_factory, std::move(partition_computer), snapshot_manager, snapshot_commit, + ctx->IgnoreEmptyCommit(), ctx->AppendCommitCheckConflict(), schema, + std::move(schema_id_loader), manifest_file, manifest_list, index_manifest_file, + expire_snapshots, GetSchemaManager(), std::move(scan_supplier)); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h new file mode 100644 index 000000000..f1b72bd3d --- /dev/null +++ b/src/paimon/core/table/file_store_table.h @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class InternalReadContext; +class IOManager; +class ManifestFile; +class ManifestList; +class RealtimeContext; +class SplitGenerator; + +/// Managed table backed by snapshots and manifests. +/// Shares operator setup across append-only and primary-key tables, selected from the schema. +class FileStoreTable : public Table { + public: + /// Creates an append-only or primary-key table according to the schema's primary keys. + /// @param specified_file_system Explicit file system, overriding option-based resolution. + /// @param schema Loaded table schema; must not be null. + /// @param branch Branch used by all operators and metadata managers. + /// @param fs_scheme_map URI scheme routing for option-based file system resolution. + static Result> Create( + const std::shared_ptr& specified_file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid, const std::string& branch, + const std::map& fs_scheme_map = {}); + + /// Binds a resolved store without replacing the loaded schema's options. + static Result> Create( + std::unique_ptr store, const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid, + const std::map& fs_scheme_map = {}); + + std::string CurrentBranch() const { + return store_->CurrentBranch(); + } + + std::shared_ptr GetSchemaManager() const { + return store_->GetSchemaManager(); + } + + std::shared_ptr GetSnapshotManager() const { + return store_->GetSnapshotManager(); + } + + std::string Name() const override { + return identifier_.GetObjectName(); + } + + std::string FullName() const override { + return identifier_.GetFullName(); + } + + Result> NewScan(std::unique_ptr context) const override; + + Result> NewRead(std::unique_ptr context) const override; + + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; + + private: + Result ResolveOptions(const std::map& options) const; + /// Rejects paths, identifiers or UUIDs that contradict this table's identity. + /// Missing context fields are allowed. + /// @param kind Operation name used in errors. + /// @param context_table_id Catalog commit UUID; nullopt for writes. + Status CheckContextNamesThisTable(const std::string& kind, const std::string& context_root_path, + const std::optional& context_identifier, + const std::optional& context_table_id) const; + + /// Creates a reader for the internal context's schema and primary keys. + Result> CreateTableRead( + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const; + + protected: + FileStoreTable(std::unique_ptr store, const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid); + + /// Restores real-time write progress from the latest snapshot. + static Status RestoreRealtimeCommittedProgress( + const std::shared_ptr& realtime_context, + const std::shared_ptr& snapshot_manager, const CoreOptions& options); + + static std::unique_ptr WithTablePath(std::unique_ptr&& scan, + const std::string& root_path); + + /// Creates the table-specific writer. + /// @param io_manager Optional spill manager. + /// @param ignore_previous_files Whether to start over instead of appending. + virtual Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; + + virtual Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const = 0; + + virtual Result> NewSplitGenerator( + const CoreOptions& core_options) const = 0; + + virtual CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const = 0; + + std::unique_ptr store_; + + /// URI routing supplied when this table was loaded. + std::map fs_scheme_map_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index a039baa4b..166062421 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -21,6 +21,7 @@ #include #include "fmt/format.h" +#include "paimon/commit_context.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/string_utils.h" @@ -29,8 +30,21 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/format/format_table_file_store_commit.h" +#include "paimon/core/table/format/format_table_file_store_write.h" +#include "paimon/core/table/format/format_table_read.h" +#include "paimon/core/table/format/format_table_scan.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" namespace paimon { @@ -45,10 +59,7 @@ FormatTable::FormatTable(const std::shared_ptr& file_system, const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata) - : file_system_(file_system), - location_(location), - identifier_(identifier), - schema_(schema), + : Table(file_system, location, schema, identifier, ""), options_(options), format_(format), file_compression_(file_compression), @@ -192,12 +203,12 @@ Result> FormatTable::Copy( for (const auto& [key, value] : dynamic_options) { options[key] = value; } - return Create(table->file_system_, table->location_, table->identifier_, table->schema_, + return Create(table->file_system_, table->root_path_, table->identifier_, table->GetSchema(), table->location_carries_paimon_metadata_, options); } const std::vector& FormatTable::PartitionKeys() const { - return schema_->PartitionKeys(); + return GetSchema()->PartitionKeys(); } std::string FormatTable::FullName() const { @@ -205,7 +216,152 @@ std::string FormatTable::FullName() const { } Result> FormatTable::GetArrowSchema() const { - return schema_->GetArrowSchema(); + return GetSchema()->GetArrowSchema(); +} + +namespace { +Result> NewFormatTableWrite( + const std::shared_ptr& table, const WriteContext& ctx) { + if (ctx.GetCatalog() != nullptr) { + return Status::Invalid( + "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " + "format table"); + } + if (ctx.IsStreamingMode()) { + return Status::NotImplemented( + "a format table has no snapshots, so there is nothing a streaming write could " + "commit against"); + } + if (ctx.GetRealtimeContext() != nullptr) { + return Status::NotImplemented("a format table has no real-time store to write into"); + } + if (!ctx.GetWriteSchema().empty()) { + return Status::NotImplemented( + "a format table write takes the table's own columns; a write schema naming a " + "subset of them is not supported yet"); + } + // A write id prefixes a postpone-bucket writer's files so that one compaction reader can + // put them back in order. A format table has no buckets, so it would identify nothing. + if (ctx.GetWriteId().has_value()) { + return Status::NotImplemented( + "a format table has no buckets, so a write id would name nothing"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, + FormatTableFileStoreWrite::Create(table, ctx.GetMemoryPool())); + return std::unique_ptr(std::move(format_write)); +} + +Result> NewFormatTableScan(const std::shared_ptr& table, + const std::shared_ptr& context) { + // Anything a `ScanContext` carries that a format table cannot honour is refused rather than + // silently dropped. + if (context->IsStreamingMode()) { + return Status::NotImplemented( + "a format table has no snapshots, so there is nothing for a streaming scan to follow"); + } + if (context->GetRealtimeContext() != nullptr) { + return Status::NotImplemented( + "a format table has no real-time store to union with what is on disk"); + } + if (context->GetGlobalIndexResult() != nullptr) { + return Status::NotImplemented("a format table carries no global index"); + } + std::map partition_filter; + std::shared_ptr filters = context->GetScanFilters(); + if (filters != nullptr) { + if (filters->GetPredicate() != nullptr) { + return Status::NotImplemented( + "a format table scan does not take a predicate: its files carry no statistics to " + "skip by, and pruning by the partition columns a predicate names is not " + "implemented yet; give the partition values as a partition filter instead"); + } + if (filters->GetBucketFilter()) { + return Status::NotImplemented("a format table has no buckets to filter by"); + } + const std::vector>& partition_filters = + filters->GetPartitionFilters(); + if (partition_filters.size() > 1) { + return Status::NotImplemented("a format table scan takes at most one partition filter"); + } + if (partition_filters.size() == 1) { + partition_filter = partition_filters.front(); + } + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + FormatTableScan::Create(table, partition_filter, context->GetLimit())); + return std::unique_ptr(std::move(scan)); +} + +Result> NewFormatTableRead(const std::shared_ptr& table, + const std::shared_ptr& context) { + if (context->EnablePreciseFileIndexEvaluate()) { + return Status::NotImplemented( + "Precise file-index evaluation is not supported for format tables"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + FormatTableRead::Create(table, context)); + return std::unique_ptr(std::move(read)); +} + +Result> NewFormatTableCommit( + const std::shared_ptr& table, const CommitContext& ctx) { + if (!ctx.IgnoreEmptyCommit()) { + return Status::NotImplemented( + "a format table cannot record an empty commit: keeping one means writing a " + "snapshot that adds no files, and there are no snapshots here"); + } + if (ctx.UseRESTCatalogCommit()) { + return Status::NotImplemented( + "a format table commits by renaming files into place, not by sending a snapshot " + "to a rest catalog"); + } + if (ctx.GetTableId()) { + return Status::NotImplemented( + "a format table sends no commit table request, so there is nowhere to put a table id"); + } + if (ctx.GetCatalog() != nullptr) { + return Status::NotImplemented( + "a format table keeps no snapshot for a catalog to take, so it commits by renaming " + "files into place rather than through the catalog it was loaded from"); + } + if (ctx.AppendCommitCheckConflict()) { + return Status::NotImplemented( + "a format table has no manifests to check a concurrent commit against"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, + FormatTableFileStoreCommit::Create(table)); + return std::unique_ptr(std::move(format_commit)); +} +} // namespace +Result> FormatTable::ForContext( + const std::map& options) const { + return Copy(std::shared_ptr(new FormatTable(*this)), options); +} +Result> FormatTable::NewScan(std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "scan")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + std::shared_ptr context = std::move(ctx); + return NewFormatTableScan(table, context); +} +Result> FormatTable::NewRead(std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + std::shared_ptr context = std::move(ctx); + return NewFormatTableRead(table, context); +} +Result> FormatTable::NewWrite( + std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + std::shared_ptr context = std::move(ctx); + return NewFormatTableWrite(table, *context); +} +Result> FormatTable::NewCommit( + std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + std::shared_ptr context = std::move(ctx); + return NewFormatTableCommit(table, *context); } } // namespace paimon diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index 40e4c3ebd..f1e1c7aab 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -56,8 +56,10 @@ #include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/predicate/full_text_search.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/record_batch.h" #include "paimon/scan_context.h" @@ -644,6 +646,40 @@ TEST(FormatTableTest, TestCreateReadsOptions) { ASSERT_EQ(table->FullName(), "db.tbl"); } +TEST(FormatTableTest, TestReadRejectsSearchForDirectAndStaticEntrypoints) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + CreateTable(dir->GetFileSystem(), dir->Str(), {})); + const std::shared_ptr
loaded_table = table; + for (bool vector_search : {false, true}) { + for (int32_t entrypoint : {0, 1, 2}) { + SCOPED_TRACE(vector_search); + SCOPED_TRACE(entrypoint); + ReadContextBuilder builder = + entrypoint == 2 ? ReadContextBuilder(table) : ReadContextBuilder(dir->Str()); + builder.EnablePreciseFileIndexEvaluate(true); + if (vector_search) { + builder.SetVectorSearch(std::make_shared( + "embedding", 3, std::vector{1.0f, 2.0f}, nullptr, nullptr, + VectorSearch::DistanceType::COSINE, std::map{})); + } else { + builder.SetFullTextSearch( + std::make_shared("name", R"({"match":{"query":"paimon"}})", 3)); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + auto read = entrypoint == 0 ? loaded_table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); + ASSERT_NOK_WITH_MSG(read, + "Precise file-index evaluation is not supported for format tables"); + ASSERT_TRUE(read.status().IsNotImplemented()); + } + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ReadContextBuilder(dir->Str()).Finish()); + ASSERT_OK(loaded_table->NewRead(std::move(context))); +} + TEST(FormatTableTest, TestFileCompressionComesFromCoreOptions) { // The resolution order itself is `CoreOptionsTest.TestFormatTableFileCompression`'s business; // what matters here is that the table asks for it rather than resolving compression again. @@ -752,11 +788,10 @@ TEST(FormatTableTest, TestCatalogManagedPartitionsAreRejected) { // And the same refusal when it arrives at the call rather than in the schema. Validating the // schema's own options alone would let this one through and then drop it, leaving a caller // who asked for catalog-managed partitions with a scan that read the directories anyway. - Result> dynamic_table = - FormatTable::Create(off_dir->GetFileSystem(), off_dir->Str(), Identifier("db", "tbl"), - off_table->LatestSchema(), - /*location_carries_paimon_metadata=*/true, - {{Options::METASTORE_PARTITIONED_TABLE, "true"}}); + Result> dynamic_table = FormatTable::Create( + off_dir->GetFileSystem(), off_dir->Str(), Identifier("db", "tbl"), off_table->GetSchema(), + /*location_carries_paimon_metadata=*/true, + {{Options::METASTORE_PARTITIONED_TABLE, "true"}}); ASSERT_FALSE(dynamic_table.ok()); ASSERT_TRUE(dynamic_table.status().IsNotImplemented()) << dynamic_table.status().ToString(); } @@ -2719,7 +2754,7 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { "a format table sends no commit table request"); auto format_catalog = std::make_shared(); - format_catalog->SetTableSchema(table->LatestSchema()); + format_catalog->SetTableSchema(table->GetSchema()); CommitContextBuilder catalog_builder(dir->Str(), "test-user"); catalog_builder.WithCatalog(format_catalog, Identifier("db", "fmt")); ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_context, catalog_builder.Finish()); diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp new file mode 100644 index 000000000..936fc29c2 --- /dev/null +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/primary_key_file_store_table.h" + +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/global_index/btree/btree_defs.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/mergetree/compact/lookup_merge_function.h" +#include "paimon/core/mergetree/compact/merge_function.h" +#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_write.h" +#include "paimon/core/options/merge_engine.h" +#include "paimon/core/postpone/postpone_bucket_file_store_write.h" +#include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/source/merge_tree_split_generator.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/utils/primary_key_table_utils.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { +namespace { +Result HasHistoricalPrimaryKeyBTreeDefinition( + const std::shared_ptr& schema_manager, int64_t current_schema_id) { + if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); + for (int64_t schema_id : schema_ids) { + if (schema_id >= current_schema_id) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, + schema_manager->ReadSchema(schema_id)); + const auto& historical_options = historical_schema->Options(); + auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); + if (option != historical_options.end() && + !StringUtils::IsNullOrWhitespaceOnly(option->second)) { + return true; + } + } + return false; +} +} // namespace + +struct KeyValue; +template +class MergeFunctionWrapper; + +PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable(std::unique_ptr store, + const std::shared_ptr& schema, + const Identifier& identifier, + const std::string& uuid) + : FileStoreTable(std::move(store), schema, identifier, uuid) {} + +Result> PrimaryKeyFileStoreTable::CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const { + const std::shared_ptr& file_store_path_factory = path_factory; + auto schema = std::static_pointer_cast(schema_); + auto schema_manager = GetSchemaManager(); + const std::string& branch = CurrentBranch(); + FileStore file_store(root_path_, options); + + // pk table + if (ctx.GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); + if (ignore_previous_files) { + return Status::NotImplemented("PK realtime requires restore from the latest snapshot"); + } + if (!ctx.GetWriteSchema().empty()) { + return Status::NotImplemented("PK realtime does not support a custom write schema"); + } + PAIMON_RETURN_NOT_OK( + RestoreRealtimeCommittedProgress(ctx.GetRealtimeContext(), snapshot_manager, options)); + } + if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { + return PostponeBucketFileStoreWrite::Create( + snapshot_manager, schema_manager, ctx.GetCommitUser(), root_path_, schema, arrow_schema, + partition_schema, io_manager, options, ctx.IsStreamingMode(), + ctx.IgnoreNumBucketCheck(), ctx.GetWriteId(), fs_scheme_map_, ctx.GetExecutor(), + ctx.GetMemoryPool(), file_system_); + } + if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in key value table", options.GetBucket())); + } + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, + schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_primary_key_fields, + options.SequenceFieldSortOrderIsAscending())); + auto primary_keys = schema->PrimaryKeys(); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, + PrimaryKeyTableUtils::CreateMergeFunction(arrow_schema, primary_keys, + options, ctx.GetMemoryPool())); + if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { + // don't wrap first row, it is already OK + merge_function = std::make_unique(std::move(merge_function)); + } + std::shared_ptr> merge_function_wrapper = + std::make_shared(std::move(merge_function)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr sequence_fields_comparator, + PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); + + std::shared_ptr dv_maintainer_factory; + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + bool has_btree_index = false; + for (const PrimaryKeyIndexDefinition& definition : + primary_key_index_definitions.Definitions()) { + if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { + has_btree_index = true; + break; + } + } + std::optional latest_snapshot; + bool has_existing_index_manifest = false; + if (!has_btree_index) { + PAIMON_ASSIGN_OR_RAISE( + bool has_historical_btree_index, + HasHistoricalPrimaryKeyBTreeDefinition(schema_manager, schema->Id())); + if (has_historical_btree_index) { + PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); + has_existing_index_manifest = + latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); + } + } + std::shared_ptr index_file_handler; + if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE( + index_file_handler, + file_store.NewIndexFileHandler(ResolveBucketMode(options.GetBucket(), schema), + file_store_path_factory, ctx.GetMemoryPool())); + } + bool has_restored_btree_index = false; + if (!has_btree_index && has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE( + std::vector restored_btree_entries, + index_file_handler->Scan( + latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { + return entry.index_file->IndexType() == BtreeDefs::kIdentifier && + IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); + })); + has_restored_btree_index = !restored_btree_entries.empty(); + } + if (options.DeletionVectorsEnabled()) { + dv_maintainer_factory = std::make_shared(index_file_handler); + } + std::shared_ptr + primary_key_index_maintainer_factory; + if (has_btree_index || has_restored_btree_index) { + primary_key_index_maintainer_factory = + std::make_shared( + root_path_, branch, schema, primary_key_index_definitions.Definitions(), + file_store_path_factory, index_file_handler, options, io_manager, + ctx.EnableMultiThreadSpill(), ctx.GetExecutor(), ctx.GetMemoryPool()); + } + + std::shared_ptr realtime_schema_layout; + if (ctx.GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr layout, + RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); + realtime_schema_layout = std::move(layout); + } + + return std::make_unique( + file_store_path_factory, snapshot_manager, schema_manager, ctx.GetCommitUser(), root_path_, + schema, arrow_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, + primary_key_index_maintainer_factory, io_manager, key_comparator, + sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, + ctx.IsStreamingMode(), ctx.IgnoreNumBucketCheck(), ctx.EnableMultiThreadSpill(), + ctx.GetRealtimeContext(), ctx.GetExecutor(), ctx.GetMemoryPool()); +} + +Result> PrimaryKeyFileStoreTable::NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const { + // TODO(liancheng.lsz): support fallback branch in scan + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + std::unique_ptr scan; + PAIMON_ASSIGN_OR_RAISE( + scan, KeyValueFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), manifest_list, + manifest_file, table_schema, arrow_schema, + context->GetScanFilters(), core_options, executor, + memory_pool)); + return WithTablePath(std::move(scan), root_path_); +} + +Result> PrimaryKeyFileStoreTable::NewSplitGenerator( + const CoreOptions& core_options) const { + auto source_split_target_size = core_options.GetSourceSplitTargetSize(); + auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); + // TODO(liancheng.lsz): support evolution + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + table_schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, + table_schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); + return std::make_unique( + source_split_target_size, source_split_open_file_cost, + core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), key_comparator); +} + +CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) const { + const std::shared_ptr& schema_manager = GetSchemaManager(); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h new file mode 100644 index 000000000..c27335e91 --- /dev/null +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class FileStoreScan; +class IOManager; +class SplitGenerator; +class TableSchema; + +/// Managed table with primary keys; uses the shared FileStoreTable operator flows. +class PrimaryKeyFileStoreTable : public FileStoreTable { + public: + PrimaryKeyFileStoreTable(std::unique_ptr store, + const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid); + + private: + Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const override; + + Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const override; + + Result> NewSplitGenerator( + const CoreOptions& core_options) const override; + + CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 2e38731c6..3433cd670 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -39,10 +39,12 @@ #include "paimon/core/operation/abstract_split_read.h" #include "paimon/core/operation/split_read.h" #include "paimon/core/schema/schema_manager.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" #include "paimon/defs.h" +#include "paimon/fs/local/local_file_system.h" #include "paimon/predicate/full_text_match.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/literal.h" @@ -137,30 +139,51 @@ TEST(TableReadTest, TestFullTextMatchReadEntryAndScoreValidation) { auto scalar = PredicateBuilder::Equal(/*field_index=*/1, "f1", FieldType::INT, Literal(1)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({scalar, full_text})); const std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder builder(table_path); - builder.EnablePreciseFileIndexEvaluate(true) - .SetReadFieldNames({"f0", "f1"}) - .SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK(TableRead::Create(std::move(context))); + auto fs = std::make_shared(); + SchemaManager manager(fs, table_path); + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, manager.ReadSchema(0)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"), "", + "main")); + for (bool table_entry : {false, true}) { + ReadContextBuilder builder(table_path); + builder.EnablePreciseFileIndexEvaluate(true) + .SetReadFieldNames({"f0", "f1"}) + .SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); + ASSERT_OK(table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context))); - builder.EnablePreciseFileIndexEvaluate(true) - .SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}) - .SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(context)), - "Score-producing FullTextMatch cannot appear under OR"); + builder.EnablePreciseFileIndexEvaluate(true) + .SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}) + .SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(context, builder.Finish()); + ASSERT_NOK_WITH_MSG(table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)), + "Score-producing FullTextMatch cannot appear under OR"); + } } TEST(TableReadTest, TestPerFieldScoreReadEntry) { const std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; auto search = std::make_shared("f0", R"({"match":{"query":"apple"}})", /*limit=*/1); - ReadContextBuilder builder(path); - builder.EnablePreciseFileIndexEvaluate(true).SetFullTextSearch(search).SetReadFieldNames( - {"f1", "_f0_INDEX_SCORE"}); - ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK(TableRead::Create(std::move(context))); + auto fs = std::make_shared(); + SchemaManager manager(fs, path); + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, manager.ReadSchema(0)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"), "", + "main")); + for (bool table_entry : {false, true}) { + ReadContextBuilder builder(path); + builder.EnablePreciseFileIndexEvaluate(true).SetFullTextSearch(search).SetReadFieldNames( + {"f1", "_f0_INDEX_SCORE"}); + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); + ASSERT_OK(table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context))); + } } TEST(TableReadTest, TestMixedFilePredicateProducesScoresThroughPublicReadEntry) { diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 8de9742e1..0967aefc2 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -42,8 +42,10 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -357,16 +359,21 @@ std::unique_ptr CreateChangelogBatchReader( AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, + std::string data_table_name, std::map options) - : fs_(std::move(fs)), + : AuditLogSystemTable(std::move(fs), std::move(table_path), std::move(table_schema), + std::move(options), ComposeName(data_table_name, kName)) {} + +AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, + std::shared_ptr table_schema, + std::map options, + std::string name) + : SystemTable(fs, table_path, std::move(name)), + fs_(std::move(fs)), table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} -std::string AuditLogSystemTable::Name() const { - return kName; -} - Result> AuditLogSystemTable::ArrowSchema() const { std::shared_ptr row_kind_field = DataField::ConvertDataFieldToArrowField(SpecialFields::RowKind()); @@ -383,17 +390,19 @@ Result> AuditLogSystemTable::ArrowSchema() const } Result> AuditLogSystemTable::NewScan( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { return Status::NotImplemented("audit_log system table predicate pushdown is not supported"); } std::shared_ptr scan_filter = context->GetScanFilters(); ScanContextBuilder builder(table_path_); - builder.SetOptions(options_) - .WithStreamingMode(context->IsStreamingMode()) + builder.WithStreamingMode(context->IsStreamingMode()) + .SetOptions(context->GetOptions()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + .WithFileSystem(GetFileSystem()) .WithCache(context->GetCache()); if (scan_filter) { if (scan_filter->GetBucketFilter()) { @@ -407,11 +416,16 @@ Result> AuditLogSystemTable::NewScan( builder.SetLimit(context->GetLimit().value()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, builder.Finish()); - return TableScan::Create(std::move(data_context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, options_)); + return data_table->NewScan(std::move(data_context)); } Result> AuditLogSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return NewChangelogRead(context, std::make_shared()); } @@ -430,8 +444,11 @@ Result> AuditLogSystemTable::NewChangelogRead( PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, CreateDataReadContext(context, read_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, read_options, + context->GetFileSystemSchemeToIdentifierMap())); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_read, - TableRead::Create(std::move(data_context))); + data_table->NewRead(std::move(data_context))); auto* key_value_read = dynamic_cast(data_read.get()); if (!key_value_read) { return Status::Invalid("audit_log system table requires key-value table read"); @@ -449,14 +466,16 @@ Result> AuditLogSystemTable::CreateDataReadContext( const std::map& read_options) const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr base_read_schema, BaseReadSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); + auto options = context->GetOptions(); + if (core_options.TableReadSequenceNumberEnabled()) { + options[Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED] = "true"; + } ReadContextBuilder builder(table_path_); - builder.SetOptions(read_options) - .SetReadFieldNames(base_read_schema->field_names()) - .WithBranch(core_options.GetBranch()) + builder.SetReadFieldNames(base_read_schema->field_names()) + .SetOptions(options) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) - .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) + .WithFileSystem(GetFileSystem()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) .SetPrefetchMaxParallelNum(context->GetPrefetchMaxParallelNum()) diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index 780f5fa72..bc15c385f 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -54,18 +54,21 @@ class AuditLogSystemTable : public SystemTable { public: static constexpr const char* kName = "audit_log"; + /// @param data_table_name Logical table name, including any branch suffix. AuditLogSystemTable(std::shared_ptr fs, std::string table_path, - std::shared_ptr table_schema, + std::shared_ptr table_schema, std::string data_table_name, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; - Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; protected: + /// Constructor for derived changelog views with a different system-table name. + AuditLogSystemTable(std::shared_ptr fs, std::string table_path, + std::shared_ptr table_schema, + std::map options, std::string name); + Result> NewChangelogRead( const std::shared_ptr& context, std::shared_ptr converter) const; diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index 88541194d..7b73ce1dc 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -31,6 +31,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/table/source/table_read.h" @@ -72,13 +73,10 @@ std::shared_ptr CreateBinlogBatchConverter() { BinlogSystemTable::BinlogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, + std::string data_table_name, std::map options) : AuditLogSystemTable(std::move(fs), std::move(table_path), std::move(table_schema), - std::move(options)) {} - -std::string BinlogSystemTable::Name() const { - return kName; -} + std::move(options), ComposeName(data_table_name, kName)) {} Result> BinlogSystemTable::ArrowSchema() const { std::shared_ptr rowkind_field = @@ -98,7 +96,10 @@ Result> BinlogSystemTable::ArrowSchema() const { } Result> BinlogSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return NewChangelogRead(context, CreateBinlogBatchConverter()); } diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index 3b4f45bd0..fcdcb84b6 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -36,14 +36,13 @@ class BinlogSystemTable : public AuditLogSystemTable { public: static constexpr const char* kName = "binlog"; + /// @param data_table_name Logical table name, including any branch suffix. BinlogSystemTable(std::shared_ptr fs, std::string table_path, - std::shared_ptr table_schema, + std::shared_ptr table_schema, std::string data_table_name, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewRead(std::unique_ptr context) const override; }; } // namespace paimon diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 647da76c3..29b71768f 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -134,11 +134,15 @@ struct CatalogTableInfo { // Match Java CatalogUtils::listAllTables: tolerate databases or tables removed concurrently, but // propagate all other catalog and schema errors instead of returning incomplete system-table rows. -Result> LoadAllDataTables(const Catalog& catalog) { +Result> LoadAllDataTables(const Catalog* catalog, + const std::string& table_name) { + if (catalog == nullptr) { + return Status::NotImplemented("global system table requires catalog context: ", table_name); + } std::vector result; - PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog.ListDatabases()); + PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog->ListDatabases()); for (const std::string& database : databases) { - Result> tables_result = catalog.ListTables(database); + Result> tables_result = catalog->ListTables(database); if (!tables_result.ok()) { if (tables_result.status().IsNotExist()) { continue; @@ -147,7 +151,7 @@ Result> LoadAllDataTables(const Catalog& catalog) } for (const std::string& table : tables_result.value()) { Identifier identifier(database, table); - Result> schema_result = catalog.LoadTableSchema(identifier); + Result> schema_result = catalog->LoadTableSchema(identifier); if (!schema_result.ok()) { if (schema_result.status().IsNotExist()) { continue; @@ -300,7 +304,9 @@ Result> GlobalSystemTableLoader::Load( return Status::NotImplemented("global system table requires catalog context: ", table_name); } - return entry.factory(context); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, entry.factory(context)); + PAIMON_RETURN_NOT_OK(table->AdoptOwnSchema()); + return table; } } return Status::NotImplemented("unsupported global system table: ", table_name); @@ -324,11 +330,7 @@ Result> GlobalSystemTableLoader::GetSupportedTableNames // ============================================================================= CatalogOptionsSystemTable::CatalogOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/catalog_options"), context_(std::move(context)) {} - -std::string CatalogOptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/catalog_options", kName), context_(std::move(context)) {} Result> CatalogOptionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -360,11 +362,8 @@ Result> CatalogOptionsSystemTable::BuildRows() const { // ============================================================================= AllTableOptionsSystemTable::AllTableOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/all_table_options"), context_(std::move(context)) {} - -std::string AllTableOptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/all_table_options", kName), + context_(std::move(context)) {} Result> AllTableOptionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -380,7 +379,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { for (const auto& [key, value] : table.schema->Options()) { GenericRow row(schema->num_fields()); @@ -399,11 +398,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { // ============================================================================= TablesSystemTable::TablesSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/tables"), context_(std::move(context)) {} - -std::string TablesSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/tables", kName), context_(std::move(context)) {} Result> TablesSystemTable::ArrowSchema() const { return arrow::schema({ @@ -429,7 +424,7 @@ Result> TablesSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { const std::shared_ptr& data_schema = table.schema; @@ -471,11 +466,7 @@ Result> TablesSystemTable::BuildRows() const { // ============================================================================= PartitionsSystemTable::PartitionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/partitions"), context_(std::move(context)) {} - -std::string PartitionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/partitions", kName), context_(std::move(context)) {} Result> PartitionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -495,7 +486,7 @@ Result> PartitionsSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { const std::shared_ptr& data_schema = table.schema; Identifier id(table.database_name, table.table_name); diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h index 3d6936262..22d8a0262 100644 --- a/src/paimon/core/table/system/global_system_tables.h +++ b/src/paimon/core/table/system/global_system_tables.h @@ -33,7 +33,10 @@ class FileSystem; /// Context passed to global system table constructors, providing catalog-level /// access for enumerating databases, tables, and reading metadata. struct GlobalSystemTableContext { - const Catalog* catalog = nullptr; // non-owning pointer + /// Catalog used when loading rows; borrowed for synchronous schema loading. + const Catalog* catalog = nullptr; + /// Keeps the catalog alive for tables returned by Catalog::GetTable(). + std::shared_ptr catalog_owner; std::shared_ptr fs; std::string warehouse; std::map catalog_options; @@ -48,7 +51,6 @@ class CatalogOptionsSystemTable : public InMemorySystemTable { explicit CatalogOptionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -64,7 +66,6 @@ class AllTableOptionsSystemTable : public InMemorySystemTable { explicit AllTableOptionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -80,7 +81,6 @@ class TablesSystemTable : public InMemorySystemTable { explicit TablesSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -96,7 +96,6 @@ class PartitionsSystemTable : public InMemorySystemTable { explicit PartitionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -113,6 +112,7 @@ class GlobalSystemTableLoader { static Result IsSupported(const std::string& table_name, const std::map& catalog_options); + /// Loads a global system table. Views that enumerate tables require a catalog in context. static Result> Load(const std::string& table_name, const GlobalSystemTableContext& context); diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index 8d6111d79..08c7d7726 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -29,8 +29,10 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/core/io/generic_row_to_arrow_array_converter.h" #include "paimon/core/table/system/system_table_scan.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/memory/memory_pool.h" #include "paimon/read_context.h" +#include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/source/table_read.h" @@ -104,19 +106,28 @@ class InMemorySystemTableRead : public TableRead { } // namespace -InMemorySystemTable::InMemorySystemTable(std::string table_path) - : table_path_(std::move(table_path)) {} +InMemorySystemTable::InMemorySystemTable(const std::shared_ptr& fs, + std::string table_path, std::string name) + : SystemTable(fs, table_path, std::move(name)) {} Result> InMemorySystemTable::NewScan( - const std::shared_ptr& /*context*/) const { - return std::make_unique(table_path_); + std::unique_ptr /*context*/) const { + return std::make_unique(GetRootPath()); } Result> InMemorySystemTable::NewRead( - const std::shared_ptr& context) const { - return std::make_unique( - checked_pointer_cast(shared_from_this()), - context->GetMemoryPool()); + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + ForRead(context->GetOptions())); + return std::make_unique(std::move(table), context->GetMemoryPool()); +} + +Result> InMemorySystemTable::ForRead( + const std::map& /*options*/) const { + return checked_pointer_cast(shared_from_this()); } } // namespace paimon diff --git a/src/paimon/core/table/system/in_memory_system_table.h b/src/paimon/core/table/system/in_memory_system_table.h index f9fcebdb3..d0909f360 100644 --- a/src/paimon/core/table/system/in_memory_system_table.h +++ b/src/paimon/core/table/system/in_memory_system_table.h @@ -19,6 +19,7 @@ #pragma once +#include #include #include #include @@ -28,27 +29,30 @@ namespace paimon { +class FileSystem; + /// Base class for system tables whose result can be materialized as a single in-memory /// RecordBatch. /// /// It provides the common singleton split scan and one-shot batch reader. class InMemorySystemTable : public SystemTable { public: - explicit InMemorySystemTable(std::string table_path); + InMemorySystemTable(const std::shared_ptr& fs, std::string table_path, + std::string name); Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + std::unique_ptr /*context*/) const override; + Result> NewRead(std::unique_ptr context) const override; virtual Result> BuildRows() const = 0; protected: + /// Creates an independent read view when dynamic options affect materialized rows. + virtual Result> ForRead( + const std::map& options) const; + const std::string& TablePath() const { - return table_path_; + return GetRootPath(); } - - private: - std::string table_path_; }; } // namespace paimon diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index 8db1c41cf..19c650ad7 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -55,6 +55,7 @@ #include "paimon/core/manifest/manifest_file.h" #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/snapshot.h" @@ -199,6 +200,27 @@ Result CreateCoreOptions(const MetadataSystemTableContext& context) return CoreOptions::FromMap(context.options, context.fs); } +Result WithReadOptions( + const MetadataSystemTableContext& context, + const std::map& overrides) { + auto branch = overrides.find(Options::BRANCH); + if (branch != overrides.end() && + BranchManager::NormalizeBranch(branch->second) != context.branch) { + return Status::Invalid("context branch contradicts the loaded table"); + } + MetadataSystemTableContext merged = context; + for (const auto& [key, value] : overrides) { + merged.options[key] = value; + } + PAIMON_ASSIGN_OR_RAISE(TableType type, TableTypeDefine::FromOptions(merged.options)); + PAIMON_ASSIGN_OR_RAISE(TableType bound_type, TableTypeDefine::FromOptions(context.options)); + if (type != bound_type) { + return Status::Invalid("context type contradicts the loaded table"); + } + merged.options[Options::BRANCH] = context.branch; + return merged; +} + Result> CreatePathFactory( const MetadataSystemTableContext& context, const CoreOptions& core_options, const std::shared_ptr& pool) { @@ -517,13 +539,11 @@ Result> WriteColsValue( } // namespace -OptionsSystemTable::OptionsSystemTable(std::string table_path, +OptionsSystemTable::OptionsSystemTable(const std::shared_ptr& fs, + std::string table_path, std::shared_ptr table_schema) - : InMemorySystemTable(std::move(table_path)), table_schema_(std::move(table_schema)) {} - -std::string OptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), + table_schema_(std::move(table_schema)) {} Result> OptionsSystemTable::ArrowSchema() const { return arrow::schema({arrow::field("key", arrow::utf8(), /*nullable=*/false), @@ -545,13 +565,9 @@ Result> OptionsSystemTable::BuildRows() const { SnapshotsSystemTable::SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string SnapshotsSystemTable::Name() const { - return kName; -} - Result> SnapshotsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("snapshot_id", arrow::int64(), /*nullable=*/false), @@ -607,13 +623,9 @@ Result> SnapshotsSystemTable::BuildRows() const { SchemasSystemTable::SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string SchemasSystemTable::Name() const { - return kName; -} - Result> SchemasSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("schema_id", arrow::int64(), /*nullable=*/false), @@ -663,13 +675,9 @@ Result> SchemasSystemTable::BuildRows() const { TagsSystemTable::TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string TagsSystemTable::Name() const { - return kName; -} - Result> TagsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("tag_name", arrow::utf8(), /*nullable=*/false), @@ -713,13 +721,9 @@ Result> TagsSystemTable::BuildRows() const { BranchesSystemTable::BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string BranchesSystemTable::Name() const { - return kName; -} - Result> BranchesSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("branch_name", arrow::utf8(), /*nullable=*/false), @@ -752,13 +756,9 @@ Result> BranchesSystemTable::BuildRows() const { ConsumersSystemTable::ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string ConsumersSystemTable::Name() const { - return kName; -} - Result> ConsumersSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("consumer_id", arrow::utf8(), /*nullable=*/false), @@ -787,14 +787,10 @@ ManifestsSystemTable::ManifestsSystemTable(std::shared_ptr fs, std:: std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} -std::string ManifestsSystemTable::Name() const { - return kName; -} - Result> ManifestsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("file_name", arrow::utf8(), /*nullable=*/false), @@ -809,6 +805,14 @@ Result> ManifestsSystemTable::ArrowSchema() const }); } +Result> ManifestsSystemTable::ForRead( + const std::map& options) const { + PAIMON_ASSIGN_OR_RAISE(MetadataSystemTableContext context, WithReadOptions(context_, options)); + auto table = std::make_shared(*this); + table->context_ = std::move(context); + return std::shared_ptr(std::move(table)); +} + Result> ManifestsSystemTable::BuildRows() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CreateCoreOptions(context_)); @@ -857,14 +861,10 @@ Result> ManifestsSystemTable::BuildRows() const { FilesSystemTable::FilesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} -std::string FilesSystemTable::Name() const { - return kName; -} - Result> FilesSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("partition", arrow::utf8(), /*nullable=*/true), @@ -891,6 +891,14 @@ Result> FilesSystemTable::ArrowSchema() const { }); } +Result> FilesSystemTable::ForRead( + const std::map& options) const { + PAIMON_ASSIGN_OR_RAISE(MetadataSystemTableContext context, WithReadOptions(context_, options)); + auto table = std::make_shared(*this); + table->context_ = std::move(context); + return std::shared_ptr(std::move(table)); +} + Result> FilesSystemTable::BuildRows() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); std::shared_ptr pool = GetDefaultPool(); diff --git a/src/paimon/core/table/system/metadata_system_tables.h b/src/paimon/core/table/system/metadata_system_tables.h index e3961e476..cfb8c6faf 100644 --- a/src/paimon/core/table/system/metadata_system_tables.h +++ b/src/paimon/core/table/system/metadata_system_tables.h @@ -35,9 +35,9 @@ class OptionsSystemTable : public InMemorySystemTable { public: static constexpr const char* kName = "options"; - OptionsSystemTable(std::string table_path, std::shared_ptr table_schema); + OptionsSystemTable(const std::shared_ptr& fs, std::string table_path, + std::shared_ptr table_schema); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -62,7 +62,6 @@ class SnapshotsSystemTable : public InMemorySystemTable { SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -77,7 +76,6 @@ class SchemasSystemTable : public InMemorySystemTable { SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -92,7 +90,6 @@ class TagsSystemTable : public InMemorySystemTable { TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -107,7 +104,6 @@ class BranchesSystemTable : public InMemorySystemTable { BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -123,7 +119,6 @@ class ConsumersSystemTable : public InMemorySystemTable { ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -140,11 +135,13 @@ class ManifestsSystemTable : public InMemorySystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; private: + Result> ForRead( + const std::map& options) const override; + MetadataSystemTableContext context_; }; @@ -157,11 +154,13 @@ class FilesSystemTable : public InMemorySystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; private: + Result> ForRead( + const std::map& options) const override; + MetadataSystemTableContext context_; }; diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 8d5696447..b2d4bf99e 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -26,7 +26,9 @@ #include "arrow/c/bridge.h" #include "paimon/common/types/data_field.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/read_optimized_scan_options.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -35,17 +37,16 @@ namespace paimon { -ReadOptimizedSystemTable::ReadOptimizedSystemTable(std::string table_path, +ReadOptimizedSystemTable::ReadOptimizedSystemTable(const std::shared_ptr& fs, + std::string table_path, std::shared_ptr table_schema, + std::string data_table_name, std::map options) - : table_path_(std::move(table_path)), + : SystemTable(fs, table_path, ComposeName(data_table_name, kName)), + table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} -std::string ReadOptimizedSystemTable::Name() const { - return kName; -} - Result> ReadOptimizedSystemTable::ArrowSchema() const { return DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields()); } @@ -57,18 +58,20 @@ std::map ReadOptimizedSystemTable::ReadOptimizedOption } Result> ReadOptimizedSystemTable::NewScan( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetRealtimeContext() && !table_schema_->PrimaryKeys().empty()) { return Status::NotImplemented( "PK real-time union read does not support read-optimized scans"); } auto options = ReadOptimizedOptions(); ScanContextBuilder builder(table_path_); - builder.SetOptions(options) - .WithStreamingMode(context->IsStreamingMode()) + builder.WithStreamingMode(context->IsStreamingMode()) + .SetOptions(context->GetOptions()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + .WithFileSystem(GetFileSystem()) .WithCache(context->GetCache()); if (context->GetLimit().has_value()) { builder.SetLimit(context->GetLimit().value()); @@ -83,36 +86,26 @@ Result> ReadOptimizedSystemTable::NewScan( if (context->GetGlobalIndexResult()) { builder.SetGlobalIndexResult(context->GetGlobalIndexResult()); } - if (context->GetSpecificTableSchema().has_value()) { - builder.SetTableSchema(context->GetSpecificTableSchema().value()); - } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, builder.Finish()); - return TableScan::Create(std::move(base_context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, options)); + return data_table->NewScan(std::move(base_context)); } Result> ReadOptimizedSystemTable::CreateDataReadContext( const std::shared_ptr& context) const { - auto options = options_; - std::string branch = context->GetBranch(); - // SystemTableLoader injects Options::BRANCH when parsing paths such as `T$branch_dev$ro`. - auto branch_iter = options.find(Options::BRANCH); - if (branch_iter != options.end()) { - branch = branch_iter->second; - } ReadContextBuilder builder(table_path_); - builder.SetOptions(options) - .WithBranch(branch) - .SetPredicate(context->GetPredicate()) + builder.SetPredicate(context->GetPredicate()) .EnablePredicateFilter(context->EnablePredicateFilter()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) .SetPrefetchMaxParallelNum(context->GetPrefetchMaxParallelNum()) .EnableMultiThreadRowToBatch(context->EnableMultiThreadRowToBatch()) .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber()) + .SetOptions(context->GetOptions()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) - .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) + .WithFileSystem(GetFileSystem()) .SetReadAheadCacheEnabled(context->ReadAheadCacheEnabled()) .WithCacheConfig(context->GetCacheConfig()) .SetWarmupLevel(context->GetWarmupLevel()) @@ -126,17 +119,20 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, c_read_schema.get())); builder.SetReadSchema(std::move(c_read_schema)); } - if (context->GetSpecificTableSchema().has_value()) { - builder.SetTableSchema(context->GetSpecificTableSchema().value()); - } return builder.Finish(); } Result> ReadOptimizedSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, CreateDataReadContext(context)); - return TableRead::Create(std::move(base_context)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr data_table, + CreateDataTable(table_schema_, options_, context->GetFileSystemSchemeToIdentifierMap())); + return data_table->NewRead(std::move(base_context)); } } // namespace paimon diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h index 3123fe03b..164b2c79c 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.h +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -26,6 +26,7 @@ #include "paimon/core/table/system/system_table.h" namespace paimon { +class FileSystem; class TableSchema; /// System table for `T$ro`, exposing read-optimized data. @@ -33,15 +34,14 @@ class ReadOptimizedSystemTable : public SystemTable { public: static constexpr const char* kName = "ro"; - ReadOptimizedSystemTable(std::string table_path, std::shared_ptr table_schema, + /// @param data_table_name Logical table name, including any branch suffix. + ReadOptimizedSystemTable(const std::shared_ptr& fs, std::string table_path, + std::shared_ptr table_schema, std::string data_table_name, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; - Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; private: /// Build the context for the data table underneath this read-optimized view. diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 5a1a1de00..34c06489c 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -26,26 +26,38 @@ #include #include +#include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" +#include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/metadata_system_tables.h" #include "paimon/core/table/system/read_optimized_system_table.h" +#include "paimon/core/table/system/system_table_schema.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/read_context.h" #include "paimon/status.h" +#include "paimon/write_context.h" namespace paimon { namespace { using SystemTableFactory = std::function>( const std::shared_ptr&, const std::string&, const std::shared_ptr&, - const std::map&)>; + const std::map&, const std::string&)>; struct SystemTableRegistryEntry { std::string name; @@ -70,81 +82,84 @@ std::string LoadBranch(const std::map& options) { const std::vector& SystemTableRegistry() { static const std::vector registry = { {OptionsSystemTable::kName, - [](const std::shared_ptr& /*fs*/, const std::string& table_path, + [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& /*dynamic_options*/) - -> Result> { - return std::make_shared(table_path, table_schema); + const std::map& /*dynamic_options*/, + const std::string& /*data_table_name*/) -> Result> { + return std::make_shared(fs, table_path, table_schema); }}, {AuditLogSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& data_table_name) -> Result> { return std::make_shared( - fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, data_table_name, + MergeOptions(table_schema, dynamic_options)); }}, {BinlogSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& data_table_name) -> Result> { return std::make_shared( - fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, data_table_name, + MergeOptions(table_schema, dynamic_options)); }}, {ReadOptimizedSystemTable::kName, - [](const std::shared_ptr& /*fs*/, const std::string& table_path, + [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& data_table_name) -> Result> { return std::make_shared( - table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, data_table_name, + MergeOptions(table_schema, dynamic_options)); }}, {SnapshotsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {SchemasSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {TagsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {BranchesSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {ConsumersSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {ManifestsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); auto branch = LoadBranch(options); return std::make_shared(fs, table_path, std::move(branch), @@ -153,8 +168,8 @@ const std::vector& SystemTableRegistry() { {FilesSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); auto branch = LoadBranch(options); return std::make_shared(fs, table_path, std::move(branch), @@ -176,6 +191,57 @@ std::optional FindSystemTableFactory(const std::string& syst } // namespace +SystemTable::SystemTable(const std::shared_ptr& fs, const std::string& table_path, + std::string name) + : Table(fs, table_path, /*schema=*/nullptr, + Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/""), + name_(std::move(name)) {} + +std::string SystemTable::ComposeName(const std::string& data_table_name, const std::string& kind) { + return data_table_name + Catalog::SYSTEM_TABLE_SPLITTER + kind; +} + +Status SystemTable::CheckNoSearch(const std::shared_ptr& context) { + if (context->EnablePreciseFileIndexEvaluate()) { + return Status::NotImplemented( + "Precise file-index evaluation is not supported for system tables"); + } + return Status::OK(); +} + +Status SystemTable::AdoptOwnSchema() { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, ArrowSchema()); + schema_ = std::make_shared(std::move(arrow_schema)); + return Status::OK(); +} + +Result> SystemTable::CreateDataTable( + const std::shared_ptr& table_schema, + const std::map& options, + const std::map& fs_scheme_map_overrides) const { + auto fs_scheme_map = fs_scheme_map_; + for (const auto& [scheme, identifier] : fs_scheme_map_overrides) { + fs_scheme_map[scheme] = identifier; + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, + table_schema->MergeOptions(options)); + const std::string branch = BranchManager::NormalizeBranch(LoadBranch(merged_schema->Options())); + return FileStoreTable::Create( + GetFileSystem(), GetRootPath(), std::shared_ptr(std::move(merged_schema)), + CatalogUtils::IdentifierFromPath(GetRootPath(), branch), "", branch, fs_scheme_map); +} + +Result> SystemTable::NewWrite( + std::unique_ptr /*context*/) const { + return Status::NotImplemented(Name(), " system table is read-only: writing is not supported"); +} + +Result> SystemTable::NewCommit( + std::unique_ptr /*context*/) const { + return Status::NotImplemented(Name(), + " system table is read-only: committing for it is not supported"); +} + bool SystemTableLoader::IsSupported(const std::string& system_table_name) { return FindSystemTableFactory(system_table_name).has_value(); } @@ -183,10 +249,20 @@ bool SystemTableLoader::IsSupported(const std::string& system_table_name) { Result> SystemTableLoader::Load( const std::string& system_table_name, const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) { + const std::map& dynamic_options, const std::string& data_table_name, + const std::map& fs_scheme_map) { std::optional factory = FindSystemTableFactory(system_table_name); if (factory) { - return factory.value()(fs, table_path, table_schema, dynamic_options); + const std::string effective_name = + data_table_name.empty() ? PathUtil::GetName(table_path) : data_table_name; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + factory.value()(fs, table_path, table_schema, dynamic_options, effective_name)); + system_table->name_ = + SystemTable::ComposeName(effective_name, StringUtils::ToLowerCase(system_table_name)); + system_table->fs_scheme_map_ = fs_scheme_map; + PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); + return system_table; } return Status::NotImplemented("unsupported system table: ", system_table_name); } @@ -223,7 +299,8 @@ Result> SystemTableLoader::TryParsePath(const std Result> SystemTableLoader::LoadFromPath( const std::shared_ptr& fs, const std::string& path, - const std::map& dynamic_options) { + const std::map& dynamic_options, + const std::map& fs_scheme_map) { PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, TryParsePath(path)); if (!system_table_path) { return Status::Invalid("path is not a system table path: ", path); @@ -237,7 +314,6 @@ Result> SystemTableLoader::LoadFromPath( // The warehouse is the grandparent of the sys/
path context.warehouse = PathUtil::GetParentDirPath(PathUtil::GetParentDirPath(path)); context.catalog_options = dynamic_options; - // Note: context.catalog is intentionally left as nullptr here. // Global tables loaded from path do not have a Catalog reference and // cannot enumerate databases/tables. Only tables that don't require // catalog enumeration (e.g. catalog_options) will work in this path. @@ -255,7 +331,12 @@ Result> SystemTableLoader::LoadFromPath( if (parsed.branch) { options[Options::BRANCH] = parsed.branch.value(); } - return Load(parsed.system_table_name, fs, parsed.table_path, latest_schema.value(), options); + // Preserve branch-qualified logical names when deriving identifiers from paths. + const Identifier data_identifier(Identifier::kUnknownDatabase, + PathUtil::GetName(parsed.table_path), + parsed.branch.value_or(std::string())); + return Load(parsed.system_table_name, fs, parsed.table_path, latest_schema.value(), options, + data_identifier.GetObjectName(), fs_scheme_map); } } // namespace paimon diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 39e1da31b..73420559b 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -25,17 +25,20 @@ #include #include "arrow/api.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/result.h" #include "paimon/status.h" #include "paimon/type_fwd.h" namespace paimon { class FileSystem; +class FileStoreTable; class ReadContext; class ScanContext; -class TableScan; -class TableRead; class TableSchema; +class SystemTableSchema; +struct GlobalSystemTableContext; /// Parsed information for a table-scoped system table path. struct SystemTablePath { @@ -49,19 +52,55 @@ struct SystemTablePath { bool is_global = false; }; -/// Base interface for table-scoped system tables such as `T$options` and `T$snapshots`. -/// -/// Implementations expose a read-only schema and create their own scan/read objects. -class SystemTable : public std::enable_shared_from_this { +/// Read-only table-scoped or global system view. +/// Table-scoped names are $; global names are . FullName() equals Name(). +class SystemTable : public Table, public std::enable_shared_from_this { public: - virtual ~SystemTable() = default; + std::string Name() const override { + return name_; + } + + std::string FullName() const override { + return name_; + } - virtual std::string Name() const = 0; + /// Returns this view's schema, rather than its data table's schema. virtual Result> ArrowSchema() const = 0; - virtual Result> NewScan( - const std::shared_ptr& context) const = 0; - virtual Result> NewRead( - const std::shared_ptr& context) const = 0; + + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; + + protected: + /// @param fs Data table file system. + /// @param table_path Data table root path. + /// @param name System table name. + SystemTable(const std::shared_ptr& fs, const std::string& table_path, + std::string name); + + /// Composes $, preserving any branch suffix. + static std::string ComposeName(const std::string& data_table_name, const std::string& kind); + + /// Rejects TopK and full-text search requests, which system-table readers cannot evaluate. + static Status CheckNoSearch(const std::shared_ptr& context); + + /// Initializes the system schema after construction, when ArrowSchema() can be called. + Status AdoptOwnSchema(); + + /// Builds the data table using this view's data schema and options. + Result> CreateDataTable( + const std::shared_ptr& table_schema, + const std::map& options, + const std::map& fs_scheme_map_overrides = {}) const; + + friend class SystemTableLoader; + friend class GlobalSystemTableLoader; + + private: + std::string name_; + std::map fs_scheme_map_; }; /// Loads system table implementations from parsed table identifiers or table paths. @@ -74,13 +113,16 @@ class SystemTableLoader { static Result> Load( const std::string& system_table_name, const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options); + const std::map& dynamic_options, + const std::string& data_table_name = "", + const std::map& fs_scheme_map = {}); static Result> TryParsePath(const std::string& path); static Result> LoadFromPath( const std::shared_ptr& fs, const std::string& path, - const std::map& dynamic_options); + const std::map& dynamic_options, + const std::map& fs_scheme_map = {}); }; } // namespace paimon diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 7591e53bf..9f837e4a1 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -77,11 +77,11 @@ TEST(SystemTableTest, TestChangelogArrowSchemaReturnsInvalidOptions) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, CreateTableSchemaForTest(options)); - AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); ASSERT_NOK_WITH_MSG(audit_log.ArrowSchema(), "Invalid Config [table-read.sequence-number.enabled: invalid]"); - BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); ASSERT_NOK_WITH_MSG(binlog.ArrowSchema(), "Invalid Config [table-read.sequence-number.enabled: invalid]"); } @@ -92,7 +92,7 @@ TEST(SystemTableTest, TestBinlogArrowSchemaWithSequenceNumber) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, CreateTableSchemaForTest(options)); - BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, binlog.ArrowSchema()); ASSERT_EQ(schema->field_names(), @@ -219,7 +219,7 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, "/tmp/table", table_schema, /*dynamic_options=*/{})); - ASSERT_EQ(system_table->Name(), ReadOptimizedSystemTable::kName); + ASSERT_EQ(system_table->Name(), "table$ro"); ASSERT_OK_AND_ASSIGN(std::shared_ptr arrow_schema, system_table->ArrowSchema()); ASSERT_EQ(arrow_schema->field_names(), (std::vector{"pk", "v"})); @@ -247,9 +247,10 @@ TEST(SystemTableTest, TestNewReadPropagatesWarmupLevel) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, CreateTableSchemaForTest(options)); - AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); - BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); - ReadOptimizedSystemTable read_optimized("/tmp/table", table_schema, options); + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, "table", + options); for (WarmupLevel level : {WarmupLevel::NONE, WarmupLevel::RAW, WarmupLevel::DECODED}) { ReadContextBuilder builder("/tmp/table"); diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index ec131ef9e..c51b8400b 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -18,194 +18,7 @@ #include "paimon/catalog/table.h" -#include -#include -#include -#include -#include -#include -#include -#include - -#include "arrow/c/bridge.h" -#include "fmt/format.h" -#include "paimon/catalog/catalog.h" -#include "paimon/commit_context.h" -#include "paimon/common/global_index/btree/btree_defs.h" -#include "paimon/common/predicate/predicate_validator.h" -#include "paimon/common/types/data_field.h" -#include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/common/utils/binary_row_partition_computer.h" -#include "paimon/common/utils/fields_comparator.h" -#include "paimon/common/utils/options_utils.h" -#include "paimon/common/utils/string_utils.h" -#include "paimon/core/catalog/catalog_snapshot_commit.h" -#include "paimon/core/catalog/catalog_utils.h" -#include "paimon/core/catalog/commit_lock.h" -#include "paimon/core/catalog/renaming_snapshot_commit.h" -#include "paimon/core/catalog/snapshot_commit.h" -#include "paimon/core/catalog/version_managed_catalog.h" -#include "paimon/core/core_options.h" -#include "paimon/core/disk/io_manager.h" -#include "paimon/core/index/index_file_handler.h" -#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" -#include "paimon/core/index/pk/primary_key_index_definitions.h" -#include "paimon/core/manifest/index_manifest_entry.h" -#include "paimon/core/manifest/index_manifest_file.h" -#include "paimon/core/mergetree/compact/lookup_merge_function.h" -#include "paimon/core/mergetree/compact/merge_function.h" -#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" -#include "paimon/core/operation/append_only_file_store_scan.h" -#include "paimon/core/operation/append_only_file_store_write.h" -#include "paimon/core/operation/commit/realtime_commit_properties.h" -#include "paimon/core/operation/data_evolution_file_store_scan.h" -#include "paimon/core/operation/expire_snapshots.h" -#include "paimon/core/operation/file_store_commit_impl.h" -#include "paimon/core/operation/file_store_scan.h" -#include "paimon/core/operation/internal_read_context.h" -#include "paimon/core/operation/key_value_file_store_scan.h" -#include "paimon/core/operation/key_value_file_store_write.h" -#include "paimon/core/options/merge_engine.h" -#include "paimon/core/postpone/postpone_bucket_file_store_write.h" -#include "paimon/core/realtime/realtime_context_impl.h" -#include "paimon/core/realtime/realtime_schema_layout.h" -#include "paimon/core/realtime/realtime_utils.h" -#include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/schema_validation.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/table/bucket_mode.h" -#include "paimon/core/table/file_store.h" -#include "paimon/core/table/format/format_table_file_store_commit.h" -#include "paimon/core/table/format/format_table_file_store_write.h" -#include "paimon/core/table/format/format_table_loader.h" -#include "paimon/core/table/format/format_table_read.h" -#include "paimon/core/table/format/format_table_scan.h" -#include "paimon/core/table/source/abstract_table_scan.h" -#include "paimon/core/table/source/append_only_split_generator.h" -#include "paimon/core/table/source/append_only_table_read.h" -#include "paimon/core/table/source/data_evolution_batch_scan.h" -#include "paimon/core/table/source/data_evolution_split_generator.h" -#include "paimon/core/table/source/data_table_batch_scan.h" -#include "paimon/core/table/source/data_table_stream_scan.h" -#include "paimon/core/table/source/fallback_table_read.h" -#include "paimon/core/table/source/key_value_table_read.h" -#include "paimon/core/table/source/merge_tree_split_generator.h" -#include "paimon/core/table/source/primary_key_index_batch_scan.h" -#include "paimon/core/table/source/read_optimized_scan_options.h" -#include "paimon/core/table/source/realtime_table_scan.h" -#include "paimon/core/table/source/snapshot/snapshot_reader.h" -#include "paimon/core/table/source/split_generator.h" -#include "paimon/core/table/system/system_table.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/core/utils/field_mapping.h" -#include "paimon/core/utils/file_store_path_factory.h" -#include "paimon/core/utils/primary_key_table_utils.h" -#include "paimon/core/utils/snapshot_manager.h" -#include "paimon/defs.h" -#include "paimon/file_store_commit.h" -#include "paimon/file_store_write.h" -#include "paimon/format/file_format.h" -#include "paimon/fs/file_system.h" -#include "paimon/read_context.h" -#include "paimon/realtime/realtime_context.h" -#include "paimon/result.h" -#include "paimon/scan_context.h" -#include "paimon/schema/schema.h" -#include "paimon/status.h" -#include "paimon/table/format/format_table.h" -#include "paimon/table/source/table_read.h" -#include "paimon/table/source/table_scan.h" -#include "paimon/write_context.h" - -namespace arrow { -class Schema; -} // namespace arrow - namespace paimon { -class DataSplit; -class Executor; -struct KeyValue; -class MemoryPool; -template -class MergeFunctionWrapper; - -namespace { - -Status RestoreRealtimeCommittedProgress(const std::shared_ptr& realtime_context, - const std::shared_ptr& snapshot_manager, - const CoreOptions& options) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, - RealtimeContextImpl::Cast(realtime_context)); - PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_ASSIGN_OR_RAISE( - RealtimeOffsetMap realtime_committed_offsets, - RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); - PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( - latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); - } - return Status::OK(); -} - -/// Maps a `WriteContext` onto `FormatTableFileStoreWrite`, refusing by name what a format table -/// cannot honour rather than silently dropping it. -Result> NewFormatTableWrite( - const std::shared_ptr& table, const WriteContext& ctx) { - if (ctx.GetCatalog() != nullptr) { - return Status::Invalid( - "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " - "format table"); - } - if (ctx.IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing a streaming write could " - "commit against"); - } - if (ctx.GetRealtimeContext() != nullptr) { - return Status::NotImplemented("a format table has no real-time store to write into"); - } - if (!ctx.GetWriteSchema().empty()) { - return Status::NotImplemented( - "a format table write takes the table's own columns; a write schema naming a " - "subset of them is not supported yet"); - } - // A write id prefixes a postpone-bucket writer's files so that one compaction reader can - // put them back in order. A format table has no buckets, so it would identify nothing. - if (ctx.GetWriteId().has_value()) { - return Status::NotImplemented( - "a format table has no buckets, so a write id would name nothing"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, - FormatTableFileStoreWrite::Create(table, ctx.GetMemoryPool())); - return std::unique_ptr(std::move(format_write)); -} - -Result HasHistoricalPrimaryKeyBTreeDefinition( - const std::shared_ptr& schema_manager, int64_t current_schema_id) { - if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { - return false; - } - PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); - for (int64_t schema_id : schema_ids) { - if (schema_id >= current_schema_id) { - continue; - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, - schema_manager->ReadSchema(schema_id)); - const auto& historical_options = historical_schema->Options(); - auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); - if (option != historical_options.end() && - !StringUtils::IsNullOrWhitespaceOnly(option->second)) { - return true; - } - } - return false; -} - -} // namespace - Table::Table(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid) @@ -214,1144 +27,4 @@ Table::Table(const std::shared_ptr& file_system, const std::string& schema_(schema), identifier_(identifier), uuid_(uuid) {} - -Result> Table::NewScan(std::unique_ptr context) const { - return TableScan::Create(std::move(context)); -} - -Result> Table::NewRead(std::unique_ptr context) const { - return TableRead::Create(std::move(context)); -} - -Result> Table::NewWrite( - std::unique_ptr context) const { - return FileStoreWrite::Create(std::move(context)); -} - -Result> Table::NewCommit( - std::unique_ptr context) const { - return FileStoreCommit::Create(std::move(context)); -} - -Result> FileStoreWrite::Create(std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("write context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableWrite(given_table, *ctx); - } - - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && - ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); - } - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs, - ctx->GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx->GetBranch(); - if (ctx->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); - } - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } - // Only the main branch may have a current schema the catalog alone holds. A branch publishes - // its own under `branch/branch-`, which is where the load below and a read of the branch - // both read it, so a catalog answering for no branch schema does not fail a write to one. - std::optional catalog_table_schema; - if (ctx->GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog write requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); - } - } - // A format table writes plain data files into a directory, so it never reaches the manifest - // path below. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch, - ctx->GetOptions(), catalog_table_schema, schema_manager.get(), - &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableWrite(format_table, *ctx); - } - // The schema the dispatch above already read through `schema_manager`, rather than a second - // read of the same file. - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths above are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; - PAIMON_ASSIGN_OR_RAISE( - CoreOptions options, - CoreOptions::FromMap(opts, specific_fs, ctx->GetFileSystemSchemeToIdentifierMap())); - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); - if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) { - return Status::NotImplemented( - "Dedicated VECTOR storage does not yet support custom write schemas or partial writes"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); - - FileStore file_store(ctx->GetRootPath(), options); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr file_store_path_factory, - file_store.NewPathFactory(arrow_schema, schema->PartitionKeys(), - options.GetWriteFileFormat(/*level=*/0)->Identifier(), - ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_manager, - file_store.NewSnapshotManager(ctx->GetCatalog(), catalog_identifier)); - std::shared_ptr io_manager; - const auto& io_temp_dir = ctx->GetTempDirectory(); - if (!io_temp_dir.empty()) { - io_manager = std::make_shared(io_temp_dir, options.GetFileSystem()); - } - - bool ignore_previous_files = ctx->IgnorePreviousFiles(); - if (ctx->GetRealtimeContext() && !options.RealtimeEnabled()) { - return Status::Invalid("real-time write requires realtime.enabled=true"); - } - if (ctx->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, - RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); - PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); - } - if (schema->PrimaryKeys().empty()) { - // append table - const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); - bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); - if (options.GetBucket() == -1) { - need_dv_maintainer_factory = false; - ignore_previous_files = true; - } else if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in append table", options.GetBucket())); - } - if (ctx->GetRealtimeContext()) { - if (!ctx->GetWriteSchema().empty()) { - return Status::Invalid("real-time append write does not support a custom schema"); - } - if (!realtime_deduplicate) { - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - } else if (realtime_deduplicate) { - if (!ctx->GetWriteSchema().empty() || ignore_previous_files || options.WriteOnly()) { - return Status::Invalid( - "deduplicate compaction requires the full schema, restored files and " - "write-only=false"); - } - } - std::shared_ptr write_schema = arrow_schema; - const auto& write_field_names = ctx->GetWriteSchema(); - if (!write_field_names.empty()) { - arrow::FieldVector write_fields; - write_fields.reserve(write_field_names.size()); - for (const auto& field_name : write_field_names) { - auto field = arrow_schema->GetFieldByName(field_name); - if (!field) { - // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER - return Status::Invalid( - fmt::format("write field {} does not exist in table schema", field_name)); - } - write_fields.push_back(field); - } - write_schema = arrow::schema(write_fields); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - const RealtimeStoreMode mode = realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE - : RealtimeStoreMode::APPEND_ONLY; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, - RealtimeSchemaLayout::Create(mode, write_schema)); - realtime_schema_layout = std::move(layout); - } - - std::shared_ptr dv_maintainer_factory; - if (need_dv_maintainer_factory) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr index_file_handler, - file_store.NewIndexFileHandler(ResolveBucketMode(options.GetBucket(), schema), - file_store_path_factory, ctx->GetMemoryPool())); - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - - auto file_store_write = std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, write_schema, realtime_schema_layout, - partition_schema, dv_maintainer_factory, io_manager, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetRealtimeContext(), - ctx->GetExecutor(), ctx->GetMemoryPool()); - if (ctx->GetRealtimeContext() && realtime_deduplicate) { - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_RETURN_NOT_OK( - file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); - } - } - return std::unique_ptr(std::move(file_store_write)); - } else { - // pk table - if (ctx->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); - if (ignore_previous_files) { - return Status::NotImplemented( - "PK realtime requires restore from the latest snapshot"); - } - if (!ctx->GetWriteSchema().empty()) { - return Status::NotImplemented("PK realtime does not support a custom write schema"); - } - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { - return PostponeBucketFileStoreWrite::Create( - snapshot_manager, schema_manager, ctx->GetCommitUser(), ctx->GetRootPath(), schema, - arrow_schema, partition_schema, io_manager, options, ctx->IsStreamingMode(), - ctx->IgnoreNumBucketCheck(), ctx->GetWriteId(), - ctx->GetFileSystemSchemeToIdentifierMap(), ctx->GetExecutor(), ctx->GetMemoryPool(), - specific_fs); - } - if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in key value table", options.GetBucket())); - } - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, - schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_primary_key_fields, - options.SequenceFieldSortOrderIsAscending())); - auto primary_keys = schema->PrimaryKeys(); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, - PrimaryKeyTableUtils::CreateMergeFunction( - arrow_schema, primary_keys, options, ctx->GetMemoryPool())); - if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { - // don't wrap first row, it is already OK - merge_function = std::make_unique(std::move(merge_function)); - } - std::shared_ptr> merge_function_wrapper = - std::make_shared(std::move(merge_function)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr sequence_fields_comparator, - PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); - - std::shared_ptr dv_maintainer_factory; - PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, - PrimaryKeyIndexDefinitions::Create(*schema)); - bool has_btree_index = false; - for (const PrimaryKeyIndexDefinition& definition : - primary_key_index_definitions.Definitions()) { - if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { - has_btree_index = true; - break; - } - } - std::optional latest_snapshot; - bool has_existing_index_manifest = false; - if (!has_btree_index) { - PAIMON_ASSIGN_OR_RAISE( - bool has_historical_btree_index, - HasHistoricalPrimaryKeyBTreeDefinition(schema_manager, schema->Id())); - if (has_historical_btree_index) { - PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); - has_existing_index_manifest = - latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); - } - } - std::shared_ptr index_file_handler; - if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - index_file_handler, - file_store.NewIndexFileHandler(ResolveBucketMode(options.GetBucket(), schema), - file_store_path_factory, ctx->GetMemoryPool())); - } - bool has_restored_btree_index = false; - if (!has_btree_index && has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - std::vector restored_btree_entries, - index_file_handler->Scan( - latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { - return entry.index_file->IndexType() == BtreeDefs::kIdentifier && - IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); - })); - has_restored_btree_index = !restored_btree_entries.empty(); - } - if (options.DeletionVectorsEnabled()) { - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - std::shared_ptr - primary_key_index_maintainer_factory; - if (has_btree_index || has_restored_btree_index) { - primary_key_index_maintainer_factory = - std::make_shared( - ctx->GetRootPath(), branch, schema, primary_key_index_definitions.Definitions(), - file_store_path_factory, index_file_handler, options, io_manager, - ctx->EnableMultiThreadSpill(), ctx->GetExecutor(), ctx->GetMemoryPool()); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr layout, - RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); - realtime_schema_layout = std::move(layout); - } - - return std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, realtime_schema_layout, partition_schema, - dv_maintainer_factory, primary_key_index_maintainer_factory, io_manager, key_comparator, - sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->EnableMultiThreadSpill(), - ctx->GetRealtimeContext(), ctx->GetExecutor(), ctx->GetMemoryPool()); - } -} - -namespace { - -class TableScanImpl { - public: - static Result> CreateFileStoreScan( - const std::shared_ptr& path_factory, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const std::shared_ptr& executor, const std::shared_ptr& memory_pool, - const ScanContext* context) { - auto fs = core_options.GetFileSystem(); - std::string branch = BranchManager::NormalizeBranch(core_options.GetBranch()); - FileStore file_store(context->GetPath(), core_options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_manager, - file_store.NewSnapshotManager(/*catalog=*/nullptr, - /*catalog_identifier=*/std::nullopt)); - // TODO(liancheng.lsz): support fallback branch in scan - auto schema_manager = std::make_shared(fs, context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, - file_store.NewManifestList(path_factory, memory_pool)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - file_store.NewManifestFile(path_factory, partition_schema, memory_pool)); - std::unique_ptr scan; - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - PAIMON_ASSIGN_OR_RAISE( - scan, DataEvolutionFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, AppendOnlyFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, - context->GetScanFilters(), core_options, - executor, memory_pool)); - } - return WithTablePath(std::move(scan), context); - } - - static std::unique_ptr WithTablePath(std::unique_ptr&& scan, - const ScanContext* context) { - scan->WithTablePath(context->GetPath()); - return std::move(scan); - } - - static Result> CreateSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const ScanContext* context) { - auto source_split_target_size = core_options.GetSourceSplitTargetSize(); - auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.BlobSplitByFileSize()); - } - BucketMode bucket_mode = (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE - : BucketMode::HASH_FIXED); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, bucket_mode); - } else { - // TODO(liancheng.lsz): support evolution - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - table_schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, - table_schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), - key_comparator); - } - } -}; - -/// @param loaded_schema The table's schema, when the caller already read it, or null when it has -/// to be read here. `TableScan::Create()` reads one to dispatch on its table type. -Result> NewDataTableScan( - const std::shared_ptr& context, const std::shared_ptr& loaded_schema); - -/// Maps a `ScanContext` onto `FormatTableScan`, which takes one partition filter and a limit and -/// nothing else. -/// -/// A format table has no buckets, so a bucket filter has nothing to act on. A predicate is -/// refused too, which is narrower than Java: `FormatReadBuilder.newScan()` hands the partition -/// half of one to the scan to prune directories with, which this does not do yet. -Result> NewFormatTableScan(const std::shared_ptr& table, - const std::shared_ptr& context) { - // Anything a `ScanContext` carries that a format table cannot honour is refused rather than - // silently dropped. - if (context->IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing for a streaming scan to follow"); - } - if (context->GetRealtimeContext() != nullptr) { - return Status::NotImplemented( - "a format table has no real-time store to union with what is on disk"); - } - if (context->GetGlobalIndexResult() != nullptr) { - return Status::NotImplemented("a format table carries no global index"); - } - std::map partition_filter; - std::shared_ptr filters = context->GetScanFilters(); - if (filters != nullptr) { - if (filters->GetPredicate() != nullptr) { - return Status::NotImplemented( - "a format table scan does not take a predicate: its files carry no statistics to " - "skip by, and pruning by the partition columns a predicate names is not " - "implemented yet; give the partition values as a partition filter instead"); - } - if (filters->GetBucketFilter()) { - return Status::NotImplemented("a format table has no buckets to filter by"); - } - const std::vector>& partition_filters = - filters->GetPartitionFilters(); - if (partition_filters.size() > 1) { - return Status::NotImplemented("a format table scan takes at most one partition filter"); - } - if (partition_filters.size() == 1) { - partition_filter = partition_filters.front(); - } - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - FormatTableScan::Create(table, partition_filter, context->GetLimit())); - return std::unique_ptr(std::move(scan)); -} - -} // namespace - -Result> TableScan::Create(std::unique_ptr context) { - if (context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - std::shared_ptr shared_context = std::move(context); - // A table the caller already loaded says what it is, so nothing is read to find out. - if (shared_context->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTable::Copy(shared_context->GetFormatTable(), shared_context->GetOptions())); - return NewFormatTableScan(format_table, shared_context); - } - // load schema - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(shared_context->GetOptions(), - shared_context->GetSpecificFileSystem(), {})); - if (shared_context->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(tmp_options.GetBranch())); - } - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(shared_context->GetPath())); - if (system_table_path) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), shared_context->GetPath(), - shared_context->GetOptions())); - return system_table->NewScan(shared_context); - } - // A format table is planned by listing directories, so it never reaches the manifest path - // below. One `TableScan` interface serves both, as Java Paimon serves both through one - // `ReadBuilder`. - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), shared_context->GetPath(), - BranchManager::NormalizeBranch(tmp_options.GetBranch()), - shared_context->GetOptions(), - shared_context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableScan(format_table, shared_context); - } - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableScan(shared_context, latest_schema); -} - -namespace { - -Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, - const ScanContext& context, bool read_optimized) { - if (!context.GetRealtimeContext()) { - return Status::OK(); - } - if (!core_options.RealtimeEnabled()) { - return Status::Invalid("real-time scan requires realtime.enabled=true"); - } - if (core_options.GetBucket() <= 0) { - return Status::Invalid("real-time union read requires fixed bucket mode"); - } - if (core_options.DataEvolutionEnabled()) { - return Status::Invalid("real-time union read does not support data evolution"); - } - if (!table_schema.PrimaryKeys().empty()) { - if (read_optimized) { - return Status::NotImplemented( - "PK real-time union read does not support read-optimized scans"); - } - PAIMON_RETURN_NOT_OK( - PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); - } - if (context.IsStreamingMode()) { - return Status::Invalid("real-time union read currently supports batch scans only"); - } - if (context.GetLimit()) { - return Status::Invalid("real-time union read does not support scan limit pushdown"); - } - if (context.GetGlobalIndexResult()) { - return Status::Invalid("real-time union read does not support global index splits"); - } - StartupMode startup_mode = core_options.GetStartupMode(); - const bool specified_snapshot = - startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); - if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || - specified_snapshot)) { - return Status::Invalid( - "real-time union read requires the latest snapshot or an explicit snapshot id"); - } - return Status::OK(); -} - -Result> NewDataTableScan( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); - std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema, - schema_manager.Latest()); - if (latest_table_schema == std::nullopt) { - return Status::Invalid("not found latest schema"); - } - table_schema = latest_table_schema.value(); - } - PAIMON_ASSIGN_OR_RAISE(bool read_optimized, - OptionsUtils::GetValueFromMap(context->GetOptions(), - kReadOptimizedScanOption, false)); - // merge options - auto options = table_schema->Options(); - for (const auto& [key, value] : context->GetOptions()) { - options[key] = value; - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, context->GetSpecificFileSystem(), {})); - core_options.WithCache(context->GetCache()); - - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); - - PAIMON_RETURN_NOT_OK( - ValidateRealtimeScan(*table_schema, core_options, *context, read_optimized)); - // validate options - if (core_options.GetBucket() == -1) { - if (!table_schema->PrimaryKeys().empty()) { - return Status::NotImplemented(fmt::format( - "do not support pk table bucket={} in scan process", core_options.GetBucket())); - } - } else if (core_options.GetBucket() < 1 && - !SchemaValidation::IsPostponeBucketTable(*table_schema, core_options.GetBucket())) { - return Status::Invalid( - fmt::format("do not support bucket={} in scan process", core_options.GetBucket())); - } - - // validate schema and scan filter - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { - PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( - *arrow_schema, context->GetScanFilters()->GetPredicate(), - /*validate_field_idx=*/false)); - } - FileStore file_store(context->GetPath(), core_options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, - file_store.NewPathFactory(arrow_schema, table_schema->PartitionKeys(), - core_options.GetFileFormat()->Identifier(), - context->GetMemoryPool())); - - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_store_scan, - TableScanImpl::CreateFileStoreScan( - path_factory, arrow_schema, table_schema, core_options, - context->GetExecutor(), context->GetMemoryPool(), context.get())); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr split_generator, - TableScanImpl::CreateSplitGenerator(table_schema, core_options, context.get())); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr index_file_handler, - file_store.NewIndexFileHandler(ResolveBucketMode(core_options.GetBucket(), table_schema), - path_factory, context->GetMemoryPool())); - auto snapshot_reader = std::make_shared( - file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler)); - const bool pk_table = !table_schema->PrimaryKeys().empty(); - if (read_optimized && pk_table && context->IsStreamingMode()) { - return Status::NotImplemented( - "read-optimized system table does not support streaming scan for primary key table"); - } - if (context->IsStreamingMode()) { - return std::make_unique(core_options, snapshot_reader); - } - auto batch_scan = std::make_unique( - /*pk_table=*/pk_table, core_options, snapshot_reader, read_optimized, context->GetLimit(), - /*realtime_pk_scan=*/pk_table && context->GetRealtimeContext() != nullptr); - if (context->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, - RealtimeContextImpl::Cast(context->GetRealtimeContext())); - return std::make_unique( - std::move(batch_scan), pk_table, realtime_context, path_factory, - snapshot_reader->GetSnapshotManager(), core_options.GetFileSystem(), - context->GetScanFilters(), core_options.GetRealtimeReadViewTtlMillis()); - } - if (core_options.DataEvolutionEnabled()) { - return std::make_unique( - context->GetPath(), snapshot_reader, std::move(batch_scan), table_schema, - context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), - context->GetExecutor()); - } - if (pk_table && !read_optimized && core_options.GlobalIndexEnabled()) { - PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, - PrimaryKeyIndexDefinitions::Create(*table_schema)); - if (!definitions.ScalarDefinitions().empty()) { - return PrimaryKeyIndexBatchScan::Create(snapshot_reader, std::move(batch_scan), - table_schema, path_factory, core_options, - context->GetMemoryPool()); - } - } - return batch_scan; -} - -} // namespace - -namespace { - -/// @param loaded_schema The schema of `branch`, when the caller already read it, or null when it -/// has to be read here: `TableRead::Create()` reads one to dispatch on the table type, and -/// reading it again would cost a second listing and read. -Result> CreateInternalReadContext( - const std::shared_ptr& context, const std::string& branch, - const std::shared_ptr& loaded_schema) { - if (context->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); - } - std::map tmp_options = context->GetOptions(); - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, - CoreOptions::FromMap(tmp_options, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - SchemaManager schema_manager(tmp_core_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - schema_manager.Latest()); - if (!latest_schema) { - return Status::Invalid(fmt::format("schema file not found in path {}, branch {}", - context->GetPath(), branch)); - } - table_schema = latest_schema.value(); - } - assert(table_schema); - - // merge options - auto options = table_schema->Options(); - for (const auto& [key, value] : tmp_options) { - options[key] = value; - } - if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { - options[Options::BRANCH] = branch; - } - return InternalReadContext::Create(context, table_schema, options); -} - -Result> CreateTableRead( - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor) { - const auto& core_options = internal_context->GetCoreOptions(); - const auto& table_schema = internal_context->GetTableSchema(); - if (internal_context->GetRealtimeContext() && !core_options.RealtimeEnabled()) { - return Status::Invalid("real-time read requires realtime.enabled=true"); - } - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - FileStore file_store(internal_context->GetPath(), core_options); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - file_store.NewPathFactory(arrow_schema, table_schema->PartitionKeys(), - core_options.GetFileFormat()->Identifier(), memory_pool)); - - if (internal_context->GetPrimaryKeys().empty()) { - return std::make_unique(path_factory, internal_context, memory_pool, - executor); - } - return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); -} - -Result> NewDataTableRead( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, - CreateInternalReadContext(context, context->GetBranch(), loaded_schema)); - - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr table_read, - CreateTableRead(internal_context, context->GetMemoryPool(), context->GetExecutor())); - - std::optional scan_fallback_branch = - internal_context->GetCoreOptions().GetScanFallbackBranch(); - if (!scan_fallback_branch || - StringUtils::IsNullOrWhitespaceOnly(scan_fallback_branch.value())) { - return std::move(table_read); - } - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr fallback_context, - CreateInternalReadContext(context, /*branch=*/scan_fallback_branch.value(), - /*loaded_schema=*/nullptr)); - - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr fallback_table_read, - CreateTableRead(fallback_context, context->GetMemoryPool(), context->GetExecutor())); - const std::shared_ptr& arrow_pool = internal_context->GetArrowMemoryPool(); - return std::make_unique(std::move(table_read), - std::move(fallback_table_read), arrow_pool); -} - -/// Maps a `ReadContext` onto `FormatTableRead`, which reads everything it needs out of the context -/// itself: the columns, the predicate, the pool and executor, and what a file is opened with. It -/// refuses by name what a format table cannot honour. -Result> NewFormatTableRead(const std::shared_ptr& table, - const std::shared_ptr& context) { - if (context->EnablePreciseFileIndexEvaluate()) { - return Status::NotImplemented( - "Precise file-index evaluation is not supported for format tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - FormatTableRead::Create(table, context)); - return std::unique_ptr(std::move(read)); -} - -} // namespace - -Result> TableRead::Create(std::unique_ptr ctx) { - std::shared_ptr context = std::move(ctx); - if (context == nullptr) { - return Status::Invalid("read context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - if (context->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(context->GetBranch())); - } - // A table the caller already loaded says what it is, so nothing is read to find out. - if (context->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); - return NewFormatTableRead(format_table, context); - } - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_core_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(context->GetPath())); - if (system_table_path) { - if (context->EnablePreciseFileIndexEvaluate()) { - return Status::NotImplemented( - "Precise file-index evaluation is not supported for system tables"); - } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetOptions())); - return system_table->NewRead(context); - } - - // A format table has no manifest and a file another engine wrote carries no field ids, so - // which files there are and how a row is put back together is its own. How a file is opened - // is not: both paths go through `DataFileReaderFactory`. - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetBranch(), context->GetOptions(), - context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableRead(format_table, context); - } - - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableRead(context, latest_schema); -} - -namespace { - -CommitScanner::ScanSupplier CreateAppendScanSupplier( - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - -CommitScanner::ScanSupplier CreatePkScanSupplier( - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - -bool IsCatalogCommit(const CommitContext& ctx) { - return AsVersionManaged(ctx.GetCatalog()) != nullptr; -} - -Result> NewSnapshotCommit( - const CommitContext& ctx, const std::optional& catalog_identifier, - const CoreOptions& options, const std::shared_ptr& snapshot_manager) { - if (IsCatalogCommit(ctx)) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr lock, - AsVersionManaged(ctx.GetCatalog()) - ->CreateCommitLock(catalog_identifier.value(), ctx.GetTableId(), - ctx.GetCommitUser(), options)); - // Reuse the caller's UUID so table recreation cannot redirect this commit. - return std::shared_ptr(std::make_shared( - ctx.GetCatalog(), catalog_identifier.value(), ctx.GetTableId(), std::move(lock))); - } - if (ctx.UseRESTCatalogCommit()) { - if (options.RestCommitAcquireLock()) { - return Status::Invalid(fmt::format( - "'{}' needs a commit through a REST catalog with WithCatalog(); a commit table " - "request built for the caller to send cannot hold a commit lease", - Options::REST_COMMIT_ACQUIRE_LOCK)); - } - return std::shared_ptr( - std::make_shared(ctx.GetTableId())); - } - // Catalogs without version management may still supply a table ID. - if (ctx.GetTableId() && ctx.GetCatalog() == nullptr) { - return Status::Invalid( - "a commit which publishes the snapshot by writing it to the table directory sends no " - "commit table request, so there is nowhere to put a table id; commit through a " - "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); - } - return std::shared_ptr( - std::make_shared(options.GetFileSystem(), snapshot_manager)); -} - -/// Creates a format-table commit, rejecting non-default options that require snapshots. -Result> NewFormatTableCommit( - const std::shared_ptr& table, const CommitContext& ctx) { - if (!ctx.IgnoreEmptyCommit()) { - return Status::NotImplemented( - "a format table cannot record an empty commit: keeping one means writing a " - "snapshot that adds no files, and there are no snapshots here"); - } - if (ctx.UseRESTCatalogCommit()) { - return Status::NotImplemented( - "a format table commits by renaming files into place, not by sending a snapshot " - "to a rest catalog"); - } - if (ctx.GetTableId()) { - return Status::NotImplemented( - "a format table sends no commit table request, so there is nowhere to put a table id"); - } - if (ctx.GetCatalog() != nullptr) { - return Status::NotImplemented( - "a format table keeps no snapshot for a catalog to take, so it commits by renaming " - "files into place rather than through the catalog it was loaded from"); - } - if (ctx.AppendCommitCheckConflict()) { - return Status::NotImplemented( - "a format table has no manifests to check a concurrent commit against"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, - FormatTableFileStoreCommit::Create(table)); - return std::unique_ptr(std::move(format_commit)); -} - -} // namespace - -Result> FileStoreCommit::Create( - std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("commit context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableCommit(given_table, *ctx); - } - - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - // Use catalog credentials unless the caller supplied a file system. - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - // Use the credentials of this table, which a catalog issuing per-table temporary - // ones only hands out through GetTableFileSystem. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); - const std::string& root_path = ctx->GetRootPath(); - // The branch the context was built for, already validated by `CommitContextBuilder`. - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - ctx->GetIdentifier(), ctx->GetOptions(), - /*explicit_branch=*/std::nullopt, "commit")); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } - // Catalog-managed schemas may be absent from the table directory, but only on the main branch: - // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail - // a commit to one. - std::optional catalog_table_schema; - FileStoreCommitImpl::SchemaIdLoader schema_id_loader; - if (ctx->GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); - catalog_table_schema = std::move(schema_json); - } - std::shared_ptr catalog = ctx->GetCatalog(); - Identifier identifier = catalog_identifier.value(); - schema_id_loader = [catalog, identifier]() -> Result { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - catalog->LoadTableSchema(identifier)); - std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); - if (data_schema == nullptr) { - return Status::Invalid( - "the catalog did not hand back a data table schema, which has the id a " - "snapshot records"); - } - return data_schema->Id(); - }; - } - // A format table commits by renaming files into place, so it never reaches the snapshot path - // below. The managed path here reads the commit's branch, so this reads the same one: the two - // must not dispatch on different schemas. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), root_path, branch); - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTableLoader::TryLoad( - tmp_options.GetFileSystem(), root_path, branch, ctx->GetOptions(), - catalog_table_schema, schema_manager.get(), &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableCommit(format_table, *ctx); - } - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths below are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; - std::shared_ptr arrow_schema = - DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, specific_fs)); - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); - assert(options.GetFileSystem()); - assert(options.GetFileFormat()); - PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); - PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path)); - // Catalog commits do not require atomic rename on the object store. - if (is_object_store && !ctx->UseRESTCatalogCommit() && !IsCatalogCommit(*ctx) && - opts.find("enable-object-store-commit-in-inte-test") == opts.end()) { - return Status::NotImplemented( - "commit operation does not support object store file system for now"); - } - FileStore file_store(root_path, options); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr partition_computer, - file_store.NewPartitionComputer(arrow_schema, schema->PartitionKeys(), - ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - file_store.NewPathFactory(arrow_schema, schema->PartitionKeys(), - options.GetFileFormat()->Identifier(), ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_manager, - file_store.NewSnapshotManager(ctx->GetCatalog(), catalog_identifier)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, - file_store.NewManifestList(path_factory, ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - file_store.NewManifestFile(path_factory, partition_schema, ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr index_manifest_file, - file_store.NewIndexManifestFile(ResolveBucketMode(options.GetBucket(), schema), - path_factory, ctx->GetMemoryPool())); - - auto expire_snapshots = std::make_shared( - snapshot_manager, path_factory, manifest_list, manifest_file, index_manifest_file, - options.GetFileSystem(), options.GetExpireConfig(), options.RealtimeEnabled(), - ctx->GetExecutor()); - - CommitScanner::ScanSupplier scan_supplier; - if (schema->PrimaryKeys().empty()) { - scan_supplier = CreateAppendScanSupplier(snapshot_manager, schema_manager, manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); - } else { - scan_supplier = CreatePkScanSupplier(snapshot_manager, schema_manager, manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); - } - - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_commit, - NewSnapshotCommit(*ctx, catalog_identifier, options, snapshot_manager)); - - return std::make_unique( - ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path, ctx->GetCommitUser(), - options, path_factory, std::move(partition_computer), snapshot_manager, snapshot_commit, - ctx->IgnoreEmptyCommit(), ctx->AppendCommitCheckConflict(), schema, - std::move(schema_id_loader), manifest_file, manifest_list, index_manifest_file, - expire_snapshots, schema_manager, std::move(scan_supplier)); -} - } // namespace paimon diff --git a/src/paimon/core/table/table_factory.cpp b/src/paimon/core/table/table_factory.cpp new file mode 100644 index 000000000..2f12551b1 --- /dev/null +++ b/src/paimon/core/table/table_factory.cpp @@ -0,0 +1,378 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/table_factory.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/table.h" +#include "paimon/commit_context.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/realtime/realtime_utils.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/format/format_table_loader.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/table/format/format_table.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" + +namespace paimon { +namespace { +Result> ManagedTable( + const std::string& path, const std::string& branch, const std::shared_ptr& schema, + CoreOptions options, const std::optional& identifier, + const std::optional& table_id, + const std::map& fs_scheme_map) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + FileStoreTable::Create(std::make_unique(path, std::move(options)), schema, + identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)), + table_id.value_or(""), fs_scheme_map)); + return std::shared_ptr
(std::move(table)); +} +} // namespace + +Result> TableFactory::Create(const WriteContext* ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + + // A table the caller already loaded says what it is, so nothing is read to find out. + if (ctx->GetFormatTable() != nullptr) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, + FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); + return std::shared_ptr
(given_table); + } + + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && + ctx->GetCatalog() != nullptr) { + if (!ctx->GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the write uses the credentials of this table. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx->GetOptions(), specific_fs, + ctx->GetFileSystemSchemeToIdentifierMap())); + std::string branch = ctx->GetBranch(); + if (ctx->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); + } + std::optional catalog_identifier; + if (ctx->GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE( + Identifier branch_identifier, + CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); + catalog_identifier.emplace(std::move(branch_identifier)); + } + // Only the main branch may have a current schema the catalog alone holds. A branch publishes + // its own under `branch/branch-`, which is where the load below and a read of the branch + // both read it, so a catalog answering for no branch schema does not fail a write to one. + std::optional catalog_table_schema; + if (ctx->GetCatalog() != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog write requires a table identifier"); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); + } + } + // A format table writes plain data files into a directory, so it never reaches the manifest + // path below. + auto schema_manager = + std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); + std::shared_ptr latest_schema; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr format_table, + FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch, + ctx->GetOptions(), catalog_table_schema, schema_manager.get(), + &latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr
(format_table); + } + // The schema the dispatch above already read through `schema_manager`, rather than a second + // read of the same file. + if (latest_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); + } + const std::shared_ptr& schema = latest_schema; + auto opts = schema->Options(); + for (const auto& [key, value] : ctx->GetOptions()) { + opts[key] = value; + } + // The metadata paths above are built for this branch, so what reads through these options, + // whose cache keys are scoped by it, has to read the same one. + opts[Options::BRANCH] = branch; + PAIMON_ASSIGN_OR_RAISE( + CoreOptions options, + CoreOptions::FromMap(opts, specific_fs, ctx->GetFileSystemSchemeToIdentifierMap())); + return ManagedTable(ctx->GetRootPath(), branch, schema, std::move(options), + ctx->GetIdentifier(), std::nullopt, + ctx->GetFileSystemSchemeToIdentifierMap()); +} + +Result> TableFactory::Create(const CommitContext* ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + + // A table the caller already loaded says what it is, so nothing is read to find out. + if (ctx->GetFormatTable() != nullptr) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, + FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); + return std::shared_ptr
(given_table); + } + + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + // Use catalog credentials unless the caller supplied a file system. + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetCatalog() != nullptr) { + if (!ctx->GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + // Use the credentials of this table, which a catalog issuing per-table temporary + // ones only hands out through GetTableFileSystem. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); + const std::string& root_path = ctx->GetRootPath(); + // The branch the context was built for, already validated by `CommitContextBuilder`. + PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( + ctx->GetIdentifier(), ctx->GetOptions(), + /*explicit_branch=*/std::nullopt, "commit")); + std::optional catalog_identifier; + if (ctx->GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE( + Identifier branch_identifier, + CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); + catalog_identifier.emplace(std::move(branch_identifier)); + } + // Catalog-managed schemas may be absent from the table directory, but only on the main branch: + // a branch publishes its own under `branch/branch-`, which is where the load below and a + // read of the branch both read it, so a catalog answering for no branch schema does not fail + // a commit to one. + std::optional catalog_table_schema; + if (ctx->GetCatalog() != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, + ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); + catalog_table_schema = std::move(schema_json); + } + } + + // A format table commits by renaming files into place, so it never reaches the snapshot path + // below. The managed path here reads the commit's branch, so this reads the same one: the two + // must not dispatch on different schemas. + auto schema_manager = + std::make_shared(tmp_options.GetFileSystem(), root_path, branch); + std::shared_ptr latest_schema; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + FormatTableLoader::TryLoad( + tmp_options.GetFileSystem(), root_path, branch, ctx->GetOptions(), + catalog_table_schema, schema_manager.get(), &latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr
(format_table); + } + if (latest_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); + } + const std::shared_ptr& schema = latest_schema; + auto opts = schema->Options(); + for (const auto& [key, value] : ctx->GetOptions()) { + opts[key] = value; + } + // The metadata paths below are built for this branch, so what reads through these options, + // whose cache keys are scoped by it, has to read the same one. + opts[Options::BRANCH] = branch; + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, specific_fs)); + return ManagedTable(ctx->GetRootPath(), branch, schema, std::move(options), + ctx->GetIdentifier(), ctx->GetTableId(), {}); +} + +Result> TableFactory::Create(const ScanContext* context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + + // A table the caller already loaded says what it is, so nothing is read to find out. + if (context->GetFormatTable() != nullptr) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); + return std::shared_ptr
(format_table); + } + // load schema + PAIMON_ASSIGN_OR_RAISE( + CoreOptions tmp_options, + CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); + if (context->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(tmp_options.GetBranch())); + } + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(context->GetPath())); + if (system_table_path) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), context->GetPath(), + context->GetOptions())); + return std::shared_ptr
(system_table); + } + // A format table is planned by listing directories, so it never reaches the manifest path + // below. One `TableScan` interface serves both, as Java Paimon serves both through one + // `ReadBuilder`. + std::shared_ptr latest_schema; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr format_table, + FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), context->GetPath(), + BranchManager::NormalizeBranch(tmp_options.GetBranch()), + context->GetOptions(), context->GetSpecificTableSchema(), + /*schema_manager=*/nullptr, &latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr
(format_table); + } + if (!latest_schema) { + return Status::Invalid("not found latest schema"); + } + auto options_map = latest_schema->Options(); + for (const auto& [key, value] : context->GetOptions()) { + options_map[key] = value; + } + PAIMON_ASSIGN_OR_RAISE( + std::string branch, + BranchManager::ResolveBranch(std::nullopt, options_map, std::nullopt, "scan")); + if (options_map.count(Options::BRANCH) != 0) { + options_map[Options::BRANCH] = branch; + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, + CoreOptions::FromMap(options_map, context->GetSpecificFileSystem(), {})); + return ManagedTable(context->GetPath(), branch, latest_schema, std::move(options), + context->GetIdentifier(), std::nullopt, {}); +} + +Result> TableFactory::Create(const ReadContext* context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + + // A table the caller already loaded says what it is, so nothing is read to find out. + if (context->GetFormatTable() != nullptr) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); + return std::shared_ptr
(format_table); + } + PAIMON_ASSIGN_OR_RAISE( + CoreOptions tmp_core_options, + CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), + context->GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(context->GetPath())); + if (system_table_path) { + if (context->EnablePreciseFileIndexEvaluate()) { + return Status::NotImplemented( + "Precise file-index evaluation is not supported for system tables"); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), context->GetPath(), + context->GetOptions(), + context->GetFileSystemSchemeToIdentifierMap())); + return std::shared_ptr
(system_table); + } + + // A format table has no manifest and a file another engine wrote carries no field ids, so + // which files there are and how a row is put back together is its own. How a file is opened + // is not: both paths go through `DataFileReaderFactory`. + std::shared_ptr latest_schema; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr format_table, + FormatTableLoader::TryLoad(tmp_core_options.GetFileSystem(), context->GetPath(), + context->GetBranch(), context->GetOptions(), + context->GetSpecificTableSchema(), + /*schema_manager=*/nullptr, &latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr
(format_table); + } + + if (!latest_schema) { + return Status::Invalid("not found latest schema"); + } + auto options_map = latest_schema->Options(); + for (const auto& [key, value] : context->GetOptions()) { + options_map[key] = value; + } + PAIMON_ASSIGN_OR_RAISE( + std::string branch, + BranchManager::ResolveBranch(std::nullopt, options_map, std::nullopt, "read")); + if (options_map.count(Options::BRANCH) != 0) { + options_map[Options::BRANCH] = branch; + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, + CoreOptions::FromMap(options_map, context->GetSpecificFileSystem(), + context->GetFileSystemSchemeToIdentifierMap())); + return ManagedTable(context->GetPath(), branch, latest_schema, std::move(options), + context->GetIdentifier(), std::nullopt, + context->GetFileSystemSchemeToIdentifierMap()); +} + +Result> TableScan::Create(std::unique_ptr context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); + return table->NewScan(std::move(context)); +} + +Result> TableRead::Create(std::unique_ptr context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); + return table->NewRead(std::move(context)); +} + +Result> FileStoreWrite::Create( + std::unique_ptr context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "write")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); + return table->NewWrite(std::move(context)); +} + +Result> FileStoreCommit::Create( + std::unique_ptr context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); + return table->NewCommit(std::move(context)); +} +} // namespace paimon diff --git a/src/paimon/core/table/table_factory.h b/src/paimon/core/table/table_factory.h new file mode 100644 index 000000000..7c560b752 --- /dev/null +++ b/src/paimon/core/table/table_factory.h @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +#include "paimon/result.h" +namespace paimon { +class Table; +class ScanContext; +class ReadContext; +class WriteContext; +class CommitContext; +/// Compatibility loading for the existing static operator factories. +class TableFactory { + public: + static Result> Create(const ScanContext* context); + static Result> Create(const ReadContext* context); + static Result> Create(const WriteContext* context); + static Result> Create(const CommitContext* context); +}; +} // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 4792a8461..5d8961f47 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -18,6 +18,7 @@ #include "paimon/catalog/table.h" +#include #include #include #include @@ -27,18 +28,34 @@ #include "arrow/api.h" #include "gtest/gtest.h" #include "paimon/commit_context.h" +#include "paimon/common/catalog/catalog_context.h" #include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/catalog/file_system_catalog.h" +#include "paimon/core/core_options.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/append_only_file_store_table.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/primary_key_file_store_table.h" +#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/source/fallback_table_read.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/table/table_factory.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/realtime/realtime_context.h" #include "paimon/scan_context.h" #include "paimon/schema/schema.h" +#include "paimon/table/format/format_table.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -57,9 +74,11 @@ Result> CreateManagedTable(const std::shared_ptr created, schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{})); - return std::make_shared
( - file_system, path, - checked_pointer_cast(std::shared_ptr(std::move(created))), identifier); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + FileStoreTable::Create(file_system, path, std::shared_ptr(std::move(created)), + identifier, "", branch)); + return std::shared_ptr
(std::move(table)); } } // namespace @@ -94,7 +113,7 @@ TEST(TableTest, TestGetTableWithUnknownDatabase) { EXPECT_EQ(table->Name(), "tbl1"); EXPECT_EQ(table->FullName(), "tbl1"); - std::shared_ptr latest_schema = table->LatestSchema(); + std::shared_ptr latest_schema = table->GetSchema(); ASSERT_NE(latest_schema, nullptr); auto data_schema = std::dynamic_pointer_cast(latest_schema); ASSERT_TRUE(data_schema != nullptr); @@ -123,11 +142,11 @@ TEST(TableTest, TestGetTableWithBranch) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "tbl1$branch_b1"))); ASSERT_EQ("tbl1$branch_b1", table->Name()); - ASSERT_EQ((std::vector{"b1_c0"}), table->LatestSchema()->FieldNames()); + ASSERT_EQ((std::vector{"b1_c0"}), table->GetSchema()->FieldNames()); for (const auto& main_identifier : {Identifier("db1", "tbl1$branch_main"), Identifier("db1", "tbl1$branch_MAIN")}) { ASSERT_OK_AND_ASSIGN(table, catalog.GetTable(main_identifier)); - ASSERT_EQ((std::vector{"main_c0"}), table->LatestSchema()->FieldNames()); + ASSERT_EQ((std::vector{"main_c0"}), table->GetSchema()->FieldNames()); } Status missing_branch = catalog.GetTable(Identifier("db1", "tbl1$branch_missing")).status(); @@ -164,8 +183,12 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { SCOPED_TRACE(main_identifier.ToString()); // A catalog that loaded the schema itself calls the constructor directly, which a // rest catalog is shaped like. The alias resolves to the main branch as well. - Table table(fs, dir->Str(), schema, main_identifier); - ASSERT_OK_AND_ASSIGN(std::string branch, table.CurrentBranch()); + ASSERT_OK_AND_ASSIGN(std::string resolved_branch, main_identifier.GetBranchNameOrDefault()); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, dir->Str(), std::dynamic_pointer_cast(schema), + main_identifier, "", resolved_branch)); + std::string branch = table->CurrentBranch(); EXPECT_EQ("main", branch); } } @@ -178,14 +201,14 @@ TEST(TableTest, TestCurrentBranch) { ASSERT_OK_AND_ASSIGN( std::shared_ptr
main_table, CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); - ASSERT_OK_AND_ASSIGN(std::string branch, main_table->CurrentBranch()); + std::string branch = std::dynamic_pointer_cast(main_table)->CurrentBranch(); EXPECT_EQ(branch, "main"); // The identifier a table was opened with names the branch it is for. ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", Identifier("db", "tbl", "dev"), /*branch=*/"dev")); - ASSERT_OK_AND_ASSIGN(branch, branch_table->CurrentBranch()); + branch = std::dynamic_pointer_cast(branch_table)->CurrentBranch(); EXPECT_EQ(branch, "dev"); } @@ -197,24 +220,16 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); auto catalog = std::make_shared(); - catalog->SetTableSchema(table->LatestSchema()); - - // A catalog that hands out no per-table file system opens the table without one, and one - // that does opens it with the file system issued for that table. - ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog->GetTable(identifier)); - EXPECT_EQ(opened->GetFileSystem(), nullptr); + catalog->SetTableSchema(table->GetSchema()); catalog->SetTableFileSystem(dir->GetFileSystem()); - ASSERT_OK_AND_ASSIGN(opened, catalog->GetTable(identifier)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog->GetTable(identifier)); EXPECT_EQ(opened->GetFileSystem(), dir->GetFileSystem()); // This mock serves no table location, so the table it opens carries none. EXPECT_EQ(opened->GetRootPath(), ""); } -TEST(TableTest, TestGetTableRejectsATableTypeThatIsNotAManagedTable) { - // A `Table` describes a managed paimon table. Handing back another type as one would promise - // snapshots and manifests it never had, so the failure belongs at the call that opens the - // table rather than at the first read. +TEST(TableTest, TestGetTableDispatchesByTableType) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); @@ -230,16 +245,43 @@ TEST(TableTest, TestGetTableRejectsATableTypeThatIsNotAManagedTable) { format_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {{"type", "format-table"}, {"file.format", "parquet"}})); - ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db", "format")), "Cannot open format table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
format_table, + catalog.GetTable(Identifier("db", "format"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(format_table)); - // A managed table with no `type` option at all is what a `Table` is for, and still opens. const std::string managed_dir = dir->Str() + "/db.db/managed"; ASSERT_OK(fs->Mkdirs(managed_dir)); SchemaManager managed_schema_manager(fs, managed_dir); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr managed_schema, managed_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {})); - ASSERT_OK(catalog.GetTable(Identifier("db", "managed"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
managed_table, + catalog.GetTable(Identifier("db", "managed"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(managed_table)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
system_table, + catalog.GetTable(Identifier("db", "managed$options"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(system_table)); + ASSERT_EQ((std::vector{"key", "value"}), system_table->GetSchema()->FieldNames()); + ASSERT_EQ(nullptr, std::dynamic_pointer_cast(system_table->GetSchema())); + for (bool direct : {false, true}) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ReadContextBuilder(managed_dir + "$options") + .EnablePreciseFileIndexEvaluate(true) + .Finish()); + auto read = direct ? system_table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); + ASSERT_NOK_WITH_MSG(read, + "Precise file-index evaluation is not supported for system tables"); + } + ASSERT_NOK_WITH_MSG(system_table->NewWrite(nullptr), "read-only"); + ASSERT_NOK_WITH_MSG(system_table->NewCommit(nullptr), "read-only"); + + const std::string pk_dir = dir->Str() + "/db.db/pk"; + SchemaManager pk_schema_manager(fs, pk_dir); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr pk_schema, + pk_schema_manager.CreateTable(schema, {}, {"id"}, {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
pk_table, catalog.GetTable(Identifier("db", "pk"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(pk_table)); } TEST(TableTest, TestGetTableFailedWithNonExistSchema) { @@ -313,7 +355,7 @@ TEST(TableTest, TestNewWriteCreatesAFileStoreWrite) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); // A catalog write reads the current schema from the catalog on the main branch. - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(table->GetSchema()); ASSERT_OK_AND_ASSIGN( std::unique_ptr ctx, @@ -332,7 +374,7 @@ TEST(TableTest, TestNewCommitCreatesAFileStoreCommit) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); // Like a write, a catalog commit reads the current schema from the catalog. - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(table->GetSchema()); ASSERT_OK_AND_ASSIGN( std::unique_ptr ctx, @@ -366,7 +408,7 @@ TEST(TableTest, TestCreateBuildsTheOperator) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); // A catalog write and commit read the current schema from the catalog on the main branch. - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(table->GetSchema()); // `TableScan::Create()` and friends build the operator straight from the context, which // carries the catalog and identifier it resolves the table through. @@ -393,4 +435,410 @@ TEST(TableTest, TestCreateBuildsTheOperator) { FileStoreCommit::Create(std::move(commit_ctx))); } +TEST(TableTest, TestLoadedSchemaAndPathAreUsedWithoutReloading) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"))); + auto schema = table->GetSchema(); + ASSERT_OK(fs->Delete(dir->Str() + "/schema", true)); + ASSERT_EQ(schema, table->GetSchema()); + ASSERT_EQ((std::vector{"id"}), schema->FieldNames()); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read_context, + ReadContextBuilder(dir->Str() + "/other").SetReadFieldNames({"id"}).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str() + "/other").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_TRUE(plan->Splits().empty()); +} + +TEST(TableTest, TestContextOptionsCannotChangeLoadedTypeOrBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); + for (const auto& options : {std::map{{Options::TYPE, "format-table"}}, + std::map{{Options::BRANCH, "dev"}}}) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str()).SetOptions(options).Finish()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), + "contradicts the loaded table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(dir->Str()).SetOptions(options).Finish()); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), + "contradicts the loaded table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + WriteContextBuilder(dir->Str(), "user").SetOptions(options).Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), + "contradicts the loaded table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + CommitContextBuilder(dir->Str(), "user").SetOptions(options).Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), + "contradicts the loaded table"); + } +} + +TEST(TableTest, TestOperationOptionsDoNotMutateTheLoadedTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); + auto schema = std::dynamic_pointer_cast(table->GetSchema()); + ASSERT_NE(nullptr, schema); + auto options = schema->Options(); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr invalid_context, + ScanContextBuilder(dir->Str()).SetOptions({{Options::BUCKET, "0"}}).Finish()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(invalid_context)), "bucket=0"); + ASSERT_EQ(options, schema->Options()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + table->NewScan(std::move(context))); +} + +TEST(TableTest, TestPathIdentifierPreservesBranch) { + Identifier catalog_table = CatalogUtils::IdentifierFromPath("/warehouse/db.db/tbl/", "dev"); + ASSERT_EQ("db", catalog_table.GetDatabaseName()); + ASSERT_EQ("tbl$branch_dev", catalog_table.GetObjectName()); + for (const std::string& path : {"tbl", "/warehouse/db.db.extra/tbl"}) { + Identifier table = CatalogUtils::IdentifierFromPath(path, "dev"); + ASSERT_EQ(Identifier::kUnknownDatabase, table.GetDatabaseName()); + ASSERT_EQ("tbl$branch_dev", table.GetObjectName()); + } +} + +TEST(TableTest, TestGlobalSystemTableRetainsCatalog) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr unique_catalog, + Catalog::Create(dir->Str(), {}, dir->GetFileSystem())); + std::shared_ptr catalog(std::move(unique_catalog)); + std::weak_ptr weak_catalog = catalog; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("sys", "tables"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(table)); + ASSERT_EQ(nullptr, std::dynamic_pointer_cast(table->GetSchema())); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(table->GetRootPath()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(table->GetRootPath()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); + catalog.reset(); + table.reset(); + ASSERT_FALSE(weak_catalog.expired()); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + ASSERT_TRUE(BatchReader::IsEofBatch(batch)); + reader->Close(); + reader.reset(); + read.reset(); + ASSERT_TRUE(weak_catalog.expired()); +} + +TEST(TableTest, TestExplicitContextBranchMustMatchLoadedTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + for (const std::string& table_branch : {"main", "dev"}) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/" + table_branch, + Identifier("db", "tbl", table_branch), table_branch)); + for (const std::optional& requested : + {std::optional(), std::optional("main"), + std::optional("dev"), std::optional(" ")}) { + SCOPED_TRACE(table_branch + ":" + requested.value_or("unset")); + WriteContextBuilder write_builder(table->GetRootPath(), "user"); + ReadContextBuilder read_builder(table->GetRootPath()); + if (requested) { + write_builder.WithBranch(*requested); + read_builder.WithBranch(*requested); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + auto write = table->NewWrite(std::move(write_context)); + auto read = table->NewRead(std::move(read_context)); + if (requested && BranchManager::NormalizeBranch(*requested) != table_branch) { + ASSERT_NOK_WITH_MSG(write, "context branch contradicts the loaded table"); + ASSERT_NOK_WITH_MSG(read, "context branch contradicts the loaded table"); + } else { + ASSERT_OK(write); + ASSERT_OK(read); + } + } + } +} + +TEST(TableTest, TestPreloadedSchemaBranchMatchesScanAndRead) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/table"; + ASSERT_TRUE(TestUtil::CopyDirectory( + std::filesystem::path( + GetDataDir() + "/parquet/append_table_with_rt_branch.db/append_table_with_rt_branch"), + std::filesystem::path(path))); + SnapshotManager main_snapshots(fs, path); + ASSERT_OK_AND_ASSIGN(Snapshot original, main_snapshots.LoadSnapshot(1)); + rapidjson::Document snapshot_json; + snapshot_json.SetObject(); + snapshot_json.CopyFrom(original.ToJson(&snapshot_json.GetAllocator()), + snapshot_json.GetAllocator()); + snapshot_json[Snapshot::FIELD_ID].SetInt64(2); + std::string json; + ASSERT_TRUE(RapidJsonUtil::ToJson(snapshot_json, &json)); + ASSERT_OK(fs->AtomicStore(main_snapshots.SnapshotPath(2), json)); + ASSERT_OK(main_snapshots.CommitLatestHint(2)); + SchemaManager schemas(fs, path, "rt"); + ASSERT_OK_AND_ASSIGN(std::optional> schema, schemas.Latest()); + ASSERT_TRUE(schema); + ASSERT_OK_AND_ASSIGN(std::unique_ptr preloaded, + schema.value()->MergeOptions({{Options::BRANCH, "rt"}})); + ASSERT_OK_AND_ASSIGN(std::string schema_json, preloaded->ToJsonString()); + for (const auto& overrides : {std::map{}, + std::map{{Options::BRANCH, "rt"}}, + std::map{{Options::BRANCH, "main"}}}) { + const std::string branch = overrides.empty() ? "rt" : overrides.at(Options::BRANCH); + SCOPED_TRACE(branch); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan_context, + ScanContextBuilder(path).SetTableSchema(schema_json).SetOptions(overrides).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(branch == "rt" ? 1 : 2, plan->SnapshotId()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read_context, + ReadContextBuilder(path).SetTableSchema(schema_json).SetOptions(overrides).Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + TableFactory::Create(read_context.get())); + ASSERT_EQ(branch, std::dynamic_pointer_cast(loaded)->CurrentBranch()); + // Explicit non-main loads keep using that branch's persisted schema. + const auto& expected_options = overrides.count(Options::BRANCH) != 0 && branch == "rt" + ? schema.value()->Options() + : preloaded->Options(); + ASSERT_EQ(expected_options, + std::dynamic_pointer_cast(loaded->GetSchema())->Options()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + loaded->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, + ReadResultCollector::CollectResult(std::move(reader))); + SnapshotManager snapshots(fs, path, branch); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, snapshots.LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_EQ(snapshot->TotalRecordCount(), rows->length()); + } + ASSERT_OK_AND_ASSIGN( + std::unique_ptr explicit_main, + ReadContextBuilder(path).SetTableSchema(schema_json).WithBranch("main").Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, TableFactory::Create(explicit_main.get())); + ASSERT_EQ("main", std::dynamic_pointer_cast(loaded)->CurrentBranch()); +} + +TEST(TableTest, TestResolvedStoreUsesOptionsBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::BRANCH, "dev"}}, dir->GetFileSystem())); + auto store = std::make_unique(dir->Str(), options); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr loaded, + FileStoreTable::Create(std::move(store), + std::dynamic_pointer_cast(table->GetSchema()), + Identifier("db", "tbl", "dev"), "")); + ASSERT_EQ("dev", loaded->CurrentBranch()); + ASSERT_EQ("dev", loaded->GetSnapshotManager()->Branch()); +} + +TEST(TableTest, TestLoadedTableReadRetainsOperationSchemeMap) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::string path = dir->Str() + "/db.db/tbl"; + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
created, + CreateManagedTable(dir->GetFileSystem(), path, Identifier("db", "tbl"))); + FileSystemCatalog catalog(dir->GetFileSystem(), dir->Str(), {}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db", "tbl"))); + const std::map scheme_map = {{"custom", "local"}}; + for (bool direct : {false, true}) { + SCOPED_TRACE(direct); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr context, + ReadContextBuilder(path).WithFileSystemSchemeToIdentifierMap(scheme_map).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + direct ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context))); + auto* append_read = dynamic_cast(read.get()); + ASSERT_NE(nullptr, append_read); + ASSERT_EQ( + scheme_map, + append_read->context_->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); + if (direct) { + ASSERT_EQ(table->GetFileSystem(), + append_read->context_->GetCoreOptions().GetFileSystem()); + } + } +} + +TEST(TableTest, TestReadMergesSchemeMapsForMainAndFallback) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main, + CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"))); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
dev, + CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl", "dev"), "dev")); + const std::map table_map = {{"base", "local"}, {"shared", "mock_fs"}}; + const std::map read_map = {{"custom", "local"}, {"shared", "local"}}; + const std::map expected = { + {"base", "local"}, {"custom", "local"}, {"shared", "local"}}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, dir->Str(), + std::dynamic_pointer_cast(main->GetSchema()), + Identifier("db", "tbl"), "", "main", table_map)); + for (bool fallback : {false, true}) { + SCOPED_TRACE(fallback); + ReadContextBuilder builder("unused_path"); + builder.WithFileSystemSchemeToIdentifierMap(read_map); + if (fallback) { + builder.AddOption(Options::SCAN_FALLBACK_BRANCH, "dev"); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(context))); + std::vector reads; + if (fallback) { + auto* fallback_read = dynamic_cast(read.get()); + ASSERT_NE(nullptr, fallback_read); + reads = {fallback_read->main_table_.get(), fallback_read->fallback_table_.get()}; + } else { + reads = {read.get()}; + } + for (TableRead* data_read : reads) { + auto* append_read = dynamic_cast(data_read); + ASSERT_NE(nullptr, append_read); + const auto& internal = append_read->context_; + ASSERT_EQ(expected, + internal->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); + ASSERT_EQ(fs, internal->GetCoreOptions().GetFileSystem()); + ASSERT_EQ(dir->Str(), internal->GetPath()); + } + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ReadContextBuilder("unused_path").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(context))); + auto* append_read = dynamic_cast(read.get()); + ASSERT_NE(nullptr, append_read); + ASSERT_EQ(table_map, + append_read->context_->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); +} + +TEST(TableTest, TestSystemTablePathRetainsSchemeMapForDependencyReads) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::shared_ptr
data_table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); + const std::map scheme_map = {{"file", "local"}}; + for (bool reload_context : {false, true}) { + SCOPED_TRACE(reload_context); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ReadContextBuilder(dir->Str() + "$ro") + .WithFileSystemSchemeToIdentifierMap(scheme_map) + .Finish()); + std::unique_ptr read; + if (reload_context) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, TableFactory::Create(context.get())); + ASSERT_OK_AND_ASSIGN(context, ReadContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(read, table->NewRead(std::move(context))); + } else { + ASSERT_OK_AND_ASSIGN(read, TableRead::Create(std::move(context))); + } + auto* append_read = dynamic_cast(read.get()); + ASSERT_NE(nullptr, append_read); + const auto& internal = append_read->context_; + ASSERT_EQ(scheme_map, internal->GetFileSystemSchemeToIdentifierMap()); + ASSERT_EQ(scheme_map, + internal->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); + } +} + +TEST(TableTest, TestSystemTableReadMergesSchemeMapsWithoutReplacingFileSystem) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
data_table, + CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"))); + const std::map table_map = {{"base", "local"}}; + const std::map read_map = {{"file", "local"}}; + const std::map expected = {{"base", "local"}, {"file", "local"}}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + SystemTableLoader::Load("ro", fs, dir->Str(), + std::dynamic_pointer_cast(data_table->GetSchema()), {}, + "tbl", table_map)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr context, + ReadContextBuilder("unused_path").WithFileSystemSchemeToIdentifierMap(read_map).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(context))); + auto* append_read = dynamic_cast(read.get()); + ASSERT_NE(nullptr, append_read); + const auto& internal = append_read->context_; + ASSERT_EQ(fs, internal->GetCoreOptions().GetFileSystem()); + ASSERT_EQ(dir->Str(), internal->GetPath()); + ASSERT_EQ(expected, internal->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); +} + +TEST(TableTest, TestRealtimeFallbackOnlyAllowsMainBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + auto arrow_schema = arrow::schema({arrow::field("id", arrow::int32(), false)}); + const std::map table_options = {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}}; + SchemaManager main_schemas(fs, dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_schema, + main_schemas.CreateTable(arrow_schema, {}, {}, table_options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, dir->Str(), std::shared_ptr(std::move(main_schema)), + Identifier("db", "tbl"), "", "main")); + SchemaManager dev_schemas(fs, dir->Str(), "dev"); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr dev_schema, + dev_schemas.CreateTable(arrow_schema, {}, {}, table_options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime, RealtimeContext::Create()); + for (bool realtime_enabled : {false, true}) { + for (const std::string& fallback : {"main", "dev"}) { + SCOPED_TRACE(fallback + (realtime_enabled ? ":realtime" : ":disk")); + ReadContextBuilder builder(dir->Str()); + builder.SetOptions({{Options::SCAN_FALLBACK_BRANCH, fallback}}); + if (realtime_enabled) { + builder.WithRealtimeContext(realtime).AddOption(Options::REALTIME_ENABLED, "true"); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + auto read = table->NewRead(std::move(context)); + if (realtime_enabled && fallback != "main") { + ASSERT_NOK_WITH_MSG(read, "real-time operations only support the main branch"); + } else { + ASSERT_OK(read); + } + } + } +} + } // namespace paimon::test diff --git a/src/paimon/core/utils/context_utils.h b/src/paimon/core/utils/context_utils.h new file mode 100644 index 000000000..5e52e64a0 --- /dev/null +++ b/src/paimon/core/utils/context_utils.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "fmt/format.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// Shared operator-context checks. +class ContextUtils { + public: + ContextUtils() = delete; + ~ContextUtils() = delete; + + /// Requires a context, memory pool and executor; kind labels errors. + template + static Status CheckOperatorContext(const ContextPointer& context, const char* kind) { + if (context == nullptr) { + return Status::Invalid(fmt::format("{} context is null pointer", kind)); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + return Status::OK(); + } +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 273804d0d..8f23150f1 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -32,8 +32,10 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" @@ -448,19 +450,88 @@ Result> RestCatalog::LoadFormatTable( /*dynamic_options=*/{}); } +Result> RestCatalog::LoadSystemTable( + const Identifier& identifier, const Identifier& load_identifier, + const std::optional& branch) const { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::string data_table_path; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, + LoadDataTableSchema(load_identifier, branch, &data_table_path, + /*table_id=*/nullptr)); + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, + GetTableFileSystem(identifier)); + // Use the catalog name, including any branch, even when data lives under a UUID directory. + PAIMON_ASSIGN_OR_RAISE( + Identifier data_identifier, + CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); + return SystemTableLoader::Load(system_table_name.value(), table_file_system, data_table_path, + latest_schema, dynamic_options, data_identifier.GetObjectName(), + fs_scheme_to_identifier_map_); +} + Result> RestCatalog::GetTable(const Identifier& identifier) const { - // Read UUID, path and schema together to avoid mixing table generations after recreation. + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, + GlobalSystemTableLoader::IsSupported(identifier.GetObjectName(), + api_->GetMergedOptions())); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::shared_ptr self = weak_from_this().lock(); + if (self == nullptr) { + return Status::Invalid( + "a global system table retains its catalog, so the catalog must be owned by " + "a std::shared_ptr before it is asked for one"); + } + GlobalSystemTableContext context; + context.catalog = self.get(); + context.catalog_owner = std::move(self); + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = api_->GetMergedOptions(); + return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + if (is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + return LoadSystemTable(identifier, load_identifier, NormalizeBranch(std::move(branch))); + } + // Load UUID, location and schema together to avoid mixing table generations. + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + branch = NormalizeBranch(std::move(branch)); + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); std::string uuid; std::string table_path; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - LoadTableSchema(identifier, &uuid, &table_path)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + LoadDataTableSchema(load_identifier, branch, &table_path, &uuid)); + std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); + if (data_schema != nullptr) { + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(data_schema->Options())); + if (is_format_table) { + return FormatTable::Create(fs_, table_path, identifier, data_schema, + /*location_carries_paimon_metadata=*/false, + /*dynamic_options=*/{}); + } + } PAIMON_RETURN_NOT_OK( CatalogUtils::CheckManagedTableType(identifier, schema, "Catalog::GetTable")); // The table holds the file system its own data is served by, which a server issuing data // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - return std::make_shared
(table_file_system, table_path, schema, identifier, uuid); + return FileStoreTable::Create(table_file_system, table_path, schema, identifier, uuid, + branch.value_or(Identifier::kDefaultMainBranch), + fs_scheme_to_identifier_map_); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 2f57e912d..fc1db3d55 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -35,6 +35,7 @@ struct ArrowSchema; namespace paimon { +class SystemTable; class FileSystem; class TableSchema; @@ -124,6 +125,9 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { const Identifier& identifier) const override; private: + Result> LoadSystemTable( + const Identifier& identifier, const Identifier& load_identifier, + const std::optional& branch) const; RestCatalog(std::shared_ptr api, const std::shared_ptr& fs, const std::string& warehouse, bool data_token_enabled, bool fs_explicitly_supplied, const std::map& fs_scheme_to_identifier_map); diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 147a908ce..0f710bb47 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -49,15 +49,21 @@ #include "paimon/core/core_options.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" #include "paimon/rest/mock_rest_server.h" #include "paimon/rest/rest_api.h" +#include "paimon/scan_context.h" #include "paimon/schema/schema.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" #include "paimon/testing/utils/snapshot_test_helper.h" #include "paimon/testing/utils/testharness.h" @@ -810,6 +816,32 @@ TEST_F(RestCatalogTest, CatalogFactoryMetastoreDispatch) { ASSERT_NOK_WITH_MSG(status, "unsupported metastore"); } +TEST_F(RestCatalogTest, GlobalSystemTableRetainsFactoryCatalog) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr unique_catalog, + Catalog::Create(kWarehouse, options_)); + std::shared_ptr catalog(std::move(unique_catalog)); + std::weak_ptr weak_catalog = catalog; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("sys", "tables"))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(table->GetRootPath()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(table->GetRootPath()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); + catalog.reset(); + table.reset(); + ASSERT_FALSE(weak_catalog.expired()); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + ASSERT_TRUE(BatchReader::IsEofBatch(batch)); + reader->Close(); + reader.reset(); + read.reset(); + ASSERT_TRUE(weak_catalog.expired()); +} + TEST_F(RestCatalogTest, CreateWithWrongTokenFails) { options_[CatalogOptions::TOKEN] = "wrong-token"; ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "not authorized"); @@ -918,8 +950,7 @@ TEST_F(RestCatalogTest, TableOperations) { std::lock_guard lock(state_->mutex); ASSERT_EQ(requests_before_get_table + 1, state_->get_table_requests); } - std::shared_ptr schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, schema); ASSERT_EQ((std::vector{"f0", "f1"}), schema->FieldNames()); ASSERT_EQ((std::vector{"f0"}), schema->PrimaryKeys()); @@ -937,7 +968,7 @@ TEST_F(RestCatalogTest, TableOperations) { // timeMillis is backed by the server's audit "updatedAt" instead of the current // time, keeping the conversion deterministic std::shared_ptr table_schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, table_schema); ASSERT_EQ(123, table_schema->TimeMillis()); @@ -959,7 +990,7 @@ TEST_F(RestCatalogTest, TableOperations) { } ASSERT_OK_AND_ASSIGN(std::shared_ptr
commented_table, catalog->GetTable(Identifier("db1", "commented"))); - ASSERT_EQ("a table comment", commented_table->LatestSchema()->Comment().value_or("")); + ASSERT_EQ("a table comment", commented_table->GetSchema()->Comment().value_or("")); ASSERT_OK(catalog->RenameTable(identifier, Identifier("db1", "t2"), /*ignore_if_not_exists=*/false)); @@ -1062,8 +1093,8 @@ TEST_F(RestCatalogTest, SystemTableSchema) { } ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); ASSERT_EQ("t1$options", table->Name()); - ASSERT_EQ((std::vector{"key", "value"}), table->LatestSchema()->FieldNames()); - ASSERT_EQ("db1.t1$options", table->Uuid()); + ASSERT_EQ((std::vector{"key", "value"}), table->GetSchema()->FieldNames()); + ASSERT_EQ("t1$options", table->Uuid()); { std::lock_guard lock(state_->mutex); ASSERT_EQ(requests_before + 1, state_->get_table_requests); @@ -1105,8 +1136,7 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(branch_identifier)); ASSERT_EQ("t1$branch_b1", table->Name()); - std::shared_ptr schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, schema); // the branch's own schema and schema id, not the main table's ASSERT_EQ((std::vector{"b0"}), schema->FieldNames()); @@ -1118,7 +1148,7 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_table, catalog->GetTable(Identifier("db1", "t1$branch_main"))); std::shared_ptr main_schema = - std::dynamic_pointer_cast(main_table->LatestSchema()); + std::dynamic_pointer_cast(main_table->GetSchema()); ASSERT_NE(nullptr, main_schema); ASSERT_EQ((std::vector{"f0", "f1"}), main_schema->FieldNames()); ASSERT_EQ(0, main_schema->Options().count(Options::BRANCH)); @@ -1128,13 +1158,14 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_case_table, catalog->GetTable(Identifier("db1", "t1$branch_MAIN"))); std::shared_ptr main_case_schema = - std::dynamic_pointer_cast(main_case_table->LatestSchema()); + std::dynamic_pointer_cast(main_case_table->GetSchema()); ASSERT_NE(nullptr, main_case_schema); ASSERT_EQ((std::vector{"f0", "f1"}), main_case_schema->FieldNames()); ASSERT_EQ(0, main_case_schema->Options().count(Options::BRANCH)); // the branch the alias spells resolves to the main branch, not to a branch // directory named after the alias - ASSERT_OK_AND_ASSIGN(std::string main_case_branch, main_case_table->CurrentBranch()); + std::string main_case_branch = + std::dynamic_pointer_cast(main_case_table)->CurrentBranch(); ASSERT_EQ("main", main_case_branch); // a system table on a branch resolves against the branch's data table: the system @@ -1275,7 +1306,9 @@ TEST_F(RestCatalogTest, FormatTableIsLoadedInOneRequest) { // directory named "schema" there is a partition value, not metadata to skip. ASSERT_FALSE(table->LocationCarriesPaimonMetadata()); - ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "fmt")), "Cannot open format table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, + catalog->GetTable(Identifier("db1", "fmt"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(opened)); // A managed table stays out of this path, and so does a system table. ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); @@ -1389,8 +1422,7 @@ TEST_F(RestCatalogTest, NestedSchemaHighestFieldId) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("db1", "nested"))); - std::shared_ptr schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, schema); ASSERT_EQ(5, schema->Id()); // 9 lives inside the value row of the map: ROW, ARRAY element and MAP key/value @@ -1484,8 +1516,7 @@ TEST_F(RestCatalogTest, PartitionKeysRoundTrip) { /*ignore_if_exists=*/false)); // the partition keys survive both the create request and the load response conversion ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("db1", "pt"))); - std::shared_ptr loaded = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr loaded = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, loaded); ASSERT_EQ((std::vector{"f1"}), loaded->PartitionKeys()); } diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 1a2f0b7d7..5f8a8d52f 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -34,7 +34,9 @@ #include "paimon/core/catalog/commit_lock.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/partition/partition_statistics.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/result.h" #include "paimon/schema/schema.h" #include "paimon/status.h" @@ -115,9 +117,19 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { // Serves the per-table file system when one was set, matching `GetTableFileSystem` // without recording the request, so a test can still tell the two apart. No location: // this mock's `GetTableLocation` is the unsupported stub. - return std::make_shared
( - table_file_system_ != nullptr ? table_file_system_ : file_system_, - /*root_path=*/"", table_schema_, identifier, table_uuid_); + if (!table_schema_) { + return Status::NotExist("table schema not set"); + } + PAIMON_ASSIGN_OR_RAISE(std::string json, table_schema_->GetJsonSchema()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr schema, + TableSchema::CreateFromJson(json)); + PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + FileStoreTable::Create( + table_file_system_ != nullptr ? table_file_system_ : file_system_, "", + std::shared_ptr(std::move(schema)), identifier, table_uuid_, branch)); + return std::shared_ptr
(std::move(table)); } void SetTableSchema(const std::shared_ptr& table_schema) { diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index ca889342c..e68411a94 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -38,6 +38,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/factories/io_hook.h" @@ -49,6 +50,7 @@ #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/read_ahead_cache.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/snapshot.h" @@ -924,6 +926,49 @@ TEST(SystemTableReadInteTest, TestReadMetadataSystemTablesAtPinnedSnapshot) { "snapshot 3 does not exist"); ASSERT_NOK_WITH_MSG(ReadSystemTable(table_path + "$manifests", missing_options), "snapshot 3 does not exist"); + + FileSystemCatalog catalog(dir->GetFileSystem(), dir->Str(), {}); + for (const std::string& kind : {"files", "manifests"}) { + SCOPED_TRACE(kind); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog.GetTable(Identifier("foo", "bar$" + kind))); + const auto bound_schema = table->GetSchema(); + auto create_reader = [&table](const std::map& read_options) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan_context, + ScanContextBuilder("unused_path").SetOptions(read_options).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + table->NewScan(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr read_context, + ReadContextBuilder("unused_path").SetOptions(read_options).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + table->NewRead(std::move(read_context))); + return read->CreateReader(plan->Splits()); + }; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr pinned_reader, + create_reader(pinned_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr latest_reader, create_reader({})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr missing_reader, + create_reader(missing_options)); + ASSERT_NOK_WITH_MSG(ReadResultCollector::CollectResult(std::move(missing_reader)), + "snapshot 3 does not exist"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr pinned, + ReadResultCollector::CollectResult(std::move(pinned_reader))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr latest, + ReadResultCollector::CollectResult(std::move(latest_reader))); + ASSERT_TRUE(pinned->Equals(kind == "files" ? pinned_files : pinned_manifests)); + ASSERT_TRUE(latest->Equals(kind == "files" ? latest_files : latest_manifests)); + ASSERT_EQ(bound_schema, table->GetSchema()); + for (const auto& overrides : + {std::map{{Options::BRANCH, "dev"}}, + std::map{{Options::TYPE, "format-table"}}}) { + ASSERT_NOK_WITH_MSG(create_reader(overrides), "contradicts the loaded table"); + } + } } TEST(SystemTableReadInteTest, TestReadOptimizedSystemTable) { @@ -1688,6 +1733,57 @@ TEST(SystemTableReadInteTest, TestReadAuditLogSystemTableWithSequenceNumber) { ])"); } +TEST(SystemTableReadInteTest, TestChangelogSequenceNumberCannotBeDisabledByReadOptions) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), + arrow::field("v", arrow::int32())}; + const std::map options = { + {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "1"}, + {Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, "true"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(dir->Str(), arrow::schema(fields), {}, {"pk"}, options, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["a", 1]])", {}, 0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), 0, std::nullopt)); + FileSystemCatalog catalog(dir->GetFileSystem(), dir->Str(), {}); + const std::map read_options = { + {Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "false"}}; + for (const std::string& kind : {"audit_log", "binlog"}) { + SCOPED_TRACE(kind); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog.GetTable(Identifier("foo", "bar$" + kind))); + for (bool direct : {false, true}) { + SCOPED_TRACE(direct); + const std::string path = table->GetRootPath() + "$" + kind; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(path).SetOptions(read_options).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + direct ? table->NewScan(std::move(scan_context)) + : TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(path).SetOptions(read_options).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + direct ? table->NewRead(std::move(read_context)) + : TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadResultCollector::CollectResult(std::move(reader))); + auto array = SingleStructChunk(result); + ASSERT_EQ((std::vector{"rowkind", "_SEQUENCE_NUMBER", "pk", "v"}), + StructFieldNames(array)); + AssertStructArrayEqualsJson(array, kind == "audit_log" ? R"([["+I", 0, "a", 1]])" + : R"([["+I", 0, ["a"], [1]]])"); + } + } +} + TEST(SystemTableReadInteTest, TestReadBinlogSystemTable) { arrow::FieldVector fields = { arrow::field("pk", arrow::utf8()), @@ -4249,18 +4345,18 @@ Result> ReadGlobalSystemTable( ScanContextBuilder scan_context_builder(sys_path); scan_context_builder.SetOptions(options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr scan_context, + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - system_table->NewScan(scan_context)); + system_table->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); ReadContextBuilder read_context_builder(sys_path); read_context_builder.SetOptions(options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - system_table->NewRead(read_context)); + system_table->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, From 7f6bc02bd8046ecb19bb5a4cd0fb72d170325d4c Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 6 Nov 2026 15:38:23 +0000 Subject: [PATCH 2/3] fix(table): resolve hierarchy CI warnings --- src/paimon/core/table/format/format_table.cpp | 2 +- src/paimon/core/table/table_test.cpp | 6 +++--- test/inte/read_inte_test.cpp | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index 166062421..dbd42952a 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -335,7 +335,7 @@ Result> NewFormatTableCommit( } // namespace Result> FormatTable::ForContext( const std::map& options) const { - return Copy(std::shared_ptr(new FormatTable(*this)), options); + return Copy(std::make_shared(*this), options); } Result> FormatTable::NewScan(std::unique_ptr ctx) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "scan")); diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 5d8961f47..a02f72d3f 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -509,7 +509,7 @@ TEST(TableTest, TestPathIdentifierPreservesBranch) { Identifier catalog_table = CatalogUtils::IdentifierFromPath("/warehouse/db.db/tbl/", "dev"); ASSERT_EQ("db", catalog_table.GetDatabaseName()); ASSERT_EQ("tbl$branch_dev", catalog_table.GetObjectName()); - for (const std::string& path : {"tbl", "/warehouse/db.db.extra/tbl"}) { + for (const std::string path : {"tbl", "/warehouse/db.db.extra/tbl"}) { Identifier table = CatalogUtils::IdentifierFromPath(path, "dev"); ASSERT_EQ(Identifier::kUnknownDatabase, table.GetDatabaseName()); ASSERT_EQ("tbl$branch_dev", table.GetObjectName()); @@ -549,7 +549,7 @@ TEST(TableTest, TestGlobalSystemTableRetainsCatalog) { TEST(TableTest, TestExplicitContextBranchMustMatchLoadedTable) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - for (const std::string& table_branch : {"main", "dev"}) { + for (const std::string table_branch : {"main", "dev"}) { ASSERT_OK_AND_ASSIGN( std::shared_ptr
table, CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/" + table_branch, @@ -823,7 +823,7 @@ TEST(TableTest, TestRealtimeFallbackOnlyAllowsMainBranch) { dev_schemas.CreateTable(arrow_schema, {}, {}, table_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime, RealtimeContext::Create()); for (bool realtime_enabled : {false, true}) { - for (const std::string& fallback : {"main", "dev"}) { + for (const std::string fallback : {"main", "dev"}) { SCOPED_TRACE(fallback + (realtime_enabled ? ":realtime" : ":disk")); ReadContextBuilder builder(dir->Str()); builder.SetOptions({{Options::SCAN_FALLBACK_BRANCH, fallback}}); diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index e68411a94..43642fd92 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -928,7 +928,7 @@ TEST(SystemTableReadInteTest, TestReadMetadataSystemTablesAtPinnedSnapshot) { "snapshot 3 does not exist"); FileSystemCatalog catalog(dir->GetFileSystem(), dir->Str(), {}); - for (const std::string& kind : {"files", "manifests"}) { + for (const std::string kind : {"files", "manifests"}) { SCOPED_TRACE(kind); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("foo", "bar$" + kind))); @@ -1753,7 +1753,7 @@ TEST(SystemTableReadInteTest, TestChangelogSequenceNumberCannotBeDisabledByReadO FileSystemCatalog catalog(dir->GetFileSystem(), dir->Str(), {}); const std::map read_options = { {Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "false"}}; - for (const std::string& kind : {"audit_log", "binlog"}) { + for (const std::string kind : {"audit_log", "binlog"}) { SCOPED_TRACE(kind); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("foo", "bar$" + kind))); From 2de582d505ef4a3fbdb688be0406f8717966ad92 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Sat, 7 Nov 2026 00:35:04 +0800 Subject: [PATCH 3/3] refactor(context): separate table loading and dynamic options from contexts --- benchmark/benchmark_suite.cpp | 57 +- docs/source/api/catalog.rst | 3 + docs/source/api/format_table.rst | 22 +- docs/source/user_guide/catalog.rst | 174 +- docs/source/user_guide/commit.rst | 2 +- docs/source/user_guide/file_index.rst | 4 +- docs/source/user_guide/format_table.rst | 42 +- docs/source/user_guide/global_index.rst | 33 +- docs/source/user_guide/manifest_cache.rst | 2 +- .../user_guide/manifest_entry_cache.rst | 12 +- .../user_guide/parquet_metadata_cache.rst | 4 +- docs/source/user_guide/read.rst | 5 +- docs/source/user_guide/realtime.rst | 29 +- docs/source/user_guide/write.rst | 4 +- examples/clean_demo.cpp | 24 +- examples/read_write_demo.cpp | 36 +- include/paimon/catalog/table.h | 15 +- include/paimon/catalog/table_load_options.h | 46 + include/paimon/commit_context.h | 136 +- include/paimon/file_store_commit.h | 11 +- include/paimon/file_store_write.h | 7 +- include/paimon/read_context.h | 161 +- include/paimon/scan_context.h | 102 +- include/paimon/table/format/format_table.h | 14 +- include/paimon/table/source/table_read.h | 7 +- include/paimon/table/source/table_scan.h | 7 +- include/paimon/write_context.h | 147 +- .../append_compact_coordinator_test.cpp | 21 +- src/paimon/core/catalog/catalog.cpp | 3 +- .../core/catalog/file_system_catalog.cpp | 19 +- src/paimon/core/catalog/file_system_catalog.h | 4 +- .../core/catalog/file_system_catalog_test.cpp | 18 +- .../global_index/global_index_write_task.cpp | 20 +- ...eted_primary_key_index_maintainer_test.cpp | 74 +- .../pksorted/pk_sorted_data_file_reader.cpp | 12 +- ..._changelog_merge_tree_compact_rewriter.cpp | 9 +- .../lookup_merge_tree_compact_rewriter.cpp | 9 +- ...erge_tree_compact_manager_factory_test.cpp | 10 +- .../compact/merge_tree_compact_rewriter.cpp | 9 +- .../merge_tree_compact_rewriter_test.cpp | 9 +- src/paimon/core/mergetree/lookup_levels.cpp | 9 +- .../append_only_file_store_scan_test.cpp | 47 +- .../append_only_file_store_write.cpp | 9 +- .../append_only_file_store_write_test.cpp | 111 +- ...eduplicate_compaction_changes_provider.cpp | 10 +- src/paimon/core/operation/commit_context.cpp | 160 +- .../core/operation/commit_context_test.cpp | 164 +- .../data_evolution_split_read_test.cpp | 94 +- .../operation/file_store_commit_impl_test.cpp | 1509 +++++++----- .../core/operation/file_store_commit_test.cpp | 57 +- .../core/operation/file_store_write_test.cpp | 171 +- .../core/operation/internal_read_context.cpp | 47 +- .../core/operation/internal_read_context.h | 26 +- .../operation/internal_read_context_test.cpp | 320 ++- .../key_value_file_store_write_test.cpp | 200 +- .../operation/merge_file_split_read_test.cpp | 329 ++- .../precise_file_index_read_test.cpp | 202 +- .../operation/raw_file_split_read_test.cpp | 219 +- src/paimon/core/operation/read_context.cpp | 175 +- .../core/operation/read_context_test.cpp | 229 +- src/paimon/core/operation/scan_context.cpp | 144 +- .../core/operation/scan_context_test.cpp | 78 +- src/paimon/core/operation/write_context.cpp | 175 +- .../core/operation/write_context_test.cpp | 163 +- src/paimon/core/snapshot_file_scan_test.cpp | 31 +- src/paimon/core/table/file_store_table.cpp | 195 +- src/paimon/core/table/file_store_table.h | 37 +- src/paimon/core/table/format/format_table.cpp | 36 +- .../core/table/format/format_table_test.cpp | 553 +++-- .../core/table/sink/commit_message_test.cpp | 18 +- .../table/source/append_count_reader_test.cpp | 21 +- .../table/source/pk_count_reader_test.cpp | 25 +- .../source/scan_exception_safety_test.cpp | 20 +- .../core/table/source/table_read_test.cpp | 224 +- .../core/table/source/table_scan_test.cpp | 78 +- .../table/system/audit_log_system_table.cpp | 34 +- .../table/system/audit_log_system_table.h | 11 +- .../core/table/system/binlog_system_table.cpp | 5 +- .../core/table/system/binlog_system_table.h | 4 +- .../table/system/in_memory_system_table.cpp | 8 +- .../table/system/in_memory_system_table.h | 7 +- .../system/read_optimized_system_table.cpp | 23 +- .../system/read_optimized_system_table.h | 8 +- src/paimon/core/table/system/system_table.cpp | 15 +- src/paimon/core/table/system/system_table.h | 9 +- .../core/table/system/system_table_test.cpp | 5 +- src/paimon/core/table/table_factory.cpp | 473 ++-- src/paimon/core/table/table_factory.h | 19 +- src/paimon/core/table/table_test.cpp | 551 ++++- src/paimon/core/utils/blob_view_lookup.cpp | 30 +- src/paimon/core/utils/branch_manager.cpp | 3 + src/paimon/core/utils/branch_manager.h | 4 +- src/paimon/core/utils/branch_manager_test.cpp | 29 + src/paimon/rest/rest_catalog.cpp | 6 +- src/paimon/rest/rest_catalog_test.cpp | 53 +- src/paimon/testing/mock/mock_catalog.h | 15 + src/paimon/testing/utils/test_helper.h | 74 +- test/inte/append_compaction_inte_test.cpp | 25 +- test/inte/blob_table_inte_test.cpp | 61 +- test/inte/clean_inte_test.cpp | 208 +- test/inte/data_evolution_table_test.cpp | 194 +- test/inte/global_index_test.cpp | 77 +- test/inte/nested_column_pruning_inte_test.cpp | 118 +- test/inte/paimon_read_compat_inte_test.cpp | 27 +- test/inte/pk_compaction_inte_test.cpp | 184 +- .../primary_key_sorted_index_inte_test.cpp | 42 +- test/inte/read_inte_test.cpp | 834 ++++--- test/inte/realtime_write_inte_test.cpp | 595 +++-- test/inte/scan_and_read_inte_test.cpp | 2062 +++++++++++------ test/inte/scan_inte_test.cpp | 740 ++++-- test/inte/write_and_read_inte_test.cpp | 320 ++- .../write_and_read_inte_with_index_test.cpp | 108 +- test/inte/write_inte_test.cpp | 445 ++-- 113 files changed, 8452 insertions(+), 6183 deletions(-) create mode 100644 include/paimon/catalog/table_load_options.h diff --git a/benchmark/benchmark_suite.cpp b/benchmark/benchmark_suite.cpp index 0a48a9173..307ee6abc 100644 --- a/benchmark/benchmark_suite.cpp +++ b/benchmark/benchmark_suite.cpp @@ -37,6 +37,7 @@ #include "benchmark/cli_option_parsing.h" #include "paimon/api.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/testing/utils/testharness.h" #if __has_include("parquet/arrow/reader.h") @@ -434,25 +435,33 @@ Status WriteSourceDataToWriter(paimon::FileStoreWrite* writer, const SourceDataS Status WriteAndCommit(const std::string& table_path, const std::map& options, const SourceDataSpec& source_spec) { - paimon::WriteContextBuilder write_builder(table_path, "benchmark-writer"); + paimon::TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + paimon::WriteContextBuilder write_builder("benchmark-writer"); + write_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_ctx, + AddContext(write_builder.Finish(), "create write context")); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr write_ctx, - AddContext(write_builder.SetOptions(options).Finish(), "create write context")); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, - AddContext(paimon::FileStoreWrite::Create(std::move(write_ctx)), - "create file store writer")); + std::unique_ptr writer, + AddContext(paimon::FileStoreWrite::Create(write_builder_load_options, std::move(write_ctx), + write_builder_options), + "create file store writer")); PAIMON_RETURN_NOT_OK(WriteSourceDataToWriter(writer.get(), source_spec)); PAIMON_ASSIGN_OR_RAISE(std::vector> messages, AddContext(writer->PrepareCommit(), "prepare commit")); - paimon::CommitContextBuilder commit_builder(table_path, "benchmark-writer"); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr commit_ctx, - AddContext(commit_builder.SetOptions(options).Finish(), "create commit context")); + paimon::TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + paimon::CommitContextBuilder commit_builder("benchmark-writer"); + commit_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_ctx, + AddContext(commit_builder.Finish(), "create commit context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr committer, - AddContext(paimon::FileStoreCommit::Create(std::move(commit_ctx)), "create committer")); + AddContext(paimon::FileStoreCommit::Create(commit_builder_load_options, + std::move(commit_ctx), commit_builder_options), + "create committer")); PAIMON_RETURN_NOT_OK(AddContext(committer->Commit(messages), "commit write")); return Status::OK(); } @@ -562,20 +571,26 @@ Result GetOrCreateSharedReadTable(const std::string Result ReadRows(const std::string& table_path, const std::map& options, int32_t prefetch_parallel_num) { - paimon::ScanContextBuilder scan_builder(table_path); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan_ctx, - AddContext(scan_builder.SetOptions(options).Finish(), "create scan context")); + paimon::TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + paimon::ScanContextBuilder scan_builder; + scan_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_ctx, + AddContext(scan_builder.Finish(), "create scan context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr scanner, - AddContext(paimon::TableScan::Create(std::move(scan_ctx)), "create scanner")); + AddContext(paimon::TableScan::Create(scan_builder_load_options, std::move(scan_ctx), + scan_builder_options), + "create scanner")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, AddContext(scanner->CreatePlan(), "create plan")); - paimon::ReadContextBuilder read_builder(table_path); + paimon::TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + paimon::ReadContextBuilder read_builder; constexpr int32_t kPrefetchBatchCount = 600; - read_builder.SetOptions(options) - .EnablePrefetch(true) + read_builder_options = options; + read_builder.EnablePrefetch(true) .SetPrefetchBatchCount(kPrefetchBatchCount) .SetPrefetchMaxParallelNum(prefetch_parallel_num) .EnableMultiThreadRowToBatch(GetRowToBatchThreadNumber() > 1) @@ -584,7 +599,9 @@ Result ReadRows(const std::string& table_path, AddContext(read_builder.Finish(), "create read context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr reader, - AddContext(paimon::TableRead::Create(std::move(read_ctx)), "create table reader")); + AddContext(paimon::TableRead::Create(read_builder_load_options, std::move(read_ctx), + read_builder_options), + "create table reader")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, AddContext(reader->CreateReader(plan->Splits()), "create batch reader")); diff --git a/docs/source/api/catalog.rst b/docs/source/api/catalog.rst index df247f4a8..df19a17c6 100644 --- a/docs/source/api/catalog.rst +++ b/docs/source/api/catalog.rst @@ -37,5 +37,8 @@ Interface .. doxygenclass:: paimon::Table :members: +.. doxygenstruct:: paimon::TableLoadOptions + :members: + .. doxygenstruct:: paimon::CatalogOptions :members: diff --git a/docs/source/api/format_table.rst b/docs/source/api/format_table.rst index 1ec964375..7464205f7 100644 --- a/docs/source/api/format_table.rst +++ b/docs/source/api/format_table.rst @@ -35,18 +35,10 @@ and :cpp:class:`paimon::FileStoreCommit`. See :doc:`../user_guide/format_table`. Contexts ======== -Each of those entry points builds its context from a table path, and the schema under that path -says what kind of table it is. A caller that already holds a ``FormatTable`` hands it over instead, -through the constructor each context builder has for one: - -* :cpp:class:`paimon::ScanContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::ReadContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::WriteContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::CommitContextBuilder`\ ``(const std::shared_ptr&)`` - -This is the only way to reach a format table whose schema does not live under its own location, -such as one a REST catalog serves: nothing under the location says that it is a format table, nor -that what sits below it is data rather than metadata. The table knows both, so a setting that would -answer either question again - ``SetTableSchema()``, ``WithFileSystem()`` and a branch among them - -is refused by these builders rather than quietly ignored. Options given at the call still win over -the ones the table carries, as they do everywhere else. +Static entry points load the table through :cpp:struct:`paimon::TableLoadOptions`. +Operation Contexts provide execution settings; dynamic options are a separate +argument. A caller holding a loaded ``FormatTable`` creates operators directly +with its ``NewScan()``, ``NewRead()``, ``NewWrite()`` or ``NewCommit()`` methods. +This also works when a catalog keeps the schema outside the table location. +The loaded table supplies its schema and file system, and dynamic options passed +to ``New*()`` override the stored options. diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 684a990e7..0a6cf1168 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -121,64 +121,60 @@ through the regular ``Catalog`` API, and table snapshots can be listed through Committing through the catalog ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A REST catalog reports ``Catalog::SupportsVersionManagement() == true`` and -publishes snapshots through the server. Pass the catalog and table identifier -to ``CommitContextBuilder::WithCatalog`` to commit on local or object storage -without an atomic snapshot-file rename: +publishes snapshots through the server. A loaded table retains its catalog +services and UUID. Writers and committers created from that table use the same +identity and per-table credentials: .. code-block:: cpp - // Writers and committers share ownership of the catalog. std::shared_ptr shared_catalog(std::move(catalog)); - - // Load the table's identity before preparing changes. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, shared_catalog->GetTable(paimon::Identifier("db", "tbl"))); - - paimon::CommitContextBuilder builder(table_path, "commit-user"); - builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); - // CatalogUuid() stays null when the server supplies no id. - if (std::optional table_uuid = table->CatalogUuid()) { - builder.WithTableId(table_uuid.value()); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, + paimon::WriteContextBuilder("commit-user").Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, + table->NewWrite(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, + paimon::CommitContextBuilder("commit-user").Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - paimon::FileStoreCommit::Create(std::move(context))); + table->NewCommit(std::move(commit_context))); PAIMON_RETURN_NOT_OK(commit->Commit(commit_messages)); -Configure native writers with the same catalog and identifier. They load the -current schema and latest snapshot from the catalog, including committed -offsets when a real-time writer is created: +``Catalog::GetTable()`` returns a common ``Table`` interface for managed, format +and system tables. Managed tables select append-only or primary-key behavior +from the loaded schema; system tables reject writes and commits. -.. code-block:: cpp +``NewCommit()``, ``NewWrite()``, ``NewRead()`` and ``NewScan()`` take an operation +Context and an optional dynamic options map. They use the table's bound type, +path, schema, file system and catalog services. ``GetSchema()`` returns the +bound schema without metadata I/O; reload through the catalog to refresh it. +Branch or type options that contradict a loaded managed table are rejected. +See the :ref:`Table API `. - paimon::WriteContextBuilder write_builder(table_path, "commit-user"); - write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, - write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, - paimon::FileStoreWrite::Create(std::move(write_context))); +The static factories instead take ``TableLoadOptions``, an operation Context and +an optional dynamic options map. Loading parameters are independent of the +Context; finishing a Context performs no table metadata I/O: -``Catalog::GetTable()`` returns a common ``Table`` interface for managed, format and -system tables. The managed implementation selects append-only or primary-key behavior -from the loaded schema; system tables reject writes and commits. +.. code-block:: cpp -``table->NewCommit()``, ``NewWrite()``, ``NewRead()`` and ``NewScan()`` use the table's -bound type, path, schema and file system. ``GetSchema()`` returns that schema without -metadata I/O; reload through the catalog to refresh it. Contexts still provide execution -settings, dynamic options and optional catalog services. Branch or type options that -contradict a loaded managed table are rejected. The legacy static ``Create()`` methods -load the table from their context before delegating to these methods. -See the :ref:`Table API `. + paimon::TableLoadOptions load_options; + load_options.catalog = shared_catalog; + load_options.identifier.emplace("db", "tbl"); + // Optional expected UUID, validated against the catalog during loading. + load_options.table_id = table->CatalogUuid(); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, + paimon::CommitContextBuilder("commit-user").Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, + paimon::FileStoreCommit::Create(load_options, std::move(context))); -An explicit ``WithBranch()`` or branch-qualified identifier participates in the same -branch check. Omitting a branch allows the loaded table to supply it. The static -scan and read factories use an explicit context branch first, then the branch in -the schema options, and finally main. +For file-system loading, set ``TableLoadOptions::path``. Optional fields specify +an explicit file system, URI scheme routing or a serialized Paimon table schema. +The serialized table schema is separate from the Arrow projection in +``ReadContextBuilder::SetReadSchema()``. Set the branch through the identifier +or the dynamic ``branch`` option; conflicting values are rejected. -Global system tables under ``sys`` retain their catalog for deferred row loading. -Before calling ``GetTable()`` for one, place the catalog in a ``std::shared_ptr``, -as in the example above. ``LoadTableSchema()`` can inspect the schema without -retaining the catalog. +Global system tables under ``sys`` retain their catalog for deferred row +loading. Keep the catalog in a ``std::shared_ptr`` before loading these tables. For real-time writes, also set ``WithStreamingMode(true)`` and ``WithRealtimeContext(...)`` with ``realtime.enabled=true``. After @@ -189,14 +185,12 @@ already advanced the latest snapshot, the requested snapshot must be published under the table path. Recreating the writer restores offsets from the catalog's current snapshot. -Both builders use the file system of the table being written or committed for -manifests and data, so a catalog that issues temporary credentials per table -serves them through ``Catalog::GetTableFileSystem``. ``WithFileSystem`` overrides -it; writers also allow ``WithFileSystemSchemeToIdentifierMap`` to override -file-system selection. For format tables, use ``WriteContextBuilder(FormatTable)`` -instead. +Table loading uses ``Catalog::GetTableFileSystem`` for per-table credentials. +An explicit ``TableLoadOptions::file_system`` or +``fs_scheme_to_identifier_map`` controls file-system resolution instead. +Loaded format tables create operators directly with their ``New*()`` methods. -A file system passed to ``WithFileSystem`` is used exactly as it is: neither the +A file system passed in ``TableLoadOptions::file_system`` is used exactly as it is: neither the built-in file systems nor the table's own credentials are involved, so a custom one has to authenticate its own accesses. Draw credentials that expire from a ``CredentialProvider`` you build with ``CredentialProviderFactory``: hold the @@ -210,8 +204,9 @@ returns credentials that are still valid and reloads them before they expire. "oss://bucket/tbl", options)); // MyFileSystem merges provider->GetCredentials() over its own options per access. auto my_fs = std::make_shared(provider); - write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")) - .WithFileSystem(my_fs); + load_options.catalog = shared_catalog; + load_options.identifier.emplace("db", "tbl"); + load_options.file_system = my_fs; The snapshot being committed carries a uuid generated on the client, and the commit names the snapshot it is based on by that snapshot's uuid, so the server @@ -227,7 +222,7 @@ a lease for each publication attempt. Both default to ``false``; enabling the table property alone does not make writers acquire, and asking to acquire without it fails when the commit is created. A writer with an active lease keeps the others out, including writers that do not acquire one. The lease is bound to the -table or branch, to the table UUID, which has to be passed to ``WithTableId``, +table or branch, to the table UUID retained by the loaded table, and to the exact commit user, so give each logical writer its own commit user and keep it across recovery. @@ -239,10 +234,11 @@ keep it across recovery. if (!table_uuid) { return paimon::Status::Invalid("a commit lease needs the table UUID"); } - paimon::CommitContextBuilder builder(table_path, "writer-1"); - builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")) - .WithTableId(table_uuid.value()) - .AddOption(paimon::Options::REST_COMMIT_ACQUIRE_LOCK, "true"); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, + paimon::CommitContextBuilder("writer-1").Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, + table->NewCommit(std::move(context), + {{paimon::Options::REST_COMMIT_ACQUIRE_LOCK, "true"}})); Each attempt acquires its own lease before reading the latest snapshot, waits while another writer holds it until ``commit.timeout`` without using up @@ -267,7 +263,7 @@ under it. Ending an attempt that published nothing only stops renewing the lease, without waiting for a renewal in flight, and the lease stays held until it expires. -Leases need ``WithCatalog``: a commit that only builds the commit table request +Leases need a table loaded through a catalog: a commit that only builds the commit table request for the caller to send cannot hold one, so asking to acquire is rejected there. The snapshot UUID checks above still apply. @@ -285,8 +281,9 @@ A catalog response of ``{"snapshot": null}`` means the table has no snapshot. Only a catalog reporting that snapshot loading is unsupported falls back to file-system lookup; other catalog errors propagate to the caller. -Read ``Table::CatalogUuid()`` before preparing changes and pass it to -``WithTableId`` so the server can reject commits to a dropped and recreated table. +Load the table before preparing changes so its retained ``CatalogUuid()`` lets +the server reject commits to a dropped and recreated table. Static commit +loading accepts the expected UUID through ``TableLoadOptions::table_id``. ``Table::Uuid()`` can fall back to the table name and must not supply this id. An unset table id is sent as null and subject to server validation. Each commit attempt to the main branch reloads the catalog's current schema id for the new @@ -334,47 +331,24 @@ so its restored file references are visible to the expiration operation. Reading through the catalog ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To read a native table with the per-table temporary credentials a catalog issues, -pass the catalog and table identifier to ``ReadContextBuilder::WithCatalog`` or -``ScanContextBuilder::WithCatalog``. This is a shorthand for asking the catalog -for that table's file system and passing it in through ``WithFileSystem``: +Load the table through a shared catalog and create read and scan operators from it: .. code-block:: cpp - // Readers and scanners share ownership of the catalog. - std::shared_ptr shared_catalog(std::move(catalog)); - - paimon::ScanContextBuilder scan_builder(table_path); - scan_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + shared_catalog->GetTable(paimon::Identifier("db", "tbl"))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - scan_builder.Finish()); + paimon::ScanContextBuilder().Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(std::move(scan_context))); - - paimon::ReadContextBuilder read_builder(table_path); - read_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + table->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, - read_builder.Finish()); + paimon::ReadContextBuilder().Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - paimon::TableRead::Create(std::move(read_context))); - -When ``Finish()`` builds the context, it asks the catalog for the table's file -system through ``Catalog::GetTableFileSystem`` and uses it as-is for the schema, -snapshots, manifests and data under the table path, so it signs every access with -the table's credentials and reloads them as they expire. An explicit -``WithFileSystem`` takes precedence, so the catalog is not asked. A catalog that -issues no per-table credentials returns its catalog-level file system, so this is -also how you read with the catalog's own object-store credentials. For a format -table, use ``ReadContextBuilder(FormatTable)`` or ``ScanContextBuilder(FormatTable)`` -instead; ``WithCatalog`` is rejected because the table already carries the file -system it was loaded through. - -The C++ REST catalog covers the database, table, snapshot and commit operations -of the ``Catalog`` API. The parts of the Java REST catalog that have no C++ -counterpart yet — altering a database or a table, views, functions, partitions, -tags, branch management and consumers — are not supported. Creating, deleting -and merging a branch is what branch management covers; committing to a branch -that already exists is supported, as described below. + table->NewRead(std::move(read_context))); + +The table retains its file system, catalog configuration and identifier for +reads of dependency tables. REST dependency reads preserve an existing +``X-Paimon-Read-Via`` header or derive it from the outer table identifier. Committing to a branch ~~~~~~~~~~~~~~~~~~~~~~ @@ -383,19 +357,17 @@ This library creates no branch: one has to exist, with its schema published unde Naming the branch ^^^^^^^^^^^^^^^^^ -A branch is addressed by the identifier of ``WithCatalog``, which +A branch is addressed by ``TableLoadOptions::identifier`` or the loaded table identifier, which ``paimon::Identifier("db", "tbl", "dev")`` builds as ``tbl$branch_dev``, so that the catalog answers for the branch: the snapshot it loads is the branch's, and the commit request is addressed to that branch by that identifier rather than by -anything in its body. The ``branch`` option and, for writers, ``WithBranch`` may -name the same branch as well; naming two different branches is rejected rather +anything in its body. The dynamic ``branch`` option may name the same branch as well; naming two different branches is rejected rather than resolved, and ``tbl$branch_main``, ``branch=main`` and an empty name all -mean the main branch. Naming a branch only in the option or ``WithBranch`` while +mean the main branch. Naming a branch only in the option while the identifier names the bare table is rejected, as the catalog would then answer for the table while the files were written for the branch. -Names a catalog cannot tell apart from another object are rejected where a -builder sees them: a name differing from ``main`` by case alone, such as +Names a catalog cannot tell apart from another object are rejected during loading: a name differing from ``main`` by case alone, such as ``tbl$branch_MAIN`` or ``branch=MAIN``, which a catalog reads as the main branch while its snapshots go to a directory of their own; and a name holding a ``$``, such as ``branch=dev$options``, which ``tbl$branch_dev$options`` reads back as @@ -441,7 +413,7 @@ from a store Paimon has no client for — reach a file system through a ``REGISTER_PAIMON_FACTORY``. Build a provider with ``CredentialProviderFactory::Get``, wrap it in a ``FileSystem`` of your own that calls ``CredentialProvider::GetCredentials`` as it signs each access, and pass -that file system in through ``Catalog::Create`` or a builder's ``WithFileSystem``. +that file system in through ``Catalog::Create`` or ``TableLoadOptions::file_system``. A file system passed this way is used as-is, so the credentials are reloaded as they expire without it being rebuilt. diff --git a/docs/source/user_guide/commit.rst b/docs/source/user_guide/commit.rst index 121462cb9..c7dd46c72 100644 --- a/docs/source/user_guide/commit.rst +++ b/docs/source/user_guide/commit.rst @@ -129,7 +129,7 @@ Snapshot publication depends on the commit mode: published through an atomic rename. After publication, the committer writes the ``LATEST`` hint file to reduce list operations when fetching the latest Snapshot. -* **Catalog-mediated commit:** When ``CommitContextBuilder::WithCatalog()`` +* **Catalog-mediated commit:** When the Table is loaded through a catalog supplies a catalog that manages table versions, the committer sends the Snapshot and change statistics to that catalog for publication. This path does not rename a local Snapshot file or write a local ``LATEST`` hint. diff --git a/docs/source/user_guide/file_index.rst b/docs/source/user_guide/file_index.rst index 44955689e..a0cc67cf7 100644 --- a/docs/source/user_guide/file_index.rst +++ b/docs/source/user_guide/file_index.rst @@ -58,7 +58,7 @@ positions in that schema. For example, the following constructs PAIMON_ASSIGN_OR_RAISE( std::shared_ptr filter, paimon::PredicateBuilder::Or({text, price})); - paimon::ReadContextBuilder builder(table_path); + paimon::ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"body", "price"}) .SetPredicate(filter); @@ -82,7 +82,7 @@ predicate by projecting ``_body_INDEX_SCORE``: PAIMON_ASSIGN_OR_RAISE( std::shared_ptr text, paimon::PredicateBuilder::FullTextMatch(0, request)); - paimon::ReadContextBuilder builder(table_path); + paimon::ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"body", "_body_INDEX_SCORE"}) .SetPredicate(text); diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 5bda38fcf..b5f77936c 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -45,33 +45,27 @@ interface. Its ``NewScan``, ``NewRead``, ``NewWrite`` and ``NewCommit`` methods loaded schema, location and file system. :cpp:func:`Catalog::GetFormatTable` also returns the concrete ``FormatTable`` interface for callers using the legacy context constructors. -Reading and writing go through the entry points every other table uses: ``TableScan::Create``, -``TableRead::Create``, ``FileStoreWrite::Create`` and ``FileStoreCommit::Create``, each from its -usual context builder. Each reads the table's schema - from under the table path, or from the one -the context carries - and dispatches to a format table when its ``type`` says so. - -That path needs a schema under the table's own location, which is what every table a file system -catalog serves has. A table loaded from a catalog that keeps the schema elsewhere, such as the -REST catalog, has none: nothing under the location says that it is a format table, nor that -``schema`` and ``branch`` below it are data rather than metadata. Such a table is read and written -by handing the loaded ``FormatTable`` to the context instead of a path - every context builder has -a constructor taking one - which is what Java does through ``FormatTable.newReadBuilder()`` and +Static ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create`` +and ``FileStoreCommit::Create`` load tables through ``TableLoadOptions``. They +read the schema from the table path or ``table_schema_json`` and dispatch by its +type. Operation Contexts contain execution settings; dynamic options are a +separate final argument. + +A format table whose schema lives only in a catalog is used through its loaded +Table handle, as Java uses ``FormatTable.newReadBuilder()`` and ``newBatchWriteBuilder()``: .. code-block:: cpp PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, catalog->GetFormatTable(paimon::Identifier("db", "tbl"))); - paimon::ScanContextBuilder scan_builder(table); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - scan_builder.Finish()); + paimon::ScanContextBuilder().Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); -The table already carries its schema and the file system it was loaded through, so a context built -from one refuses ``SetTableSchema()``, ``WithFileSystem()``, -``WithFileSystemSchemeToIdentifierMap()``, ``WithCatalog()`` and a branch. Options -given at the call still win over the ones the schema stored, as they do everywhere else. +The loaded table supplies its schema and file system. Dynamic options passed to +``New*()`` override stored options. A batch comes back in the table's column order, or the projection's when the read names one, behind the ``_VALUE_KIND`` field every ``BatchReader`` carries. Which columns a *file* is asked for @@ -129,17 +123,15 @@ A setting the format path cannot act on is refused by name rather than quietly d * ``WriteContextBuilder::WithWriteId()``, which prefixes a postpone-bucket writer's files so one compaction reader can put them back in order; a format table has no buckets. * ``CommitContextBuilder::IgnoreEmptyCommit(false)``, ``UseRESTCatalogCommit(true)``, - ``WithTableId()``, ``WithCatalog()`` and ``AppendCommitCheckConflict(true)``. Keeping an empty - commit means writing a snapshot that adds no files, a rest-catalog commit sends that snapshot to - a catalog, a table id names the table in the request that commit sends, committing through a - catalog hands it a snapshot to take, and the conflict check reads the manifests of concurrent - commits - none of which exist here. Each is refused only when set away from its default, so an + ``TableLoadOptions::table_id`` and ``AppendCommitCheckConflict(true)``. Keeping an empty + commit means writing a snapshot that adds no files, a REST commit sends that snapshot to + a catalog, a table id names the table in that request, and the conflict check reads the + manifests of concurrent commits - none of which exist here. Each is refused only when set away from its default, so an ordinary commit is unaffected. * the ``branch`` option, whatever names it: a format table keeps no metadata to branch, and its data is the files under its one location, which is where a read or a write would go whichever branch was asked for. It is refused wherever it comes from - the schema, a catalog that parsed - ``tbl$branch_dev``, or the options of a single call - and so is ``WithBranch()`` on a context - built from a format table. A branch quietly ignored would be the wrong answer everywhere and a + ``tbl$branch_dev``, or the dynamic options of a single call. A branch quietly ignored would be the wrong answer everywhere and a destructive one for an overwrite, which would clear the data it was told not to touch. The value naming the main branch is the default and is left alone. * A scan predicate or bucket filter, as above, and more than one partition filter: a scan descends diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index cafaabd50..9b4fbeb15 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -223,10 +223,15 @@ to both the index scan and the data scan: reader->VisitFullTextSearch(std::make_shared( "f0", R"({"match":{"query":"paimon lake"}})", /*limit=*/10))); - paimon::ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(paimon::Options::SCAN_MODE, "from-snapshot") - .AddOption(paimon::Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)) - .SetGlobalIndexResult(result); + paimon::TableLoadOptions load_options{table_path}; + paimon::ScanContextBuilder scan_builder; + const std::map scan_options = { + {paimon::Options::SCAN_MODE, "from-snapshot"}, + {paimon::Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)}}; + scan_builder.SetGlobalIndexResult(result); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + paimon::TableScan::Create(load_options, std::move(context), scan_options)); **Configuration:** @@ -314,20 +319,21 @@ not supported yet. search_builder.ExecuteLocal()); // Read the rows found, with their scores. - paimon::ScanContextBuilder scan_builder(table_path); + paimon::TableLoadOptions load_options{table_path}; + paimon::ScanContextBuilder scan_builder; scan_builder.SetGlobalIndexResult(result); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(std::move(scan_context))); + paimon::TableScan::Create(load_options, std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_plan, scan->CreatePlan()); - paimon::ReadContextBuilder read_builder(table_path); + paimon::ReadContextBuilder read_builder; read_builder.SetReadFieldNames({"id", "content", "_INDEX_SCORE"}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - paimon::TableRead::Create(std::move(read_context))); + paimon::TableRead::Create(load_options, std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(data_plan->Splits())); @@ -376,12 +382,17 @@ outlive the snapshot. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, search_read->Read(*plan)); - paimon::ScanContextBuilder scan_builder(table_path); + paimon::TableLoadOptions load_options{table_path}; + paimon::ScanContextBuilder scan_builder; scan_builder.SetGlobalIndexResult(result); + std::map scan_options; if (plan->SnapshotId()) { - scan_builder.AddOption(paimon::Options::SCAN_MODE, "from-snapshot") - .AddOption(paimon::Options::SCAN_SNAPSHOT_ID, std::to_string(*plan->SnapshotId())); + scan_options[paimon::Options::SCAN_MODE] = "from-snapshot"; + scan_options[paimon::Options::SCAN_SNAPSHOT_ID] = std::to_string(*plan->SnapshotId()); } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + paimon::TableScan::Create(load_options, std::move(context), scan_options)); ``ExecuteLocal()`` plans and searches in one process. To plan in one process and search in others, plan with ``NewFullTextScan()``, ship the splits with ``FullTextSearchSplit::Serialize()`` and diff --git a/docs/source/user_guide/manifest_cache.rst b/docs/source/user_guide/manifest_cache.rst index f6dd034af..7b76e136d 100644 --- a/docs/source/user_guide/manifest_cache.rst +++ b/docs/source/user_guide/manifest_cache.rst @@ -86,7 +86,7 @@ Example: std::make_shared(), std::make_shared()); - paimon::ScanContextBuilder scan_builder(table_path); + paimon::ScanContextBuilder scan_builder; scan_builder.WithCache(cache); Passing ``nullptr`` or omitting ``ScanContextBuilder::WithCache()`` leaves diff --git a/docs/source/user_guide/manifest_entry_cache.rst b/docs/source/user_guide/manifest_entry_cache.rst index 4d9fed056..ab55b230d 100644 --- a/docs/source/user_guide/manifest_entry_cache.rst +++ b/docs/source/user_guide/manifest_entry_cache.rst @@ -78,13 +78,15 @@ under .. code-block:: cpp auto cache = std::make_shared(128 * 1024 * 1024); - ScanContextBuilder context_builder(table_path); + TableLoadOptions load_options{table_path}; + ScanContextBuilder context_builder; PAIMON_ASSIGN_OR_RAISE( std::unique_ptr scan_context, - context_builder - .WithCache(cache) - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .Finish()); + context_builder.WithCache(cache).Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + TableScan::Create(load_options, std::move(scan_context), + {{Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"}})); .. note:: diff --git a/docs/source/user_guide/parquet_metadata_cache.rst b/docs/source/user_guide/parquet_metadata_cache.rst index 514f1b64c..80c7da0fd 100644 --- a/docs/source/user_guide/parquet_metadata_cache.rst +++ b/docs/source/user_guide/parquet_metadata_cache.rst @@ -91,10 +91,10 @@ Example: std::make_shared(), std::make_shared()); - paimon::ScanContextBuilder scan_builder(table_path); + paimon::ScanContextBuilder scan_builder; scan_builder.WithCache(cache); - paimon::ReadContextBuilder read_builder(table_path); + paimon::ReadContextBuilder read_builder; read_builder.WithCache(cache); Passing ``nullptr`` or omitting ``WithCache()`` leaves Parquet metadata caching diff --git a/docs/source/user_guide/read.rst b/docs/source/user_guide/read.rst index d55a141c1..76543e074 100644 --- a/docs/source/user_guide/read.rst +++ b/docs/source/user_guide/read.rst @@ -49,10 +49,11 @@ In a function returning ``Status`` or ``Result``, use the ordinary scan API: .. code-block:: cpp - ScanContextBuilder builder(table_path); + TableLoadOptions load_options{table_path}; + ScanContextBuilder builder; builder.WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, TableScan::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, TableScan::Create(load_options, std::move(context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); * The pool controls only allocations routed through it, not all planning memory diff --git a/docs/source/user_guide/realtime.rst b/docs/source/user_guide/realtime.rst index 5033d547a..52ba22f60 100644 --- a/docs/source/user_guide/realtime.rst +++ b/docs/source/user_guide/realtime.rst @@ -283,34 +283,34 @@ and an input ``batch`` built as described above. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime, RealtimeContext::Create()); - WriteContextBuilder write_builder(table_path, commit_user); - write_builder.SetOptions(options) - .WithStreamingMode(true) + TableLoadOptions load_options{table_path}; + WriteContextBuilder write_builder(commit_user); + write_builder.WithStreamingMode(true) .WithRealtimeContext(realtime); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(load_options, std::move(write_context), options)); PAIMON_RETURN_NOT_OK(writer->Write(std::move(batch))); To query before preparing or committing: .. code-block:: cpp - ScanContextBuilder scan_builder(table_path); - scan_builder.SetOptions(options).WithRealtimeContext(realtime); + ScanContextBuilder scan_builder; + scan_builder.WithRealtimeContext(realtime); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(load_options, std::move(scan_context), options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); - ReadContextBuilder read_builder(table_path); - read_builder.SetOptions(options).WithRealtimeContext(realtime); + ReadContextBuilder read_builder; + read_builder.WithRealtimeContext(realtime); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableRead::Create(load_options, std::move(read_context), options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); // Consume NextBatch() or NextBatchWithBitmap(), then Close() the reader. @@ -328,11 +328,11 @@ Checkpoint with progress-aware APIs: PAIMON_ASSIGN_OR_RAISE(std::vector progress, writer->PrepareCommitWithProgress(checkpoint_id)); - CommitContextBuilder commit_builder(table_path, commit_user); + CommitContextBuilder commit_builder(commit_user); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_builder.SetOptions(options).Finish()); + commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(load_options, std::move(commit_context), options)); PAIMON_ASSIGN_OR_RAISE( int64_t snapshot_id, commit->CommitWithProgress(progress, checkpoint_id, /*watermark=*/std::nullopt)); @@ -365,7 +365,8 @@ deduplicate ``deleted_offsets`` bitmap. Preserve the complete progress when coordinating checkpoints. Snapshot conflicts are retried internally against the latest snapshot; commit identifiers must follow the streaming commit protocol. With a version-managed catalog, configure the writer and committer -with the same ``WithCatalog()`` catalog and identifier. +from the same catalog-loaded Table, or use the same catalog and identifier in +``TableLoadOptions``. Read Views and Consistency -------------------------- diff --git a/docs/source/user_guide/write.rst b/docs/source/user_guide/write.rst index 97c4dd30b..2166643eb 100644 --- a/docs/source/user_guide/write.rst +++ b/docs/source/user_guide/write.rst @@ -241,7 +241,7 @@ produce a correct ``Snapshot``, which commonly includes (but is not limited to): file-system commits for both table types on non-object-store paths. Object-store paths need a commit that does not rename: either a catalog that manages the versions of its tables, such as the REST catalog, passed to - ``CommitContextBuilder::WithCatalog()``, or ``UseRESTCatalogCommit(true)``, + a table loaded through a version-managed catalog, or ``UseRESTCatalogCommit(true)``, which builds the commit request for the caller to send instead of publishing the snapshot itself. The two are mutually exclusive. A catalog without version management still commits by renaming, so passing one is rejected on @@ -286,7 +286,7 @@ Operational Flow passes the objects to ``FileStoreCommit`` for either an Append or PK table. For a table of a catalog which owns its versions, such as a REST catalog, it passes that catalog and the table identifier to - ``WriteContextBuilder::WithCatalog`` and ``CommitContextBuilder::WithCatalog``. + a shared catalog-loaded Table or matching ``TableLoadOptions``. The writer loads the latest snapshot from the catalog, and the current schema from it as well when it writes to the main branch; a write aimed at another branch reads the schema published under ``branch/branch-`` instead. On diff --git a/examples/clean_demo.cpp b/examples/clean_demo.cpp index b2888a86a..c5e4e1ecb 100644 --- a/examples/clean_demo.cpp +++ b/examples/clean_demo.cpp @@ -26,6 +26,7 @@ #include "arrow/ipc/api.h" #include "paimon/api.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/orphan_files_cleaner.h" namespace fs = std::filesystem; @@ -47,27 +48,36 @@ Status CleanOrphanFiles(const std::string& table_path, int64_t older_than_ms) { Status DropPartition(const std::string& table_path, const std::vector>& partitions) { - CommitContextBuilder commit_context_builder(table_path, /*commit_user=*/"commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder(/*commit_user=*/"commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr committer, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr committer, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); PAIMON_RETURN_NOT_OK(committer->DropPartition(partitions, /*commit_identifier=*/10)); return Status::OK(); } Status ExpireSnapshot(const std::string& table_path) { - CommitContextBuilder commit_context_builder(table_path, /*commit_user=*/"commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder(/*commit_user=*/"commit_user_1"); std::map commit_options = { {Options::SNAPSHOT_NUM_RETAINED_MAX, "2"}, {Options::SNAPSHOT_NUM_RETAINED_MIN, "1"}, {Options::SNAPSHOT_TIME_RETAINED, "1ms"}, {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}}; + commit_context_builder_options = commit_options; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_context_builder.SetOptions(commit_options).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr committer, - FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr committer, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); PAIMON_RETURN_NOT_OK(committer->Expire()); return Status::OK(); diff --git a/examples/read_write_demo.cpp b/examples/read_write_demo.cpp index df46fcc54..02e24fa95 100644 --- a/examples/read_write_demo.cpp +++ b/examples/read_write_demo.cpp @@ -27,6 +27,7 @@ #include "paimon/api.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/table.h" +#include "paimon/catalog/table_load_options.h" arrow::Result> PrepareData(const arrow::FieldVector& fields) { arrow::StringBuilder f0_builder; @@ -91,11 +92,13 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, std::string commit_user = "some_commit_user"; // write - paimon::WriteContextBuilder context_builder(table_path, commit_user); + std::map context_builder_options; + paimon::WriteContextBuilder context_builder(commit_user); + context_builder_options = options; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, - table->NewWrite(std::move(write_context))); + table->NewWrite(std::move(write_context), context_builder_options)); // prepare data auto struct_array = PrepareData(fields); @@ -115,27 +118,34 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, writer->PrepareCommit()); // commit - paimon::CommitContextBuilder commit_context_builder(table_path, commit_user); + std::map commit_context_builder_options; + paimon::CommitContextBuilder commit_context_builder(commit_user); + commit_context_builder_options = options; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr committer, - table->NewCommit(std::move(commit_context))); + commit_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr committer, + table->NewCommit(std::move(commit_context), commit_context_builder_options)); PAIMON_RETURN_NOT_OK(committer->Commit(commit_message)); // scan - paimon::ScanContextBuilder scan_context_builder(table_path); + std::map scan_context_builder_options; + paimon::ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - scan_context_builder.SetOptions(options).Finish()); + scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scanner, - table->NewScan(std::move(scan_context))); + table->NewScan(std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scanner->CreatePlan()); // read - paimon::ReadContextBuilder read_context_builder(table_path); + std::map read_context_builder_options; + paimon::ReadContextBuilder read_context_builder; + read_context_builder_options = options; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, - read_context_builder.SetOptions(options).Finish()); + read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - table->NewRead(std::move(read_context))); + table->NewRead(std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); arrow::ArrayVector result_array_vector; diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index aed1e34cc..a219b35d8 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -71,8 +71,7 @@ class PAIMON_EXPORT Table { return uuid_.empty() ? FullName() : uuid_; } - /// Returns the metastore UUID, or null if unavailable, for - /// `CommitContextBuilder::WithTableId()`. + /// Returns the metastore UUID, or null if unavailable. std::optional CatalogUuid() const { return uuid_.empty() ? std::nullopt : std::optional(uuid_); } @@ -85,19 +84,23 @@ class PAIMON_EXPORT Table { /// Creates a scan using this table's schema and options. virtual Result> NewScan( - std::unique_ptr context) const = 0; + std::unique_ptr context, + const std::map& dynamic_options = {}) const = 0; /// Creates a reader using this table's schema and options. virtual Result> NewRead( - std::unique_ptr context) const = 0; + std::unique_ptr context, + const std::map& dynamic_options = {}) const = 0; /// Creates a writer using this table's schema and options. virtual Result> NewWrite( - std::unique_ptr context) const = 0; + std::unique_ptr context, + const std::map& dynamic_options = {}) const = 0; /// Creates a commit using this table's schema and options. virtual Result> NewCommit( - std::unique_ptr context) const = 0; + std::unique_ptr context, + const std::map& dynamic_options = {}) const = 0; protected: /// @param file_system File system the table's data is accessed through. diff --git a/include/paimon/catalog/table_load_options.h b/include/paimon/catalog/table_load_options.h new file mode 100644 index 000000000..760d7ac62 --- /dev/null +++ b/include/paimon/catalog/table_load_options.h @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/visibility.h" + +namespace paimon { +class Catalog; +class FileSystem; + +/// Parameters used only while loading a table for a static operator factory. +/// A catalog requires an identifier; a path is required for file-system loading. +struct PAIMON_EXPORT TableLoadOptions { + std::string path; + std::shared_ptr file_system; + std::map fs_scheme_to_identifier_map; + /// Serialized Paimon table schema, independent of the reader's Arrow projection. + std::optional table_schema_json; + std::shared_ptr catalog; + std::optional identifier; + /// Expected catalog UUID, checked before preparing writes or commits. + std::optional table_id; +}; +} // namespace paimon diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index ffd40ac07..c0f68049f 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -19,20 +19,15 @@ #pragma once -#include #include -#include #include -#include "paimon/catalog/identifier.h" #include "paimon/result.h" #include "paimon/type_fwd.h" #include "paimon/visibility.h" namespace paimon { -class Catalog; class Executor; -class FormatTable; class MemoryPool; /// `CommitContext` is some configuration for commit operations. @@ -42,31 +37,12 @@ class MemoryPool; /// @see CommitContextBuilder class PAIMON_EXPORT CommitContext { public: - /// Retained for source compatibility. Prefer `CommitContextBuilder` for input validation. - CommitContext(const std::string& root_path, const std::string& commit_user, - bool ignore_empty_commit, bool use_rest_catalog_commit, - bool append_commit_check_conflict, const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table); - - CommitContext(const std::string& root_path, const std::string& commit_user, - bool ignore_empty_commit, bool use_rest_catalog_commit, - const std::shared_ptr& catalog, - const std::optional& identifier, - const std::optional& table_id, bool append_commit_check_conflict, + CommitContext(const std::string& commit_user, bool ignore_empty_commit, + bool use_rest_catalog_commit, bool append_commit_check_conflict, const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table); + const std::shared_ptr& executor); ~CommitContext(); - const std::string& GetRootPath() const { - return root_path_; - } - const std::string& GetCommitUser() const { return commit_user_; } @@ -79,21 +55,6 @@ class PAIMON_EXPORT CommitContext { return use_rest_catalog_commit_; } - /// Returns the configured catalog, or null if unset. - const std::shared_ptr& GetCatalog() const { - return catalog_; - } - - /// Returns the table identifier supplied with the catalog. - const std::optional& GetIdentifier() const { - return identifier_; - } - - /// Returns the catalog table UUID sent as `tableId` in commit requests. - const std::optional& GetTableId() const { - return table_id_; - } - bool AppendCommitCheckConflict() const { return append_commit_check_conflict_; } @@ -106,73 +67,22 @@ class PAIMON_EXPORT CommitContext { return executor_; } - std::shared_ptr GetSpecificFileSystem() const { - return specific_file_system_; - } - - const std::map& GetOptions() const { - return options_; - } - - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - private: - std::string root_path_; std::string commit_user_; bool ignore_empty_commit_; bool use_rest_catalog_commit_; - std::shared_ptr catalog_; - std::optional identifier_; - std::optional table_id_; bool append_commit_check_conflict_; std::shared_ptr memory_pool_; std::shared_ptr executor_; - std::shared_ptr specific_file_system_; - std::map options_; - std::shared_ptr format_table_; }; /// `CommitContextBuilder` used to build a `CommitContext`, has input validation. class PAIMON_EXPORT CommitContextBuilder { public: - /// Constructs a `CommitContextBuilder` with required parameters. - /// @param root_path The root path of the Paimon table. - /// @param commit_user The user identifier for the commit operation. - CommitContextBuilder(const std::string& root_path, const std::string& commit_user); - - /// Constructs a `CommitContextBuilder` for a format table that is already loaded: the only way - /// to commit to one whose schema lives in a metastore rather than under its location, such as - /// a table a REST catalog serves. The table carries what such a location does not say, so - /// `WithFileSystem()` is refused here rather than ignored. - /// - /// There is no commit user: a format table keeps no snapshot to record one in. - /// - /// @param table The format table to commit to, as `Catalog::GetFormatTable()` hands it back. - explicit CommitContextBuilder(const std::shared_ptr& table); + explicit CommitContextBuilder(const std::string& commit_user); ~CommitContextBuilder(); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - CommitContextBuilder& SetOptions(const std::map& options); - - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. - /// - /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` - /// instead. - /// @param key The option key. - /// @param value The option value. - /// @return Reference to this builder for method chaining. - CommitContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Sets whether to ignore empty commits (default is true). /// When set to true, commits that don't contain any actual data changes will be ignored. /// @param ignore_empty_commit True to ignore empty commits, false otherwise. @@ -180,39 +90,12 @@ class PAIMON_EXPORT CommitContextBuilder { CommitContextBuilder& IgnoreEmptyCommit(bool ignore_empty_commit); /// Builds a REST commit request for the caller to send (default is false). - /// Use `WithCatalog()` for catalog-managed commits; the two modes are mutually exclusive. + /// A table loaded through a catalog commits through that catalog; the two modes are exclusive. /// @note Temporary interface, will be removed in the future. /// @param use_rest_catalog_commit True to build the request without sending it. /// @return Reference to this builder for method chaining. CommitContextBuilder& UseRESTCatalogCommit(bool use_rest_catalog_commit); - /// Loads the current schema from `catalog`. Catalogs with version management also load and - /// commit snapshots, rebasing on conflicts. Other catalogs use file-system snapshot commits. - /// Mutually exclusive with `UseRESTCatalogCommit()`. - /// - /// Data and manifests remain on the file system. Recovery, rollback and row-id checks require - /// historical snapshots there; row-id checks also require the inspected data files' schemas. - /// `Expire()` manages file-system snapshots after confirming that the catalog's current - /// snapshot and retained history are published there. Configure writers with the same catalog - /// via `WriteContextBuilder::WithCatalog()`. - /// A branch is addressed by the identifier, as `tbl$branch_dev`, so that the catalog answers - /// for that branch. Its schema is read from the branch directory rather than from the catalog, - /// just as a read of that branch reads it. Configure the writer with the same identifier. - /// @param catalog Non-null catalog, kept alive by this context. - /// @param identifier Table to commit to, naming a branch of it as `tbl$branch_dev`. - /// @return Reference to this builder for method chaining. - CommitContextBuilder& WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier); - - /// Sets the catalog table UUID sent as `tableId` to detect table recreation (default is unset). - /// - /// Capture `Table::CatalogUuid()` before preparing changes. `Table::Uuid()` may fall back to - /// the table name. An unset ID is sent as null and validated by the server. - /// Requires `WithCatalog()` or `UseRESTCatalogCommit()`. - /// @param table_id Catalog UUID of the table to commit to. - /// @return Reference to this builder for method chaining. - CommitContextBuilder& WithTableId(const std::string& table_id); - /// Sets whether append commits should perform conflict checking (default is false). /// @param append_commit_check_conflict True to enable append conflict checks. /// @return Reference to this builder for method chaining. @@ -228,15 +111,6 @@ class PAIMON_EXPORT CommitContextBuilder { /// @return Reference to this builder for method chaining. CommitContextBuilder& WithExecutor(const std::shared_ptr& executor); - /// Sets a custom file system instance to be used for all file operations in this commit - /// context. - /// This bypasses the global file system registry and uses the provided implementation directly. - /// - /// @param file_system The file system to use. - /// @return Reference to this builder for method chaining. - /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) - CommitContextBuilder& WithFileSystem(const std::shared_ptr& file_system); - /// Build and return a `CommitContext` instance with input validation. /// @return Result containing the constructed `CommitContext` or an error status. Result> Finish(); diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index fcb094e05..8b1658390 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -26,6 +26,7 @@ #include #include +#include "paimon/catalog/table_load_options.h" #include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" @@ -46,16 +47,20 @@ class CommitMessage; /// dropping partitions, and retrieving commit metrics. /// /// @note Direct file-system commits support append-only and primary-key tables on non-object-store -/// paths. `CommitContextBuilder::WithCatalog()` also supports object stores when the catalog +/// paths. `TableLoadOptions::catalog` also supports object stores when the catalog /// manages versions. `UseRESTCatalogCommit()` only prepares a request for the caller to send. class PAIMON_EXPORT FileStoreCommit { public: /// Create an instance of `FileStoreCommit`. /// + /// @param load_options Table identity, metadata and file-system loading parameters. + /// @param dynamic_options Overrides merged on top of the table options. /// @param context A unique pointer to the `CommitContext` used for commit operations. /// /// @return A Result containing a unique pointer to the `FileStoreCommit` instance. - static Result> Create(std::unique_ptr context); + static Result> Create( + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options = {}); virtual ~FileStoreCommit() = default; @@ -93,7 +98,7 @@ class PAIMON_EXPORT FileStoreCommit { /// An error is terminal for the writer state which produced `realtime_commits`. The caller must /// discard its `RealtimeContext` and `FileStoreWrite`, load the current latest snapshot's /// durable offsets, recreate both objects, and replay input from those exclusive offsets. - /// Conflicts from `CommitContextBuilder::WithCatalog()` are retried internally. With + /// Conflicts from `TableLoadOptions::catalog` are retried internally. With /// `UseRESTCatalogCommit()`, the caller sends the request and handles server conflicts. /// Concurrent rollback or partition deletion must be fenced by the upstream coordinator. /// Configure catalog writers with `WriteContextBuilder::WithCatalog()` too. diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 5e6b253c5..c391b2a42 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -25,6 +25,7 @@ #include #include +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_message.h" #include "paimon/defs.h" #include "paimon/executor.h" @@ -45,10 +46,14 @@ class PAIMON_EXPORT FileStoreWrite { public: /// Create an instance of `FileStoreWrite`. /// + /// @param load_options Table identity, metadata and file-system loading parameters. + /// @param dynamic_options Overrides merged on top of the table options. /// @param context A unique pointer to the `WriteContext` used for write operations. /// /// @return A Result containing a unique pointer to the `FileStoreWrite` instance. - static Result> Create(std::unique_ptr context); + static Result> Create( + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options = {}); virtual ~FileStoreWrite() = default; diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 4ca68dd9c..a65272e40 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -19,15 +19,12 @@ #pragma once #include -#include #include -#include #include #include #include "arrow/c/abi.h" #include "paimon/cache/cache.h" -#include "paimon/catalog/identifier.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" #include "paimon/type_fwd.h" @@ -35,13 +32,10 @@ #include "paimon/visibility.h" namespace paimon { -class Catalog; class Executor; struct FullTextSearch; -class FormatTable; class MemoryPool; class Predicate; -class FileSystem; class RealtimeContext; struct VectorSearch; @@ -52,8 +46,7 @@ struct VectorSearch; /// @see ReadContextBuilder class PAIMON_EXPORT ReadContext { public: - ReadContext(const std::string& path, const std::string& branch, - const std::vector& read_field_names, + ReadContext(const std::vector& read_field_names, const std::vector& read_field_ids, const std::shared_ptr& predicate, const std::shared_ptr& vector_search, @@ -61,36 +54,14 @@ class PAIMON_EXPORT ReadContext { bool enable_precise_file_index_evaluate, bool enable_predicate_filter, bool enable_prefetch, bool enable_late_materializing, uint32_t prefetch_batch_count, uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch, - uint32_t row_to_batch_thread_number, const std::optional& table_schema, - const std::shared_ptr& memory_pool, + uint32_t row_to_batch_thread_number, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, - const std::map& options, bool read_ahead_cache_enabled, - const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, - const std::shared_ptr& catalog, - const std::optional& identifier); + bool read_ahead_cache_enabled, const CacheConfig& cache_config, + const std::shared_ptr& cache, WarmupLevel warmup_level); ~ReadContext(); - const std::string& GetPath() const { - return path_; - } - - const std::string& GetBranch() const { - return branch_; - } - - const std::map& GetFileSystemSchemeToIdentifierMap() const { - return fs_scheme_to_identifier_map_; - } - - const std::map& GetOptions() const { - return options_; - } - const std::vector& GetReadFieldNames() const { return read_field_names_; } @@ -141,18 +112,13 @@ class PAIMON_EXPORT ReadContext { uint32_t GetRowToBatchThreadNumber() const { return row_to_batch_thread_number_; } - const std::optional& GetSpecificTableSchema() const { - return table_schema_; - } + std::shared_ptr GetMemoryPool() const { return memory_pool_; } std::shared_ptr GetExecutor() const { return executor_; } - std::shared_ptr GetSpecificFileSystem() const { - return specific_file_system_; - } /// Returns the context used to resolve process-local real-time split tickets. std::shared_ptr GetRealtimeContext() const { @@ -171,26 +137,10 @@ class PAIMON_EXPORT ReadContext { return cache_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - WarmupLevel GetWarmupLevel() const { return warmup_level_; } - /// Returns the catalog supplying table metadata, or null if unset. - const std::shared_ptr& GetCatalog() const { - return catalog_; - } - - /// Returns the table identifier supplied with the catalog. - const std::optional& GetIdentifier() const { - return identifier_; - } - /// Whether a read schema (C ArrowSchema) for nested column pruning was provided. bool HasReadSchema() const { return read_schema_ != nullptr && read_schema_->release != nullptr; @@ -208,8 +158,6 @@ class PAIMON_EXPORT ReadContext { void SetReadSchema(std::unique_ptr schema); private: - std::string path_; - std::string branch_; std::vector read_field_names_; std::vector read_field_ids_; std::shared_ptr predicate_; @@ -223,20 +171,13 @@ class PAIMON_EXPORT ReadContext { uint32_t prefetch_max_parallel_num_; bool enable_multi_thread_row_to_batch_; uint32_t row_to_batch_thread_number_; - std::optional table_schema_; std::shared_ptr memory_pool_; std::shared_ptr executor_; - std::shared_ptr specific_file_system_; - std::map fs_scheme_to_identifier_map_; std::shared_ptr realtime_context_; - std::map options_; bool read_ahead_cache_enabled_; CacheConfig cache_config_; std::shared_ptr cache_; - std::shared_ptr format_table_; WarmupLevel warmup_level_; - std::shared_ptr catalog_; - std::optional identifier_; // Owns schema resources and releases ArrowSchema::release in destructor. std::unique_ptr read_schema_; }; @@ -244,18 +185,7 @@ class PAIMON_EXPORT ReadContext { /// `ReadContextBuilder` used to build a `ReadContext`, has input validation. class PAIMON_EXPORT ReadContextBuilder { public: - /// Constructs a `ReadContextBuilder` with required parameters. - /// @param path The root path of the table. - explicit ReadContextBuilder(const std::string& path); - - /// Constructs a `ReadContextBuilder` for a format table that is already loaded: the only way - /// to read one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()`, `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()` and a - /// branch are refused here rather than ignored. - /// - /// @param table The format table to read, as `Catalog::GetFormatTable()` hands it back. - explicit ReadContextBuilder(const std::shared_ptr& table); + ReadContextBuilder(); ~ReadContextBuilder(); @@ -335,23 +265,6 @@ class PAIMON_EXPORT ReadContextBuilder { /// When set, read_field_ids and read_field_names are ignored. ReadContextBuilder& SetReadSchema(std::unique_ptr read_schema); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - ReadContextBuilder& SetOptions(const std::map& options); - - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. - /// - /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` - /// instead. - /// @param key The option key. - /// @param value The option value. - /// @return Reference to this builder for method chaining. - ReadContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set a predicate for filtering data during reading. /// /// The predicate is used for both partition pruning and data filtering. @@ -514,68 +427,6 @@ class PAIMON_EXPORT ReadContextBuilder { ReadContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// Set the table schema as a string to avoid schema loading I/O operations. - /// - /// This optimization allows the reader to use a pre-loaded schema instead of - /// reading it from the table metadata, which can improve performance especially - /// in scenarios with many small read operations. - /// - /// @param table_schema String representation of the table schema. - /// @return Reference to this builder for method chaining. - /// @note The user must ensure that the schema string is valid and matches the table. - /// @note If not set, the schema will be loaded from the table path. - ReadContextBuilder& SetTableSchema(const std::string& table_schema); - - /// Set the specific branch to read from in a versioned table. - /// - /// Paimon supports branching for data versioning and time travel queries. - /// This method allows reading from a specific branch instead of the main branch. - /// - /// The `branch` option names the branch too, as it does for a scan of one branch; naming two - /// different branches is refused rather than silently resolved. - /// - /// @param branch Name of the branch to read from. - /// @return Reference to this builder for method chaining. - /// @note Default branch is "main" if not specified. An empty name is the main branch. - ReadContextBuilder& WithBranch(const std::string& branch); - - /// Sets a mapping from URI schemes (e.g., "file", "oss") to registered file system - /// identifiers. This allows selecting different pre-registered file system implementations - /// based on the URI scheme at runtime. - /// - /// @param fs_scheme_to_identifier_map Map from URI scheme (like "oss") to the corresponding - /// file system identifier. - /// @return Reference to this builder for method chaining. - /// @note - /// - This method is intended for environments where multiple file systems are pre-registered. - /// - The specified identifiers must correspond to file systems that have been registered at - /// compile time or initialization. - /// - Cannot be used together with `WithFileSystem()`. - /// - If not set, use default file system (configured in `Options::FILE_SYSTEM`). - /// Example: - /// builder.WithFileSystemSchemeToIdentifierMap({{"oss", "jindo"}, {"file", "local"}}); - /// - ReadContextBuilder& WithFileSystemSchemeToIdentifierMap( - const std::map& fs_scheme_to_identifier_map); - - /// Sets a custom file system instance to be used for all file operations in this read context. - /// This bypasses the global file system registry and uses the provided implementation directly. - /// - /// @param file_system The file system to use. - /// @return Reference to this builder for method chaining. - /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) - ReadContextBuilder& WithFileSystem(const std::shared_ptr& file_system); - - /// Reads a native table through its own file system - including the per-table temporary - /// credentials a catalog that issues them hands out through `Catalog::GetTableFileSystem`. - /// This is a shorthand for `WithFileSystem(catalog->GetTableFileSystem(identifier))`; an - /// explicit `WithFileSystem()` takes precedence, so the catalog is not asked. - /// @param catalog Non-null catalog, read when `Finish()` builds the context. - /// @param identifier The native table to read. - /// @return Reference to this builder for method chaining. - ReadContextBuilder& WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier); - /// Inject a cache for read operations. Passing nullptr disables cache. /// @return Reference to this builder for method chaining. ReadContextBuilder& WithCache(const std::shared_ptr& cache); diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index efee52743..57b6843bf 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -26,7 +26,6 @@ #include #include "paimon/cache/cache.h" -#include "paimon/catalog/identifier.h" #include "paimon/global_index/global_index_result.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" @@ -35,9 +34,7 @@ namespace paimon { class ScanContextBuilder; class ScanFilter; -class Catalog; class Executor; -class FormatTable; class MemoryPool; class Predicate; @@ -48,26 +45,15 @@ class Predicate; /// @see ScanContextBuilder class PAIMON_EXPORT ScanContext { public: - ScanContext(const std::string& path, bool is_streaming_mode, std::optional limit, + ScanContext(bool is_streaming_mode, std::optional limit, const std::shared_ptr& scan_filter, const std::shared_ptr& global_index_result, const std::shared_ptr& realtime_context, const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::optional& table_schema, - const std::map& options, - const std::shared_ptr& cache, - const std::shared_ptr& format_table, - const std::shared_ptr& catalog, - const std::optional& identifier); + const std::shared_ptr& executor, const std::shared_ptr& cache); ~ScanContext(); - const std::string& GetPath() const { - return path_; - } - bool IsStreamingMode() const { return is_streaming_mode_; } @@ -79,9 +65,6 @@ class PAIMON_EXPORT ScanContext { std::shared_ptr GetScanFilters() const { return scan_filters_; } - const std::map& GetOptions() const { - return options_; - } std::shared_ptr GetMemoryPool() const { return memory_pool_; @@ -99,36 +82,11 @@ class PAIMON_EXPORT ScanContext { return realtime_context_; } - std::shared_ptr GetSpecificFileSystem() const { - return specific_file_system_; - } - - const std::optional& GetSpecificTableSchema() const { - return table_schema_; - } - std::shared_ptr GetCache() const { return cache_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - - /// Returns the catalog supplying table metadata, or null if unset. - const std::shared_ptr& GetCatalog() const { - return catalog_; - } - - /// Returns the table identifier supplied with the catalog. - const std::optional& GetIdentifier() const { - return identifier_; - } - private: - std::string path_; bool is_streaming_mode_; std::optional limit_; std::shared_ptr scan_filters_; @@ -136,13 +94,7 @@ class PAIMON_EXPORT ScanContext { std::shared_ptr realtime_context_; std::shared_ptr memory_pool_; std::shared_ptr executor_; - std::shared_ptr specific_file_system_; - std::optional table_schema_; - std::map options_; std::shared_ptr cache_; - std::shared_ptr format_table_; - std::shared_ptr catalog_; - std::optional identifier_; }; /// Filter configuration for table scan operations @@ -174,17 +126,7 @@ class PAIMON_EXPORT ScanFilter { /// `ScanContextBuilder` used to build a `ScanContext`, has input validation. class PAIMON_EXPORT ScanContextBuilder { public: - /// Constructs a `ScanContextBuilder` with required parameters. - /// @param path The root path of the table. - explicit ScanContextBuilder(const std::string& path); - - /// Constructs a `ScanContextBuilder` for a format table that is already loaded: the only way - /// to scan one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()` and `WithFileSystem()` are refused here rather than ignored. - /// - /// @param table The format table to scan, as `Catalog::GetFormatTable()` hands it back. - explicit ScanContextBuilder(const std::shared_ptr& table); + ScanContextBuilder(); ~ScanContextBuilder(); /// If limit is not set, it defaults to unlimited. @@ -208,16 +150,6 @@ class PAIMON_EXPORT ScanContextBuilder { ScanContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// The options added or set in `ScanContextBuilder` have high priority and will be merged with - /// the options in table schema. - ScanContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - ScanContextBuilder& SetOptions(const std::map& options); - /// Set whether the scan is in streaming mode. /// @note if not set, is_streaming_mode = false ScanContextBuilder& WithStreamingMode(bool is_streaming_mode); @@ -234,34 +166,6 @@ class PAIMON_EXPORT ScanContextBuilder { /// @return Reference to this builder for method chaining. /// @note if not set, executor is default executor ScanContextBuilder& WithExecutor(const std::shared_ptr& executor); - /// Sets a custom file system instance to be used for all file operations in this scan context. - /// This bypasses the global file system registry and uses the provided implementation directly. - /// @param file_system The file system to use. - /// @return Reference to this builder for method chaining. - /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) - ScanContextBuilder& WithFileSystem(const std::shared_ptr& file_system); - - /// Plans a native table through its own file system - including the per-table temporary - /// credentials a catalog that issues them hands out through `Catalog::GetTableFileSystem`. - /// This is a shorthand for `WithFileSystem(catalog->GetTableFileSystem(identifier))`; an - /// explicit `WithFileSystem()` takes precedence, so the catalog is not asked. - /// @param catalog Non-null catalog, read when `Finish()` builds the context. - /// @param identifier The native table to scan. - /// @return Reference to this builder for method chaining. - ScanContextBuilder& WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier); - - /// Set the table schema as a string to avoid schema loading I/O operations. - /// - /// This optimization allows the scanner to use a pre-loaded schema instead of - /// reading it from the table metadata, which can improve performance especially - /// in scenarios with many small scan operations. - /// - /// @param table_schema String representation of the table schema. - /// @return Reference to this builder for method chaining. - /// @note The user must ensure that the schema string is valid and matches the table. - /// @note If not set, the schema will be loaded from the table path. - ScanContextBuilder& SetTableSchema(const std::string& table_schema); /// Inject a cache for scan operations. Passing nullptr disables cache. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index 917fcb46c..136b4aa99 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -118,12 +118,18 @@ class PAIMON_EXPORT FormatTable : public Table { ~FormatTable() override; - Result> NewScan(std::unique_ptr context) const override; - Result> NewRead(std::unique_ptr context) const override; + Result> NewScan( + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; + Result> NewRead( + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; Result> NewWrite( - std::unique_ptr context) const override; + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; Result> NewCommit( - std::unique_ptr context) const override; + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; /// Directory the data files live in. const std::string& Location() const { diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index c77b323da..46485459a 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -22,6 +22,7 @@ #include #include +#include "paimon/catalog/table_load_options.h" #include "paimon/executor.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" @@ -44,9 +45,13 @@ class PAIMON_EXPORT TableRead { /// Create an instance of `TableRead`. /// + /// @param load_options Table identity, metadata and file-system loading parameters. + /// @param dynamic_options Overrides merged on top of the table options. /// @param context A unique pointer to the `ReadContext` used for read operations. /// @return A Result containing a unique pointer to the `TableRead` instance. - static Result> Create(std::unique_ptr context); + static Result> Create( + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options = {}); /// Creates a `BatchReader` instance for reading data. /// diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index 8bae74c41..e02a059f9 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -24,6 +24,7 @@ #include #include +#include "paimon/catalog/table_load_options.h" #include "paimon/result.h" #include "paimon/table/source/plan.h" #include "paimon/table/source/scan_metrics.h" @@ -38,9 +39,13 @@ class PAIMON_EXPORT TableScan { public: /// Create an instance of `TableScan`. /// + /// @param load_options Table identity, metadata and file-system loading parameters. + /// @param dynamic_options Overrides merged on top of the table options. /// @param context A unique pointer to the `ScanContext` used for scan operations. /// @return A Result containing a unique pointer to the `TableScan` instance. - static Result> Create(std::unique_ptr context); + static Result> Create( + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options = {}); virtual ~TableScan() = default; diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index e0a940eb0..f7237377b 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -19,21 +19,17 @@ #pragma once #include -#include #include #include #include #include -#include "paimon/catalog/identifier.h" #include "paimon/result.h" #include "paimon/type_fwd.h" #include "paimon/visibility.h" namespace paimon { -class Catalog; class Executor; -class FormatTable; class MemoryPool; /// `WriteContext` is some configuration for write operations. @@ -43,38 +39,20 @@ class MemoryPool; /// @see WriteContextBuilder class PAIMON_EXPORT WriteContext { public: - WriteContext(const std::string& root_path, const std::string& commit_user, - bool is_streaming_mode, bool ignore_num_bucket_check, bool ignore_previous_files, + WriteContext(const std::string& commit_user, bool is_streaming_mode, + bool ignore_num_bucket_check, bool ignore_previous_files, bool enable_multi_thread_spill, const std::optional& write_id, - const std::string& branch, const std::vector& write_schema, + const std::vector& write_schema, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::string& temp_directory, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_to_identifier_map, - const std::shared_ptr& realtime_context, - const std::map& options, - const std::shared_ptr& format_table, - const std::shared_ptr& catalog = nullptr, - const std::optional& identifier = std::nullopt); + const std::shared_ptr& realtime_context); ~WriteContext(); - const std::string& GetRootPath() const { - return root_path_; - } - const std::string& GetCommitUser() const { return commit_user_; } - const std::map& GetFileSystemSchemeToIdentifierMap() const { - return fs_scheme_to_identifier_map_; - } - - const std::map& GetOptions() const { - return options_; - } - bool IsStreamingMode() const { return is_streaming_mode_; } @@ -91,10 +69,6 @@ class PAIMON_EXPORT WriteContext { return write_id_; } - const std::string& GetBranch() const { - return branch_; - } - const std::vector& GetWriteSchema() const { return write_schema_; } @@ -111,10 +85,6 @@ class PAIMON_EXPORT WriteContext { return temp_directory_; } - std::shared_ptr GetSpecificFileSystem() const { - return specific_file_system_; - } - bool EnableMultiThreadSpill() const { return enable_multi_thread_spill_; } @@ -124,26 +94,8 @@ class PAIMON_EXPORT WriteContext { return realtime_context_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - - /// Returns the catalog supplying table metadata, or null if unset. - const std::shared_ptr& GetCatalog() const { - return catalog_; - } - - /// Returns the table identifier supplied with the catalog. - const std::optional& GetIdentifier() const { - return identifier_; - } - private: - std::string root_path_; std::string commit_user_; - std::string branch_; bool is_streaming_mode_; bool ignore_num_bucket_check_; bool ignore_previous_files_; @@ -153,53 +105,16 @@ class PAIMON_EXPORT WriteContext { std::shared_ptr memory_pool_; std::shared_ptr executor_; std::string temp_directory_; - std::shared_ptr specific_file_system_; - std::map fs_scheme_to_identifier_map_; std::shared_ptr realtime_context_; - std::map options_; - std::shared_ptr format_table_; - std::shared_ptr catalog_; - std::optional identifier_; }; /// `WriteContextBuilder` used to build a `WriteContext`, has input validation. class PAIMON_EXPORT WriteContextBuilder { public: - /// Constructs a `WriteContextBuilder` with required parameters. - /// @param root_path The root path of the table. - /// @param commit_user The user identifier for commit operations. - WriteContextBuilder(const std::string& root_path, const std::string& commit_user); - - /// Constructs a `WriteContextBuilder` for a format table that is already loaded: the only way - /// to write one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()`, `WithCatalog()` and a branch - /// are refused here rather than ignored. - /// - /// There is no commit user: a format table keeps no snapshot to record one in. - /// - /// @param table The format table to write to, as `Catalog::GetFormatTable()` hands it back. - explicit WriteContextBuilder(const std::shared_ptr& table); + explicit WriteContextBuilder(const std::string& commit_user); ~WriteContextBuilder(); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - WriteContextBuilder& SetOptions(const std::map& options); - - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. - /// - /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` - /// instead. - /// @param key The option key. - /// @param value The option value. - /// @return Reference to this builder for method chaining. - WriteContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set whether to enable streaming mode (default is false) /// @return Reference to this builder for method chaining. WriteContextBuilder& WithStreamingMode(bool is_streaming_mode); @@ -237,69 +152,17 @@ class PAIMON_EXPORT WriteContextBuilder { /// @return Reference to this builder for method chaining. WriteContextBuilder& WithWriteId(int32_t write_id); - /// Write to specific branch, default is main. - /// - /// The identifier of `WithCatalog()` and the `branch` option may name the branch too; naming - /// two different ones is refused rather than silently resolved. An empty name is the main - /// branch. - /// @param branch Name of the branch to write to. - /// @return Reference to this builder for method chaining. - WriteContextBuilder& WithBranch(const std::string& branch); - /// For data evolution, user can write partial specific fields from table schema. /// If not set, write all fields in table. /// @return Reference to this builder for method chaining. WriteContextBuilder& WithWriteSchema(const std::vector& write_schema); - /// Sets a custom file system instance to be used for all file operations in this write context. - /// This bypasses the global file system registry and uses the provided implementation directly. - /// - /// @param file_system The file system to use. - /// @return Reference to this builder for method chaining. - /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) - WriteContextBuilder& WithFileSystem(const std::shared_ptr& file_system); - - /// Loads a native table's schema and, for catalogs with version management, its latest - /// snapshot. Use the same catalog and identifier as `CommitContextBuilder` to restore and - /// refresh real-time progress. - /// - /// A branch is addressed by the identifier, as `tbl$branch_dev`, so that the catalog answers - /// for that branch. Its schema is read from the branch directory rather than from the catalog, - /// just as a read of that branch reads it. - /// Data, manifests and historical metadata remain on the file system. The catalog supplies - /// that file system unless overridden by `WithFileSystem()` or - /// `WithFileSystemSchemeToIdentifierMap()`. - /// @param catalog Non-null catalog, kept alive while the writer uses its snapshot loader. - /// @param identifier The native table to write to. - /// @return Reference to this builder for method chaining. - WriteContextBuilder& WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier); - /// Enables the real-time write path with the provided shared context. /// @param realtime_context Non-null context that owns the real-time stores. /// @return Reference to this builder for method chaining. WriteContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// Sets a mapping from URI schemes (e.g., "file", "oss") to registered file system - /// identifiers. This allows selecting different pre-registered file system implementations - /// based on the URI scheme at runtime. - /// - /// @param fs_scheme_to_identifier_map Map from URI scheme (like "oss") to the corresponding - /// file system identifier. - /// @return Reference to this builder for method chaining. - /// @note - /// - This method is intended for environments where multiple file systems are pre-registered. - /// - The specified identifiers must correspond to file systems that have been registered at - /// compile time or initialization. - /// - Cannot be used together with `WithFileSystem()`. - /// - If not set, use default file system (configured in `Options::FILE_SYSTEM`). - /// Example: - /// builder.WithFileSystemSchemeToIdentifierMap({{"oss", "jindo"}, {"file", "local"}}); - /// - WriteContextBuilder& WithFileSystemSchemeToIdentifierMap( - const std::map& fs_scheme_to_identifier_map); - /// Set the thread number for write buffer spill operations. (default is 0) /// If <= 0, threading is disabled for spill IPC read/write. /// If > 0, sets arrow CPU thread pool capacity for spill operations. diff --git a/src/paimon/core/append/append_compact_coordinator_test.cpp b/src/paimon/core/append/append_compact_coordinator_test.cpp index acdf59cb0..df11f129c 100644 --- a/src/paimon/core/append/append_compact_coordinator_test.cpp +++ b/src/paimon/core/append/append_compact_coordinator_test.cpp @@ -29,6 +29,7 @@ #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/path_util.h" @@ -79,11 +80,15 @@ class AppendCompactCoordinatorTest : public ::testing::Test { const std::map& partition, int32_t bucket, const std::shared_ptr& write_array, int64_t commit_identifier) { - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(false); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); auto record_batch = std::make_unique( @@ -97,11 +102,15 @@ class AppendCompactCoordinatorTest : public ::testing::Test { Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { - CommitContextBuilder commit_builder(table_path, "commit_user_1"); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); return file_store_commit->Commit(commit_msgs); } diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 978c20e9d..dddbf3864 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -67,7 +67,8 @@ Result> Catalog::Create( } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system, fs_scheme_to_identifier_map)); - return std::make_unique(core_options.GetFileSystem(), root_path, options); + return std::make_unique(core_options.GetFileSystem(), root_path, options, + fs_scheme_to_identifier_map); } bool Catalog::SupportsVersionManagement() const { diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 4aeac4d53..a8b882531 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -57,8 +57,10 @@ struct ArrowSchema; namespace paimon { FileSystemCatalog::FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, - const std::map& catalog_options) + const std::map& catalog_options, + const std::map& fs_scheme_map) : fs_(fs), + fs_scheme_map_(fs_scheme_map), warehouse_(warehouse), catalog_options_(catalog_options), logger_(Logger::GetLogger("FileSystemCatalog")) {} @@ -323,9 +325,10 @@ Result> FileSystemCatalog::LoadTableSchema( dynamic_options[Options::BRANCH] = branch.value(); } PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::Load(system_table_name.value(), fs_, table_path, - latest_schema.value(), dynamic_options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + SystemTableLoader::Load(system_table_name.value(), fs_, table_path, + latest_schema.value(), dynamic_options, "", fs_scheme_map_)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, system_table->ArrowSchema()); return std::make_shared(std::move(arrow_schema)); @@ -361,7 +364,7 @@ Result> FileSystemCatalog::LoadSystemTable( PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, latest_schema.value(), dynamic_options, - data_identifier.GetObjectName(), {}); + data_identifier.GetObjectName(), fs_scheme_map_); } Result> FileSystemCatalog::GetTable(const Identifier& identifier) const { @@ -409,8 +412,12 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); + std::shared_ptr catalog = std::const_pointer_cast(weak_from_this().lock()); + if (!catalog) { + catalog = std::make_shared(*this); + } return FileStoreTable::Create(fs_, table_path, *latest_schema, identifier, /*uuid=*/"", branch, - {}); + fs_scheme_map_, catalog); } Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index 37bb23ea6..fba7519e0 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -42,7 +42,8 @@ class Logger; class FileSystemCatalog : public Catalog { public: FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, - const std::map& catalog_options); + const std::map& catalog_options, + const std::map& fs_scheme_map = {}); Status CreateDatabase(const std::string& db_name, const std::map& options, @@ -110,6 +111,7 @@ class FileSystemCatalog : public Catalog { const std::vector& external_paths); std::shared_ptr fs_; + std::map fs_scheme_map_; std::string warehouse_; std::map catalog_options_; diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 2409942fc..5887a0b04 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -26,6 +26,7 @@ #include "arrow/c/helpers.h" #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/path_util.h" @@ -1479,13 +1480,18 @@ TEST(FileSystemCatalogTest, TestClaimingVersionManagementWithoutImplementingItIs ASSERT_NOK_WITH_MSG(CheckVersionManagementImplemented(claiming), "does not implement VersionManagedCatalog"); - CommitContextBuilder claiming_builder(dir->Str(), "commit_user_1"); + TableLoadOptions claiming_builder_load_options{dir->Str()}; + std::map claiming_builder_options; + CommitContextBuilder claiming_builder("commit_user_1"); + claiming_builder_options = options; + claiming_builder_load_options.catalog = claiming; + claiming_builder_load_options.identifier.emplace(Identifier("db", "tbl")); ASSERT_OK_AND_ASSIGN(std::unique_ptr claiming_context, - claiming_builder.SetOptions(options) - .WithCatalog(claiming, Identifier("db", "tbl")) - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(claiming_context)), - "does not implement VersionManagedCatalog"); + claiming_builder.Finish()); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(claiming_builder_load_options, std::move(claiming_context), + claiming_builder_options), + "does not implement VersionManagedCatalog"); std::shared_ptr plain = std::make_shared(core_options.GetFileSystem(), dir->Str(), options); diff --git a/src/paimon/core/global_index/global_index_write_task.cpp b/src/paimon/core/global_index/global_index_write_task.cpp index cb11fd920..594f4b4b3 100644 --- a/src/paimon/core/global_index/global_index_write_task.cpp +++ b/src/paimon/core/global_index/global_index_write_task.cpp @@ -26,6 +26,7 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" #include "arrow/type.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/mem_utils.h" @@ -174,16 +175,19 @@ Result> CreateBatchReader( const std::string& table_path, const std::vector& read_field_names, const std::shared_ptr& indexed_split, const CoreOptions& core_options, const std::shared_ptr& pool) { - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(core_options.ToMap()) - .WithFileSystem(core_options.GetFileSystem()) - .EnablePrefetch(true) - .WithMemoryPool(pool) - .SetReadFieldNames(read_field_names); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = core_options.ToMap(); + read_context_builder_load_options.file_system = core_options.GetFileSystem(); + read_context_builder.EnablePrefetch(true).WithMemoryPool(pool).SetReadFieldNames( + read_field_names); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); return table_read->CreateReader(indexed_split); } diff --git a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp index 81803ed90..c2d0e3d7a 100644 --- a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp +++ b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp @@ -33,6 +33,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/global_index/indexed_split_impl.h" @@ -320,15 +321,18 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam> CreateWriter(bool with_temp_directory = true, bool write_buffer_spillable = true) const { - WriteContextBuilder builder(table_path_, kCommitUser); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + WriteContextBuilder builder(kCommitUser); std::map writer_options = options_; writer_options[Options::WRITE_BUFFER_SPILLABLE] = write_buffer_spillable ? "true" : "false"; - builder.SetOptions(writer_options).WithStreamingMode(true); + builder_options = writer_options; + builder.WithStreamingMode(true); if (with_temp_directory) { builder.WithTempDirectory(directory_->Str()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreWrite::Create(std::move(context)); + return FileStoreWrite::Create(builder_load_options, std::move(context), builder_options); } Result>> WriteAndPrepare( @@ -354,11 +358,15 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam>& messages, int64_t commit_identifier) const { - CommitContextBuilder builder(table_path_, kCommitUser); - builder.SetOptions(options_); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(kCommitUser); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); return commit->Commit(messages, commit_identifier); } @@ -454,11 +462,15 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted std::shared_ptr value_predicate = PredicateBuilder::Equal( /*field_index=*/1, "value", FieldType::STRING, Literal(FieldType::STRING, indexed_value.data(), indexed_value.size())); - ScanContextBuilder scan_builder(table_path_); - scan_builder.SetPredicate(value_predicate).AddOption(Options::GLOBAL_INDEX_ENABLED, "true"); + TableLoadOptions scan_builder_load_options{table_path_}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::GLOBAL_INDEX_ENABLED] = "true"; + scan_builder.SetPredicate(value_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); int32_t indexed_split_count = 0; int32_t data_split_count = 0; @@ -472,16 +484,19 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted ASSERT_GT(indexed_split_count, 0); ASSERT_EQ(0, data_split_count); - ReadContextBuilder read_builder(table_path_); + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; // The source build above keeps lazy decoding enabled. Disable it only for the final data-row // assertion, whose predicate reader is outside the maintenance path covered by this test. + read_builder_options["orc.read.enable-lazy-decoding"] = "false"; read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) - .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -686,11 +701,15 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF std::shared_ptr value_predicate = PredicateBuilder::Equal( /*field_index=*/1, "value", FieldType::STRING, Literal(FieldType::STRING, indexed_value.data(), indexed_value.size())); - ScanContextBuilder scan_builder(table_path_); - scan_builder.SetPredicate(value_predicate).AddOption(Options::GLOBAL_INDEX_ENABLED, "true"); + TableLoadOptions scan_builder_load_options{table_path_}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::GLOBAL_INDEX_ENABLED] = "true"; + scan_builder.SetPredicate(value_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); int32_t indexed_split_count = 0; int32_t data_split_count = 0; @@ -704,14 +723,17 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF ASSERT_EQ(0, indexed_split_count); ASSERT_GT(data_split_count, 0); - ReadContextBuilder read_builder(table_path_); + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options["orc.read.enable-lazy-decoding"] = "false"; read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) - .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -763,13 +785,15 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, DeletesPayloadAfterLastDefinitionI PathUtil::JoinPath(table_path_, "schema/schema-1"), schema_json, /*overwrite=*/false)); - WriteContextBuilder builder(table_path_, kCommitUser); - builder.SetOptions(evolved_options) - .WithStreamingMode(true) - .WithTempDirectory(directory_->Str()); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + WriteContextBuilder builder(kCommitUser); + builder_options = evolved_options; + builder.WithStreamingMode(true).WithTempDirectory(directory_->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK(writer->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN(std::vector> cleanup_messages, writer->PrepareCommit(/*wait_compaction=*/true, diff --git a/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp b/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp index 1407c55b7..5fe88f8ba 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp @@ -48,18 +48,18 @@ Result> PkSortedDataFileReader::Create( const std::shared_ptr& executor, const std::shared_ptr& pool) { std::map read_options = options.ToMap(); read_options[Options::BRANCH] = branch; - ReadContextBuilder builder(root_path); - builder.SetReadFieldIds({field_id}) - .SetOptions(read_options) - .WithBranch(branch) - .WithFileSystem(options.GetFileSystem()) + ReadContextBuilder builder; + builder + .SetReadFieldIds({field_id}) + .WithExecutor(executor) .WithMemoryPool(pool) .EnablePrefetch(false) .EnablePredicateFilter(false); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shared_internal_context, - InternalReadContext::Create(read_context, table_schema, read_options)); + InternalReadContext::Create(read_context, table_schema, read_options, + options.GetFileSystem(), root_path)); return std::unique_ptr( new PkSortedDataFileReader(path_factory, shared_internal_context, pool, executor)); } diff --git a/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp index 9fc0744ef..8cf896665 100644 --- a/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp @@ -65,10 +65,8 @@ FullChangelogMergeTreeCompactRewriter::Create( std::shared_ptr write_schema = SpecialFields::CompleteSequenceAndValueKindField(data_schema); - ReadContextBuilder read_context_builder(path_factory_cache->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) - .EnablePrefetch(true) + ReadContextBuilder read_context_builder; + read_context_builder.EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); @@ -76,7 +74,8 @@ FullChangelogMergeTreeCompactRewriter::Create( read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + options.GetFileSystem(), path_factory_cache->RootPath())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, path_factory_cache->GetOrCreatePathFactory(options.GetFileFormat()->Identifier())); diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp index 390f95d84..88b355f72 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp @@ -73,10 +73,8 @@ LookupMergeTreeCompactRewriter::Create( auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(data_schema); // TODO(xinyu.lxy): set executor - ReadContextBuilder read_context_builder(path_factory_cache->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) - .EnablePrefetch(true) + ReadContextBuilder read_context_builder; + read_context_builder.EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); @@ -84,7 +82,8 @@ LookupMergeTreeCompactRewriter::Create( read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + options.GetFileSystem(), path_factory_cache->RootPath())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, path_factory_cache->GetOrCreatePathFactory(options.GetFileFormat()->Identifier())); diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp index 223abafed..04fe07b1c 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp @@ -31,6 +31,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/path_util.h" @@ -115,9 +116,11 @@ class MergeTreeCompactManagerFactoryWriteTest : public ::testing::Test { table_options, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("test"); if (!context_options.empty()) { - context_builder.SetOptions(context_options); + context_builder_options = context_options; } if (with_io_manager) { context_builder.WithTempDirectory(dir->Str()); @@ -125,7 +128,8 @@ class MergeTreeCompactManagerFactoryWriteTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, context_builder.Finish()); - return FileStoreWrite::Create(std::move(write_context)); + return FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options); } Result> CreateSingleStringFileStoreWrite( diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp index afe50ddfb..7c5e2e24a 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp @@ -73,10 +73,8 @@ Result> MergeTreeCompactRewriter::Crea auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(data_schema); // TODO(xinyu.lxy): set executor - ReadContextBuilder read_context_builder(path_factory_cache->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) - .EnablePrefetch(true) + ReadContextBuilder read_context_builder; + read_context_builder.EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); @@ -84,7 +82,8 @@ Result> MergeTreeCompactRewriter::Crea read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + options.GetFileSystem(), path_factory_cache->RootPath())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, path_factory_cache->GetOrCreatePathFactory(options.GetFileFormat()->Identifier())); diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp index 27424a1ae..73c634ddc 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp @@ -21,6 +21,7 @@ #include "arrow/api.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/table/special_fields.h" #include "paimon/core/mergetree/compact/interval_partition.h" @@ -57,10 +58,14 @@ class MergeTreeCompactRewriterTest : public testing::Test { Result>> GenerateSortedRuns( const std::string& table_path, const std::shared_ptr& table_schema, int32_t bucket, const std::map& partition) const { - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetBucketFilter(bucket).SetPartitionFilter({partition}); PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); auto splits = result_plan->Splits(); EXPECT_EQ(1, splits.size()); diff --git a/src/paimon/core/mergetree/lookup_levels.cpp b/src/paimon/core/mergetree/lookup_levels.cpp index 7d7aecedf..2618c2f7e 100644 --- a/src/paimon/core/mergetree/lookup_levels.cpp +++ b/src/paimon/core/mergetree/lookup_levels.cpp @@ -57,10 +57,8 @@ Result>> LookupLevels::Create( auto partition_schema = DataField::ConvertDataFieldsToArrowSchema(partition_fields); // TODO(xinyu.lxy): set executor - ReadContextBuilder read_context_builder(path_factory->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) - .EnablePrefetch(true) + ReadContextBuilder read_context_builder; + read_context_builder.EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); @@ -68,7 +66,8 @@ Result>> LookupLevels::Create( read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr internal_read_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + options.GetFileSystem(), path_factory->RootPath())); auto split_read = std::make_unique(path_factory, internal_read_context, pool, CreateDefaultExecutor()); diff --git a/src/paimon/core/operation/append_only_file_store_scan_test.cpp b/src/paimon/core/operation/append_only_file_store_scan_test.cpp index b8dfbfced..be7dd0406 100644 --- a/src/paimon/core/operation/append_only_file_store_scan_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_scan_test.cpp @@ -30,6 +30,7 @@ #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/cache/cache.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" #include "paimon/common/data/serializer/binary_row_serializer.h" @@ -598,13 +599,18 @@ TEST(AppendOnlyFileStoreScanTest, TestReconstructPredicateWithNonCastedFields) { TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) { TimezoneGuard guard("Asia/Shanghai"); std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); auto pool = GetDefaultPool(); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); auto typed_table_scan = dynamic_cast(table_scan.get()); ASSERT_TRUE(typed_table_scan); @@ -644,12 +650,17 @@ TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) { TEST(AppendOnlyFileStoreScanTest, TestScanDurationMetric) { TimezoneGuard guard("Asia/Shanghai"); std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); auto typed_table_scan = dynamic_cast(table_scan.get()); ASSERT_TRUE(typed_table_scan); @@ -745,12 +756,16 @@ std::shared_ptr BuildScan(const std::string& table_path, const std::shared_ptr& predicate = nullptr, bool manifest_entry_lazy_decode_enabled = true, int32_t max_snapshots = 8) { - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, - fmt::format("{}", max_snapshots)) - .AddOption(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED, - manifest_entry_lazy_decode_enabled ? "true" : "false") + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = + fmt::format("{}", max_snapshots); + context_builder_options[Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED] = + manifest_entry_lazy_decode_enabled ? "true" : "false"; + context_builder + .WithCache(cache); if (bucket) { context_builder.SetBucketFilter(bucket.value()); @@ -759,7 +774,9 @@ std::shared_ptr BuildScan(const std::string& table_path, context_builder.SetPredicate(predicate); } EXPECT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); - EXPECT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + EXPECT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); auto typed_table_scan = dynamic_cast(table_scan.get()); EXPECT_TRUE(typed_table_scan); return typed_table_scan->snapshot_reader_->scan_; diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index 666260267..e8a3cca84 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -370,10 +370,8 @@ Result> AppendOnlyFileStoreWrite::CreateFilesReader const BinaryRow& partition, int32_t bucket, DeletionVector::Factory dv_factory, const std::vector>& files, const std::optional& veto_reason) const { - ReadContextBuilder context_builder(root_path_); - context_builder.SetOptions(options_.ToMap()) - .WithFileSystem(options_.GetFileSystem()) - .EnablePrefetch(true) + ReadContextBuilder context_builder; + context_builder.EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool_); @@ -397,7 +395,8 @@ Result> AppendOnlyFileStoreWrite::CreateFilesReader options[kParquetReadEnableDictionaryPassthrough] = "false"; } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_read_context, - InternalReadContext::Create(read_context, table_schema_, options)); + InternalReadContext::Create(read_context, table_schema_, options, + options_.GetFileSystem(), root_path_)); auto read = std::make_unique(file_store_path_factory_, internal_read_context, pool_, compact_executor_); diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index bfeaefced..b8c28e2cc 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -36,6 +36,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" @@ -111,11 +112,15 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { const std::string& table_path, const std::shared_ptr& schema, const std::map& options, const std::string& json, int64_t commit_identifier) const { - WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options; + EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); EXPECT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(builder_load_options, std::move(write_context), + builder_options)); EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( /*wait_compaction=*/false, commit_identifier)); @@ -128,11 +133,15 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { const std::map& options, const std::vector& write_schema, const std::string& json, int64_t commit_identifier) const { - WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options).WithWriteSchema(write_schema); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options; + builder.WithWriteSchema(write_schema); EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); EXPECT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(builder_load_options, std::move(write_context), + builder_options)); EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( /*wait_compaction=*/false, commit_identifier)); @@ -142,11 +151,15 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { void Commit(const std::string& table_path, const std::map& options, const std::vector>& commit_msgs) const { - CommitContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + CommitContextBuilder builder(commit_user_); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(builder_load_options, + std::move(commit_context), builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); } @@ -225,10 +238,13 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestWriteWithInvalidBatch) { /*primary_keys=*/{}, /*options=*/{}, /*ignore_if_exists=*/false)); - WriteContextBuilder builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), commit_user_); + TableLoadOptions builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(builder_load_options, std::move(write_context), + builder_options)); ASSERT_NOK_WITH_MSG(file_store_write->PrepareCommitWithProgress(/*commit_identifier=*/0), "PrepareCommitWithProgress is only supported by a real-time writer"); ASSERT_NOK_WITH_MSG(file_store_write->Write(nullptr), "batch is null pointer"); @@ -245,11 +261,14 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestWriteWithInvalidBatch) { /*primary_keys=*/{}, /*options=*/{}, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - commit_user_); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder(commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); auto array = std::make_shared(); arrow::StringBuilder builder; for (size_t j = 0; j < 100; j++) { @@ -290,11 +309,16 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) { ASSERT_OK_AND_ASSIGN(std::shared_ptr batch_realtime_context, RealtimeContext::Create()); - WriteContextBuilder batch_builder(table_path, commit_user_); - batch_builder.SetOptions(options).WithRealtimeContext(batch_realtime_context); + TableLoadOptions batch_builder_load_options{table_path}; + std::map batch_builder_options; + WriteContextBuilder batch_builder(commit_user_); + batch_builder_options = options; + batch_builder.WithRealtimeContext(batch_realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_context, batch_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_writer, - FileStoreWrite::Create(std::move(batch_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_writer, + FileStoreWrite::Create(batch_builder_load_options, std::move(batch_context), + batch_builder_options)); ASSERT_NOK_WITH_MSG(batch_writer->PrepareCommitWithProgress(/*commit_identifier=*/0), "PrepareCommitWithProgress requires streaming mode"); ASSERT_OK(batch_writer->Close()); @@ -302,10 +326,15 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); ASSERT_OK(file_store_write->Write(MakeBatch(realtime_schema, R"([ [10, 1, "a"], @@ -378,14 +407,18 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) } TEST_F(AppendOnlyFileStoreWriteTest, TestGetMaxSequenceNumberFromMultiPartition) { - WriteContextBuilder builder( + TableLoadOptions builder_load_options; + builder_load_options.path = paimon::test::GetDataDir() + - "/orc/multi_partition_append_table.db/multi_partition_append_table/", - commit_user_); + "/orc/multi_partition_append_table.db/multi_partition_append_table/"; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options["file.format"] = "orc"; + builder_options["manifest.format"] = "orc"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr write_context, - builder.AddOption("file.format", "orc").AddOption("manifest.format", "orc").Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + auto file_store_write, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); auto write = dynamic_cast(file_store_write.get()); auto pool = GetDefaultPool(); { @@ -445,10 +478,15 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMapAdaptsAcrossRollingFi std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([ [1, [["a", 1], ["b", 2]]] ])"))); @@ -553,10 +591,15 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMultipleMapColumnsAdaptA CreateTable(dir->Str(), logical_schema, options); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([ [1, [["a", 1], ["b", 2]], [["c", 3], ["d", 4], ["e", 5], ["f", 6]]] ])"))); diff --git a/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp b/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp index da115ca2b..a81c2ac9a 100644 --- a/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp +++ b/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp @@ -242,15 +242,13 @@ Result> RealtimeDeduplicateCompactionChangesProvi } PAIMON_ASSIGN_OR_RAISE(result->files_to_delete_after_commit, CollectIntermediateFilesToDelete(snapshot_buckets)); - ReadContextBuilder builder(path_factory_->RootPath()); + ReadContextBuilder builder; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, - builder.SetOptions(options_.ToMap()) - .WithFileSystem(options_.GetFileSystem()) - .WithMemoryPool(pool_) - .Finish()); + builder.WithMemoryPool(pool_).Finish()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema_, options_.ToMap())); + InternalReadContext::Create(read_context, table_schema_, options_.ToMap(), + options_.GetFileSystem(), path_factory_->RootPath())); RawFileSplitRead read(path_factory_, context, pool_, executor_); const auto schema = DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields()); const auto offset_schema = diff --git a/src/paimon/core/operation/commit_context.cpp b/src/paimon/core/operation/commit_context.cpp index e61ec950f..ce26f221c 100644 --- a/src/paimon/core/operation/commit_context.cpp +++ b/src/paimon/core/operation/commit_context.cpp @@ -19,61 +19,26 @@ #include "paimon/commit_context.h" -#include #include #include -#include "fmt/format.h" -#include "paimon/catalog/catalog.h" -#include "paimon/common/utils/path_util.h" -#include "paimon/core/catalog/catalog_utils.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { -CommitContext::CommitContext(const std::string& root_path, const std::string& commit_user, - bool ignore_empty_commit, bool use_rest_catalog_commit, - bool append_commit_check_conflict, +CommitContext::CommitContext(const std::string& commit_user, bool ignore_empty_commit, + bool use_rest_catalog_commit, bool append_commit_check_conflict, const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table) - : CommitContext(root_path, commit_user, ignore_empty_commit, use_rest_catalog_commit, - /*catalog=*/nullptr, /*identifier=*/std::nullopt, /*table_id=*/std::nullopt, - append_commit_check_conflict, memory_pool, executor, specific_file_system, - options, format_table) {} - -CommitContext::CommitContext(const std::string& root_path, const std::string& commit_user, - bool ignore_empty_commit, bool use_rest_catalog_commit, - const std::shared_ptr& catalog, - const std::optional& identifier, - const std::optional& table_id, - bool append_commit_check_conflict, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table) - : root_path_(root_path), - commit_user_(commit_user), + const std::shared_ptr& executor) + : commit_user_(commit_user), ignore_empty_commit_(ignore_empty_commit), use_rest_catalog_commit_(use_rest_catalog_commit), - catalog_(catalog), - identifier_(identifier), - table_id_(table_id), append_commit_check_conflict_(append_commit_check_conflict), memory_pool_(memory_pool), - executor_(executor), - specific_file_system_(specific_file_system), - options_(options), - format_table_(format_table) {} + executor_(executor) {} CommitContext::~CommitContext() = default; @@ -84,65 +49,27 @@ class CommitContextBuilder::Impl { void Reset() { ignore_empty_commit_ = true; use_rest_catalog_commit_ = false; - catalog_.reset(); - identifier_.reset(); - table_id_.reset(); append_commit_check_conflict_ = false; memory_pool_ = GetDefaultPool(); executor_ = CreateDefaultExecutor(); - specific_file_system_.reset(); - options_.clear(); } private: - std::string root_path_; - /// Kept across `Reset()`, as `root_path_` is: both name the table this builder builds for, - /// rather than a setting of one commit to it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; std::string commit_user_; bool ignore_empty_commit_ = true; bool use_rest_catalog_commit_ = false; - std::shared_ptr catalog_; - std::optional identifier_; - std::optional table_id_; bool append_commit_check_conflict_ = false; std::shared_ptr memory_pool_ = GetDefaultPool(); std::shared_ptr executor_ = CreateDefaultExecutor(); - std::shared_ptr specific_file_system_; - std::map options_; }; -CommitContextBuilder::CommitContextBuilder(const std::string& root_path, - const std::string& commit_user) +CommitContextBuilder::CommitContextBuilder(const std::string& commit_user) : impl_(std::make_unique()) { - impl_->root_path_ = root_path; impl_->commit_user_ = commit_user; } -CommitContextBuilder::CommitContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->root_path_ = table->Location(); - } -} - CommitContextBuilder::~CommitContextBuilder() = default; -CommitContextBuilder& CommitContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - -CommitContextBuilder& CommitContextBuilder::SetOptions( - const std::map& opts) { - impl_->options_ = opts; - return *this; -} - CommitContextBuilder& CommitContextBuilder::IgnoreEmptyCommit(bool ignore_empty_commit) { impl_->ignore_empty_commit_ = ignore_empty_commit; return *this; @@ -153,18 +80,6 @@ CommitContextBuilder& CommitContextBuilder::UseRESTCatalogCommit(bool use_rest_c return *this; } -CommitContextBuilder& CommitContextBuilder::WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier) { - impl_->catalog_ = catalog; - impl_->identifier_.emplace(identifier); - return *this; -} - -CommitContextBuilder& CommitContextBuilder::WithTableId(const std::string& table_id) { - impl_->table_id_ = table_id; - return *this; -} - CommitContextBuilder& CommitContextBuilder::AppendCommitCheckConflict( bool append_commit_check_conflict) { impl_->append_commit_check_conflict_ = append_commit_check_conflict; @@ -183,69 +98,10 @@ CommitContextBuilder& CommitContextBuilder::WithExecutor( return *this; } -CommitContextBuilder& CommitContextBuilder::WithFileSystem( - const std::shared_ptr& file_system) { - impl_->specific_file_system_ = file_system; - return *this; -} - Result> CommitContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot commit with null format table"); - } - if (impl_->catalog_ == nullptr && impl_->identifier_) { - return Status::Invalid("cannot commit through a null catalog"); - } - if (impl_->identifier_) { - // Before the branch is read out of the identifier: the branch of `tbl$branch_dev$options` - // reads back as `dev`, so a commit built for such a name would be aimed at another branch. - PAIMON_RETURN_NOT_OK( - CatalogUtils::CheckNotSystemTable(impl_->identifier_.value(), "commit")); - } - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - impl_->identifier_, impl_->options_, - /*explicit_branch=*/std::nullopt, "commit")); - if (!BranchManager::IsMainBranch(branch)) { - // A commit which goes to a catalog, or is handed back as a request for one, names its - // branch by an object name, so the branch has to be one that reads back as itself. - // Refused here rather than at the commit, which writes metadata before publishing it. - if (impl_->catalog_ != nullptr || impl_->use_rest_catalog_commit_) { - PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); - } - if (impl_->catalog_ != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::optional identifier_branch, - impl_->identifier_.value().GetBranchName()); - if (!identifier_branch) { - PAIMON_ASSIGN_OR_RAISE(std::string table_name, - impl_->identifier_.value().GetDataTableName()); - return Status::Invalid(fmt::format( - "a commit through a catalog addresses a branch by the table identifier, so " - "name branch '{}' there as '{}$branch_{}' rather than only in the '{}' option", - branch, table_name, branch, Options::BRANCH)); - } - } - } - if (impl_->catalog_ != nullptr && impl_->use_rest_catalog_commit_) { - return Status::Invalid( - "a commit either goes to the catalog of WithCatalog() or is handed back as a request " - "for the caller to send, which is what UseRESTCatalogCommit() asks for"); - } - // The table already carries the file system it was loaded through, and from a source this - // cannot see behind, so a second answer is refused rather than silently dropped. - if (impl_->format_table_ != nullptr && impl_->specific_file_system_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "cannot be used with one"); - } - PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_)); - if (impl_->root_path_.empty()) { - return Status::Invalid("root path is empty"); - } auto ctx = std::make_unique( - impl_->root_path_, impl_->commit_user_, impl_->ignore_empty_commit_, - impl_->use_rest_catalog_commit_, impl_->catalog_, impl_->identifier_, impl_->table_id_, - impl_->append_commit_check_conflict_, impl_->memory_pool_, impl_->executor_, - impl_->specific_file_system_, impl_->options_, impl_->format_table_); + impl_->commit_user_, impl_->ignore_empty_commit_, impl_->use_rest_catalog_commit_, + impl_->append_commit_check_conflict_, impl_->memory_pool_, impl_->executor_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/commit_context_test.cpp b/src/paimon/core/operation/commit_context_test.cpp index 51fcab399..6eb8752b3 100644 --- a/src/paimon/core/operation/commit_context_test.cpp +++ b/src/paimon/core/operation/commit_context_test.cpp @@ -24,6 +24,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/utils/path_util.h" #include "paimon/defs.h" #include "paimon/executor.h" @@ -34,49 +35,22 @@ namespace paimon::test { -TEST(CommitContextTest, TestLegacyConstructor) { - auto pool = GetDefaultPool(); - std::shared_ptr executor = CreateDefaultExecutor(); - auto fs = std::make_shared(); - const std::map options = {{"key", "value"}}; - CommitContext context("table_root_path", "commit_user_1", false, true, true, pool, executor, fs, - options, nullptr); - ASSERT_EQ(context.GetRootPath(), "table_root_path"); - ASSERT_EQ(context.GetCommitUser(), "commit_user_1"); - ASSERT_FALSE(context.IgnoreEmptyCommit()); - ASSERT_TRUE(context.UseRESTCatalogCommit()); - ASSERT_TRUE(context.AppendCommitCheckConflict()); - ASSERT_EQ(context.GetMemoryPool(), pool); - ASSERT_EQ(context.GetExecutor(), executor); - ASSERT_EQ(context.GetSpecificFileSystem(), fs); - ASSERT_EQ(context.GetOptions(), options); - ASSERT_EQ(context.GetCatalog(), nullptr); - ASSERT_FALSE(context.GetIdentifier()); - ASSERT_FALSE(context.GetTableId()); -} - TEST(CommitContextTest, TestDefaultValue) { - CommitContextBuilder builder("table_root_path", "commit_user_1"); + CommitContextBuilder builder("commit_user_1"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto expected_root_path, PathUtil::NormalizePath("table_root_path")); - ASSERT_EQ(ctx->GetRootPath(), expected_root_path); ASSERT_EQ(ctx->GetCommitUser(), "commit_user_1"); ASSERT_TRUE(ctx->IgnoreEmptyCommit()); ASSERT_FALSE(ctx->UseRESTCatalogCommit()); - ASSERT_EQ(ctx->GetCatalog(), nullptr); - ASSERT_FALSE(ctx->GetIdentifier().has_value()); - ASSERT_EQ(ctx->GetTableId(), std::nullopt); ASSERT_FALSE(ctx->AppendCommitCheckConflict()); ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); - ASSERT_FALSE(ctx->GetSpecificFileSystem()); - ASSERT_TRUE(ctx->GetOptions().empty()); } TEST(CommitContextTest, TestSetContent) { - CommitContextBuilder builder("table_root_path", "commit_user_1"); + CommitContextBuilder builder("commit_user_1"); auto memory_pool = GetDefaultPool(); std::shared_ptr executor = CreateDefaultExecutor(); @@ -84,150 +58,18 @@ TEST(CommitContextTest, TestSetContent) { ASSERT_OK_AND_ASSIGN(auto ctx, builder.IgnoreEmptyCommit(false) .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") .AppendCommitCheckConflict(true) .WithMemoryPool(memory_pool) .WithExecutor(executor) - .WithFileSystem(fs) - .AddOption("key", "value") .Finish()); ASSERT_OK_AND_ASSIGN(auto expected_root_path, PathUtil::NormalizePath("table_root_path")); - ASSERT_EQ(ctx->GetRootPath(), expected_root_path); ASSERT_EQ(ctx->GetCommitUser(), "commit_user_1"); ASSERT_FALSE(ctx->IgnoreEmptyCommit()); ASSERT_TRUE(ctx->UseRESTCatalogCommit()); - ASSERT_EQ(ctx->GetTableId(), std::optional("table-uuid")); ASSERT_TRUE(ctx->AppendCommitCheckConflict()); ASSERT_EQ(ctx->GetMemoryPool(), memory_pool); ASSERT_EQ(ctx->GetExecutor(), executor); - ASSERT_EQ(ctx->GetSpecificFileSystem(), fs); - - std::map expected_options = {{"key", "value"}}; - ASSERT_EQ(ctx->GetOptions(), expected_options); -} - -TEST(CommitContextTest, TestWithCatalog) { - auto catalog = std::make_shared(); - CommitContextBuilder builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto ctx, - builder.WithCatalog(catalog, Identifier("db1", "t1")).WithTableId("table-uuid").Finish()); - ASSERT_EQ(ctx->GetCatalog(), catalog); - ASSERT_TRUE(ctx->GetIdentifier().has_value()); - ASSERT_EQ(ctx->GetIdentifier().value(), Identifier("db1", "t1")); - ASSERT_EQ(ctx->GetTableId(), std::optional("table-uuid")); - - CommitContextBuilder null_catalog_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(null_catalog_builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), - "cannot commit through a null catalog"); - - CommitContextBuilder both_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(both_builder.WithCatalog(catalog, Identifier("db1", "t1")) - .UseRESTCatalogCommit(true) - .Finish(), - "either goes to the catalog"); - - ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); - ASSERT_EQ(next_ctx->GetCatalog(), nullptr); - ASSERT_FALSE(next_ctx->GetIdentifier().has_value()); - ASSERT_EQ(next_ctx->GetTableId(), std::nullopt); -} - -TEST(CommitContextTest, TestCatalogAddressesBranchByIdentifier) { - auto catalog = std::make_shared(); - - CommitContextBuilder branch_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto branch_ctx, - branch_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")).Finish()); - ASSERT_EQ(branch_ctx->GetIdentifier().value(), Identifier("db1", "t1$branch_dev")); - CommitContextBuilder agreeing_builder("table_root_path", "commit_user_1"); - ASSERT_OK(agreeing_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .AddOption(Options::BRANCH, "dev") - .Finish()); - CommitContextBuilder constructed_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto constructed_ctx, - constructed_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).Finish()); - ASSERT_EQ(constructed_ctx->GetIdentifier().value(), Identifier("db1", "t1$branch_dev")); - CommitContextBuilder main_builder("table_root_path", "commit_user_1"); - ASSERT_OK(main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) - .AddOption(Options::BRANCH, "main") - .Finish()); - - CommitContextBuilder mixed_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(mixed_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .AddOption(Options::BRANCH, "main") - .Finish(), - "but both 'dev' and 'main' were named"); - CommitContextBuilder mixed_option_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - mixed_option_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) - .AddOption(Options::BRANCH, "dev") - .Finish(), - "but both 'main' and 'dev' were named"); - - CommitContextBuilder option_only_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(option_only_builder.WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "dev") - .Finish(), - "name branch 'dev' there as 't1$branch_dev'"); - - CommitContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - upper_main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_MAIN")).Finish(), - "a catalog names branch 'MAIN' as it names the main branch"); - - CommitContextBuilder splitter_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(splitter_builder.WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "dev$options") - .Finish(), - "a branch a catalog addresses cannot contain '$'"); - - CommitContextBuilder system_table_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - system_table_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev$options")).Finish(), - "Cannot 'commit' for system table"); -} - -TEST(CommitContextTest, TestBranch) { - CommitContextBuilder file_system_builder("table_root_path", "commit_user_1"); - ASSERT_OK(file_system_builder.AddOption(Options::BRANCH, "dev").Finish()); - - CommitContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto upper_main_ctx, - upper_main_builder.AddOption(Options::BRANCH, "MAIN").Finish()); - ASSERT_EQ(upper_main_ctx->GetOptions().at(Options::BRANCH), "MAIN"); - CommitContextBuilder splitter_builder("table_root_path", "commit_user_1"); - ASSERT_OK(splitter_builder.AddOption(Options::BRANCH, "dev$options").Finish()); - - CommitContextBuilder rest_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto rest_ctx, - rest_builder.UseRESTCatalogCommit(true).AddOption(Options::BRANCH, "dev").Finish()); - ASSERT_TRUE(rest_ctx->UseRESTCatalogCommit()); - ASSERT_EQ(rest_ctx->GetOptions().at(Options::BRANCH), "dev"); - CommitContextBuilder rest_upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(rest_upper_main_builder.UseRESTCatalogCommit(true) - .AddOption(Options::BRANCH, "MAIN") - .Finish(), - "a catalog names branch 'MAIN' as it names the main branch"); - - CommitContextBuilder escaping_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(escaping_builder.AddOption(Options::BRANCH, "dev/../../outside").Finish(), - "branch name cannot contain path separators"); -} - -TEST(CommitContextTest, TestSetOptionsOverridesAddedOptions) { - CommitContextBuilder builder("table_root_path", "commit_user_1"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(ctx->GetOptions(), expected_options); } } // namespace paimon::test diff --git a/src/paimon/core/operation/data_evolution_split_read_test.cpp b/src/paimon/core/operation/data_evolution_split_read_test.cpp index 4cdfce10b..fb71a4b7f 100644 --- a/src/paimon/core/operation/data_evolution_split_read_test.cpp +++ b/src/paimon/core/operation/data_evolution_split_read_test.cpp @@ -29,6 +29,7 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/file_index/bitmap/bitmap_file_index.h" @@ -411,10 +412,16 @@ TEST_F(DataEvolutionSplitReadTest, TestBlobFallbackValidatesRangesBeforeOpeningF {Options::DATA_EVOLUTION_ENABLED, "true"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, TableSchema::Create(0, schema, {}, {}, options)); - ReadContextBuilder builder(dir->Str()); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); DataEvolutionSplitRead split_read(/*path_factory=*/nullptr, context, GetDefaultPool(), /*executor=*/nullptr); auto blob = @@ -449,10 +456,16 @@ TEST_F(DataEvolutionSplitReadTest, TestEmptyBlobReaders) { {Options::DATA_EVOLUTION_ENABLED, "true"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, TableSchema::Create(0, schema, {}, {}, options)); - ReadContextBuilder builder(dir->Str()); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); ASSERT_OK_AND_ASSIGN( std::shared_ptr path_factory, FileStorePathFactory::Create(dir->Str(), schema, {}, "default", "parquet", "data-", @@ -1106,11 +1119,18 @@ TEST_F(DataEvolutionSplitReadTest, TestSequentialVectorReadReusesMainReader) { std::vector{"_ROW_ID", "_SEQUENCE_NUMBER"}}) { auto counting_fs = std::make_shared( PathUtil::JoinPath(bucket_path, main->file_name)); - ReadContextBuilder builder(dir->Str()); - builder.WithFileSystem(counting_fs).EnablePrefetch(false).SetReadFieldNames(projection); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ReadContextBuilder builder; + builder_load_options.file_system = counting_fs; + builder.EnablePrefetch(false).SetReadFieldNames(projection); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); auto split_read = std::make_unique(path_factory, context, GetDefaultPool(), executor); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, split_read->CreateReader(split)); @@ -1131,12 +1151,19 @@ TEST_F(DataEvolutionSplitReadTest, TestSequentialVectorReadReusesMainReader) { // A selection confined to the first VECTOR file must not include rows from the second. { - ReadContextBuilder builder(dir->Str()); - builder.WithFileSystem(fs).EnablePrefetch(false).SetReadFieldNames( + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ReadContextBuilder builder; + builder_load_options.file_system = fs; + builder.EnablePrefetch(false).SetReadFieldNames( {"id", "embedding", "_ROW_ID", "_SEQUENCE_NUMBER"}); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); DataEvolutionSplitRead split_read(path_factory, context, GetDefaultPool(), executor); auto selected_split = std::make_shared(split, std::vector{Range(0, 1)}); @@ -1171,12 +1198,19 @@ TEST_F(DataEvolutionSplitReadTest, TestSequentialVectorReadReusesMainReader) { {vector1, legacy_main, vector0}); ASSERT_OK_AND_ASSIGN(std::shared_ptr legacy_split, legacy_split_builder.RawConvertible(false).Build()); - ReadContextBuilder builder(dir->Str()); - builder.WithFileSystem(fs).EnablePrefetch(false).SetReadFieldNames( + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ReadContextBuilder builder; + builder_load_options.file_system = fs; + builder.EnablePrefetch(false).SetReadFieldNames( {"id", "embedding", "_ROW_ID", "_SEQUENCE_NUMBER"}); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, evolved_schema, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr context, + InternalReadContext::Create( + read_context, evolved_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); DataEvolutionSplitRead split_read(path_factory, context, GetDefaultPool(), executor); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, split_read.CreateReader(legacy_split)); @@ -1392,10 +1426,16 @@ TEST_F(DataEvolutionSplitReadTest, TestFileIndexSelectionOnlyPushesDownNormalGro {Options::FILE_INDEX_READ_ENABLED, "true"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, TableSchema::Create(0, schema, {}, {}, options)); - ReadContextBuilder builder(dir->Str()); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); DataEvolutionSplitRead split_read(/*path_factory=*/nullptr, context, pool, /*executor=*/nullptr); @@ -1566,10 +1606,16 @@ TEST_F(DataEvolutionSplitReadTest, TestGroupBitmapCombinesWithRowRangesAndDvInFi {Options::DATA_EVOLUTION_ENABLED, "true"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, TableSchema::Create(0, schema, {}, {}, options)); - ReadContextBuilder builder(dir->Str()); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); DataEvolutionSplitRead split_read(/*path_factory=*/nullptr, context, GetDefaultPool(), /*executor=*/nullptr); split_read.file_indexes_evaluated_ = true; diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index be1482052..211a7fea9 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -37,6 +37,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" #include "paimon/common/data/binary_row.h" @@ -280,8 +281,8 @@ class FileStoreCommitImplTest : public testing::Test { } std::shared_ptr CreateCatalogServingSchemaId( - int64_t schema_id) const { - std::shared_ptr catalog = CreateCatalog(); + int64_t schema_id, const std::string& table_uuid = "mock-table-uuid") const { + std::shared_ptr catalog = CreateCatalog(true, table_uuid); catalog->SetTableSchema(TableSchemaWithId(schema_id)); return catalog; } @@ -324,11 +325,14 @@ class FileStoreCommitImplTest : public testing::Test { } void CommitAsCompetitor(int32_t round, int64_t commit_identifier) { - CommitContextBuilder builder(table_path_, "competitor"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr competitor, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("competitor"); + builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr competitor, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK(competitor->Commit(CommitMessagesOfRound(round), commit_identifier)); } @@ -344,19 +348,22 @@ class FileStoreCommitImplTest : public testing::Test { Result> CreateCatalogCommit( const CatalogCommitSpec& spec) const { - CommitContextBuilder builder(spec.root_path.empty() ? table_path_ : spec.root_path, - spec.commit_user); - builder.AddOption(Options::FILE_SYSTEM, "local") - .WithCatalog(spec.catalog, spec.identifier.value()) - .AppendCommitCheckConflict(spec.append_commit_check_conflict); + TableLoadOptions builder_load_options{spec.root_path.empty() ? table_path_ + : spec.root_path}; + std::map builder_options; + CommitContextBuilder builder(spec.commit_user); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_load_options.catalog = spec.catalog; + builder_load_options.identifier.emplace(spec.identifier.value()); + builder.AppendCommitCheckConflict(spec.append_commit_check_conflict); if (spec.table_id) { - builder.WithTableId(spec.table_id.value()); + builder_load_options.table_id = spec.table_id.value(); } for (const auto& [key, value] : spec.options) { - builder.AddOption(key, value); + builder_options[key] = value; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreCommit::Create(std::move(context)); + return FileStoreCommit::Create(builder_load_options, std::move(context), builder_options); } Result> CreateCatalogCommit( @@ -413,16 +420,20 @@ class FileStoreCommitImplTest : public testing::Test { void CheckExpireIsRefusedWithoutDeleting(const std::string& branch, const std::string& expected_msg) { - CommitContextBuilder builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, branch) - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::BRANCH] = branch; + builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; + builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder + + .Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK(PrepareFirstCommitMessageFiles()); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); ASSERT_OK(commit->DropPartition({{{"f1", "10"}}}, /*commit_identifier=*/2)); @@ -611,13 +622,16 @@ class FileStoreCommitImplTest : public testing::Test { }; TEST_F(FileStoreCommitImplTest, TestCommit) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -639,15 +653,18 @@ TEST_F(FileStoreCommitImplTest, TestCommit) { TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { TimezoneGuard guard("Asia/Shanghai"); - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_load_options.table_id = "table-uuid"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); + context_builder.UseRESTCatalogCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = CommitMessagesOfRound(1); ASSERT_NOK_WITH_MSG(commit->GetLastCommitTableRequest(), @@ -697,10 +714,14 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { TEST_F(FileStoreCommitImplTest, TestCommitCarriesBaseSnapshotUuid) { std::vector> msgs = CommitMessagesOfRound(1); - CommitContextBuilder base_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + TableLoadOptions base_builder_load_options{table_path_}; + std::map base_builder_options; + CommitContextBuilder base_builder("commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(base_builder_load_options, std::move(base_context), + base_builder_options)); ASSERT_OK(base_commit->Commit(msgs, 1)); std::string snapshot_path = PathUtil::JoinPath(table_path_, "snapshot/snapshot-1"); @@ -708,13 +729,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitCarriesBaseSnapshotUuid) { ASSERT_TRUE(base_snapshot.Uuid().has_value()); const std::string base_uuid = base_snapshot.Uuid().value(); - CommitContextBuilder rest_builder(table_path_, "commit_user_1"); + TableLoadOptions rest_builder_load_options{table_path_}; + std::map rest_builder_options; + CommitContextBuilder rest_builder("commit_user_1"); + rest_builder_options[Options::FILE_SYSTEM] = "local"; + rest_builder_load_options.table_id = "table-uuid"; ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_context, - rest_builder.AddOption(Options::FILE_SYSTEM, "local") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto rest_commit, FileStoreCommit::Create(std::move(rest_context))); + rest_builder.UseRESTCatalogCommit(true).Finish()); + ASSERT_OK_AND_ASSIGN(auto rest_commit, + FileStoreCommit::Create(rest_builder_load_options, std::move(rest_context), + rest_builder_options)); ASSERT_OK(rest_commit->Commit(msgs, 2)); ASSERT_OK_AND_ASSIGN(std::string request_str, rest_commit->GetLastCommitTableRequest()); @@ -739,7 +763,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitCarriesBaseSnapshotUuid) { } TEST_F(FileStoreCommitImplTest, TestCatalogCommitTakesTheSnapshot) { - auto catalog = CreateCatalog(); + auto catalog = CreateCatalog(true, "table-uuid-1"); CatalogCommitSpec spec; spec.catalog = catalog; spec.table_id = "table-uuid-1"; @@ -754,7 +778,7 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitTakesTheSnapshot) { ASSERT_EQ(call.table_uuid, std::optional("table-uuid-1")); ASSERT_EQ(call.base_snapshot_uuid, std::nullopt); ASSERT_EQ(call.snapshot.Id(), 1); - ASSERT_EQ(catalog->GetTableCalls(), 0u); + ASSERT_EQ(catalog->GetTableCalls(), 1u); ASSERT_GT(catalog->LoadTableSchemaCalls(), 0u); ASSERT_OK_AND_ASSIGN(std::string request_str, commit->GetLastCommitTableRequest()); @@ -774,25 +798,30 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitTakesTheSnapshot) { } TEST_F(FileStoreCommitImplTest, TestCommitToBranch) { - CommitContextBuilder missing_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr missing_context, - missing_builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(missing_context)), - "not found latest schema in branch dev"); + TableLoadOptions missing_builder_load_options{table_path_}; + std::map missing_builder_options; + CommitContextBuilder missing_builder("commit_user_1"); + missing_builder_options[Options::FILE_SYSTEM] = "local"; + missing_builder_options[Options::BRANCH] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr missing_context, missing_builder.Finish()); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(missing_builder_load_options, std::move(missing_context), + missing_builder_options), + "not found latest schema in branch dev"); SchemaManager branch_schema_manager(file_system_, table_path_, "dev"); ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); - CommitContextBuilder builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::BRANCH] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); std::string branch_snapshots = PathUtil::JoinPath(table_path_, "branch/branch-dev/snapshot"); @@ -818,15 +847,17 @@ TEST_F(FileStoreCommitImplTest, TestRequestOnlyCommitToBranch) { ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); - CommitContextBuilder builder(table_path_, "commit_user_1"); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::BRANCH] = "dev"; + builder_load_options.table_id = "table-uuid"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + builder.UseRESTCatalogCommit(true).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); ASSERT_OK_AND_ASSIGN(std::string request_str, commit->GetLastCommitTableRequest()); @@ -937,13 +968,15 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitToBranchTakesTheBranchSnapshot) ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); - CommitContextBuilder base_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .Finish()); + TableLoadOptions base_builder_load_options{table_path_}; + std::map base_builder_options; + CommitContextBuilder base_builder("commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; + base_builder_options[Options::BRANCH] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, base_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr base_commit, - FileStoreCommit::Create(std::move(base_context))); + FileStoreCommit::Create(base_builder_load_options, std::move(base_context), + base_builder_options)); ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); SnapshotManager branch_snapshot_manager(file_system_, table_path_, "dev"); ASSERT_OK_AND_ASSIGN(std::optional base_snapshot, @@ -952,7 +985,7 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitToBranchTakesTheBranchSnapshot) ASSERT_EQ(base_snapshot.value().Id(), 1); ASSERT_TRUE(base_snapshot.value().Uuid().has_value()); - auto catalog = CreateCatalog(); + auto catalog = CreateCatalog(true, "table-uuid-1"); catalog->SetHeldSnapshot(base_snapshot.value()); std::shared_ptr log = InstallCommitLock(catalog.get()); CatalogCommitSpec spec; @@ -989,7 +1022,8 @@ TEST_F(FileStoreCommitImplTest, TestBranchCommitReadsTheSchemaFromTheBranch) { ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); - std::shared_ptr catalog = CreateCatalogServingSchemaId(7); + std::shared_ptr catalog = + CreateCatalogServingSchemaId(7, "table-uuid-1"); CatalogCommitSpec spec; spec.catalog = catalog; spec.identifier.emplace("db", "tbl$branch_dev"); @@ -1002,7 +1036,7 @@ TEST_F(FileStoreCommitImplTest, TestBranchCommitReadsTheSchemaFromTheBranch) { ASSERT_EQ(catalog->CommitCalls().front().snapshot.SchemaId(), 0); ASSERT_EQ(catalog->LoadTableSchemaCalls(), 0u); - catalog->SetTableSchema(nullptr); + catalog->SetTableSchema(TableSchemaWithId(99)); ASSERT_OK_AND_ASSIGN(std::unique_ptr next_commit, CreateCatalogCommit(spec)); ASSERT_OK(next_commit->Commit(CommitMessagesOfRound(2), /*commit_identifier=*/2)); @@ -1016,7 +1050,7 @@ TEST_F(FileStoreCommitImplTest, TestBranchCommitNamesTheBranchWhenItFails) { ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); - auto catalog = CreateCatalog(Status::IOError("catalog unreachable")); + auto catalog = CreateCatalog(Status::IOError("catalog unreachable"), "table-uuid"); CatalogCommitSpec spec; spec.catalog = catalog; spec.identifier.emplace("db", "tbl$branch_dev"); @@ -1029,7 +1063,7 @@ TEST_F(FileStoreCommitImplTest, TestBranchCommitNamesTheBranchWhenItFails) { } TEST_F(FileStoreCommitImplTest, TestCatalogCommitCanonicalizesTheMainBranchIdentifier) { - auto catalog = CreateCatalog(); + auto catalog = CreateCatalog(true, "table-uuid-1"); CatalogCommitSpec spec; spec.catalog = catalog; spec.identifier.emplace("db", "tbl$branch_main"); @@ -1050,7 +1084,7 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitCanonicalizesTheMainBranchIdent } TEST_F(FileStoreCommitImplTest, TestCommitIsRefusedForATableRecreatedUnderTheSameName) { - auto catalog = CreateCatalog(true, "table-uuid-of-the-new-table"); + auto catalog = CreateCatalog(true, "table-uuid-of-the-dropped-table"); catalog->CheckTableUuid(); CatalogCommitSpec spec; spec.catalog = catalog; @@ -1058,6 +1092,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitIsRefusedForATableRecreatedUnderTheSam spec.options = {{Options::COMMIT_MAX_RETRIES, "2"}, {Options::COMMIT_MIN_RETRY_WAIT, "1ms"}}; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, CreateCatalogCommit(spec)); + catalog->SetTableUuid("table-uuid-of-the-new-table"); std::vector> msgs = CommitMessagesOfRound(1); ASSERT_NOK_WITH_MSG(commit->Commit(msgs), "no table with id table-uuid-of-the-dropped-table"); ASSERT_EQ(catalog->CommitCalls().size(), 1u); @@ -1069,7 +1104,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitIsRefusedForATableRecreatedUnderTheSam ASSERT_OK(unchecked_commit->Commit(msgs, 2)); ASSERT_EQ(catalog->CommitCalls().size(), 2u); ASSERT_EQ(catalog->CommitCalls().back().table_uuid, std::nullopt); - ASSERT_EQ(catalog->GetTableCalls(), 0u); + ASSERT_EQ(catalog->GetTableCalls(), 1u); ASSERT_OK_AND_ASSIGN(std::string request_str, unchecked_commit->GetLastCommitTableRequest()); ASSERT_OK_AND_ASSIGN(CommitTableRequest request, CommitTableRequest::FromJsonString(request_str)); @@ -1112,10 +1147,14 @@ TEST_F(FileStoreCommitImplTest, TestCatalogSchemaWinsOverTheOneUnderTheTablePath ASSERT_EQ(catalog->CommitCalls().size(), 1u); ASSERT_EQ(catalog->CommitCalls().front().snapshot.SchemaId(), 7); - CommitContextBuilder path_builder(table_path_, "commit_user_2"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr path_context, - path_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto path_commit, FileStoreCommit::Create(std::move(path_context))); + TableLoadOptions path_builder_load_options{table_path_}; + std::map path_builder_options; + CommitContextBuilder path_builder("commit_user_2"); + path_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr path_context, path_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto path_commit, + FileStoreCommit::Create(path_builder_load_options, std::move(path_context), + path_builder_options)); std::vector> other_msgs = CommitMessagesOfRound(2); ASSERT_OK(path_commit->Commit(other_msgs, 2)); ASSERT_OK_AND_ASSIGN( @@ -1182,15 +1221,17 @@ TEST_F(FileStoreCommitImplTest, TestCatalogSchemaWhichIsNotADataTableSchemaIsRef } TEST_F(FileStoreCommitImplTest, TestTableIdWithNothingToSendItInIsRefused) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local") - .WithTableId("table-uuid") - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), - "publishes the snapshot by writing it to the table directory"); - - auto catalog = CreateCatalog(); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_load_options.table_id = "table-uuid"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options), + "a table id requires a catalog or an external REST commit request"); + + auto catalog = CreateCatalog(true, "table-uuid"); catalog->SetSupportsVersionManagement(false); CatalogCommitSpec spec; spec.catalog = catalog; @@ -1216,7 +1257,7 @@ TEST_F(FileStoreCommitImplTest, TestCatalogWithoutVersionManagementWritesTheSnap TEST_F(FileStoreCommitImplTest, TestCatalogCommitRetriesWhenRefused) { for (int32_t max_retries : {0, 2}) { SCOPED_TRACE(max_retries); - auto catalog = CreateCatalog(false); + auto catalog = CreateCatalog(false, "table-uuid-2"); CatalogCommitSpec spec; spec.catalog = catalog; spec.table_id = "table-uuid-2"; @@ -1251,7 +1292,7 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitRetriesWhenRefused) { } TEST_F(FileStoreCommitImplTest, TestCatalogCommitHoldsALockScopePerAttempt) { - auto catalog = CreateCatalog(false); + auto catalog = CreateCatalog(false, "table-uuid-2"); std::shared_ptr log = InstallCommitLock(catalog.get()); CatalogCommitSpec spec; spec.catalog = catalog; @@ -1359,18 +1400,21 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithoutAUsableLockIsNotCreated) { ASSERT_NOK_WITH_MSG(CreateCatalogCommit(catalog), "no commit lease for this table"); ASSERT_TRUE(catalog->CommitCalls().empty()); - CommitContextBuilder builder(table_path_, "commit_user_1"); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::REST_COMMIT_ACQUIRE_LOCK] = "true"; + builder_load_options.table_id = "table-uuid"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::REST_COMMIT_ACQUIRE_LOCK, "true") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(context)), "cannot hold a commit lease"); + builder.UseRESTCatalogCommit(true).Finish()); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options), + "cannot hold a commit lease"); } TEST_F(FileStoreCommitImplTest, TestUnknownCatalogOutcomeKeepsTheManifests) { - auto catalog = CreateCatalog(Status::IOError("catalog unreachable")); + auto catalog = CreateCatalog(Status::IOError("catalog unreachable"), "table-uuid"); CatalogCommitSpec spec; spec.catalog = catalog; spec.table_id = "table-uuid"; @@ -1406,7 +1450,7 @@ TEST_F(FileStoreCommitImplTest, TestUnknownCatalogOutcomeKeepsTheManifests) { TEST_F(FileStoreCommitImplTest, TestCatalogCommitRebasesOnTheWinnerOfEachRace) { std::vector> msgs = CommitMessagesOfRound(1); - auto catalog = CreateCatalog(false); + auto catalog = CreateCatalog(false, "table-uuid-3"); int32_t competitor_round = 0; catalog->SetOnCommit([&]() { ++competitor_round; @@ -1662,10 +1706,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitReadsThroughTheCatalogsFileSystem) { auto catalog = CreateCatalog(); catalog->SetFileSystem(fs); - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_load_options.catalog = catalog; + context_builder_load_options.identifier.emplace(Identifier("db", "tbl")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1))); ASSERT_EQ(catalog->CommitCalls().size(), 1u); @@ -1698,12 +1747,16 @@ TEST_F(FileStoreCommitImplTest, TestAGivenFileSystemWinsOverTheCatalogs) { auto catalog = CreateCatalog(); catalog->SetFileSystem(catalog_fs); - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.WithCatalog(catalog, Identifier("db", "tbl")) - .WithFileSystem(given_fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_load_options.catalog = catalog; + context_builder_load_options.identifier.emplace(Identifier("db", "tbl")); + context_builder_load_options.file_system = given_fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1))); ASSERT_EQ(catalog->CommitCalls().size(), 1u); @@ -1975,7 +2028,7 @@ TEST_F(FileStoreCommitImplTest, TestRowIdConflictChecksSnapshotBoundsAndHistory) } TEST_F(FileStoreCommitImplTest, TestCatalogSnapshotLoaderFailurePropagates) { - auto catalog = CreateCatalog(); + auto catalog = CreateCatalog(true, "table-uuid"); catalog->SetLoadSnapshotStatus( Status::IOError("catalog unreachable while loading the snapshot")); CatalogCommitSpec spec; @@ -1989,14 +2042,19 @@ TEST_F(FileStoreCommitImplTest, TestCatalogSnapshotLoaderFailurePropagates) { } TEST_F(FileStoreCommitImplTest, TestSnapshotSequenceMaxPropertyMergedOnCommit) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::WRITE_SEQUENCE_NUMBER_INIT_MODE] = "snapshot"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + .Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -2036,16 +2094,21 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes) ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path, {})); - CommitContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::COMMIT_MAX_RETRIES, "10") - .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") - .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") - .WithFileSystem(fs) - .Finish()); - - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::COMMIT_MAX_RETRIES] = "10"; + context_builder_options[Options::COMMIT_MIN_RETRY_WAIT] = "1ms"; + context_builder_options[Options::COMMIT_MAX_RETRY_WAIT] = "1ms"; + context_builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::string latest_hint = PathUtil::JoinPath(table_path, "snapshot/LATEST"); auto* mock_fs = dynamic_cast(fs.get()); @@ -2077,15 +2140,20 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path, {})); - CommitContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") - .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") - .WithFileSystem(fs) - .Finish()); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::COMMIT_MIN_RETRY_WAIT] = "1ms"; + context_builder_options[Options::COMMIT_MAX_RETRY_WAIT] = "1ms"; + context_builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::string latest_hint = PathUtil::JoinPath(table_path, "snapshot/LATEST"); auto* mock_fs = dynamic_cast(fs.get()); EXPECT_CALL(*mock_fs, ReadFile(testing::_, testing::_)) @@ -2149,17 +2217,22 @@ TEST_F(FileStoreCommitImplTest, TestRealtimeCommitRebasesAfterSnapshotConflict) ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path_, {})); - CommitContextBuilder realtime_builder(table_path_, "realtime_commit_user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr realtime_context, - realtime_builder.AddOption(Options::REALTIME_ENABLED, "true") - .AddOption(Options::BUCKET, "2") - .AddOption(Options::COMMIT_MAX_RETRIES, "1") - .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") - .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr realtime_commit, - FileStoreCommit::Create(std::move(realtime_context))); + TableLoadOptions realtime_builder_load_options{table_path_}; + std::map realtime_builder_options; + CommitContextBuilder realtime_builder("realtime_commit_user"); + realtime_builder_options[Options::REALTIME_ENABLED] = "true"; + realtime_builder_options[Options::BUCKET] = "2"; + realtime_builder_options[Options::COMMIT_MAX_RETRIES] = "1"; + realtime_builder_options[Options::COMMIT_MIN_RETRY_WAIT] = "1ms"; + realtime_builder_options[Options::COMMIT_MAX_RETRY_WAIT] = "1ms"; + realtime_builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr realtime_context, realtime_builder + + .Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr realtime_commit, + FileStoreCommit::Create(realtime_builder_load_options, std::move(realtime_context), + realtime_builder_options)); auto realtime_commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(realtime_commit))); @@ -2176,11 +2249,16 @@ TEST_F(FileStoreCommitImplTest, TestRealtimeCommitRebasesAfterSnapshotConflict) OffsetRange(/*begin=*/0, /*end=*/1)}); } - CommitContextBuilder concurrent_builder(table_path_, "concurrent_commit_user"); + TableLoadOptions concurrent_builder_load_options{table_path_}; + std::map concurrent_builder_options; + CommitContextBuilder concurrent_builder("concurrent_commit_user"); + concurrent_builder_load_options.file_system = file_system_; ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_context, - concurrent_builder.WithFileSystem(file_system_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_commit, - FileStoreCommit::Create(std::move(concurrent_context))); + concurrent_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr concurrent_commit, + FileStoreCommit::Create(concurrent_builder_load_options, std::move(concurrent_context), + concurrent_builder_options)); auto* mock_fs = dynamic_cast(fs.get()); ASSERT_NE(nullptr, mock_fs); @@ -2224,13 +2302,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path, {})); - CommitContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(fs) - .Finish()); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::string new_snapshot_6 = PathUtil::JoinPath(table_path, "snapshot/snapshot-6"); auto* mock_fs = dynamic_cast(fs.get()); EXPECT_CALL(*mock_fs, AtomicStore(testing::StrEq(new_snapshot_6), testing::_)) @@ -2251,13 +2332,17 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua bool exist, file_system_->Exists(PathUtil::JoinPath(table_path, "snapshot/snapshot-6"))); ASSERT_TRUE(exist); - CommitContextBuilder context_builder_2(table_path, "commit_user_1"); + TableLoadOptions context_builder_2_load_options{table_path}; + std::map context_builder_2_options; + CommitContextBuilder context_builder_2("commit_user_1"); + context_builder_2_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_2_load_options.file_system = fs; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context_2, - context_builder_2.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(fs) - .Finish()); + context_builder_2.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit_2, FileStoreCommit::Create(std::move(commit_context_2))); + ASSERT_OK_AND_ASSIGN(auto commit_2, FileStoreCommit::Create(context_builder_2_load_options, + std::move(commit_context_2), + context_builder_2_options)); ASSERT_OK_AND_ASSIGN(int32_t num_committed, commit_2->FilterAndCommit({{1, msgs}})); ASSERT_EQ(0, num_committed); std::string new_snapshot_7 = PathUtil::JoinPath(table_path, "snapshot/snapshot-7"); @@ -2275,14 +2360,19 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua } TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "5kb") - .AddOption(Options::MANIFEST_MERGE_MIN_COUNT, "2") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "5kb"; + context_builder_options[Options::MANIFEST_MERGE_MIN_COUNT] = "2"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); { std::vector> msgs = @@ -2338,13 +2428,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { } TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); { std::vector> msgs = @@ -2409,12 +2502,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2466,10 +2562,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestThroughCatalog) { - CommitContextBuilder base_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + TableLoadOptions base_builder_load_options{table_path_}; + std::map base_builder_options; + CommitContextBuilder base_builder("commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(base_builder_load_options, std::move(base_context), + base_builder_options)); for (int32_t round = 1; round <= 2; ++round) { ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(round), round)); } @@ -2493,10 +2593,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestThroughCatalog) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestHoldsALockScope) { - CommitContextBuilder base_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + TableLoadOptions base_builder_load_options{table_path_}; + std::map base_builder_options; + CommitContextBuilder base_builder("commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(base_builder_load_options, std::move(base_context), + base_builder_options)); for (int32_t round = 1; round <= 2; ++round) { ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(round), round)); } @@ -2525,10 +2629,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestHoldsALockScope) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNeedsTheTargetUnderTheTablePath) { - CommitContextBuilder base_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + TableLoadOptions base_builder_load_options{table_path_}; + std::map base_builder_options; + CommitContextBuilder base_builder("commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(base_builder_load_options, std::move(base_context), + base_builder_options)); for (int32_t round = 1; round <= 2; ++round) { ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(round), round)); } @@ -2542,23 +2650,29 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNeedsTheTargetUnderTheTabl } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNoLatestSnapshotReturnsError) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); ASSERT_NOK_WITH_MSG(commit->RollbackToAsLatest(/*target_snapshot_id=*/1), "Latest snapshot is null, can not roll back."); } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestTargetNotExistReturnsError) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", @@ -2587,14 +2701,19 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestDeletionVectorOnlyChange) /*ignore_if_exists=*/false)); std::string dv_table_path = PathUtil::JoinPath(test_root_, "foo.db/dv_bar"); - CommitContextBuilder context_builder(dv_table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{dv_table_path}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2664,12 +2783,15 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF // before writing. A stateful Exists() mock reproduces exactly that ordering. ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path_, {})); - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2706,12 +2828,15 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF } TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); { std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -2751,12 +2876,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) { } TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -2808,13 +2936,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { } TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); { std::vector> msgs = @@ -2926,14 +3057,17 @@ TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { } TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs; ASSERT_OK(commit->Commit(msgs)); std::shared_ptr metrics = commit->GetCommitMetrics(); @@ -2944,14 +3078,17 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { } TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -2974,14 +3111,17 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { } TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); std::vector changes; @@ -3008,14 +3148,17 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { } TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -3035,14 +3178,17 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { } TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); std::vector changes; @@ -3367,18 +3513,23 @@ TEST_F(FileStoreCommitImplTest, TestCatalogExpireWithoutPublishedSnapshots) { } TEST_F(FileStoreCommitImplTest, TestDropPartitionAndExpireSnapshot) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .AddOption(Options::SNAPSHOT_EXPIRE_LIMIT, "30") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .IgnoreEmptyCommit(true) - .Finish()); - - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; + context_builder_options[Options::SNAPSHOT_EXPIRE_LIMIT] = "30"; + context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .IgnoreEmptyCommit(true) + .Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3439,18 +3590,23 @@ TEST_F(FileStoreCommitImplTest, TestExpireOnMainOfTableWithBranchesIsNotSupporte } TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .AddOption(Options::SNAPSHOT_EXPIRE_LIMIT, "30") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .IgnoreEmptyCommit(true) - .Finish()); - - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; + context_builder_options[Options::SNAPSHOT_EXPIRE_LIMIT] = "30"; + context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .IgnoreEmptyCommit(true) + .Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3486,14 +3642,17 @@ TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { } TEST_F(FileStoreCommitImplTest, TestTruncateTable) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3519,10 +3678,14 @@ TEST_F(FileStoreCommitImplTest, TestTruncateTable) { } TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3571,10 +3734,14 @@ TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { } TEST_F(FileStoreCommitImplTest, AbortIgnoresMissingFilesAndFailsForNonImplMessage) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3595,10 +3762,14 @@ TEST_F(FileStoreCommitImplTest, AbortIgnoresDeleteFailures) { // A delete that fails with an IO error (e.g. permission denied) must be swallowed by // best-effort cleanup, mirroring Java deleteQuietly: Abort still returns OK and does not // propagate the failure. - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -3634,13 +3805,16 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { // Truncating a table that has never been committed succeeds and produces an OVERWRITE snapshot // with no added and no deleted files: there is nothing to overwrite, but the overwrite is still // materialized. - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); ASSERT_OK(commit->TruncateTable(/*commit_identifier=*/1)); @@ -3659,14 +3833,17 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { } TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); std::vector> msgs; @@ -3679,14 +3856,19 @@ TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { } TEST_F(FileStoreCommitImplTest, TestCollectChanges) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BUCKET, "10") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::BUCKET] = "10"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3722,13 +3904,16 @@ TEST_F(FileStoreCommitImplTest, TestCollectChanges) { } TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3753,13 +3938,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { } TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3793,13 +3981,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { } TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3824,13 +4015,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) } TEST_F(FileStoreCommitImplTest, FilterAndCommit) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); std::vector data_files = { @@ -3871,13 +4065,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommit) { } TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3893,13 +4090,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { } TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3927,13 +4127,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { } TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3960,13 +4163,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { } TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3999,13 +4205,16 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { } TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4030,15 +4239,20 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::ROW_TRACKING_ENABLED, "true") - .AddOption(Options::DATA_EVOLUTION_ENABLED, "true") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::ROW_TRACKING_ENABLED] = "true"; + context_builder_options[Options::DATA_EVOLUTION_ENABLED] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4062,14 +4276,17 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { } TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4095,13 +4312,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4133,14 +4353,17 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4173,13 +4396,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4207,13 +4433,16 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); std::vector data_files = { @@ -4262,13 +4491,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { } TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4301,13 +4533,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4335,13 +4570,16 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { } TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4365,14 +4603,19 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { } TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::COMMIT_DISCARD_DUPLICATE_FILES, "true") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::COMMIT_DISCARD_DUPLICATE_FILES] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4397,12 +4640,15 @@ TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -4429,12 +4675,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { std::string tmp_table_path = tmp_dir->Str(); ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - CommitContextBuilder context_builder2(tmp_table_path, "commit_user_1"); + TableLoadOptions context_builder2_load_options{tmp_table_path}; + std::map context_builder2_options; + CommitContextBuilder context_builder2("commit_user_1"); + context_builder2_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder2_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context2, - context_builder2.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - auto commit2 = FileStoreCommit::Create(std::move(commit_context2)); + context_builder2.Finish()); + auto commit2 = FileStoreCommit::Create( + context_builder2_load_options, std::move(commit_context2), context_builder2_options); CHECK_HOOK_STATUS(commit2.status(), i); CHECK_HOOK_STATUS(commit2.value()->Commit(msgs), i); commit_run_complete = true; @@ -4458,9 +4707,12 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { ASSERT_TRUE(is_oss); // REST commit with local path should work without the object store flag - CommitContextBuilder builder(table_path_, "commit_user_1"); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("commit_user_1"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.UseRESTCatalogCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(builder_load_options, std::move(ctx), + builder_options)); std::vector> msgs = CommitMessagesOfRound(1); ASSERT_OK(commit->Commit(msgs)); @@ -4468,10 +4720,15 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { ASSERT_FALSE(json.empty()); auto catalog = CreateCatalog(); - CommitContextBuilder catalog_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto catalog_ctx, - catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_OK_AND_ASSIGN(auto catalog_commit, FileStoreCommit::Create(std::move(catalog_ctx))); + TableLoadOptions catalog_builder_load_options{table_path_}; + std::map catalog_builder_options; + CommitContextBuilder catalog_builder("commit_user_1"); + catalog_builder_load_options.catalog = catalog; + catalog_builder_load_options.identifier.emplace(Identifier("db", "tbl")); + ASSERT_OK_AND_ASSIGN(auto catalog_ctx, catalog_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto catalog_commit, + FileStoreCommit::Create(catalog_builder_load_options, + std::move(catalog_ctx), catalog_builder_options)); ASSERT_OK(catalog_commit->Commit(msgs)); ASSERT_EQ(catalog->CommitCalls().size(), 1u); @@ -4480,15 +4737,22 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { ASSERT_TRUE(path_is_object_store); auto object_store_catalog = CreateCatalog(); object_store_catalog->SetFileSystem(file_system_); - CommitContextBuilder object_store_builder(object_store_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto object_store_ctx, - object_store_builder.WithCatalog(object_store_catalog, Identifier("db", "tbl")).Finish()); - ASSERT_OK(FileStoreCommit::Create(std::move(object_store_ctx))); - - CommitContextBuilder plain_builder(object_store_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto plain_ctx, plain_builder.WithFileSystem(file_system_).Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(plain_ctx)), + TableLoadOptions object_store_builder_load_options{object_store_path}; + std::map object_store_builder_options; + CommitContextBuilder object_store_builder("commit_user_1"); + object_store_builder_load_options.catalog = object_store_catalog; + object_store_builder_load_options.identifier.emplace(Identifier("db", "tbl")); + ASSERT_OK_AND_ASSIGN(auto object_store_ctx, object_store_builder.Finish()); + ASSERT_OK(FileStoreCommit::Create(object_store_builder_load_options, + std::move(object_store_ctx), object_store_builder_options)); + + TableLoadOptions plain_builder_load_options{object_store_path}; + std::map plain_builder_options; + CommitContextBuilder plain_builder("commit_user_1"); + plain_builder_load_options.file_system = file_system_; + ASSERT_OK_AND_ASSIGN(auto plain_ctx, plain_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(plain_builder_load_options, std::move(plain_ctx), + plain_builder_options), "invalid scheme oss for local file system"); } @@ -4510,10 +4774,15 @@ TEST_F(FileStoreCommitImplTest, TestFixedBucketPKTableCommitAllowed) { std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); - CommitContextBuilder builder(pk_table_path, "test_user"); - builder.AddOption(Options::FILE_SYSTEM, "local").UseRESTCatalogCommit(true); + TableLoadOptions builder_load_options{pk_table_path}; + std::map builder_options; + CommitContextBuilder builder("test_user"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder.UseRESTCatalogCommit(true); ASSERT_OK_AND_ASSIGN(auto commit_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto committer, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto committer, + FileStoreCommit::Create(builder_load_options, std::move(commit_context), builder_options)); ASSERT_TRUE(committer != nullptr); } @@ -4543,14 +4812,19 @@ TEST_F(FileStoreCommitImplTest, ValidateCommitOptionsAllowsManifestDeleteFileDro } TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::MANIFEST_DELETE_FILE_DROP_STATS, "true") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::MANIFEST_DELETE_FILE_DROP_STATS] = "true"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(context_builder_load_options, std::move(commit_context), + context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -4572,14 +4846,19 @@ TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) { } TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::MANIFEST_DELETE_FILE_DROP_STATS, "true") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::MANIFEST_DELETE_FILE_DROP_STATS] = "true"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(context_builder_load_options, std::move(commit_context), + context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -4647,21 +4926,28 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) { } TEST_F(FileStoreCommitImplTest, DropPartitionWithEmptyPartitionsFails) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); ASSERT_NOK_WITH_MSG(commit->DropPartition({}, /*commit_identifier=*/1), "partitions list cannot be empty"); } TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4695,10 +4981,14 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) } TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4709,10 +4999,14 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) } TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4736,13 +5030,18 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) { } TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::DYNAMIC_PARTITION_OVERWRITE, "false") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::DYNAMIC_PARTITION_OVERWRITE] = "false"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4764,12 +5063,15 @@ TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) } TEST_F(FileStoreCommitImplTest, OverwriteWithChangelogFilesLogsWarning) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4809,12 +5111,15 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) /*ignore_if_exists=*/false)); std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); - CommitContextBuilder builder(pk_table_path, "test_user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::OVERWRITE_UPGRADE, "true") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions builder_load_options{pk_table_path}; + std::map builder_options; + CommitContextBuilder builder("test_user"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::OVERWRITE_UPGRADE] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, + FileStoreCommit::Create(builder_load_options, std::move(commit_context), builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4859,13 +5164,16 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) } TEST_F(FileStoreCommitImplTest, CommitWithAppendCommitCheckConflict) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AppendCommitCheckConflict(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.AppendCommitCheckConflict(true).Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -4882,12 +5190,16 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { // First snapshot is committed in the default (scan) mode, so it does NOT carry the // max-sequence-number property. { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), + context_builder_options)); std::vector> msgs1 = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", @@ -4897,13 +5209,18 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { // Second commit uses snapshot-init mode; since the previous snapshot lacks the property, // the max sequence number is recomputed by scanning the base manifests. - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::WRITE_SEQUENCE_NUMBER_INIT_MODE] = "snapshot"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4926,12 +5243,15 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { } TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4952,12 +5272,15 @@ TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) { } TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenOverwriteUpgradeDisabled) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::OVERWRITE_UPGRADE, "false") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::OVERWRITE_UPGRADE] = "false"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4985,12 +5308,15 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { /*ignore_if_exists=*/false)); std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); - CommitContextBuilder builder(pk_table_path, "test_user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::OVERWRITE_UPGRADE, "true") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions builder_load_options{pk_table_path}; + std::map builder_options; + CommitContextBuilder builder("test_user"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::OVERWRITE_UPGRADE] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, + FileStoreCommit::Create(builder_load_options, std::move(commit_context), builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5005,12 +5331,15 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { } TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelta) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5028,10 +5357,14 @@ TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelt } TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5041,10 +5374,14 @@ TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests } TEST_F(FileStoreCommitImplTest, RowIdCheckConflictSetsCheckSnapshotAndReturnsSelf) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp index a7f39628d..e034167f7 100644 --- a/src/paimon/core/operation/file_store_commit_test.cpp +++ b/src/paimon/core/operation/file_store_commit_test.cpp @@ -29,6 +29,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/utils/linked_hash_map.h" #include "paimon/common/utils/path_util.h" @@ -80,12 +81,15 @@ TEST(FileStoreCommitTest, TestCreate) { /*ignore_if_exists=*/false)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - CommitContextBuilder context_builder(table_path, "commit_user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); } @@ -95,14 +99,12 @@ TEST(FileStoreCommitTest, TestCreateWithCatalogRequiresIdentifier) { ASSERT_TRUE(dir); std::map options = {{Options::FILE_SYSTEM, "local"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(dir->Str(), options)); - auto commit_context = std::make_unique( - dir->Str(), "commit_user", /*ignore_empty_commit=*/true, - /*use_rest_catalog_commit=*/false, catalog, /*identifier=*/std::nullopt, - /*table_id=*/std::nullopt, /*append_commit_check_conflict=*/false, GetDefaultPool(), - CreateDefaultExecutor(), /*specific_file_system=*/nullptr, options, - /*format_table=*/nullptr); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), - "a catalog commit requires a table identifier"); + TableLoadOptions load_options{dir->Str()}; + load_options.catalog = catalog; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + CommitContextBuilder("commit_user").Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(load_options, std::move(context), options), + "a catalog load requires a table identifier"); } TEST(FileStoreCommitTest, TestCatalogCommitUsesPerTableFileSystem) { @@ -123,10 +125,14 @@ TEST(FileStoreCommitTest, TestCatalogCommitUsesPerTableFileSystem) { catalog->SetFileSystem(std::make_shared()); catalog->SetTableFileSystem(dir->GetFileSystem()); - CommitContextBuilder builder(table_path, "commit_user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, - builder.WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + CommitContextBuilder builder("commit_user"); + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(builder_load_options, std::move(ctx), + builder_options)); // The file system was resolved per table, not from the catalog-wide one. const std::vector& requests = catalog->TableFileSystemRequests(); @@ -161,12 +167,15 @@ TEST(FileStoreCommitTest, TestAppendDvIndexShouldUseOverwriteCommitKind) { /*ignore_if_exists=*/false)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - CommitContextBuilder context_builder(table_path, "commit_user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); LinkedHashMap dv_ranges; dv_ranges.insert_or_assign("data-file-1", diff --git a/src/paimon/core/operation/file_store_write_test.cpp b/src/paimon/core/operation/file_store_write_test.cpp index 01f46b053..8ba2b0ab5 100644 --- a/src/paimon/core/operation/file_store_write_test.cpp +++ b/src/paimon/core/operation/file_store_write_test.cpp @@ -32,6 +32,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/path_util.h" @@ -88,14 +89,16 @@ TEST(FileStoreWriteTest, TestRealtimeCatalogCommitRefreshAndRecovery) { }; auto create_writer = [&](const std::shared_ptr& realtime_context) -> Result> { - WriteContextBuilder builder(table_path, "writer"); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr ctx, - builder.WithCatalog(catalog, identifier) - .WithStreamingMode(true) - .WithRealtimeContext(realtime_context) - .AddOption(Options::FILE_SYSTEM, "unregistered-file-system") - .Finish()); - return FileStoreWrite::Create(std::move(ctx)); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder("writer"); + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(identifier); + builder_options[Options::FILE_SYSTEM] = "unregistered-file-system"; + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr ctx, + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context).Finish()); + return FileStoreWrite::Create(builder_load_options, std::move(ctx), builder_options); }; ASSERT_OK_AND_ASSIGN(auto realtime_context, RealtimeContext::Create()); ASSERT_OK_AND_ASSIGN(auto writer, create_writer(realtime_context)); @@ -106,10 +109,15 @@ TEST(FileStoreWriteTest, TestRealtimeCatalogCommitRefreshAndRecovery) { ASSERT_EQ(base_snapshot_id, std::nullopt); ASSERT_OK_AND_ASSIGN(auto progress, writer->PrepareCommitWithProgress(1)); ASSERT_EQ(progress.size(), 1u); - CommitContextBuilder commit_builder(table_path, "writer"); - ASSERT_OK_AND_ASSIGN(auto commit_ctx, - commit_builder.WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_ctx))); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("writer"); + commit_builder_load_options.catalog = catalog; + commit_builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(auto commit_ctx, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(commit_builder_load_options, std::move(commit_ctx), + commit_builder_options)); ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, commit->CommitWithProgress(progress, 1, std::nullopt)); ASSERT_EQ(snapshot_id, 1); @@ -190,13 +198,16 @@ TEST(FileStoreWriteTest, TestBucketBaseSnapshotTracksActualLazyRestore) { catalog->SetFileSystem(dir->GetFileSystem()); catalog->CheckBaseSnapshotUuid(); auto create_writer = [&](bool ignore_previous) -> Result> { - WriteContextBuilder builder(dir->Str(), "writer"); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.WithCatalog(catalog, identifier) - .WithStreamingMode(true) - .WithIgnorePreviousFiles(ignore_previous) - .Finish()); - return FileStoreWrite::Create(std::move(context)); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + WriteContextBuilder builder("writer"); + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(identifier); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr context, + builder.WithStreamingMode(true).WithIgnorePreviousFiles(ignore_previous).Finish()); + return FileStoreWrite::Create(builder_load_options, std::move(context), + builder_options); }; auto make_batch = [&](const std::string& dt, int32_t bucket, const std::string& json) -> Result> { @@ -210,10 +221,15 @@ TEST(FileStoreWriteTest, TestBucketBaseSnapshotTracksActualLazyRestore) { .SetBucket(bucket) .Finish(); }; - CommitContextBuilder commit_builder(dir->Str(), "writer"); - ASSERT_OK_AND_ASSIGN(auto commit_context, - commit_builder.WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions commit_builder_load_options{dir->Str()}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("writer"); + commit_builder_load_options.catalog = catalog; + commit_builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(auto commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_builder_load_options, + std::move(commit_context), + commit_builder_options)); auto seed_snapshot = [&](int32_t bucket, int64_t identifier) -> Status { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, create_writer(false)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, @@ -275,14 +291,16 @@ TEST(FileStoreWriteTest, TestBucketBaseSnapshotTracksActualLazyRestore) { TEST(FileStoreWriteTest, TestCreateWithInvalidInput) { auto dir = UniqueTestDirectory::Create(); - WriteContextBuilder builder(dir->Str(), "commit_user_1"); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + WriteContextBuilder builder("commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx1, builder.WithMemoryPool(nullptr).Finish()); - ASSERT_NOK(FileStoreWrite::Create(std::move(ctx1))); + ASSERT_NOK(FileStoreWrite::Create(builder_load_options, std::move(ctx1), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx2, builder.WithExecutor(nullptr).Finish()); - ASSERT_NOK(FileStoreWrite::Create(std::move(ctx2))); - ASSERT_NOK(FileStoreWrite::Create(/*context=*/nullptr)); + ASSERT_NOK(FileStoreWrite::Create(builder_load_options, std::move(ctx2), builder_options)); + ASSERT_NOK(FileStoreWrite::Create({}, /*context=*/nullptr)); } TEST(FileStoreWriteTest, TestCreateAppendTable) { @@ -303,14 +321,18 @@ TEST(FileStoreWriteTest, TestCreateAppendTable) { ASSERT_FALSE(shared_catalog->SupportsVersionManagement()); for (bool use_catalog : {false, true}) { SCOPED_TRACE(use_catalog); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - "commit_user_1"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); if (use_catalog) { - context_builder.WithCatalog(shared_catalog, Identifier("foo", "bar")); + context_builder_load_options.catalog = shared_catalog; + context_builder_load_options.identifier.emplace(Identifier("foo", "bar")); } ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); ASSERT_OK(file_store_write->Close()); } } @@ -327,12 +349,15 @@ TEST(FileStoreWriteTest, TestCatalogWriteAsksTheCatalogOnlyForTheMainBranchSchem catalog->SetTableSchema(schema); catalog->SetFileSystem(dir->GetFileSystem()); - WriteContextBuilder main_builder(dir->Str(), "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr main_context, - main_builder.WithCatalog(catalog, Identifier("db", "table$branch_main")).Finish()); + TableLoadOptions main_builder_load_options{dir->Str()}; + std::map main_builder_options; + WriteContextBuilder main_builder("commit_user_1"); + main_builder_load_options.catalog = catalog; + main_builder_load_options.identifier.emplace(Identifier("db", "table$branch_main")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_context, main_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_write, - FileStoreWrite::Create(std::move(main_context))); + FileStoreWrite::Create(main_builder_load_options, std::move(main_context), + main_builder_options)); ASSERT_OK(main_write->Close()); ASSERT_EQ(catalog->LoadTableSchemaIdentifiers().size(), 1u); ASSERT_EQ(catalog->LoadTableSchemaIdentifiers().front(), Identifier("db", "table")); @@ -341,12 +366,15 @@ TEST(FileStoreWriteTest, TestCatalogWriteAsksTheCatalogOnlyForTheMainBranchSchem ASSERT_OK(branch_schema_manager.CreateTable(logical_schema, /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{})); catalog->SetTableSchema(nullptr); - WriteContextBuilder branch_builder(dir->Str(), "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr branch_context, - branch_builder.WithCatalog(catalog, Identifier("db", "table$branch_dev")).Finish()); + TableLoadOptions branch_builder_load_options{dir->Str()}; + std::map branch_builder_options; + WriteContextBuilder branch_builder("commit_user_1"); + branch_builder_load_options.catalog = catalog; + branch_builder_load_options.identifier.emplace(Identifier("db", "table$branch_dev")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_context, branch_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_write, - FileStoreWrite::Create(std::move(branch_context))); + FileStoreWrite::Create(branch_builder_load_options, + std::move(branch_context), branch_builder_options)); ASSERT_OK(branch_write->Close()); ASSERT_EQ(catalog->LoadTableSchemaIdentifiers().size(), 1u); } @@ -369,11 +397,15 @@ TEST(FileStoreWriteTest, TestCatalogWriteUsesPerTableFileSystem) { catalog->SetFileSystem(std::make_shared()); catalog->SetTableFileSystem(dir->GetFileSystem()); - WriteContextBuilder builder(table_path, "writer"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, - builder.WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder("writer"); + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK(writer->Close()); // The file system was resolved per table, not from the catalog-wide one. @@ -397,11 +429,13 @@ TEST(FileStoreWriteTest, TestCreateAppendTableWithInvalidBucket) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{"f0", "f3"}, /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::BUCKET, "-2").Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::BUCKET] = "-2"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options), "not support bucket -2 in append table"); } @@ -423,13 +457,14 @@ TEST(FileStoreWriteTest, TestCreateAppendTableWithInvalidWriteType) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - "commit_user_1"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::BUCKET] = "-1"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::BUCKET, "-1") - .WithWriteSchema({"field_non_exist"}) - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + context_builder.WithWriteSchema({"field_non_exist"}).Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options), "write field field_non_exist does not exist in table schema"); } @@ -449,12 +484,14 @@ TEST(FileStoreWriteTest, TestCreatePrimaryKeyTable) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{"f0"}, /*primary_keys=*/{"f0", "f1", "f4"}, options, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::BUCKET, "2").Finish()); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::BUCKET] = "2"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); auto fs = std::make_shared(); SchemaManager schema_manager(fs, PathUtil::JoinPath(dir->Str(), "foo.db/bar")); @@ -483,10 +520,12 @@ TEST(FileStoreWriteTest, TestCreatePrimaryKeyTableWithInvalidBucket) { /*partition_keys=*/{"f0", "f3"}, /*primary_keys=*/{"f1", "f4"}, options, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - "commit_user_1"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options), "not support bucket -1 in key value table"); } diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 89f2cd490..b7a6fa65e 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -78,7 +78,7 @@ std::string IdentifierToFlatJson(const Identifier& identifier) { } // namespace std::shared_ptr InternalReadContext::CreateDependencyCatalogContext() const { - const auto& catalog = read_context_->GetCatalog(); + const auto& catalog = catalog_; if (!catalog) { // Preserve path-only filesystem reads using /.db/
. return std::make_shared( @@ -88,13 +88,13 @@ std::shared_ptr InternalReadContext::CreateDependencyCatalogCont // Preserve the dependency catalog's effective settings: table options, then catalog // options, then explicit read overrides. The upstream reader loads its own table options; - // ReadContextBuilder must not copy these catalog settings back into that read. + // Catalog settings are used only while loading dependencies. auto dependency_options = options_.ToMap(); const auto& catalog_options = catalog->GetOptions(); for (const auto& [key, value] : catalog_options) { dependency_options[key] = value; } - const auto& read_options = read_context_->GetOptions(); + const auto& read_options = dynamic_options_; for (const auto& [key, value] : read_options) { dependency_options[key] = value; } @@ -102,11 +102,11 @@ std::shared_ptr InternalReadContext::CreateDependencyCatalogCont std::string(RestApi::kHeaderOptionPrefix) + RestApi::kReadViaHeader; auto metastore = dependency_options.find(CatalogOptions::METASTORE); if (metastore != dependency_options.end() && - StringUtils::ToLowerCase(metastore->second) == "rest" && read_context_->GetIdentifier() && + StringUtils::ToLowerCase(metastore->second) == "rest" && identifier_ && dependency_options.count(read_via_option) == 0) { // Only fill in a missing header when the merged options select a REST dependency catalog. dependency_options[read_via_option] = - UrlUtils::EncodeString(IdentifierToFlatJson(*read_context_->GetIdentifier())); + UrlUtils::EncodeString(IdentifierToFlatJson(*identifier_)); } // A REST warehouse identifies the catalog, independent of the table's storage location. return std::make_shared(catalog->GetRootPath(), dependency_options, @@ -277,11 +277,15 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( Result> InternalReadContext::Create( const std::shared_ptr& context, const std::shared_ptr& table_schema, - const std::map& options) { + const std::map& options, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& fs_scheme_map, + const std::shared_ptr& catalog, const std::optional& identifier, + const std::map& dynamic_options) { PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - return CreateFromCoreOptions(context, table_schema, std::move(core_options)); + CoreOptions::FromMap(options, file_system, fs_scheme_map)); + return CreateForTable(context, table_schema, std::move(core_options), root_path, fs_scheme_map, + catalog, identifier, dynamic_options); } Result> InternalReadContext::CreateFromCoreOptions( @@ -490,8 +494,7 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea table_schema_(table_schema), read_schema_(read_schema), arrow_pool_(arrow_pool), - options_(options), - fs_scheme_map_(read_context->GetFileSystemSchemeToIdentifierMap()) {} + options_(options) {} Result> InternalReadContext::CreateWithSchema( const std::shared_ptr& original, @@ -507,23 +510,37 @@ Result> InternalReadContext::CreateWithSche original->options_, original->arrow_pool_)); result->path_ = original->path_; result->fs_scheme_map_ = original->fs_scheme_map_; + result->catalog_ = original->catalog_; + if (original->identifier_) { + result->identifier_.emplace(*original->identifier_); + } + result->dynamic_options_ = original->dynamic_options_; return result; } Result> InternalReadContext::CreateForTable( const std::shared_ptr& context, const std::shared_ptr& schema, CoreOptions options, const std::string& root_path, - const std::map& fs_scheme_map) { + const std::map& fs_scheme_map, + const std::shared_ptr& catalog, const std::optional& identifier, + const std::map& dynamic_options) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr result, CreateFromCoreOptions(context, schema, std::move(options))); result->path_ = root_path; result->fs_scheme_map_ = fs_scheme_map; + result->catalog_ = catalog; + if (identifier) { + result->identifier_.emplace(*identifier); + } + result->dynamic_options_ = dynamic_options; return result; } Result> InternalReadContext::CreateForFallbackBranch( const std::shared_ptr& context, const std::string& branch, const std::shared_ptr& file_system, const std::string& root_path, - const std::map& fs_scheme_map) { + const std::map& fs_scheme_map, + const std::shared_ptr& catalog, const std::optional& identifier, + const std::map& dynamic_options) { if (context->GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); } @@ -533,13 +550,13 @@ Result> InternalReadContext::CreateForFallb return Status::Invalid("fallback schema not found"); } auto options = schema.value()->Options(); - for (const auto& [key, value] : context->GetOptions()) { + for (const auto& [key, value] : dynamic_options) { options[key] = value; } options[Options::BRANCH] = branch; PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system)); return CreateForTable(context, schema.value(), std::move(core_options), root_path, - fs_scheme_map); + fs_scheme_map, catalog, identifier, dynamic_options); } } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index dd13e0f13..5f4e16269 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -24,6 +24,7 @@ #include #include +#include "paimon/catalog/identifier.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" #include "paimon/read_context.h" @@ -35,6 +36,7 @@ class Schema; } // namespace arrow namespace paimon { +class Catalog; struct CatalogContext; class Executor; class MemoryPool; @@ -59,16 +61,27 @@ class InternalReadContext { static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, - const std::map& options); + const std::map& options, + const std::shared_ptr& file_system = nullptr, const std::string& root_path = "", + const std::map& fs_scheme_map = {}, + const std::shared_ptr& catalog = nullptr, + const std::optional& identifier = std::nullopt, + const std::map& dynamic_options = {}); static Result> CreateForTable( const std::shared_ptr& context, const std::shared_ptr& schema, CoreOptions options, const std::string& root_path, - const std::map& fs_scheme_map); + const std::map& fs_scheme_map, + const std::shared_ptr& catalog = nullptr, + const std::optional& identifier = std::nullopt, + const std::map& dynamic_options = {}); static Result> CreateForFallbackBranch( const std::shared_ptr& context, const std::string& branch, const std::shared_ptr& file_system, const std::string& root_path, - const std::map& fs_scheme_map); + const std::map& fs_scheme_map, + const std::shared_ptr& catalog = nullptr, + const std::optional& identifier = std::nullopt, + const std::map& dynamic_options = {}); /// Maps score output positions in the read schema to their logical search sources. const std::map& GetIndexScoreBindings() const { @@ -88,7 +101,7 @@ class InternalReadContext { return table_schema_; } const std::string& GetPath() const { - return path_ ? *path_ : read_context_->GetPath(); + return path_; } std::shared_ptr CreateDependencyCatalogContext() const; @@ -197,7 +210,10 @@ class InternalReadContext { std::shared_ptr read_schema_; std::shared_ptr arrow_pool_; CoreOptions options_; - std::optional path_; + std::string path_; + std::shared_ptr catalog_; + std::optional identifier_; + std::map dynamic_options_; std::map fs_scheme_map_; std::map index_score_bindings_; }; diff --git a/src/paimon/core/operation/internal_read_context_test.cpp b/src/paimon/core/operation/internal_read_context_test.cpp index 2147e8242..b393ae69e 100644 --- a/src/paimon/core/operation/internal_read_context_test.cpp +++ b/src/paimon/core/operation/internal_read_context_test.cpp @@ -29,6 +29,7 @@ #include "arrow/type.h" #include "fmt/format.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/catalog_options.h" #include "paimon/common/catalog/catalog_context.h" #include "paimon/common/table/special_fields.h" @@ -52,9 +53,11 @@ namespace paimon::test { TEST(InternalReadContext, TestReadWithUnspecifiedSchema) { // no read schema is specified, read all fields std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, @@ -74,13 +77,18 @@ TEST(InternalReadContext, TestPreciseScalarEvaluation) { arrow::field("f1", arrow::utf8())}), {}, {}, {})); for (bool enabled : {false, true}) { - ReadContextBuilder builder("unused-table-path"); + TableLoadOptions builder_load_options{"unused-table-path"}; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(enabled).SetPredicate( PredicateBuilder::IsNotNull(0, "f0", FieldType::INT)); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr internal_context, - InternalReadContext::Create(context, table_schema, table_schema->Options())); + InternalReadContext::Create( + context, table_schema, table_schema->Options(), builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); ASSERT_EQ(internal_context->EnablePreciseFileIndexEvaluate(), enabled); ASSERT_FALSE(internal_context->EnablePredicateFilter()); ASSERT_TRUE(internal_context->GetCoreOptions().FileIndexReadEnabled()); @@ -92,33 +100,46 @@ TEST(InternalReadContext, TestPreciseScalarEvaluation) { } auto match = std::make_shared("f1", R"({"match":{"query":"apple"}})"); ASSERT_OK_AND_ASSIGN(auto full_text, PredicateBuilder::FullTextMatch(1, match)); - ReadContextBuilder builder("unused-table-path"); + TableLoadOptions builder_load_options{"unused-table-path"}; + std::map builder_options; + ReadContextBuilder builder; builder.SetPredicate(full_text); ASSERT_NOK_WITH_MSG(builder.Finish(), "EnablePreciseFileIndexEvaluate(true)"); builder.EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_TRUE(context->EnablePreciseFileIndexEvaluate()); - ASSERT_OK(InternalReadContext::Create(context, table_schema, table_schema->Options())); + ASSERT_OK(InternalReadContext::Create( + context, table_schema, table_schema->Options(), builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); } TEST(InternalReadContext, TestPreciseEvaluationRequiresFileIndexRead) { ASSERT_OK_AND_ASSIGN( std::shared_ptr table_schema, TableSchema::Create(0, arrow::schema({arrow::field("f0", arrow::int32())}), {}, {}, {})); - ReadContextBuilder builder("unused-table-path"); + TableLoadOptions builder_load_options{"unused-table-path"}; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); - ASSERT_NOK_WITH_MSG(InternalReadContext::Create(context, table_schema, - {{Options::FILE_INDEX_READ_ENABLED, "false"}}), - "requires file-index.read.enabled=true"); + ASSERT_NOK_WITH_MSG( + InternalReadContext::Create( + context, table_schema, {{Options::FILE_INDEX_READ_ENABLED, "false"}}, + builder_load_options.file_system, builder_load_options.path, + builder_load_options.fs_scheme_to_identifier_map, builder_load_options.catalog, + builder_load_options.identifier, builder_options), + "requires file-index.read.enabled=true"); } TEST(InternalReadContext, TestReadWithSpecifiedSchema) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, @@ -146,10 +167,13 @@ TEST(InternalReadContext, TestDependencyReadMergesTableCatalogAndReadOptions) { {CatalogOptions::URI, "http://read"}, {CatalogOptions::TOKEN, "read-token"}, {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}}; - ReadContextBuilder builder(path); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr read_context, - builder.WithCatalog(catalog, Identifier("db1", "t1")).SetOptions(read_options).Finish()); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(Identifier("db1", "t1")); + builder_options = read_options; + ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); std::map options = table_schema->Options(); options[Options::READ_BATCH_SIZE] = "64"; options[CatalogOptions::DATA_TOKEN_ENABLED] = "false"; @@ -157,8 +181,12 @@ TEST(InternalReadContext, TestDependencyReadMergesTableCatalogAndReadOptions) { for (const auto& [key, value] : read_options) { options[key] = value; } - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); auto dependency_context = internal_context->CreateDependencyCatalogContext(); auto expected_options = options; @@ -166,7 +194,7 @@ TEST(InternalReadContext, TestDependencyReadMergesTableCatalogAndReadOptions) { ASSERT_EQ(dependency_context->root_path, catalog->GetRootPath()); ASSERT_EQ(dependency_context->options, expected_options); ASSERT_EQ(dependency_context->options.at("header.X-Paimon-Read-Via"), "table-view"); - ASSERT_EQ(read_context->GetOptions(), read_options); + ASSERT_EQ(builder_options, read_options); ASSERT_EQ(catalog->GetOptions(), catalog_options); ASSERT_EQ(internal_context->GetCoreOptions().ToMap(), options); } @@ -195,20 +223,26 @@ TEST(InternalReadContext, TestDependencyReadKeepsTheOutermostView) { if (test_case.read_header) { read_options["header.X-Paimon-Read-Via"] = test_case.read_header; } - ReadContextBuilder builder(path); - ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, - builder.WithCatalog(catalog, Identifier("db1", "nested-view")) - .SetOptions(read_options) - .Finish()); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(Identifier("db1", "nested-view")); + builder_options = read_options; + ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); auto options = table_schema->Options(); options["header.X-Paimon-Read-Via"] = test_case.read_header ? test_case.read_header : "table-view"; - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); auto dependency_context = internal_context->CreateDependencyCatalogContext(); ASSERT_EQ(dependency_context->options.at("header.X-Paimon-Read-Via"), test_case.expected_header); - ASSERT_EQ(read_context->GetOptions(), read_options); + ASSERT_EQ(builder_options, read_options); ASSERT_EQ(catalog->GetOptions(), catalog_options); ASSERT_EQ(internal_context->GetCoreOptions().ToMap(), options); } @@ -252,13 +286,19 @@ TEST(InternalReadContext, TestDependencyReadHeaderUsesFinalMetastore) { read_options[CatalogOptions::METASTORE] = test_case.read_metastore; options[CatalogOptions::METASTORE] = test_case.read_metastore; } - ReadContextBuilder builder(path); - ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, - builder.WithCatalog(catalog, Identifier("db1", "t1")) - .SetOptions(read_options) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(read_context, table_schema, options)); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(Identifier("db1", "t1")); + builder_options = read_options; + ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); auto dependency_context = internal_context->CreateDependencyCatalogContext(); if (test_case.expected_metastore) { ASSERT_EQ(dependency_context->options.at(CatalogOptions::METASTORE), @@ -271,7 +311,7 @@ TEST(InternalReadContext, TestDependencyReadHeaderUsesFinalMetastore) { } else { ASSERT_EQ(dependency_context->options.count("header.X-Paimon-Read-Via"), 0u); } - ASSERT_EQ(read_context->GetOptions(), read_options); + ASSERT_EQ(builder_options, read_options); ASSERT_EQ(catalog->GetOptions(), catalog_options); ASSERT_EQ(internal_context->GetCoreOptions().ToMap(), options); } @@ -281,10 +321,10 @@ TEST(InternalReadContext, TestDependencyReadHeaderNeedsAnIdentifier) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; SchemaManager schema_manager(std::make_shared(), path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); - ReadContextBuilder builder(path); - ASSERT_OK_AND_ASSIGN(auto read_context, - builder.AddOption(CatalogOptions::METASTORE, "REST").Finish()); - ASSERT_FALSE(read_context->GetIdentifier()); + TableLoadOptions builder_load_options{path}; + ReadContextBuilder builder; + ASSERT_OK_AND_ASSIGN(auto read_context, builder.Finish()); + ASSERT_FALSE(builder_load_options.identifier); auto options = table_schema->Options(); options[CatalogOptions::METASTORE] = "REST"; ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), @@ -300,12 +340,12 @@ TEST(InternalReadContext, TestDependencyReadHeaderReportsTheBranchOfTheStartingT auto catalog = std::make_shared(); catalog->SetTableFileSystem(std::make_shared()); catalog->SetOptions({{CatalogOptions::METASTORE, "REST"}}); - ReadContextBuilder builder(path); - ASSERT_OK_AND_ASSIGN(auto read_context, - builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).Finish()); - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + ReadContextBuilder builder; + ASSERT_OK_AND_ASSIGN(auto read_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create(std::move(read_context), table_schema, table_schema->Options(), + nullptr, path, {}, catalog, Identifier("db1", "t1", "dev"))); auto dependency_context = internal_context->CreateDependencyCatalogContext(); ASSERT_EQ(dependency_context->options.at("header.X-Paimon-Read-Via"), "%7B%22database%22%3A%22db1%22%2C%22object%22%3A%22t1%24branch_dev%22%7D"); @@ -319,14 +359,18 @@ TEST(InternalReadContext, TestFileSystemSchemeMapReachesTheDependencyRead) { // surfaces as a `dfs://` dependency table that cannot be opened, far away from the cause. std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; const std::map scheme_map = {{"file", "local"}}; - ReadContextBuilder context_builder(path); - context_builder.WithFileSystemSchemeToIdentifierMap(scheme_map); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; + context_builder_load_options.fs_scheme_to_identifier_map = scheme_map; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create(std::move(read_context), table_schema, table_schema->Options(), + nullptr, path, scheme_map)); ASSERT_EQ(internal_context->GetFileSystemSchemeToIdentifierMap(), scheme_map); ASSERT_EQ(internal_context->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map, scheme_map); @@ -334,10 +378,12 @@ TEST(InternalReadContext, TestFileSystemSchemeMapReachesTheDependencyRead) { TEST(InternalReadContext, TestReadWithSpecifiedFieldId) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; context_builder.SetReadFieldIds({3, 0}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, @@ -350,13 +396,15 @@ TEST(InternalReadContext, TestReadWithSpecifiedFieldId) { TEST(InternalReadContext, TestReadWithSpecifiedFieldIdAndSchema) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; // read schema is specified, read fields in schema // will use field ids instead of field names. context_builder.SetReadFieldNames({"f0"}); context_builder.SetReadFieldIds({3, 0}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, @@ -371,11 +419,13 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { { // test simple std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames( {"f3", "f0", "_ROW_ID", "_SEQUENCE_NUMBER", "_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); auto new_options = table_schema->Options(); new_options[Options::ROW_TRACKING_ENABLED] = "true"; @@ -393,10 +443,12 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { { // test invalid case: disable row tracking while read row tracking fields std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "_ROW_ID", "_SEQUENCE_NUMBER"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_NOK_WITH_MSG(InternalReadContext::Create(std::move(read_context), table_schema, table_schema->Options()), @@ -405,10 +457,12 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { { // test invalid case: disable data evolution while read score fields std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_NOK_WITH_MSG(InternalReadContext::Create(std::move(read_context), table_schema, table_schema->Options()), @@ -423,7 +477,9 @@ TEST(InternalReadContext, TestPreciseEvaluationRejectsUnnamedScoreProjection) { for (const auto& search_type : std::vector{"none", "scalar", "vector", "full-text", "full-text-match"}) { for (const auto& projection : std::vector{"names", "ids", "schema"}) { - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true).SetReadFieldNames({"f0"}); if (projection == "names") { builder.SetReadFieldNames({"f0", "_INDEX_SCORE"}); @@ -457,7 +513,11 @@ TEST(InternalReadContext, TestPreciseEvaluationRejectsUnnamedScoreProjection) { } ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_NOK_WITH_MSG( - InternalReadContext::Create(context, table_schema, table_schema->Options()), + InternalReadContext::Create( + context, table_schema, table_schema->Options(), + builder_load_options.file_system, builder_load_options.path, + builder_load_options.fs_scheme_to_identifier_map, builder_load_options.catalog, + builder_load_options.identifier, builder_options), "not exist in table schema"); } } @@ -471,21 +531,30 @@ TEST(InternalReadContext, TestPrimaryKeyVectorSearchRequiresDeletionVectors) { std::shared_ptr search = std::make_shared("f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; builder.SetReadFieldNames({"f0", "_f3_INDEX_SCORE"}) .EnablePreciseFileIndexEvaluate(true) .SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr internal, - InternalReadContext::Create(context, table_schema, table_schema->Options())); + InternalReadContext::Create( + context, table_schema, table_schema->Options(), builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); EXPECT_TRUE(internal->HasVectorOrFullTextSearch()); EXPECT_TRUE(internal->EnablePreciseFileIndexEvaluate()); std::map options = table_schema->Options(); options[Options::DELETION_VECTORS_ENABLED] = "false"; - ASSERT_NOK_WITH_MSG(InternalReadContext::Create(context, table_schema, options), - "primary-key tables requires deletion vectors"); + ASSERT_NOK_WITH_MSG( + InternalReadContext::Create( + context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options), + "primary-key tables requires deletion vectors"); } TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { @@ -511,7 +580,9 @@ TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { std::make_shared(test_case.field_name, R"({"match":{"query":"apple"}})"); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::FullTextMatch(test_case.field_index, match)); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames(test_case.projection) .SetPredicate(predicate); @@ -519,7 +590,11 @@ TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { if (test_case.expected_error.empty()) { ASSERT_OK_AND_ASSIGN( std::shared_ptr internal, - InternalReadContext::Create(context, table_schema, table_schema->Options())); + InternalReadContext::Create( + context, table_schema, table_schema->Options(), + builder_load_options.file_system, builder_load_options.path, + builder_load_options.fs_scheme_to_identifier_map, builder_load_options.catalog, + builder_load_options.identifier, builder_options)); ASSERT_TRUE(internal->GetIndexScoreBindings().empty()); ASSERT_FALSE(internal->NeedsScores( {InternalReadContext::IndexScoreSource::Type::FULL_TEXT_MATCH, match->field_name})); @@ -528,7 +603,11 @@ TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { "CreateWithSchema does not support"); } else { ASSERT_NOK_WITH_MSG( - InternalReadContext::Create(context, table_schema, table_schema->Options()), + InternalReadContext::Create( + context, table_schema, table_schema->Options(), + builder_load_options.file_system, builder_load_options.path, + builder_load_options.fs_scheme_to_identifier_map, builder_load_options.catalog, + builder_load_options.identifier, builder_options), test_case.expected_error); } } @@ -542,7 +621,9 @@ TEST(InternalReadContext, TestPerFieldTopKScoreBindings) { for (bool vector_search : {false, true}) { for (const auto& projection_type : std::vector{"none", "names", "schema"}) { const bool project_score = projection_type != "none"; - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; std::vector projection = {"f1"}; if (project_score) { projection.push_back(vector_search ? "_f3_INDEX_SCORE" : "_f0_INDEX_SCORE"); @@ -573,7 +654,11 @@ TEST(InternalReadContext, TestPerFieldTopKScoreBindings) { ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr internal, - InternalReadContext::Create(context, table_schema, table_schema->Options())); + InternalReadContext::Create( + context, table_schema, table_schema->Options(), + builder_load_options.file_system, builder_load_options.path, + builder_load_options.fs_scheme_to_identifier_map, builder_load_options.catalog, + builder_load_options.identifier, builder_options)); ASSERT_EQ(internal->GetReadSchema()->field_names(), projection); ASSERT_TRUE(internal->NeedsScores(source)); ASSERT_FALSE(internal->NeedsScores( @@ -634,7 +719,7 @@ TEST_F(InternalReadContextScoreTest, BindsFullTextScoresByProjection) { {2, false, true}, {2, true, false}, {2, true, true}}; for (const auto& test_case : cases) { const int32_t score_count = test_case.score_count; - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; std::vector projection = {"F1", "F2"}; if (score_count >= 1) { projection.push_back(test_case.reverse_scores ? "_F1_INDEX_SCORE" : "_F2_INDEX_SCORE"); @@ -696,7 +781,7 @@ TEST_F(InternalReadContextScoreTest, BindsRootAndPredicateScores) { ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::FullTextMatch(0, match)); auto search = std::make_shared("F2", match->query, /*limit=*/10); - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; builder.SetPredicate(predicate) .EnablePreciseFileIndexEvaluate(true) .SetFullTextSearch(search) @@ -756,7 +841,7 @@ TEST_F(InternalReadContextScoreTest, RejectsAmbiguousOrConditionalScoreProjectio {valid, false, ""}}; for (const auto& test_case : cases) { for (bool project_score : {false, true}) { - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetPredicate(test_case.predicate) .SetReadFieldNames(project_score ? std::vector{"F1", "_F1_INDEX_SCORE"} @@ -790,7 +875,7 @@ TEST_F(InternalReadContextScoreTest, RejectsPredicatesOnFullTextMatchScores) { for (const auto& score_predicate : score_predicates) { ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::And({text, score_predicate})); - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"F1", "_F1_INDEX_SCORE"}) .SetPredicate(predicate); @@ -808,7 +893,7 @@ TEST_F(InternalReadContextScoreTest, RejectsNestedPredicatesOnRootSearchScores) ASSERT_OK_AND_ASSIGN(std::shared_ptr child, PredicateBuilder::Or({scalar, score})); ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::And({scalar, child})); - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetFullTextSearch(search) .SetReadFieldNames({"F1", "_F1_INDEX_SCORE"}) @@ -819,7 +904,7 @@ TEST_F(InternalReadContextScoreTest, RejectsNestedPredicatesOnRootSearchScores) } TEST_F(InternalReadContextScoreTest, RejectsPredicatesOnUnnamedIndexScores) { - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; builder.SetPredicate(PredicateBuilder::IsNull(0, "_INDEX_SCORE", FieldType::FLOAT)); ASSERT_NOK_WITH_MSG( Create(&builder), @@ -828,7 +913,7 @@ TEST_F(InternalReadContextScoreTest, RejectsPredicatesOnUnnamedIndexScores) { TEST_F(InternalReadContextScoreTest, RejectsMissingScoreSource) { for (const auto& name : {"_F1_INDEX_SCORE", "_missing_INDEX_SCORE"}) { - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; builder.SetReadFieldNames({name}); ASSERT_NOK_WITH_MSG(Create(&builder), fmt::format("Index score field '{}' has no search source", name)); @@ -840,7 +925,7 @@ TEST_F(InternalReadContextScoreTest, ResolvesScoresOnlyFromEffectiveProjection) InternalReadContext::IndexScoreSource::Type::FULL_TEXT_SEARCH, "F2"}; auto search = std::make_shared("F2", R"({"match":{"query":"apple"}})", 1); for (bool explicit_schema : {false, true}) { - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetFullTextSearch(search) .SetReadFieldNames({"_INDEX_SCORE", "_missing_INDEX_SCORE"}) @@ -870,7 +955,7 @@ TEST_F(InternalReadContextScoreTest, ResolvesScoresOnlyFromEffectiveProjection) } TEST_F(InternalReadContextScoreTest, SchemaReplacementUsesProvidedSchema) { - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::shared_ptr internal, Create(&builder)); for (const auto& schema : {arrow::schema({arrow::field("F1", arrow::utf8())}), @@ -884,10 +969,12 @@ TEST_F(InternalReadContextScoreTest, SchemaReplacementUsesProvidedSchema) { TEST(InternalReadContext, TestReadWithValueKindField) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "_VALUE_KIND", "f0"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, @@ -903,14 +990,16 @@ TEST(InternalReadContext, TestReadWithFieldIdsAndSpecialFields) { { // test simple std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; // here we use field ids instead of field names, and specify special ids for row id, // sequence number and index score. context_builder.SetReadFieldIds({3, 0, SpecialFieldIds::ROW_ID, SpecialFieldIds::SEQUENCE_NUMBER, SpecialFieldIds::INDEX_SCORE}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); auto new_options = table_schema->Options(); new_options[Options::ROW_TRACKING_ENABLED] = "true"; @@ -941,12 +1030,19 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaAndSpecialFields) { auto projected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); std::shared_ptr read_context = std::move(unique_read_context); ASSERT_NOK_WITH_MSG( - InternalReadContext::Create(read_context, table_schema, table_schema->Options()), + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + context_builder_load_options.file_system, context_builder_load_options.path, + context_builder_load_options.fs_scheme_to_identifier_map, + context_builder_load_options.catalog, context_builder_load_options.identifier, + context_builder_options), "not exist in table schema"); } @@ -959,13 +1055,20 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaAndSpecialFields) { auto projected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); std::shared_ptr read_context = std::move(unique_read_context); ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, enabled_options)); + InternalReadContext::Create( + read_context, table_schema, enabled_options, + context_builder_load_options.file_system, context_builder_load_options.path, + context_builder_load_options.fs_scheme_to_identifier_map, + context_builder_load_options.catalog, context_builder_load_options.identifier, + context_builder_options)); auto expected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields); ASSERT_TRUE(internal_context->GetReadSchema()->Equals(expected_schema)); } @@ -979,17 +1082,25 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaWithoutFieldIds) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); std::shared_ptr read_context = std::move(unique_read_context); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + context_builder_load_options.file_system, context_builder_load_options.path, + context_builder_load_options.fs_scheme_to_identifier_map, + context_builder_load_options.catalog, context_builder_load_options.identifier, + context_builder_options)); std::vector expected_fields = { DataField(3, arrow::field("f3", arrow::float64())), @@ -1011,17 +1122,25 @@ TEST(InternalReadContext, TestProjectedSchemaMetadataWhitelist) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); std::shared_ptr read_context = std::move(unique_read_context); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + context_builder_load_options.file_system, context_builder_load_options.path, + context_builder_load_options.fs_scheme_to_identifier_map, + context_builder_load_options.catalog, context_builder_load_options.identifier, + context_builder_options)); auto aligned_field = internal_context->GetReadSchema()->GetFieldByName("f0"); ASSERT_TRUE(aligned_field); @@ -1056,13 +1175,20 @@ TEST(InternalReadContext, TestMapSharedShreddingAccessRequiresSharedShreddingLay auto c_read_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*arrow::schema({access_field}), c_read_schema.get()).ok()); - ReadContextBuilder context_builder("/tmp/unused-table-path"); + TableLoadOptions context_builder_load_options{"/tmp/unused-table-path"}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadSchema(std::move(c_read_schema)); ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); std::shared_ptr read_context = std::move(unique_read_context); ASSERT_NOK_WITH_MSG( - InternalReadContext::Create(read_context, table_schema, table_schema->Options()), + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + context_builder_load_options.file_system, context_builder_load_options.path, + context_builder_load_options.fs_scheme_to_identifier_map, + context_builder_load_options.catalog, context_builder_load_options.identifier, + context_builder_options), "Selected-key MAP pushdown only supports top-level shared-shredding MAP field: tags"); } diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp index c0e94e236..26c5a0396 100644 --- a/src/paimon/core/operation/key_value_file_store_write_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp @@ -41,6 +41,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/data/shredding/map_shredding_defs.h" @@ -135,14 +136,17 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { /*primary_keys=*/{"f0"}, table_options, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("test"); if (with_temp_directory) { context_builder.WithTempDirectory(dir->Str()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, context_builder.Finish()); - return FileStoreWrite::Create(std::move(write_context)); + return FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options); } Status WriteSingleStringRow(FileStoreWrite* file_store_write, int32_t bucket, @@ -196,11 +200,15 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { const std::string& table_path, const std::shared_ptr& schema, const std::map& options, const std::string& json, int64_t commit_identifier) const { - WriteContextBuilder builder(table_path, "test"); - builder.SetOptions(options); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder("test"); + builder_options = options; + EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); EXPECT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(builder_load_options, std::move(write_context), + builder_options)); EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( /*wait_compaction=*/false, commit_identifier)); @@ -219,11 +227,15 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { void Commit(const std::string& table_path, const std::map& options, const std::vector>& commit_msgs) const { - CommitContextBuilder builder(table_path, "test"); - builder.SetOptions(options); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + CommitContextBuilder builder("test"); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(builder_load_options, + std::move(commit_context), builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); } @@ -337,10 +349,13 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { /*primary_keys=*/{"f1"}, /*options=*/{{"bucket", "1"}}, /*ignore_if_exists=*/false)); - WriteContextBuilder builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), commit_user); + TableLoadOptions builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map builder_options; + WriteContextBuilder builder(commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(builder_load_options, std::move(write_context), + builder_options)); ASSERT_NOK_WITH_MSG(file_store_write->Write(nullptr), "batch is null pointer"); } { @@ -355,11 +370,14 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { /*primary_keys=*/{"f1"}, /*options=*/{{"bucket", "-2"}}, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - commit_user); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder(commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); auto array = std::make_shared(); arrow::StringBuilder builder; for (size_t j = 0; j < 100; j++) { @@ -387,11 +405,14 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { /*primary_keys=*/{"f1"}, /*options=*/{{"bucket", "2"}}, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - commit_user); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder(commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); auto array = std::make_shared(); arrow::StringBuilder builder; for (size_t j = 0; j < 100; j++) { @@ -442,14 +463,17 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) { ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - WriteContextBuilder builder(table_path, "test"); - builder.SetOptions(options) - .WithStreamingMode(true) + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder("test"); + builder_options = options; + builder.WithStreamingMode(true) .WithRealtimeContext(realtime_context) .WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); std::unique_ptr batch = MakeBatch(realtime_schema, R"([ @@ -505,15 +529,19 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) { SCOPED_TRACE(expected_error); ASSERT_OK_AND_ASSIGN(std::shared_ptr invalid_context, RealtimeContext::Create()); - WriteContextBuilder invalid_builder(table_path, "invalid"); - invalid_builder.SetOptions(options) - .WithStreamingMode(true) + TableLoadOptions invalid_builder_load_options{table_path}; + std::map invalid_builder_options; + WriteContextBuilder invalid_builder("invalid"); + invalid_builder_options = options; + invalid_builder.WithStreamingMode(true) .WithRealtimeContext(invalid_context) .WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_write_context, invalid_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_writer, - FileStoreWrite::Create(std::move(invalid_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr invalid_writer, + FileStoreWrite::Create(invalid_builder_load_options, std::move(invalid_write_context), + invalid_builder_options)); ASSERT_OK(invalid_writer->Write(MakeBatch(realtime_schema, R"([[30, 1, "existing"]])"))); ASSERT_OK_AND_ASSIGN(std::vector prepared, invalid_writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); @@ -544,14 +572,15 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) { std::shared_ptr pool = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - WriteContextBuilder builder(table_path, "test"); - builder.SetOptions(options) - .WithStreamingMode(true) - .WithRealtimeContext(realtime_context) - .WithMemoryPool(pool); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder("test"); + builder_options = options; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context).WithMemoryPool(pool); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); const int64_t allocations_before_write = pool->allocation_count; ASSERT_OK(writer->Write(MakeBatch(realtime_schema, R"([[0, 1, "one"]])"))); @@ -567,15 +596,19 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) { std::shared_ptr rejecting_pool = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr rejecting_context, RealtimeContext::Create()); - WriteContextBuilder rejecting_builder(table_path, "rejecting"); - rejecting_builder.SetOptions(options) - .WithStreamingMode(true) + TableLoadOptions rejecting_builder_load_options{table_path}; + std::map rejecting_builder_options; + WriteContextBuilder rejecting_builder("rejecting"); + rejecting_builder_options = options; + rejecting_builder.WithStreamingMode(true) .WithRealtimeContext(rejecting_context) .WithMemoryPool(rejecting_pool); ASSERT_OK_AND_ASSIGN(std::unique_ptr rejecting_write_context, rejecting_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr rejecting_writer, - FileStoreWrite::Create(std::move(rejecting_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr rejecting_writer, + FileStoreWrite::Create(rejecting_builder_load_options, std::move(rejecting_write_context), + rejecting_builder_options)); ASSERT_OK(rejecting_writer->Write(MakeBatch(realtime_schema, "[]"))); const int64_t rejecting_allocations_before_write = rejecting_pool->allocation_count; rejecting_pool->reject_allocations = true; @@ -605,13 +638,17 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) { ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_context, RealtimeContext::Create()); - WriteContextBuilder initial_builder(table_path, "initial"); - initial_builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext( - initial_context); + TableLoadOptions initial_builder_load_options{table_path}; + std::map initial_builder_options; + WriteContextBuilder initial_builder("initial"); + initial_builder_options = options; + initial_builder.WithStreamingMode(true).WithRealtimeContext(initial_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr initial_write_context, initial_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr initial_writer, - FileStoreWrite::Create(std::move(initial_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr initial_writer, + FileStoreWrite::Create(initial_builder_load_options, std::move(initial_write_context), + initial_builder_options)); ASSERT_OK(initial_writer->Write(MakeBatch(realtime_schema, R"([[0, 0, "initial"]])"))); ASSERT_OK_AND_ASSIGN(std::vector initial_progress, initial_writer->PrepareCommitWithProgress(0)); @@ -623,22 +660,31 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) { initial_message->GetNewFilesIncrement().NewFiles()[0]->AssignSequenceNumber(max - 2, max - 2); initial_progress[0].offset_range = OffsetRange(0, max - 1); - CommitContextBuilder commit_builder(table_path, "initial"); - commit_builder.SetOptions(options); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("initial"); + commit_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr committer, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr committer, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, committer->CommitWithProgress(initial_progress, 0, std::nullopt)); ASSERT_OK(initial_writer->Close()); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - WriteContextBuilder builder(table_path, "boundary"); - builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder("boundary"); + builder_options = options; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); ASSERT_OK( writer->Write(MakeBatch(realtime_schema, fmt::format(R"([[{}, 1, "legal"]])", max - 1)))); ASSERT_OK_AND_ASSIGN(auto store_rows, ReadRealtimePrimaryKeyStoreRows(realtime_context)); @@ -747,11 +793,15 @@ TEST_F(KeyValueFileStoreWriteTest, TestSpillSimple) { {Options::WRITE_BUFFER_SPILLABLE, "true"}}, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("test"); context_builder.WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); auto key_value_file_store_write = dynamic_cast(file_store_write.get()); auto get_writer = [&](int32_t bucket) -> std::shared_ptr { auto partition_iter = key_value_file_store_write->writers_.find(BinaryRow::EmptyRow()); @@ -806,11 +856,17 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriterRestoreKeepsValueStats) { /*ignore_if_exists=*/false)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - WriteContextBuilder first_context_builder(table_path, "first-writer"); - first_context_builder.SetOptions(options); + TableLoadOptions first_context_builder_load_options{table_path}; + std::map first_context_builder_options; + WriteContextBuilder first_context_builder("first-writer"); + first_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr first_context, first_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto first_write, FileStoreWrite::Create(std::move(first_context))); + ASSERT_OK_AND_ASSIGN( + auto first_write, + FileStoreWrite::Create(first_context_builder_load_options, std::move(first_context), + first_context_builder_options)); ASSERT_OK(WriteSingleStringRow(first_write.get(), /*bucket=*/0, "alice")); ASSERT_OK_AND_ASSIGN(auto commit_messages, first_write->PrepareCommit(/*wait_compaction=*/false, 1)); @@ -823,11 +879,17 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriterRestoreKeepsValueStats) { SimpleStats::EmptyStats()); Commit(table_path, options, commit_messages); - WriteContextBuilder restored_context_builder(table_path, "restored-writer"); - restored_context_builder.SetOptions(options); + TableLoadOptions restored_context_builder_load_options{table_path}; + std::map restored_context_builder_options; + WriteContextBuilder restored_context_builder("restored-writer"); + restored_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr restored_context, restored_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto restored_write, FileStoreWrite::Create(std::move(restored_context))); + ASSERT_OK_AND_ASSIGN( + auto restored_write, + FileStoreWrite::Create(restored_context_builder_load_options, std::move(restored_context), + restored_context_builder_options)); auto key_value_write = dynamic_cast(restored_write.get()); ASSERT_NE(nullptr, key_value_write); ASSERT_OK_AND_ASSIGN(std::shared_ptr restore_files, @@ -856,11 +918,15 @@ TEST_F(KeyValueFileStoreWriteTest, TestSpillDiskQuotaExhaustedFallsBackToFlushDa {Options::WRITE_BUFFER_SPILL_MAX_DISK_SIZE, "1b"}}, /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("test"); context_builder.WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); // Disk quota is 1 byte, so spill will exhaust quota immediately and fall back to // FlushWriteBuffer (writing data files directly instead of spill temp files). @@ -904,11 +970,15 @@ TEST_F(KeyValueFileStoreWriteTest, TestMultiRoundSpillWithSameKeyDeduplication) {Options::WRITE_BUFFER_SPILLABLE, "true"}}, /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("test"); context_builder.WithTempDirectory(dir->Str()).WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); // Round 1: alice, bob, alice (duplicate key) → after dedup: alice + bob = 2 rows ASSERT_OK(WriteSingleStringRow(file_store_write.get(), /*bucket=*/0, "alice")); diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp index 2096c2aa9..ea9f5a06a 100644 --- a/src/paimon/core/operation/merge_file_split_read_test.cpp +++ b/src/paimon/core/operation/merge_file_split_read_test.cpp @@ -31,6 +31,7 @@ #include "arrow/c/abi.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/reader/concat_batch_reader.h" @@ -91,23 +92,31 @@ class MergeFileSplitReadTest : public ::testing::Test, } std::shared_ptr CreateInternalReadContext( - const std::shared_ptr& read_context, int32_t schema_id = 0) { - SchemaManager schema_manager(fs_, read_context->GetPath()); + const std::shared_ptr& read_context, + const TableLoadOptions& read_context_load_options, + const std::map& read_context_options, int32_t schema_id = 0) { + SchemaManager schema_manager(fs_, read_context_load_options.path); EXPECT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(schema_id)); - EXPECT_OK_AND_ASSIGN(auto context, InternalReadContext::Create(read_context, table_schema, - read_context->GetOptions())); + EXPECT_OK_AND_ASSIGN( + auto context, InternalReadContext::Create( + read_context, table_schema, read_context_options, + read_context_load_options.file_system, read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, + read_context_load_options.identifier, read_context_options)); return context; } - void AddOptions(ReadContextBuilder* context_builder) const { + void AddOptions(ReadContextBuilder* context_builder, + std::map& options) const { auto [use_min_heap, enable_io_prefetch, enable_multi_thread_row_to_batch] = GetParam(); if (use_min_heap) { - context_builder->AddOption(Options::SORT_ENGINE, "min-heap"); + options[Options::SORT_ENGINE] = "min-heap"; } else { - context_builder->AddOption(Options::SORT_ENGINE, "loser-tree"); + options[Options::SORT_ENGINE] = "loser-tree"; } if (enable_io_prefetch) { - context_builder->AddOption("test.enable-adaptive-prefetch-strategy", "false"); + options["test.enable-adaptive-prefetch-strategy"] = "false"; context_builder->EnablePrefetch(true); context_builder->SetPrefetchBatchCount(/*batch_count=*/3); } else { @@ -609,7 +618,10 @@ TEST_F(MergeFileSplitReadTest, TestGenerateKeyPredicates2) { TEST_P(MergeFileSplitReadTest, TestSimple) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), DataField(3, arrow::field("p1", arrow::int32())), @@ -620,13 +632,17 @@ TEST_P(MergeFileSplitReadTest, TestSimple) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN( std::unique_ptr split_read, @@ -675,7 +691,10 @@ TEST_P(MergeFileSplitReadTest, TestSimple) { TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), DataField(1, arrow::field("k1", arrow::int32())), DataField(5, arrow::field("s1", arrow::utf8())), @@ -685,11 +704,15 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"}); - context_builder.SetOptions( - {{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - std::shared_ptr internal_context = CreateInternalReadContext(read_context); + std::shared_ptr internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr split_read, CreateMergeFileSplitRead(internal_context)); @@ -742,7 +765,10 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) TEST_P(MergeFileSplitReadTest, TestLookUp) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), DataField(3, arrow::field("p1", arrow::int32())), @@ -753,14 +779,18 @@ TEST_P(MergeFileSplitReadTest, TestLookUp) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}, - {Options::FORCE_LOOKUP, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}, + {Options::FORCE_LOOKUP, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -790,15 +820,22 @@ TEST_P(MergeFileSplitReadTest, TestLookUp) { TEST_P(MergeFileSplitReadTest, TestReadWithLimits) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}, - {Options::READ_BATCH_SIZE, "1"}}); - AddOptions(&context_builder); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}, + {Options::READ_BATCH_SIZE, "1"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); @@ -816,7 +853,10 @@ TEST_P(MergeFileSplitReadTest, TestReadWithLimits) { TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), DataField(1, arrow::field("k1", arrow::int32())), @@ -827,11 +867,15 @@ TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::MERGE_ENGINE, "deduplicate"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::MERGE_ENGINE, "deduplicate"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit2())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -856,7 +900,10 @@ TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), DataField(3, arrow::field("p1", arrow::int32())), @@ -868,11 +915,12 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "s0", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + context_builder.EnableLateMaterializing(false); - AddOptions(&context_builder); + AddOptions(&context_builder, context_builder_options); // less_than will be ignore as it is partition predicate auto less_than = PredicateBuilder::GreaterThan(/*field_index=*/1, /*field_name=*/"p1", @@ -888,9 +936,12 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate_result, PredicateBuilder::And({less_than, greater_or_equal, greater_than})); context_builder.SetPredicate(predicate_result); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -919,7 +970,10 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), DataField(3, arrow::field("p1", arrow::int32())), @@ -931,10 +985,11 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "s0", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); context_builder.EnableLateMaterializing(true); // key predicate, always pushed down into the data files auto greater_or_equal = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"k1", @@ -946,9 +1001,12 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { PredicateBuilder::And({greater_or_equal, greater_than})); context_builder.SetPredicate(predicate_result); context_builder.EnablePredicateFilter(true).EnableLateMaterializing(true); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -976,7 +1034,10 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(0, arrow::field("k1", arrow::int32())), DataField(1, arrow::field("k0", arrow::int32())), @@ -991,14 +1052,18 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "k0", "p0", "p1", "s1", "s0", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context, /*schema_id=*/7); + auto internal_context = CreateInternalReadContext(read_context, read_context_load_options, + read_context_options, /*schema_id=*/7); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1028,7 +1093,10 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(8, arrow::field("v2", arrow::int32())), DataField(3, arrow::field("p1", arrow::int32())), @@ -1040,14 +1108,18 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"v2", "p1", "k0", "p0", "s0", "v0"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder, context_builder_options); + + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context, /*schema_id=*/7); + auto internal_context = CreateInternalReadContext(read_context, read_context_load_options, + read_context_options, /*schema_id=*/7); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1077,7 +1149,10 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), DataField(3, arrow::field("p1", arrow::int32())), @@ -1088,14 +1163,18 @@ TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "aggregation"}, - {"fields.v1.aggregate-function", "bool_and"}, - {"fields.v0.aggregate-function", "sum"}, - {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "aggregation"}, + {"fields.v1.aggregate-function", "bool_and"}, + {"fields.v0.aggregate-function", "sum"}, + {Options::IGNORE_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1125,7 +1204,10 @@ TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), DataField(3, arrow::field("p1", arrow::int32())), @@ -1135,14 +1217,18 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "partial-update"}, - {"fields.v1.sequence-group", "v0"}, - {"fields.v0.aggregate-function", "first_value"}, - {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "partial-update"}, + {"fields.v1.sequence-group", "v0"}, + {"fields.v0.aggregate-function", "first_value"}, + {Options::IGNORE_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1172,7 +1258,10 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), DataField(1, arrow::field("k1", arrow::int32())), @@ -1183,11 +1272,15 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions( - {{Options::MERGE_ENGINE, "partial-update"}, {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::MERGE_ENGINE, "partial-update"}, + {Options::IGNORE_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit2())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1212,7 +1305,10 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDelete) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), DataField(1, arrow::field("k1", arrow::int32())), @@ -1223,11 +1319,15 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDel ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::MERGE_ENGINE, "partial-update"}, - {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::MERGE_ENGINE, "partial-update"}, + {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit2())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1252,7 +1352,10 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDel TEST_P(MergeFileSplitReadTest, TestEmptyPlan) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), DataField(1, arrow::field("k1", arrow::int32())), @@ -1263,11 +1366,15 @@ TEST_P(MergeFileSplitReadTest, TestEmptyPlan) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::MERGE_ENGINE, "partial-update"}, - {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::MERGE_ENGINE, "partial-update"}, + {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); std::vector> empty_data_split; ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, empty_data_split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_result, @@ -1285,18 +1392,25 @@ TEST_P(MergeFileSplitReadTest, TestIOException) { DataField(6, arrow::field("v0", arrow::float64())), DataField(7, arrow::field("v1", arrow::boolean()))}; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); auto fields_with_row_kind = read_schema->fields(); fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -1336,7 +1450,10 @@ TEST_P(MergeFileSplitReadTest, TestIOException) { TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { std::string path = paimon::test::GetDataDir() + "/orc/pk_09.db/pk_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(3, arrow::field("f3", arrow::float64())), DataField(2, arrow::field("f2", arrow::int32())), DataField(0, arrow::field("f0", arrow::utf8())), @@ -1344,10 +1461,13 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"f3", "f2", "f0", "f1"}); - context_builder.SetOptions({{Options::FILE_FORMAT, "orc"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {"orc.read.enable-metrics", "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::FILE_FORMAT, "orc"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {"orc.read.enable-metrics", "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto meta1 = std::make_shared( "data-00e3ed53-16ba-4537-9264-b7dc03fefc65-0.orc", /*file_size=*/803, /*row_count=*/1, @@ -1391,7 +1511,8 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { paimon::test::GetDataDir() + "/orc/pk_09.db/pk_09/f1=10/bucket-1/", {meta1, meta2}); ASSERT_OK_AND_ASSIGN(auto data_split, builder.WithSnapshot(5).IsStreaming(false).RawConvertible(false).Build()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, {data_split})); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, ReadResultCollector::CollectResult(batch_reader.get())); diff --git a/src/paimon/core/operation/precise_file_index_read_test.cpp b/src/paimon/core/operation/precise_file_index_read_test.cpp index 640179ef9..faf139558 100644 --- a/src/paimon/core/operation/precise_file_index_read_test.cpp +++ b/src/paimon/core/operation/precise_file_index_read_test.cpp @@ -30,6 +30,7 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/file_index/bitmap/bitmap_file_index.h" #include "paimon/common/predicate/leaf_predicate_impl.h" @@ -145,30 +146,42 @@ class PreciseFileIndexReadTest : public ::testing::Test { Result> CreateSplitRead( const std::shared_ptr& read_context, + const TableLoadOptions& read_context_load_options, + const std::map& read_context_options, std::shared_ptr table_schema = nullptr) const { if (!table_schema) { SchemaManager schema_manager(std::make_shared(), - read_context->GetPath()); + read_context_load_options.path); PAIMON_ASSIGN_OR_RAISE(table_schema, schema_manager.ReadSchema(0)); } auto options = table_schema->Options(); - for (const auto& [key, value] : read_context->GetOptions()) { + for (const auto& [key, value] : read_context_options) { options[key] = value; } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, options, read_context_load_options.file_system, + read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, read_context_load_options.identifier, + read_context_options)); return std::make_unique(nullptr, context, pool_, nullptr); } Result> ReadWithFileIndex( - const std::shared_ptr& read_context, const std::shared_ptr& index, + const std::shared_ptr& read_context, + const TableLoadOptions& read_context_load_options, + const std::map& read_context_options, + const std::shared_ptr& index, const RoaringBitmap32& deleted_rows = RoaringBitmap32(), std::shared_ptr table_schema = nullptr, const std::optional>& row_ranges = std::nullopt, std::shared_ptr deletion_vector = nullptr, const std::optional& row_count = std::nullopt) const { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr split_read, - CreateSplitRead(read_context, std::move(table_schema))); + CreateSplitRead(read_context, read_context_load_options, + read_context_options, std::move(table_schema))); auto file = PrepareDataFile(); file->embedded_index = index; if (row_count) { @@ -207,11 +220,16 @@ class PreciseFileIndexReadTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr index, MockSearchFileIndex::MakeQueryIndex("f0", statistics_index_type_, matches, pool_)); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.SetReadFieldNames({"f0"}).SetPredicate(predicate).EnablePreciseFileIndexEvaluate( true); + auto context_load_options = builder_load_options; + auto context_options = builder_options; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr context, builder.Finish()); - return ReadWithFileIndex(context, index); + return ReadWithFileIndex(context, context_load_options, context_options, index); } static void CheckSearchResult(const std::shared_ptr& actual, @@ -258,14 +276,18 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateNestedAndOrWithPartitionConst "f0", statistics_index_type_, {{first_query, {{{0, 0.2f}, {1, 0.4f}}, 0.2}}, {second_query, {{{2, 0.6f}}, 0.1}}}, pool_)); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); - builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}) - .SetPredicate(predicate) - .AddOption(Options::READ_BATCH_SIZE, "1"); + builder_options[Options::READ_BATCH_SIZE] = "1"; + builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}).SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, index)); + ReadWithFileIndex(context, context_load_options, context_options, index)); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}, @@ -298,12 +320,17 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateReordersQueries) { {{wide_query, {{{0, 0.1f}, {1, 0.2f}, {2, 0.3f}}, 0.9}}, {narrow_query, {{{0, 0.4f}, {2, 0.5f}}, 0.1}}}, pool_)); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"f0", "f3"}).SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, index)); + ReadWithFileIndex(context, context_load_options, context_options, index)); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64())}, R"([["Tony",14.1]])"); @@ -449,15 +476,20 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateEmptyRowsShortCircuitExecutio {{empty_query, {{}, std::nullopt}}, {all_query, {{{0, 0.1f}, {1, 0.2f}, {2, 0.3f}}, std::nullopt}}}, pool_)); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"f0"}).SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); for (bool all_deleted : {false, true}) { statistics_->executions.clear(); ASSERT_OK_AND_ASSIGN( std::shared_ptr result, - ReadWithFileIndex(context, index, + ReadWithFileIndex(context, context_load_options, context_options, index, all_deleted ? RoaringBitmap32::From({0, 1, 2}) : RoaringBitmap32())); ASSERT_TRUE(!result || result->length() == 0); ASSERT_EQ(statistics_->executions.size(), all_deleted ? 0 : 1); @@ -507,15 +539,19 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateIndexedLeafSkipsUnselectedSca std::shared_ptr index, MockSearchFileIndex::MakeQueryIndex("f0", statistics_index_type_, {{query, {{{0, 0.25f}, {2, 0.75f}}, 0.1}}}, pool_)); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); // Single-row batches let bitmap selection skip the unselected scalar evaluation batch. - builder.SetReadFieldNames({"f0", "f3"}) - .SetPredicate(predicate) - .AddOption(Options::READ_BATCH_SIZE, "1"); + builder_options[Options::READ_BATCH_SIZE] = "1"; + builder.SetReadFieldNames({"f0", "f3"}).SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, index)); + ReadWithFileIndex(context, context_load_options, context_options, index)); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64())}, R"([["Tony",14.1]])"); @@ -552,14 +588,18 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateRechecksScalarIndexCandidates std::shared_ptr predicate, PredicateBuilder::And( {PredicateBuilder::Equal(1, "f3", FieldType::DOUBLE, Literal(13.1)), text})); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); - builder.SetReadFieldNames({"f0", "f3", "_f0_INDEX_SCORE"}) - .SetPredicate(predicate) - .AddOption(Options::READ_BATCH_SIZE, "1"); + builder_options[Options::READ_BATCH_SIZE] = "1"; + builder.SetReadFieldNames({"f0", "f3", "_f0_INDEX_SCORE"}).SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, index)); + ReadWithFileIndex(context, context_load_options, context_options, index)); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), field, arrow::field("_f0_INDEX_SCORE", arrow::float32())}, @@ -578,14 +618,20 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicatePrechecksSearchBeforeEmptyCan PredicateBuilder::Equal(1, "f1", FieldType::INT, Literal(case_id == 0 ? 20 : 10)); ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::And({scalar, text})); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); const auto deleted = case_id == 1 ? RoaringBitmap32::From({0, 1, 2}) : RoaringBitmap32(); const std::optional> ranges = case_id == 2 ? std::make_optional(std::vector()) : std::nullopt; - ASSERT_NOK_WITH_MSG(ReadWithFileIndex(context, nullptr, deleted, nullptr, ranges), + ASSERT_NOK_WITH_MSG(ReadWithFileIndex(context, context_load_options, context_options, + nullptr, deleted, nullptr, ranges), "has no File Index for search field 'f0'"); } } @@ -614,14 +660,22 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateRenamedTextAndMissingScalarCo MockSearchFileIndex::MakeQueryIndex( "f0", MockSearchFileIndex::FULL_TEXT_IDENTIFIER, {{query, {{{0, 0.25f}, {1, 0.5f}, {2, 0.75f}}, 0.5}}}, pool_)); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); - builder.SetReadFieldNames({"renamed_f0", "new_value", "f3", "_renamed_f0_INDEX_SCORE"}) - .SetOptions(options) + builder_options = options; + builder + .SetReadFieldNames({"renamed_f0", "new_value", "f3", "_renamed_f0_INDEX_SCORE"}) + .SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, index, RoaringBitmap32(), current)); + ReadWithFileIndex(context, context_load_options, context_options, index, + RoaringBitmap32(), current)); CheckSearchResult( result, {arrow::field("renamed_f0", arrow::utf8()), arrow::field("new_value", arrow::int32()), @@ -632,14 +686,23 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateRenamedTextAndMissingScalarCo ASSERT_OK_AND_ASSIGN( std::shared_ptr nonnull, PredicateBuilder::And({text, PredicateBuilder::IsNotNull(1, "new_value", FieldType::INT)})); - ReadContextBuilder nonnull_builder(SearchTablePath()); + TableLoadOptions nonnull_builder_load_options; + nonnull_builder_load_options.path = SearchTablePath(); + std::map nonnull_builder_options; + ReadContextBuilder nonnull_builder; nonnull_builder.EnablePreciseFileIndexEvaluate(true); - nonnull_builder.SetReadFieldNames({"renamed_f0", "new_value", "f3", "_renamed_f0_INDEX_SCORE"}) - .SetOptions(options) + nonnull_builder_options = options; + nonnull_builder + .SetReadFieldNames({"renamed_f0", "new_value", "f3", "_renamed_f0_INDEX_SCORE"}) + .SetPredicate(nonnull); + auto nonnull_context_load_options = nonnull_builder_load_options; + auto nonnull_context_options = nonnull_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr nonnull_context, nonnull_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr empty, - ReadWithFileIndex(nonnull_context, index, RoaringBitmap32(), current)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr empty, + ReadWithFileIndex(nonnull_context, nonnull_context_load_options, nonnull_context_options, + index, RoaringBitmap32(), current)); ASSERT_TRUE(!empty || empty->length() == 0); } @@ -666,19 +729,25 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateFullTextAndVectorScoresAfterF auto search = std::make_shared("f3", 1, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); + builder_options[Options::READ_BATCH_SIZE] = "1"; builder.SetReadFieldNames({"_f3_INDEX_SCORE", "f0", "f3", "_f0_INDEX_SCORE"}) .SetPredicate(predicate) - .SetVectorSearch(search) - .AddOption(Options::READ_BATCH_SIZE, "1"); + .SetVectorSearch(search); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); for (int32_t case_id : {0, 1, 2}) { const auto deleted = case_id == 1 ? RoaringBitmap32::From({1}) : RoaringBitmap32(); const std::optional> ranges = case_id == 2 ? std::make_optional(std::vector{Range(2, 2)}) : std::nullopt; ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, index, deleted, nullptr, ranges)); + ReadWithFileIndex(context, context_load_options, context_options, + index, deleted, nullptr, ranges)); CheckSearchResult( result, {arrow::field("_f3_INDEX_SCORE", arrow::float32()), arrow::field("f0", arrow::utf8()), @@ -691,21 +760,27 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateFullTextAndVectorScoresAfterF TEST_F(PreciseFileIndexReadTest, TestFullTextRootScoresAndLimitAfterScalarDvAndRanges) { const std::string query = R"({"match":{"query":"ranked"}})"; auto search = std::make_shared("f0", query, 1, RoaringBitmap64::From({0, 2})); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); + builder_options[Options::READ_BATCH_SIZE] = "1"; builder.SetReadFieldNames({"f0", "f3", "_f0_INDEX_SCORE"}) .SetFullTextSearch(search) - .SetPredicate(PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.5))) - .AddOption(Options::READ_BATCH_SIZE, "1"); + .SetPredicate(PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.5))); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr index, MockSearchFileIndex::MakeQueryIndex( "f0", MockSearchFileIndex::FULL_TEXT_IDENTIFIER, {{query, {{{0, 0.25f}, {1, 0.5f}, {2, 0.75f}}, std::nullopt}}}, pool_)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, index, RoaringBitmap32::From({0}), nullptr, - std::vector{Range(1, 2)})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result, + ReadWithFileIndex(context, context_load_options, context_options, index, + RoaringBitmap32::From({0}), nullptr, std::vector{Range(1, 2)})); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64()), arrow::field("_f0_INDEX_SCORE", arrow::float32())}, @@ -729,7 +804,10 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsBitmap64DeletionVec MockSearchFileIndex::Serialize(indexes, pool_)); for (const std::string& query_type : std::vector{"none", "scalar", "match", "full-text", "vector"}) { - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.SetReadFieldNames({"f0", "f3"}).EnablePreciseFileIndexEvaluate(true); if (query_type == "scalar") { builder.SetPredicate( @@ -746,6 +824,8 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsBitmap64DeletionVec "f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{})); } + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); for (bool empty_dv : {false, true}) { auto dv = std::make_shared(); @@ -756,7 +836,8 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsBitmap64DeletionVec const std::optional> ranges = empty_candidates ? std::make_optional(std::vector()) : std::nullopt; ASSERT_NOK_WITH_MSG( - ReadWithFileIndex(context, index, RoaringBitmap32(), nullptr, ranges, dv), + ReadWithFileIndex(context, context_load_options, context_options, index, + RoaringBitmap32(), nullptr, ranges, dv), "Precise file-index evaluation requires bitmap32 deletion vectors"); } } @@ -766,7 +847,10 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsBitmap64DeletionVec TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsRowsAboveBitmap32Limit) { for (const std::string& search_type : std::vector{"none", "scalar", "match", "full-text", "vector"}) { - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"f0", "f3"}); if (search_type == "scalar") { @@ -785,9 +869,12 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsRowsAboveBitmap32Li "f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{})); } + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_NOK_WITH_MSG( - ReadWithFileIndex(context, /*index=*/nullptr, RoaringBitmap32(), nullptr, + ReadWithFileIndex(context, context_load_options, context_options, /*index=*/nullptr, + RoaringBitmap32(), nullptr, /*row_ranges=*/std::nullopt, /*deletion_vector=*/nullptr, int64_t{RoaringBitmap32::MAX_VALUE} + 1), "Precise file-index evaluation requires bitmap32 physical row positions"); @@ -796,17 +883,22 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsRowsAboveBitmap32Li TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationAtBitmap32Limit) { for (bool has_predicate : {false, true}) { - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.SetReadFieldNames({"f0", "f3"}).EnablePreciseFileIndexEvaluate(true); if (has_predicate) { builder.SetPredicate( PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.0))); } + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr result, - ReadWithFileIndex(context, /*index=*/nullptr, RoaringBitmap32(), nullptr, - std::vector{Range(0, 2)}, + ReadWithFileIndex(context, context_load_options, context_options, /*index=*/nullptr, + RoaringBitmap32(), nullptr, std::vector{Range(0, 2)}, /*deletion_vector=*/nullptr, RoaringBitmap32::MAX_VALUE)); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64())}, diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp index f536696f4..bcd2a7d73 100644 --- a/src/paimon/core/operation/raw_file_split_read_test.cpp +++ b/src/paimon/core/operation/raw_file_split_read_test.cpp @@ -29,6 +29,7 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/reader/concat_batch_reader.h" #include "paimon/common/types/data_field.h" @@ -144,15 +145,24 @@ class RawFileSplitReadTest : public ::testing::Test { } Result> ReadSearchWithDeletionVector( - const std::shared_ptr& read_context, const std::string& index_type, - const RoaringBitmap32& deleted_rows, std::shared_ptr table_schema) const { - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + const std::shared_ptr& read_context, + const TableLoadOptions& read_context_load_options, + const std::map& read_context_options, + const std::string& index_type, const RoaringBitmap32& deleted_rows, + std::shared_ptr table_schema) const { + SchemaManager schema_manager(std::make_shared(), + read_context_load_options.path); if (!table_schema) { PAIMON_ASSIGN_OR_RAISE(table_schema, schema_manager.ReadSchema(0)); } PAIMON_ASSIGN_OR_RAISE( std::unique_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + read_context_load_options.file_system, read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, read_context_load_options.identifier, + read_context_options)); auto split = std::dynamic_pointer_cast(PrepareDataSplits().front()); if (!split) { return Status::Invalid("Raw search test split must be a DataSplitImpl"); @@ -171,7 +181,7 @@ class RawFileSplitReadTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( - read_context->GetPath(), schema, table_schema->PartitionKeys(), + read_context_load_options.path, schema, table_schema->PartitionKeys(), options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), external_paths, global_index_path, options.IndexFileInDataFileDir(), pool_)); @@ -197,19 +207,31 @@ class RawFileSplitReadTest : public ::testing::Test { std::string path = paimon::test::GetDataDir() + "/orc/multi_partition_append_table.db/" "multi_partition_append_table"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames(read_schema->field_names()); if (dv_factory) { - context_builder.AddOption(Options::FILE_INDEX_READ_ENABLED, "true"); + context_builder_options[Options::FILE_INDEX_READ_ENABLED] = "true"; + context_builder.SetPredicate(PredicateBuilder::IsNotNull(0, "f0", FieldType::STRING)); } + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + read_context_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + read_context_load_options.file_system, read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, read_context_load_options.identifier, + read_context_options)); auto data_splits = PrepareDataSplits(); const auto& core_options = internal_context->GetCoreOptions(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); @@ -288,17 +310,25 @@ TEST_F(RawFileSplitReadTest, TestCreateReader) { TEST_F(RawFileSplitReadTest, TestCreateReaderUsesExplicitPredicate) { const std::string path = GetDataDir() + "/orc/multi_partition_append_table.db/multi_partition_append_table"; - ReadContextBuilder builder(path); - builder.EnablePredicateFilter(true) - .SetPredicate(PredicateBuilder::Equal(0, "f0", FieldType::STRING, - Literal(FieldType::STRING, "Bob", 3))) - .AddOption(Options::FILE_INDEX_READ_ENABLED, "false"); + TableLoadOptions builder_load_options; + builder_load_options.path = path; + std::map builder_options; + ReadContextBuilder builder; + builder_options[Options::FILE_INDEX_READ_ENABLED] = "false"; + builder.EnablePredicateFilter(true).SetPredicate( + PredicateBuilder::Equal(0, "f0", FieldType::STRING, Literal(FieldType::STRING, "Bob", 3))); + auto read_context_load_options = builder_load_options; + auto read_context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); SchemaManager schemas(std::make_shared(), path); ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, schemas.ReadSchema(0)); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + read_context_load_options.file_system, read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, + read_context_load_options.identifier, read_context_options)); const CoreOptions& options = context->GetCoreOptions(); const auto schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); ASSERT_OK_AND_ASSIGN(std::vector external_paths, options.CreateExternalPaths()); @@ -544,15 +574,24 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { std::shared_ptr vector_search = std::make_shared( "f3", /*limit=*/2, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.EnablePreciseFileIndexEvaluate(true); context_builder.SetReadFieldNames({"_f3_INDEX_SCORE", "f0"}).SetVectorSearch(vector_search); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); - ASSERT_OK_AND_ASSIGN( - auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + ASSERT_OK_AND_ASSIGN(auto internal_context, + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + read_context_load_options.file_system, read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, + read_context_load_options.identifier, read_context_options)); auto data_splits = PrepareDataSplits(); auto data_split = std::dynamic_pointer_cast(data_splits.front()); @@ -594,14 +633,24 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { .ok()); ASSERT_TRUE(result->Equals(expected)); - ReadContextBuilder without_score_builder(path); + TableLoadOptions without_score_builder_load_options; + without_score_builder_load_options.path = path; + std::map without_score_builder_options; + ReadContextBuilder without_score_builder; without_score_builder.EnablePreciseFileIndexEvaluate(true); without_score_builder.SetReadFieldNames({"f0"}).SetVectorSearch(vector_search); + auto without_score_context_load_options = without_score_builder_load_options; + auto without_score_context_options = without_score_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr without_score_context, without_score_builder.Finish()); ASSERT_OK_AND_ASSIGN( auto without_score_internal_context, - InternalReadContext::Create(without_score_context, table_schema, table_schema->Options())); + InternalReadContext::Create( + without_score_context, table_schema, table_schema->Options(), + without_score_context_load_options.file_system, without_score_context_load_options.path, + without_score_context_load_options.fs_scheme_to_identifier_map, + without_score_context_load_options.catalog, + without_score_context_load_options.identifier, without_score_context_options)); auto without_score_split_read = std::make_unique( path_factory, std::move(without_score_internal_context), pool_, executor); ASSERT_OK_AND_ASSIGN(std::unique_ptr without_score_reader, @@ -624,13 +673,19 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchFiltersDeletionVectorBeforeTopK) { std::shared_ptr search = std::make_shared( "f3", /*limit=*/1, std::vector{1.0f}, user_filter, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options; + builder_load_options.path = path; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"_f3_INDEX_SCORE", "f0"}).SetVectorSearch(search); + auto read_context_load_options = builder_load_options; + auto read_context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr result, - ReadSearchWithDeletionVector(read_context, MockSearchFileIndex::VECTOR_IDENTIFIER, + ReadSearchWithDeletionVector(read_context, read_context_load_options, read_context_options, + MockSearchFileIndex::VECTOR_IDENTIFIER, RoaringBitmap32::From({0}), /*table_schema=*/nullptr)); std::shared_ptr expected; ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( @@ -650,13 +705,19 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchFiltersDeletionVectorBeforeLimit) user_filter.Add(2); auto search = std::make_shared("f3", R"({"match":{"query":"match"}})", /*limit=*/1, user_filter); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options; + builder_load_options.path = path; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"f0"}).SetFullTextSearch(search); + auto read_context_load_options = builder_load_options; + auto read_context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr result, - ReadSearchWithDeletionVector(read_context, MockSearchFileIndex::FULL_TEXT_IDENTIFIER, + ReadSearchWithDeletionVector(read_context, read_context_load_options, read_context_options, + MockSearchFileIndex::FULL_TEXT_IDENTIFIER, RoaringBitmap32::From({0}), /*table_schema=*/nullptr)); std::shared_ptr expected; ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( @@ -696,11 +757,15 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { auto search = std::make_shared("renamed_f3", 2, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options; + builder_load_options.path = path; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); - builder.SetOptions(options) - .SetReadFieldNames({"f0", "_renamed_f3_INDEX_SCORE"}) - .SetVectorSearch(search); + builder_options = options; + builder.SetReadFieldNames({"f0", "_renamed_f3_INDEX_SCORE"}).SetVectorSearch(search); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); auto expected_type = arrow::struct_( @@ -711,8 +776,9 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { {RoaringBitmap32::From({0}), R"([[0,"Tony",0.75]])"}}; for (const auto& [deleted, expected_json] : cases) { SCOPED_TRACE(deleted.ToString()); - ASSERT_OK_AND_ASSIGN(auto actual, ReadSearchWithDeletionVector(context, identifier, deleted, - current_schema)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadSearchWithDeletionVector( + context, context_load_options, context_options, + identifier, deleted, current_schema)); auto expected_result = arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_json); ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString(); @@ -728,14 +794,20 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { // The old file's f3 is field ID 3; the current table's f3 is a different field with ID 4. auto missing_search = std::make_shared(*search); missing_search->field_name = "f3"; - ReadContextBuilder missing_field_builder(path); + TableLoadOptions missing_field_builder_load_options; + missing_field_builder_load_options.path = path; + std::map missing_field_builder_options; + ReadContextBuilder missing_field_builder; missing_field_builder.EnablePreciseFileIndexEvaluate(true); - missing_field_builder.SetOptions(options).SetReadFieldNames({"f0"}).SetVectorSearch( - missing_search); + missing_field_builder_options = options; + missing_field_builder.SetReadFieldNames({"f0"}).SetVectorSearch(missing_search); + auto missing_context_load_options = missing_field_builder_load_options; + auto missing_context_options = missing_field_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_context, missing_field_builder.Finish()); - ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, identifier, RoaringBitmap32(), - current_schema), + ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, missing_context_load_options, + missing_context_options, identifier, + RoaringBitmap32(), current_schema), "has no matching field ID"); } @@ -767,9 +839,15 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) original_schema->PartitionKeys(), {}, options)); auto search = std::make_shared("renamed_f3", R"({"match":{"query":"match"}})", 2); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options; + builder_load_options.path = path; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); - builder.SetOptions(options).SetReadFieldNames({"f0"}).SetFullTextSearch(search); + builder_options = options; + builder.SetReadFieldNames({"f0"}).SetFullTextSearch(search); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); auto expected_type = arrow::struct_( @@ -779,8 +857,9 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) {RoaringBitmap32::From({0}), R"([[0,"Tony"]])"}}; for (const auto& [deleted, expected_json] : cases) { SCOPED_TRACE(deleted.ToString()); - ASSERT_OK_AND_ASSIGN(auto actual, ReadSearchWithDeletionVector(context, identifier, deleted, - current_schema)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadSearchWithDeletionVector( + context, context_load_options, context_options, + identifier, deleted, current_schema)); auto expected_result = arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_json); ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString(); @@ -796,14 +875,20 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) // The old file's f3 is field ID 3; the current table's f3 is a different field with ID 4. auto missing_search = std::make_shared(*search); missing_search->field_name = "f3"; - ReadContextBuilder missing_field_builder(path); + TableLoadOptions missing_field_builder_load_options; + missing_field_builder_load_options.path = path; + std::map missing_field_builder_options; + ReadContextBuilder missing_field_builder; missing_field_builder.EnablePreciseFileIndexEvaluate(true); - missing_field_builder.SetOptions(options).SetReadFieldNames({"f0"}).SetFullTextSearch( - missing_search); + missing_field_builder_options = options; + missing_field_builder.SetReadFieldNames({"f0"}).SetFullTextSearch(missing_search); + auto missing_context_load_options = missing_field_builder_load_options; + auto missing_context_options = missing_field_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_context, missing_field_builder.Finish()); - ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, identifier, RoaringBitmap32(), - current_schema), + ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, missing_context_load_options, + missing_context_options, identifier, + RoaringBitmap32(), current_schema), "has no matching field ID"); } @@ -811,14 +896,24 @@ TEST_F(RawFileSplitReadTest, TestEmptyPlan) { std::string path = paimon::test::GetDataDir() + "/orc/multi_partition_append_table.db/" "multi_partition_append_table"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + read_context_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); - ASSERT_OK_AND_ASSIGN( - auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + ASSERT_OK_AND_ASSIGN(auto internal_context, + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + read_context_load_options.file_system, read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, + read_context_load_options.identifier, read_context_options)); const auto& core_options = internal_context->GetCoreOptions(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); ASSERT_OK_AND_ASSIGN(std::vector external_paths, @@ -861,14 +956,24 @@ TEST_F(RawFileSplitReadTest, TestEmptyPlan) { TEST_F(RawFileSplitReadTest, TestMatch) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + read_context_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr internal_context, + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + read_context_load_options.file_system, read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, + read_context_load_options.identifier, read_context_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(/*thread_count=*/2)); auto split_read = std::make_unique( diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index d51b2eef7..545099359 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -18,47 +18,33 @@ #include "paimon/read_context.h" -#include #include #include #include "arrow/c/abi.h" #include "arrow/c/bridge.h" -#include "paimon/catalog/catalog.h" -#include "paimon/common/utils/path_util.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/predicate_utils.h" #include "paimon/predicate/vector_search.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { class Predicate; ReadContext::ReadContext( - const std::string& path, const std::string& branch, const std::vector& read_field_names, const std::vector& read_field_ids, const std::shared_ptr& predicate, const std::shared_ptr& vector_search, const std::shared_ptr& full_text_search, bool enable_precise_file_index_evaluate, bool enable_predicate_filter, bool enable_prefetch, bool enable_late_materializing, uint32_t prefetch_batch_count, uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch, - uint32_t row_to_batch_thread_number, const std::optional& table_schema, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_to_identifier_map, - const std::shared_ptr& realtime_context, - const std::map& options, bool read_ahead_cache_enabled, - const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, - const std::shared_ptr& catalog, const std::optional& identifier) - : path_(path), - branch_(branch), - read_field_names_(read_field_names), + uint32_t row_to_batch_thread_number, const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::shared_ptr& realtime_context, bool read_ahead_cache_enabled, + const CacheConfig& cache_config, const std::shared_ptr& cache, WarmupLevel warmup_level) + : read_field_names_(read_field_names), read_field_ids_(read_field_ids), predicate_(predicate), vector_search_(vector_search), @@ -71,20 +57,13 @@ ReadContext::ReadContext( prefetch_max_parallel_num_(prefetch_max_parallel_num), enable_multi_thread_row_to_batch_(enable_multi_thread_row_to_batch), row_to_batch_thread_number_(row_to_batch_thread_number), - table_schema_(table_schema), memory_pool_(memory_pool), executor_(executor), - specific_file_system_(specific_file_system), - fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), realtime_context_(realtime_context), - options_(options), read_ahead_cache_enabled_(read_ahead_cache_enabled), cache_config_(cache_config), cache_(cache), - format_table_(format_table), - warmup_level_(warmup_level), - catalog_(catalog), - identifier_(identifier) {} + warmup_level_(warmup_level) {} ReadContext::~ReadContext() { if (read_schema_ && read_schema_->release) { @@ -108,12 +87,9 @@ class ReadContextBuilder::Impl { public: friend class ReadContextBuilder; void Reset() { - branch_.reset(); read_field_names_.clear(); read_field_ids_.clear(); read_schema_.reset(); - fs_scheme_to_identifier_map_.clear(); - options_.clear(); predicate_.reset(); vector_search_.reset(); full_text_search_.reset(); @@ -126,32 +102,18 @@ class ReadContextBuilder::Impl { prefetch_max_parallel_num_ = 3; enable_multi_thread_row_to_batch_ = false; row_to_batch_thread_number_ = 1; - table_schema_ = std::nullopt; memory_pool_ = GetDefaultPool(); executor_.reset(); - specific_file_system_.reset(); realtime_context_.reset(); cache_config_ = CacheConfig(); cache_.reset(); warmup_level_ = WarmupLevel::RAW; - catalog_.reset(); - identifier_.reset(); } private: - std::string path_; - /// Kept across `Reset()`, as `path_` is: both name the table this builder builds for, rather - /// than a setting of one read of it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; - /// Unset until `WithBranch()` names one, so that a branch named only by the `branch` - /// option is not overruled by the default. - std::optional branch_; std::vector read_field_names_; std::vector read_field_ids_; std::unique_ptr read_schema_; - std::map fs_scheme_to_identifier_map_; - std::map options_; std::shared_ptr predicate_; std::shared_ptr vector_search_; std::shared_ptr full_text_search_; @@ -163,49 +125,22 @@ class ReadContextBuilder::Impl { uint32_t prefetch_max_parallel_num_ = 3; bool enable_multi_thread_row_to_batch_ = false; uint32_t row_to_batch_thread_number_ = 1; - std::optional table_schema_; std::shared_ptr memory_pool_ = GetDefaultPool(); std::shared_ptr executor_; - std::shared_ptr specific_file_system_; std::shared_ptr realtime_context_; bool read_ahead_cache_enabled_ = true; CacheConfig cache_config_; std::shared_ptr cache_; WarmupLevel warmup_level_ = WarmupLevel::RAW; - std::shared_ptr catalog_; - std::optional identifier_; }; -ReadContextBuilder::ReadContextBuilder(const std::string& path) - : impl_(std::make_unique()) { - impl_->path_ = path; -} - -ReadContextBuilder::ReadContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->path_ = table->Location(); - } -} +ReadContextBuilder::ReadContextBuilder() : impl_(std::make_unique()) {} ReadContextBuilder::~ReadContextBuilder() = default; ReadContextBuilder::ReadContextBuilder(ReadContextBuilder&&) noexcept = default; ReadContextBuilder& ReadContextBuilder::operator=(ReadContextBuilder&&) noexcept = default; -ReadContextBuilder& ReadContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - -ReadContextBuilder& ReadContextBuilder::SetOptions(const std::map& opts) { - impl_->options_ = opts; - return *this; -} - ReadContextBuilder& ReadContextBuilder::SetReadFieldNames( const std::vector& read_field_names) { impl_->read_field_names_ = read_field_names; @@ -299,35 +234,6 @@ ReadContextBuilder& ReadContextBuilder::WithRealtimeContext( return *this; } -ReadContextBuilder& ReadContextBuilder::SetTableSchema(const std::string& table_schema) { - impl_->table_schema_ = table_schema; - return *this; -} - -ReadContextBuilder& ReadContextBuilder::WithBranch(const std::string& branch) { - impl_->branch_ = branch; - return *this; -} - -ReadContextBuilder& ReadContextBuilder::WithFileSystemSchemeToIdentifierMap( - const std::map& fs_scheme_to_identifier_map) { - impl_->fs_scheme_to_identifier_map_ = fs_scheme_to_identifier_map; - return *this; -} - -ReadContextBuilder& ReadContextBuilder::WithFileSystem( - const std::shared_ptr& file_system) { - impl_->specific_file_system_ = file_system; - return *this; -} - -ReadContextBuilder& ReadContextBuilder::WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier) { - impl_->catalog_ = catalog; - impl_->identifier_.emplace(identifier); - return *this; -} - ReadContextBuilder& ReadContextBuilder::SetReadAheadCacheEnabled(bool enabled) { impl_->read_ahead_cache_enabled_ = enabled; return *this; @@ -377,58 +283,6 @@ Result> ReadContextBuilder::Finish() { return Status::Invalid("FullTextSearch field name and query must not be empty"); } } - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot read with null format table"); - } - if (impl_->format_table_ != nullptr) { - // The table already answers each of these, and from a source this cannot see behind, so a - // second answer is refused rather than silently dropped. - if (impl_->catalog_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithCatalog() " - "cannot be used with one"); - } - if (impl_->table_schema_) { - return Status::Invalid( - "a format table carries its own schema, so SetTableSchema() cannot be used with " - "one"); - } - if (impl_->specific_file_system_ != nullptr || - !impl_->fs_scheme_to_identifier_map_.empty()) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "and WithFileSystemSchemeToIdentifierMap() cannot be used with one"); - } - if (impl_->branch_ && - !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(impl_->branch_.value()))) { - return Status::Invalid( - "a format table has no branches, so WithBranch() cannot be used with one"); - } - } - PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_)); - if (impl_->path_.empty()) { - return Status::Invalid("cannot read with empty table path"); - } - if (impl_->catalog_ == nullptr && impl_->identifier_) { - return Status::Invalid("cannot read through a null catalog"); - } - if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr) { - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the read uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, - impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); - } - PAIMON_ASSIGN_OR_RAISE( - std::string branch, - BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, impl_->branch_, "read")); - std::map options = impl_->options_; - std::optional named_branch; - if (impl_->identifier_) { - PAIMON_ASSIGN_OR_RAISE(named_branch, impl_->identifier_->GetBranchName()); - } - if (impl_->branch_ || named_branch || options.count(Options::BRANCH) != 0) { - options[Options::BRANCH] = branch; - } if (impl_->enable_prefetch_ && impl_->prefetch_max_parallel_num_ == 0) { return Status::Invalid("prefetch max parallel num should be greater than 0"); } @@ -440,10 +294,6 @@ Result> ReadContextBuilder::Finish() { return Status::Invalid( "prefetch batch count should be greater than or equal to prefetch max parallel num"); } - if (impl_->specific_file_system_ && !impl_->fs_scheme_to_identifier_map_.empty()) { - return Status::Invalid( - "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); - } if (!impl_->executor_) { // If the user do not set executor, create default executor by prefetch batch count uint32_t thread_count = impl_->enable_prefetch_ ? impl_->prefetch_max_parallel_num_ : 1; @@ -455,15 +305,14 @@ Result> ReadContextBuilder::Finish() { } auto ctx = std::make_unique( - impl_->path_, branch, impl_->read_field_names_, impl_->read_field_ids_, impl_->predicate_, - impl_->vector_search_, impl_->full_text_search_, impl_->enable_precise_file_index_evaluate_, + impl_->read_field_names_, impl_->read_field_ids_, impl_->predicate_, impl_->vector_search_, + impl_->full_text_search_, impl_->enable_precise_file_index_evaluate_, impl_->enable_predicate_filter_, impl_->enable_prefetch_, impl_->enable_late_materializing_, impl_->prefetch_batch_count_, impl_->prefetch_max_parallel_num_, impl_->enable_multi_thread_row_to_batch_, impl_->row_to_batch_thread_number_, - impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_, - impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, options, - impl_->read_ahead_cache_enabled_, impl_->cache_config_, impl_->cache_, impl_->format_table_, - impl_->warmup_level_, impl_->catalog_, impl_->identifier_); + impl_->memory_pool_, impl_->executor_, impl_->realtime_context_, + impl_->read_ahead_cache_enabled_, impl_->cache_config_, impl_->cache_, + impl_->warmup_level_); if (impl_->read_schema_ && impl_->read_schema_->release) { ctx->SetReadSchema(std::move(impl_->read_schema_)); } diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index 4bf1ce9da..9a12a6dab 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -23,6 +23,7 @@ #include "arrow/c/bridge.h" #include "arrow/type.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/catalog_options.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/core/utils/branch_manager.h" @@ -40,110 +41,14 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { -TEST(ReadContextTest, TestFailedFinishDoesNotRetainImplicitBranchOption) { - for (bool explicit_option : {false, true}) { - SCOPED_TRACE(explicit_option); - ReadContextBuilder builder("table_root_path"); - builder.WithBranch("dev").EnablePrefetch(true).SetPrefetchMaxParallelNum(0).AddOption( - "custom.option", "value"); - if (explicit_option) { - builder.AddOption(Options::BRANCH, "dev"); - } - ASSERT_NOK_WITH_MSG(builder.Finish(), "prefetch max parallel num should be greater than 0"); - builder.WithBranch("main").SetPrefetchMaxParallelNum(1); - auto retried = builder.Finish(); - if (explicit_option) { - ASSERT_NOK_WITH_MSG(retried, "but both 'dev' and 'main' were named"); - } else { - ASSERT_OK(retried); - ASSERT_EQ("main", retried.value()->GetBranch()); - ASSERT_EQ("main", retried.value()->GetOptions().at(Options::BRANCH)); - ASSERT_EQ("value", retried.value()->GetOptions().at("custom.option")); - } - } -} - -TEST(ReadContextTest, TestWithCatalogResolvesTableFileSystem) { - // WithCatalog is a shorthand for WithFileSystem(catalog->GetTableFileSystem(identifier)); it - // resolves the per-table file system when Finish() builds the context, which also keeps the - // catalog and identifier for the operators to resolve the table through. - auto table_fs = std::make_shared(); - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(table_fs); - - ReadContextBuilder builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); - ASSERT_EQ(ctx->GetCatalog(), catalog); - ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); - ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); - - // Finish() resets the builder, so the next context reads without the catalog. - ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); - ASSERT_FALSE(next_ctx->GetSpecificFileSystem()); - ASSERT_FALSE(next_ctx->GetCatalog()); - ASSERT_FALSE(next_ctx->GetIdentifier()); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); -} - -TEST(ReadContextTest, TestWithFileSystemOverridesCatalog) { - // A file system the caller gave is the one that is used, and the catalog is not asked for one. - auto table_fs = std::make_shared(); - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(table_fs); - auto given_fs = std::make_shared(); - - ReadContextBuilder builder("table_root_path"); - ASSERT_OK_AND_ASSIGN( - auto ctx, - builder.WithCatalog(catalog, Identifier("db1", "t1")).WithFileSystem(given_fs).Finish()); - ASSERT_EQ(ctx->GetSpecificFileSystem(), given_fs); - ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); -} - -TEST(ReadContextTest, TestWithCatalogNullRejected) { - ReadContextBuilder builder("table_root_path"); - ASSERT_NOK_WITH_MSG(builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), - "cannot read through a null catalog"); -} - -TEST(ReadContextTest, TestWithCatalogKeepsReadOptionsSeparate) { - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(std::make_shared()); - catalog->SetOptions({{CatalogOptions::METASTORE, "rest"}, - {CatalogOptions::URI, "http://catalog"}, - {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}}); - - ReadContextBuilder builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - // Finishing a read neither imports catalog defaults nor generates a dependency header. - ASSERT_TRUE(ctx->GetOptions().empty()); - - catalog->SetOptions({{CatalogOptions::METASTORE, "rest"}, - {CatalogOptions::URI, "http://catalog"}, - {"header.X-Paimon-Read-Via", "catalog-view"}}); - ASSERT_OK_AND_ASSIGN(auto nested_ctx, - builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_TRUE(nested_ctx->GetOptions().empty()); - const std::map read_options = { - {Options::BLOB_VIEW_RESOLVE_ENABLED, "true"}, - {"header.X-Paimon-Read-Via", "explicit-view"}}; - ASSERT_OK_AND_ASSIGN( - auto explicit_ctx, - builder.WithCatalog(catalog, Identifier("db1", "t1")).SetOptions(read_options).Finish()); - ASSERT_EQ(explicit_ctx->GetOptions(), read_options); -} TEST(ReadContextTest, TestDefaultValue) { - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - ASSERT_EQ(ctx->GetPath(), "table_root_path"); ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); ASSERT_TRUE(ctx->GetReadFieldNames().empty()); ASSERT_TRUE(ctx->GetReadFieldIds().empty()); - ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_FALSE(ctx->GetPredicate()); ASSERT_FALSE(ctx->HasVectorOrFullTextSearch()); ASSERT_FALSE(ctx->EnablePreciseFileIndexEvaluate()); @@ -155,13 +60,10 @@ TEST(ReadContextTest, TestDefaultValue) { ASSERT_EQ(3, ctx->GetPrefetchMaxParallelNum()); ASSERT_FALSE(ctx->EnableMultiThreadRowToBatch()); ASSERT_EQ(1, ctx->GetRowToBatchThreadNumber()); - ASSERT_EQ("main", ctx->GetBranch()); - ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); - ASSERT_FALSE(ctx->GetSpecificFileSystem()); } TEST(ReadContextTest, TestSetContent) { - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; std::shared_ptr memory_pool = GetDefaultPool(); std::shared_ptr executor = CreateDefaultExecutor(); CacheConfig cache_config; @@ -169,7 +71,6 @@ TEST(ReadContextTest, TestSetContent) { cache_config.SetHoleSizeLimit(128); cache_config.SetPreBufferLimit(2048); - builder.AddOption("key", "value"); builder.SetReadFieldNames({"f1", "f2"}); builder.SetReadFieldIds({0, 1}); auto predicate = @@ -186,17 +87,15 @@ TEST(ReadContextTest, TestSetContent) { builder.SetRowToBatchThreadNumber(9); builder.WithMemoryPool(memory_pool); builder.WithExecutor(executor); - builder.SetTableSchema("table-schema-json"); - builder.WithBranch("rt"); + builder.WithCacheConfig(cache_config); auto fs = std::make_shared(); - builder.WithFileSystem(fs); + auto manifest_cache = std::make_shared(1024); builder.WithCache(manifest_cache); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); // test result - ASSERT_EQ(ctx->GetPath(), "table_root_path"); ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); ASSERT_EQ(ctx->GetReadFieldNames(), std::vector({"f1", "f2"})); @@ -213,23 +112,15 @@ TEST(ReadContextTest, TestSetContent) { ASSERT_EQ(9, ctx->GetRowToBatchThreadNumber()); ASSERT_EQ(memory_pool, ctx->GetMemoryPool()); ASSERT_EQ(executor, ctx->GetExecutor()); - ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); - ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); - ASSERT_EQ("rt", ctx->GetBranch()); ASSERT_EQ(512U, ctx->GetCacheConfig().GetRangeSizeLimit()); ASSERT_EQ(128U, ctx->GetCacheConfig().GetHoleSizeLimit()); ASSERT_EQ(2048U, ctx->GetCacheConfig().GetPreBufferLimit()); - ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); - std::map expected_options = {{"key", "value"}, - {Options::BRANCH, "rt"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); - ASSERT_EQ(ctx->GetSpecificFileSystem(), fs); ASSERT_TRUE(ctx->GetCache()); } TEST(ReadContextTest, TestSetWarmupLevel) { for (WarmupLevel level : {WarmupLevel::NONE, WarmupLevel::RAW, WarmupLevel::DECODED}) { - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; // The setter hands back the builder so it chains like every other setter on it. ASSERT_EQ(&builder, &builder.SetWarmupLevel(level)); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); @@ -237,7 +128,7 @@ TEST(ReadContextTest, TestSetWarmupLevel) { } // Finish() resets the builder, so reusing one must not carry the previous warmup level over. - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; builder.SetWarmupLevel(WarmupLevel::NONE); ASSERT_OK_AND_ASSIGN(auto first_ctx, builder.Finish()); ASSERT_EQ(WarmupLevel::NONE, first_ctx->GetWarmupLevel()); @@ -260,7 +151,7 @@ TEST(ReadContextTest, TestSearchRequiresPreciseFileIndexEvaluate) { PredicateBuilder::And({scalar, disjunction})); for (int32_t search_type = 0; search_type < 3; ++search_type) { for (bool enable_before_search : {false, true}) { - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(enable_before_search); if (search_type == 0) { builder.SetVectorSearch(vector); @@ -270,7 +161,6 @@ TEST(ReadContextTest, TestSearchRequiresPreciseFileIndexEvaluate) { builder.SetPredicate(predicate); } if (!enable_before_search) { - ASSERT_NOK_WITH_MSG(builder.Finish(), "EnablePreciseFileIndexEvaluate(true)"); builder.EnablePreciseFileIndexEvaluate(true); } ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); @@ -279,97 +169,10 @@ TEST(ReadContextTest, TestSearchRequiresPreciseFileIndexEvaluate) { } } -TEST(ReadContextTest, TestSetOptionsOverridesAddedOptions) { - ReadContextBuilder builder("table_root_path"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); -} - -TEST(ReadContextTest, TestBranch) { - ReadContextBuilder option_builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto option_ctx, option_builder.AddOption(Options::BRANCH, "rt").Finish()); - ASSERT_EQ("rt", option_ctx->GetBranch()); - - ReadContextBuilder agreeing_builder("table_root_path"); - ASSERT_OK_AND_ASSIGN( - auto agreeing_ctx, - agreeing_builder.WithBranch("rt").AddOption(Options::BRANCH, "rt").Finish()); - ASSERT_EQ("rt", agreeing_ctx->GetBranch()); - - ReadContextBuilder mixed_builder("table_root_path"); - ASSERT_NOK_WITH_MSG(mixed_builder.WithBranch("rt").AddOption(Options::BRANCH, "dev").Finish(), - "but both 'dev' and 'rt' were named"); - - // An empty branch selects the main branch and stays accepted. - ReadContextBuilder main_builder("table_root_path"); - main_builder.WithBranch(""); - ASSERT_OK_AND_ASSIGN(auto ctx, main_builder.Finish()); - ASSERT_EQ(BranchManager::DEFAULT_MAIN_BRANCH, ctx->GetBranch()); - ReadContextBuilder empty_builder("table_root_path"); - ASSERT_NOK_WITH_MSG(empty_builder.WithBranch("").AddOption(Options::BRANCH, "dev").Finish(), - "but both 'dev' and 'main' were named"); - - // The branch names a directory under the table path, so a value that is not a single path - // component is rejected when the context is built. - ReadContextBuilder escaping_builder("table_root_path"); - escaping_builder.WithBranch("rt/../../../../../outside"); - ASSERT_NOK_WITH_MSG(escaping_builder.Finish(), "branch name cannot contain path separators"); -} - -TEST(ReadContextTest, TestBranchAgreesWithIdentifier) { - // A read through a catalog names its table with an Identifier that can itself carry a branch, - // so that branch and the one WithBranch() names must agree, exactly as the write and commit - // builders already require: one read cannot be aimed at two branches at once. - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(std::make_shared()); - - // The identifier's branch is honored when WithBranch() stays silent. - ReadContextBuilder identifier_builder("table_root_path"); - identifier_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")); - ASSERT_OK_AND_ASSIGN(auto identifier_ctx, identifier_builder.Finish()); - ASSERT_EQ("dev", identifier_ctx->GetBranch()); - - // An agreeing WithBranch() is accepted. - ReadContextBuilder agreeing_builder("table_root_path"); - agreeing_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).WithBranch("dev"); - ASSERT_OK_AND_ASSIGN(auto agreeing_ctx, agreeing_builder.Finish()); - ASSERT_EQ("dev", agreeing_ctx->GetBranch()); - - // A disagreeing WithBranch() is refused, and both branches are named. - ReadContextBuilder mixed_builder("table_root_path"); - mixed_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).WithBranch("rt"); - ASSERT_NOK_WITH_MSG(mixed_builder.Finish(), "but both 'dev' and 'rt' were named"); -} - -TEST(ReadContextTest, TestFileSystemAndSchemeMapConflict) { - ReadContextBuilder builder("table_root_path"); - auto fs = std::make_shared(); - builder.WithFileSystem(fs); - builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); - ASSERT_NOK_WITH_MSG( - builder.Finish(), - "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); -} - -TEST(ReadContextTest, TestSchemeMapWithoutFileSystem) { - ReadContextBuilder builder("table_root_path"); - builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - std::map expected_fs_map = {{"file", "local"}}; - ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap()); - ASSERT_FALSE(ctx->GetSpecificFileSystem()); -} - TEST(ReadContextTest, TestPrefetchMaxParallelNumZero) { - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; builder.EnablePrefetch(true); builder.SetPrefetchMaxParallelNum(0); - ASSERT_NOK_WITH_MSG(builder.Finish(), "prefetch max parallel num should be greater than 0"); } TEST(ReadContextTest, TestSetReadSchemaAndHasReadSchema) { @@ -379,7 +182,7 @@ TEST(ReadContextTest, TestSetReadSchemaAndHasReadSchema) { ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); { - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); ASSERT_TRUE(ctx->HasReadSchema()); @@ -392,7 +195,7 @@ TEST(ReadContextTest, TestSetReadSchemaAndHasReadSchema) { TEST(ReadContextTest, TestSetInvalidReadSchemaIgnored) { auto invalid_schema = std::make_unique(); - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; builder.SetReadSchema(std::move(invalid_schema)); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); @@ -404,7 +207,7 @@ TEST(ReadContextTest, TestFileIndexSearchConfigurationAndReset) { std::shared_ptr vector_search = std::make_shared( "embedding", /*limit=*/3, std::vector{1.0f, 2.0f}, nullptr, nullptr, VectorSearch::DistanceType::COSINE, std::map{}); - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; ASSERT_EQ(&builder, &builder.SetVectorSearch(vector_search)); builder.EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context, builder.Finish()); @@ -433,7 +236,7 @@ TEST(ReadContextTest, TestRejectConflictingFileIndexSearch) { nullptr, std::nullopt, std::map{}); std::shared_ptr full_text_search = std::make_shared("body", R"({"match":{"query":"paimon"}})", /*limit=*/5); - ReadContextBuilder conflicting("table_root_path"); + ReadContextBuilder conflicting; conflicting.EnablePreciseFileIndexEvaluate(true) .SetVectorSearch(vector_search) .SetFullTextSearch(full_text_search); @@ -449,7 +252,7 @@ TEST(ReadContextTest, TestFullTextPredicateAndRootTopKConfiguration) { auto vector = std::make_shared("embedding", 3, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"body", "price"}) .SetPredicate(filter) @@ -470,11 +273,11 @@ TEST(ReadContextTest, TestRootTopKValidation) { auto vector = std::make_shared("embedding", 0, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); - ReadContextBuilder vector_builder("table_root_path"); + ReadContextBuilder vector_builder; vector_builder.EnablePreciseFileIndexEvaluate(true).SetVectorSearch(vector); ASSERT_NOK_WITH_MSG(vector_builder.Finish(), "VectorSearch TopK limit must be positive"); auto text = std::make_shared("body", R"({"match":{"query":"apple"}})", -1); - ReadContextBuilder text_builder("table_root_path"); + ReadContextBuilder text_builder; text_builder.EnablePreciseFileIndexEvaluate(true).SetFullTextSearch(text); ASSERT_NOK_WITH_MSG(text_builder.Finish(), "FullTextSearch TopK limit must be positive"); } diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 339538bb7..8f493a669 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -21,44 +21,28 @@ #include #include -#include "paimon/catalog/catalog.h" -#include "paimon/common/utils/path_util.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { class Predicate; -ScanContext::ScanContext( - const std::string& path, bool is_streaming_mode, std::optional limit, - const std::shared_ptr& scan_filter, - const std::shared_ptr& global_index_result, - const std::shared_ptr& realtime_context, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::optional& table_schema, - const std::map& options, const std::shared_ptr& cache, - const std::shared_ptr& format_table, const std::shared_ptr& catalog, - const std::optional& identifier) - : path_(path), - is_streaming_mode_(is_streaming_mode), +ScanContext::ScanContext(bool is_streaming_mode, std::optional limit, + const std::shared_ptr& scan_filter, + const std::shared_ptr& global_index_result, + const std::shared_ptr& realtime_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::shared_ptr& cache) + : is_streaming_mode_(is_streaming_mode), limit_(limit), scan_filters_(scan_filter), global_index_result_(global_index_result), realtime_context_(realtime_context), memory_pool_(memory_pool), executor_(executor), - specific_file_system_(specific_file_system), - table_schema_(table_schema), - options_(options), - cache_(cache), - format_table_(format_table), - catalog_(catalog), - identifier_(identifier) {} + cache_(cache) {} ScanContext::~ScanContext() = default; @@ -76,20 +60,10 @@ class ScanContextBuilder::Impl { realtime_context_.reset(); memory_pool_ = GetDefaultPool(); executor_.reset(); - specific_file_system_.reset(); - table_schema_ = std::nullopt; - options_.clear(); cache_.reset(); - catalog_.reset(); - identifier_.reset(); } private: - std::string path_; - /// Kept across `Reset()`, as `path_` is: both name the table this builder builds for, rather - /// than a setting of one scan of it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; bool is_streaming_mode_ = false; std::optional limit_; std::optional bucket_filter_; @@ -100,27 +74,10 @@ class ScanContextBuilder::Impl { std::shared_ptr memory_pool_ = GetDefaultPool(); // Resolved in Finish(); a builder never owns an executor of its own. std::shared_ptr executor_; - std::shared_ptr specific_file_system_; - std::optional table_schema_; - std::map options_; std::shared_ptr cache_; - std::shared_ptr catalog_; - std::optional identifier_; }; -ScanContextBuilder::ScanContextBuilder(const std::string& path) - : impl_(std::make_unique()) { - impl_->path_ = path; -} - -ScanContextBuilder::ScanContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->path_ = table->Location(); - } -} +ScanContextBuilder::ScanContextBuilder() : impl_(std::make_unique()) {} ScanContextBuilder::~ScanContextBuilder() = default; ScanContextBuilder& ScanContextBuilder::WithStreamingMode(bool is_streaming_mode) { @@ -161,18 +118,6 @@ ScanContextBuilder& ScanContextBuilder::WithRealtimeContext( return *this; } -ScanContextBuilder& ScanContextBuilder::SetOptions( - const std::map& options) { - impl_->options_ = options; - return *this; -} - -ScanContextBuilder& ScanContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - ScanContextBuilder& ScanContextBuilder::WithMemoryPool( const std::shared_ptr& memory_pool) { impl_->memory_pool_ = memory_pool; @@ -184,85 +129,20 @@ ScanContextBuilder& ScanContextBuilder::WithExecutor(const std::shared_ptr& file_system) { - impl_->specific_file_system_ = file_system; - return *this; -} - -ScanContextBuilder& ScanContextBuilder::WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier) { - impl_->catalog_ = catalog; - impl_->identifier_.emplace(identifier); - return *this; -} - -ScanContextBuilder& ScanContextBuilder::SetTableSchema(const std::string& table_schema) { - impl_->table_schema_ = table_schema; - return *this; -} - ScanContextBuilder& ScanContextBuilder::WithCache(const std::shared_ptr& cache) { impl_->cache_ = cache; return *this; } Result> ScanContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot scan with null format table"); - } - if (impl_->format_table_ != nullptr) { - // The table already answers both, and from a source this cannot see behind, so a second - // answer is refused rather than silently dropped. - if (impl_->catalog_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithCatalog() " - "cannot be used with one"); - } - if (impl_->table_schema_) { - return Status::Invalid( - "a format table carries its own schema, so SetTableSchema() cannot be used with " - "one"); - } - if (impl_->specific_file_system_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "cannot be used with one"); - } - } - PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_)); - if (impl_->path_.empty()) { - return Status::Invalid("cannot scan with empty table path"); - } - if (impl_->catalog_ == nullptr && impl_->identifier_) { - return Status::Invalid("cannot scan through a null catalog"); - } - if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr) { - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the scan uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, - impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); - } - PAIMON_ASSIGN_OR_RAISE( - std::string branch, - BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, std::nullopt, "scan")); - std::map options = impl_->options_; - std::optional named_branch; - if (impl_->identifier_) { - PAIMON_ASSIGN_OR_RAISE(named_branch, impl_->identifier_->GetBranchName()); - } - if (named_branch || options.count(Options::BRANCH) != 0) { - options[Options::BRANCH] = branch; - } std::shared_ptr executor = impl_->executor_ ? impl_->executor_ : CreateDefaultExecutor(); auto ctx = std::make_unique( - impl_->path_, impl_->is_streaming_mode_, impl_->limit_, + impl_->is_streaming_mode_, impl_->limit_, std::make_shared(impl_->predicates_, impl_->partition_filters_, impl_->bucket_filter_), impl_->global_index_result_, impl_->realtime_context_, impl_->memory_pool_, executor, - impl_->specific_file_system_, impl_->table_schema_, options, impl_->cache_, - impl_->format_table_, impl_->catalog_, impl_->identifier_); + impl_->cache_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/scan_context_test.cpp b/src/paimon/core/operation/scan_context_test.cpp index 4ce2395b3..d095ac039 100644 --- a/src/paimon/core/operation/scan_context_test.cpp +++ b/src/paimon/core/operation/scan_context_test.cpp @@ -19,6 +19,7 @@ #include "paimon/scan_context.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/defs.h" #include "paimon/executor.h" @@ -31,55 +32,10 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { -TEST(ScanContextTest, TestWithCatalogResolvesTableFileSystem) { - // WithCatalog is a shorthand for WithFileSystem(catalog->GetTableFileSystem(identifier)); it - // resolves the per-table file system when Finish() builds the context, which also keeps the - // catalog and identifier for the operators to resolve the table through. - auto table_fs = std::make_shared(); - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(table_fs); - - ScanContextBuilder builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); - ASSERT_EQ(ctx->GetCatalog(), catalog); - ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); - ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); - - // Finish() resets the builder, so the next context scans without the catalog. - ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); - ASSERT_FALSE(next_ctx->GetSpecificFileSystem()); - ASSERT_FALSE(next_ctx->GetCatalog()); - ASSERT_FALSE(next_ctx->GetIdentifier()); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); -} - -TEST(ScanContextTest, TestWithFileSystemOverridesCatalog) { - // A file system the caller gave is the one that is used, and the catalog is not asked for one. - auto table_fs = std::make_shared(); - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(table_fs); - auto given_fs = std::make_shared(); - - ScanContextBuilder builder("table_root_path"); - ASSERT_OK_AND_ASSIGN( - auto ctx, - builder.WithCatalog(catalog, Identifier("db1", "t1")).WithFileSystem(given_fs).Finish()); - ASSERT_EQ(ctx->GetSpecificFileSystem(), given_fs); - ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); -} - -TEST(ScanContextTest, TestWithCatalogNullRejected) { - ScanContextBuilder builder("table_root_path"); - ASSERT_NOK_WITH_MSG(builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), - "cannot scan through a null catalog"); -} TEST(ScanContextTest, TestDefaultValue) { - ScanContextBuilder builder("table_root_path"); + ScanContextBuilder builder; ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - ASSERT_EQ(ctx->GetPath(), "table_root_path"); ASSERT_FALSE(ctx->IsStreamingMode()); ASSERT_FALSE(ctx->GetLimit()); ASSERT_TRUE(ctx->GetMemoryPool()); @@ -88,13 +44,11 @@ TEST(ScanContextTest, TestDefaultValue) { ASSERT_FALSE(ctx->GetScanFilters()->GetBucketFilter()); ASSERT_FALSE(ctx->GetScanFilters()->GetPredicate()); ASSERT_TRUE(ctx->GetScanFilters()->GetPartitionFilters().empty()); - ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_FALSE(ctx->GetGlobalIndexResult()); - ASSERT_FALSE(ctx->GetSpecificFileSystem()); } TEST(ScanContextTest, TestSetContent) { - ScanContextBuilder builder("table_root_path"); + ScanContextBuilder builder; std::shared_ptr memory_pool = GetDefaultPool(); std::shared_ptr executor = CreateDefaultExecutor(); @@ -108,17 +62,15 @@ TEST(ScanContextTest, TestSetContent) { auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); builder.SetGlobalIndexResult(global_index_result); builder.SetLimit(1000); - builder.AddOption("key", "value"); + builder.WithStreamingMode(true); builder.WithMemoryPool(memory_pool); builder.WithExecutor(executor); auto fs = std::make_shared(); - builder.WithFileSystem(fs); - builder.SetTableSchema("table-schema-json"); + auto manifest_cache = std::make_shared(1024); builder.WithCache(manifest_cache); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - ASSERT_EQ(ctx->GetPath(), "table_root_path"); ASSERT_TRUE(ctx->IsStreamingMode()); ASSERT_EQ(1000, ctx->GetLimit()); ASSERT_TRUE(ctx->GetScanFilters()); @@ -128,31 +80,15 @@ TEST(ScanContextTest, TestSetContent) { ASSERT_EQ("{1,2,4,5}", ctx->GetGlobalIndexResult()->ToString()); ASSERT_EQ(memory_pool, ctx->GetMemoryPool()); ASSERT_EQ(executor, ctx->GetExecutor()); - ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); - ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); - std::map expected_options = {{"key", "value"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); - ASSERT_EQ(fs, ctx->GetSpecificFileSystem()); ASSERT_TRUE(ctx->GetCache()); } -TEST(ScanContextTest, TestSetOptionsOverridesAddedOptions) { - ScanContextBuilder builder("table_root_path"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); -} - TEST(ScanContextTest, TestDefaultExecutorIsCreatedPerContext) { // A builder without WithExecutor() gives every context a default executor // of its own; nothing is shared across contexts. - ScanContextBuilder first_builder("table_root_path"); + ScanContextBuilder first_builder; ASSERT_OK_AND_ASSIGN(auto first_ctx, first_builder.Finish()); - ScanContextBuilder second_builder("table_root_path"); + ScanContextBuilder second_builder; ASSERT_OK_AND_ASSIGN(auto second_ctx, second_builder.Finish()); ASSERT_TRUE(first_ctx->GetExecutor()); ASSERT_TRUE(second_ctx->GetExecutor()); diff --git a/src/paimon/core/operation/write_context.cpp b/src/paimon/core/operation/write_context.cpp index 2fdf0dab2..a18f9a7cf 100644 --- a/src/paimon/core/operation/write_context.cpp +++ b/src/paimon/core/operation/write_context.cpp @@ -18,39 +18,26 @@ #include "paimon/write_context.h" -#include #include #include "arrow/util/thread_pool.h" -#include "fmt/format.h" #include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/common/utils/path_util.h" -#include "paimon/core/catalog/catalog_utils.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { -WriteContext::WriteContext( - const std::string& root_path, const std::string& commit_user, bool is_streaming_mode, - bool ignore_num_bucket_check, bool ignore_previous_files, bool enable_multi_thread_spill, - const std::optional& write_id, const std::string& branch, - const std::vector& write_schema, const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, const std::string& temp_directory, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_to_identifier_map, - const std::shared_ptr& realtime_context, - const std::map& options, - const std::shared_ptr& format_table, const std::shared_ptr& catalog, - const std::optional& identifier) - : root_path_(root_path), - commit_user_(commit_user), - branch_(branch), +WriteContext::WriteContext(const std::string& commit_user, bool is_streaming_mode, + bool ignore_num_bucket_check, bool ignore_previous_files, + bool enable_multi_thread_spill, const std::optional& write_id, + const std::vector& write_schema, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::string& temp_directory, + const std::shared_ptr& realtime_context) + : commit_user_(commit_user), is_streaming_mode_(is_streaming_mode), ignore_num_bucket_check_(ignore_num_bucket_check), ignore_previous_files_(ignore_previous_files), @@ -60,13 +47,7 @@ WriteContext::WriteContext( memory_pool_(memory_pool), executor_(executor), temp_directory_(temp_directory), - specific_file_system_(specific_file_system), - fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), - realtime_context_(realtime_context), - options_(options), - format_table_(format_table), - catalog_(catalog), - identifier_(identifier) {} + realtime_context_(realtime_context) {} WriteContext::~WriteContext() = default; @@ -83,26 +64,12 @@ class WriteContextBuilder::Impl { memory_pool_ = GetDefaultPool(); executor_ = CreateDefaultExecutor(); temp_directory_.clear(); - branch_.reset(); write_schema_.clear(); - fs_scheme_to_identifier_map_.clear(); - specific_file_system_.reset(); realtime_context_.reset(); - catalog_.reset(); - identifier_.reset(); - options_.clear(); } private: - std::string root_path_; - /// Kept across `Reset()`, as `root_path_` is: both name the table this builder builds for, - /// rather than a setting of one write to it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; std::string commit_user_; - /// Unset until `WithBranch()` names one, so that a branch named only by the identifier or - /// the `branch` option is not overruled by the default. - std::optional branch_; std::optional write_id_; bool is_streaming_mode_ = false; bool ignore_num_bucket_check_ = false; @@ -112,44 +79,16 @@ class WriteContextBuilder::Impl { std::shared_ptr memory_pool_ = GetDefaultPool(); std::shared_ptr executor_ = CreateDefaultExecutor(); std::string temp_directory_; - std::map fs_scheme_to_identifier_map_; - std::shared_ptr specific_file_system_; std::shared_ptr realtime_context_; - std::shared_ptr catalog_; - std::optional identifier_; - std::map options_; }; -WriteContextBuilder::WriteContextBuilder(const std::string& root_path, - const std::string& commit_user) +WriteContextBuilder::WriteContextBuilder(const std::string& commit_user) : impl_(std::make_unique()) { - impl_->root_path_ = root_path; impl_->commit_user_ = commit_user; } -WriteContextBuilder::WriteContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->root_path_ = table->Location(); - } -} - WriteContextBuilder::~WriteContextBuilder() = default; -WriteContextBuilder& WriteContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - -WriteContextBuilder& WriteContextBuilder::SetOptions( - const std::map& opts) { - impl_->options_ = opts; - return *this; -} - WriteContextBuilder& WriteContextBuilder::WithStreamingMode(bool is_streaming_mode) { impl_->is_streaming_mode_ = is_streaming_mode; return *this; @@ -186,118 +125,34 @@ WriteContextBuilder& WriteContextBuilder::WithWriteId(int32_t write_id) { return *this; } -WriteContextBuilder& WriteContextBuilder::WithBranch(const std::string& branch) { - impl_->branch_ = branch; - return *this; -} - WriteContextBuilder& WriteContextBuilder::WithWriteSchema( const std::vector& write_schema) { impl_->write_schema_ = write_schema; return *this; } -WriteContextBuilder& WriteContextBuilder::WithFileSystemSchemeToIdentifierMap( - const std::map& fs_scheme_to_identifier_map) { - impl_->fs_scheme_to_identifier_map_ = fs_scheme_to_identifier_map; - return *this; -} - WriteContextBuilder& WriteContextBuilder::SetWriteBufferSpillThreadNumber(int32_t thread_number) { impl_->spill_thread_number_ = thread_number; return *this; } -WriteContextBuilder& WriteContextBuilder::WithFileSystem( - const std::shared_ptr& file_system) { - impl_->specific_file_system_ = file_system; - return *this; -} - WriteContextBuilder& WriteContextBuilder::WithRealtimeContext( const std::shared_ptr& realtime_context) { impl_->realtime_context_ = realtime_context; return *this; } -WriteContextBuilder& WriteContextBuilder::WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier) { - impl_->catalog_ = catalog; - impl_->identifier_.emplace(identifier); - return *this; -} - Result> WriteContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot write with null format table"); - } - if (impl_->format_table_ != nullptr) { - if (impl_->catalog_ != nullptr) { - return Status::Invalid("WithCatalog() requires a native table"); - } - // The table already answers each of these, and from a source this cannot see behind, so a - // second answer is refused rather than silently dropped. - if (impl_->specific_file_system_ != nullptr || - !impl_->fs_scheme_to_identifier_map_.empty()) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "and WithFileSystemSchemeToIdentifierMap() cannot be used with one"); - } - if (impl_->branch_ && - !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(impl_->branch_.value()))) { - return Status::Invalid( - "a format table has no branches, so WithBranch() cannot be used with one"); - } - } - PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_)); - if (impl_->root_path_.empty()) { - return Status::Invalid("root path is empty"); - } - if (impl_->catalog_ == nullptr && impl_->identifier_) { - return Status::Invalid("cannot write through a null catalog"); - } - if (impl_->identifier_) { - // Before the branch is read out of the identifier: the branch of `tbl$branch_dev$options` - // reads back as `dev`, so a write built for such a name would go to another branch. - PAIMON_RETURN_NOT_OK( - CatalogUtils::CheckNotSystemTable(impl_->identifier_.value(), "write")); - } - PAIMON_ASSIGN_OR_RAISE( - std::string branch, - BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, impl_->branch_, "write")); - std::map options = impl_->options_; - std::optional named_branch; - if (impl_->identifier_) { - PAIMON_ASSIGN_OR_RAISE(named_branch, impl_->identifier_->GetBranchName()); - } - if (impl_->branch_ || named_branch || options.count(Options::BRANCH) != 0) { - options[Options::BRANCH] = branch; - } - if (impl_->catalog_ != nullptr && !BranchManager::IsMainBranch(branch)) { - // Refused here rather than at the commit of what is written to the branch. - PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); - PAIMON_ASSIGN_OR_RAISE(std::optional identifier_branch, - impl_->identifier_->GetBranchName()); - if (!identifier_branch) { - PAIMON_ASSIGN_OR_RAISE(std::string table_name, impl_->identifier_->GetDataTableName()); - return Status::Invalid(fmt::format( - "a write through a catalog addresses a branch by the table identifier, so name " - "branch '{}' there as '{}$branch_{}'", - branch, table_name, branch)); - } - } bool enable_multi_thread_spill = impl_->spill_thread_number_ > 0; if (enable_multi_thread_spill) { PAIMON_RETURN_NOT_OK_FROM_ARROW( arrow::SetCpuThreadPoolCapacity(impl_->spill_thread_number_)); } auto ctx = std::make_unique( - impl_->root_path_, impl_->commit_user_, impl_->is_streaming_mode_, - impl_->ignore_num_bucket_check_, impl_->ignore_previous_files_, enable_multi_thread_spill, - impl_->write_id_, branch, impl_->write_schema_, impl_->memory_pool_, impl_->executor_, - impl_->temp_directory_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, - impl_->realtime_context_, options, impl_->format_table_, impl_->catalog_, - impl_->identifier_); + impl_->commit_user_, impl_->is_streaming_mode_, impl_->ignore_num_bucket_check_, + impl_->ignore_previous_files_, enable_multi_thread_spill, impl_->write_id_, + impl_->write_schema_, impl_->memory_pool_, impl_->executor_, impl_->temp_directory_, + impl_->realtime_context_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/write_context_test.cpp b/src/paimon/core/operation/write_context_test.cpp index 337254482..de15926e1 100644 --- a/src/paimon/core/operation/write_context_test.cpp +++ b/src/paimon/core/operation/write_context_test.cpp @@ -21,6 +21,7 @@ #include #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/defs.h" #include "paimon/executor.h" @@ -33,139 +34,23 @@ namespace paimon::test { -TEST(WriteContextTest, TestFailedFinishDoesNotRetainImplicitBranchOption) { - auto catalog = std::make_shared(); - WriteContextBuilder builder("table_root_path", "user"); - builder.WithCatalog(catalog, Identifier("db", "tbl")) - .WithBranch("dev") - .AddOption("custom.option", "value"); - ASSERT_NOK_WITH_MSG(builder.Finish(), "addresses a branch by the table identifier"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.WithBranch("main").Finish()); - ASSERT_EQ("main", context->GetBranch()); - ASSERT_EQ("main", context->GetOptions().at(Options::BRANCH)); - ASSERT_EQ("value", context->GetOptions().at("custom.option")); -} - TEST(WriteContextTest, TestDefaultValue) { - WriteContextBuilder builder("table_root_path", "commit_user_1"); + WriteContextBuilder builder("commit_user_1"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - ASSERT_EQ(ctx->GetRootPath(), "table_root_path"); ASSERT_EQ(ctx->GetCommitUser(), "commit_user_1"); ASSERT_FALSE(ctx->IsStreamingMode()); ASSERT_FALSE(ctx->IgnoreNumBucketCheck()); ASSERT_FALSE(ctx->IgnorePreviousFiles()); ASSERT_FALSE(ctx->EnableMultiThreadSpill()); ASSERT_EQ(ctx->GetWriteId(), std::nullopt); - ASSERT_EQ(ctx->GetBranch(), "main"); ASSERT_TRUE(ctx->GetWriteSchema().empty()); ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); ASSERT_TRUE(ctx->GetTempDirectory().empty()); - ASSERT_TRUE(ctx->GetOptions().empty()); - ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); - ASSERT_FALSE(ctx->GetSpecificFileSystem()); - ASSERT_FALSE(ctx->GetCatalog()); - ASSERT_FALSE(ctx->GetIdentifier()); -} - -TEST(WriteContextTest, TestWithCatalog) { - auto catalog = std::make_shared(); - WriteContextBuilder builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_EQ(ctx->GetCatalog(), catalog); - ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); - ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); - ASSERT_FALSE(next_ctx->GetCatalog()); - ASSERT_FALSE(next_ctx->GetIdentifier()); - - ASSERT_NOK_WITH_MSG(builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), - "cannot write through a null catalog"); -} - -TEST(WriteContextTest, TestCatalogAddressesBranchByIdentifier) { - auto catalog = std::make_shared(); - WriteContextBuilder builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .WithBranch("dev") - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_EQ(ctx->GetBranch(), "dev"); - - WriteContextBuilder identifier_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto identifier_ctx, - identifier_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")).Finish()); - ASSERT_EQ(identifier_ctx->GetBranch(), "dev"); - - for (int32_t branch_source = 0; branch_source < 2; ++branch_source) { - SCOPED_TRACE(branch_source); - WriteContextBuilder option_only_builder("table_root_path", "commit_user_1"); - option_only_builder.WithCatalog(catalog, Identifier("db1", "t1")); - if (branch_source == 0) { - option_only_builder.WithBranch("dev"); - } else { - option_only_builder.AddOption(Options::BRANCH, "dev"); - } - ASSERT_NOK_WITH_MSG(option_only_builder.Finish(), - "name branch 'dev' there as 't1$branch_dev'"); - } - - WriteContextBuilder mixed_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(mixed_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .WithBranch("release") - .Finish(), - "but both 'dev' and 'release' were named"); - - WriteContextBuilder main_builder("table_root_path", "commit_user_1"); - ASSERT_OK(main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) - .WithBranch("") - .AddOption(Options::BRANCH, "main") - .Finish()); - - WriteContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - upper_main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_MAIN")).Finish(), - "a catalog names branch 'MAIN' as it names the main branch"); - - WriteContextBuilder constructed_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto constructed_ctx, - constructed_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).Finish()); - ASSERT_EQ(constructed_ctx->GetBranch(), "dev"); - WriteContextBuilder system_table_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - system_table_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev$options")).Finish(), - "Cannot 'write' for system table"); -} - -TEST(WriteContextTest, TestBranch) { - WriteContextBuilder option_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto option_ctx, option_builder.AddOption(Options::BRANCH, "rt").Finish()); - ASSERT_EQ("rt", option_ctx->GetBranch()); - - WriteContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto upper_main_ctx, upper_main_builder.WithBranch("MAIN").Finish()); - ASSERT_EQ("MAIN", upper_main_ctx->GetBranch()); - - // An empty branch selects the main branch and stays accepted. - WriteContextBuilder main_builder("table_root_path", "commit_user_1"); - main_builder.WithBranch(""); - ASSERT_OK_AND_ASSIGN(auto ctx, main_builder.Finish()); - ASSERT_EQ(BranchManager::DEFAULT_MAIN_BRANCH, ctx->GetBranch()); - WriteContextBuilder empty_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(empty_builder.WithBranch("").AddOption(Options::BRANCH, "dev").Finish(), - "but both 'dev' and 'main' were named"); - - // The branch names a directory under the root path, so a value that is not a single path - // component is rejected when the context is built. - WriteContextBuilder escaping_builder("table_root_path", "commit_user_1"); - escaping_builder.WithBranch("rt/../../../../../outside"); - ASSERT_NOK_WITH_MSG(escaping_builder.Finish(), "branch name cannot contain path separators"); } TEST(WriteContextTest, TestSetContent) { - WriteContextBuilder builder("table_root_path", "commit_user_1"); + WriteContextBuilder builder("commit_user_1"); auto memory_pool = GetDefaultPool(); std::shared_ptr executor = CreateDefaultExecutor(); @@ -174,20 +59,15 @@ TEST(WriteContextTest, TestSetContent) { std::map fs_scheme_to_identifier_map = {{"file", "local"}, {"oss", "jindo"}}; - ASSERT_OK_AND_ASSIGN(auto ctx, - builder.WithStreamingMode(true) - .WithIgnoreNumBucketCheck(true) - .WithIgnorePreviousFiles(true) - .WithMemoryPool(memory_pool) - .WithExecutor(executor) - .WithTempDirectory("/tmp/with-all") - .WithWriteId(123) - .WithBranch("test_branch") - .WithWriteSchema(write_schema) - .WithFileSystemSchemeToIdentifierMap(fs_scheme_to_identifier_map) - .WithFileSystem(file_system) - .AddOption("key", "value") - .Finish()); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithStreamingMode(true) + .WithIgnoreNumBucketCheck(true) + .WithIgnorePreviousFiles(true) + .WithMemoryPool(memory_pool) + .WithExecutor(executor) + .WithTempDirectory("/tmp/with-all") + .WithWriteId(123) + .WithWriteSchema(write_schema) + .Finish()); ASSERT_TRUE(ctx->IsStreamingMode()); ASSERT_TRUE(ctx->IgnoreNumBucketCheck()); @@ -197,28 +77,11 @@ TEST(WriteContextTest, TestSetContent) { ASSERT_EQ(ctx->GetExecutor(), executor); ASSERT_EQ(ctx->GetTempDirectory(), "/tmp/with-all"); ASSERT_EQ(ctx->GetWriteId(), 123); - ASSERT_EQ(ctx->GetBranch(), "test_branch"); ASSERT_EQ(ctx->GetWriteSchema(), write_schema); - ASSERT_EQ(ctx->GetFileSystemSchemeToIdentifierMap(), fs_scheme_to_identifier_map); - ASSERT_EQ(ctx->GetSpecificFileSystem(), file_system); - std::map expected_options = {{"key", "value"}, - {Options::BRANCH, "test_branch"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); -} - -TEST(WriteContextTest, TestSetOptionsOverridesAddedOptions) { - WriteContextBuilder builder("table_root_path", "commit_user_1"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); } TEST(WriteContextTest, TestSetWriteBufferSpillThreadNumber) { - WriteContextBuilder builder("table_root_path", "commit_user_1"); + WriteContextBuilder builder("commit_user_1"); builder.SetWriteBufferSpillThreadNumber(2); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); diff --git a/src/paimon/core/snapshot_file_scan_test.cpp b/src/paimon/core/snapshot_file_scan_test.cpp index 3f8bba11b..e9facf193 100644 --- a/src/paimon/core/snapshot_file_scan_test.cpp +++ b/src/paimon/core/snapshot_file_scan_test.cpp @@ -32,6 +32,7 @@ #include #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/utils/path_util.h" @@ -465,14 +466,16 @@ TEST(SnapshotFileScanTest, TestCommittedRealtimeOffsetFiles) { const std::string table_path = dir->Str() + "/foo.db/bar"; ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - WriteContextBuilder write_builder(table_path, "test"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_builder.AddOption(Options::REALTIME_ENABLED, "true") - .WithStreamingMode(true) - .WithRealtimeContext(realtime_context) - .Finish()); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("test"); + write_builder_options[Options::REALTIME_ENABLED] = "true"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_context, + write_builder.WithStreamingMode(true).WithRealtimeContext(realtime_context).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); auto input_type = arrow::struct_({arrow::field("_REALTIME_OFFSET", arrow::int64(), false), schema->field(0), schema->field(1)}); ASSERT_OK_AND_ASSIGN( @@ -481,11 +484,15 @@ TEST(SnapshotFileScanTest, TestCommittedRealtimeOffsetFiles) { ASSERT_OK(writer->Write(std::move(batch))); ASSERT_OK_AND_ASSIGN(auto progress, writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); ASSERT_FALSE(progress.empty()); - CommitContextBuilder commit_builder(table_path, "test"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_builder.AddOption(Options::REALTIME_ENABLED, "true").Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("test"); + commit_builder_options[Options::REALTIME_ENABLED] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, commit->CommitWithProgress(progress, 0, std::nullopt)); ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id)); diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index d4255fd71..03dcca447 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -148,41 +148,41 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& return Status::OK(); } -bool IsCatalogCommit(const CommitContext& ctx) { - return AsVersionManaged(ctx.GetCatalog()) != nullptr; +bool IsCatalogCommit(const std::shared_ptr& catalog) { + return AsVersionManaged(catalog) != nullptr; } Result> NewSnapshotCommit( - const CommitContext& ctx, const std::optional& catalog_identifier, + const CommitContext& ctx, const std::shared_ptr& catalog, + const std::optional& table_id, const std::optional& catalog_identifier, const CoreOptions& options, const std::shared_ptr& snapshot_manager) { - if (IsCatalogCommit(ctx)) { + if (IsCatalogCommit(catalog)) { if (!catalog_identifier) { return Status::Invalid("a catalog commit requires a table identifier"); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr lock, - AsVersionManaged(ctx.GetCatalog()) - ->CreateCommitLock(catalog_identifier.value(), ctx.GetTableId(), - ctx.GetCommitUser(), options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr lock, + AsVersionManaged(catalog)->CreateCommitLock(catalog_identifier.value(), table_id, + ctx.GetCommitUser(), options)); // Reuse the caller's UUID so table recreation cannot redirect this commit. return std::shared_ptr(std::make_shared( - ctx.GetCatalog(), catalog_identifier.value(), ctx.GetTableId(), std::move(lock))); + catalog, catalog_identifier.value(), table_id, std::move(lock))); } if (ctx.UseRESTCatalogCommit()) { if (options.RestCommitAcquireLock()) { - return Status::Invalid(fmt::format( - "'{}' needs a commit through a REST catalog with WithCatalog(); a commit table " - "request built for the caller to send cannot hold a commit lease", - Options::REST_COMMIT_ACQUIRE_LOCK)); + return Status::Invalid( + fmt::format("'{}' needs a table loaded through a REST catalog; a commit table " + "request built for the caller to send cannot hold a commit lease", + Options::REST_COMMIT_ACQUIRE_LOCK)); } - return std::shared_ptr( - std::make_shared(ctx.GetTableId())); + return std::shared_ptr(std::make_shared(table_id)); } // Catalogs without version management may still supply a table ID. - if (ctx.GetTableId() && ctx.GetCatalog() == nullptr) { + if (table_id && catalog == nullptr) { return Status::Invalid( "a commit which publishes the snapshot by writing it to the table directory sends no " "commit table request, so there is nowhere to put a table id; commit through a " - "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); + "catalog or ask for the request with UseRESTCatalogCommit()"); } return std::shared_ptr( std::make_shared(options.GetFileSystem(), snapshot_manager)); @@ -203,7 +203,9 @@ Result> FileStoreTable::Create( const std::shared_ptr& specified_file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::string& branch, - const std::map& fs_scheme_map) { + const std::map& fs_scheme_map, + const std::shared_ptr& catalog, + const std::map& dynamic_options) { if (schema == nullptr) { return Status::Invalid( fmt::format("not found latest schema of {}", identifier.GetFullName())); @@ -228,13 +230,16 @@ Result> FileStoreTable::Create( CoreOptions::FromMap(options, specified_file_system, fs_scheme_map)); std::unique_ptr store = std::make_unique(root_path, std::move(core_options)); - return Create(std::move(store), effective_schema, identifier, uuid, fs_scheme_map); + return Create(std::move(store), effective_schema, identifier, uuid, fs_scheme_map, catalog, + dynamic_options); } Result> FileStoreTable::Create( std::unique_ptr store, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, - const std::map& fs_scheme_map) { + const std::map& fs_scheme_map, + const std::shared_ptr& catalog, + const std::map& dynamic_options) { if (!store || !schema) { return Status::Invalid("managed table requires a store and schema"); } @@ -250,6 +255,8 @@ Result> FileStoreTable::Create( std::make_shared(std::move(store), schema, identifier, uuid); } table->fs_scheme_map_ = fs_scheme_map; + table->catalog_ = catalog; + table->dynamic_options_ = dynamic_options; return table; } @@ -294,59 +301,6 @@ Status FileStoreTable::RestoreRealtimeCommittedProgress( return Status::OK(); } -Status FileStoreTable::CheckContextNamesThisTable( - const std::string& kind, const std::string& context_root_path, - const std::optional& context_identifier, - const std::optional& context_table_id) const { - if (!root_path_.empty()) { - PAIMON_ASSIGN_OR_RAISE(std::string context_path, - PathUtil::NormalizePath(context_root_path)); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, PathUtil::NormalizePath(root_path_)); - if (context_path != table_path) { - return Status::Invalid(fmt::format( - "the {} context names root path '{}' but this table serves '{}': the store below " - "reads and writes this table's files, so a context naming another path would " - "leave this table's metadata there and publish it as the other's", - kind, context_root_path, root_path_)); - } - } - if (context_identifier) { - PAIMON_ASSIGN_OR_RAISE(std::optional context_branch, - context_identifier.value().GetBranchName()); - if (context_branch && context_branch.value() != CurrentBranch()) { - return Status::Invalid(fmt::format( - "the {} context names branch '{}' of '{}' but this table serves branch '{}', " - "which is the one a {} of this table writes and publishes", - kind, context_branch.value(), context_identifier.value().ToString(), - CurrentBranch(), kind)); - } - PAIMON_ASSIGN_OR_RAISE( - Identifier context_table, - CatalogUtils::BranchIdentifier(context_identifier.value(), CurrentBranch())); - PAIMON_ASSIGN_OR_RAISE(Identifier served_table, - CatalogUtils::BranchIdentifier(identifier_, CurrentBranch())); - const bool names_this_table = - (served_table.GetDatabaseName() == Identifier::kUnknownDatabase || - context_table.GetDatabaseName() == served_table.GetDatabaseName()) && - context_table.GetObjectName() == served_table.GetObjectName(); - if (!names_this_table) { - return Status::Invalid(fmt::format( - "the {} context names table '{}' but this table is '{}': the store below serves " - "this table's files, so a context naming another table would write this table's " - "metadata and publish it as the other's", - kind, context_identifier.value().ToString(), identifier_.ToString())); - } - } - if (context_table_id && context_table_id != CatalogUuid()) { - return Status::Invalid(fmt::format( - "the {} context carries table id '{}' but this table's is '{}': the id tells a " - "recreated table from the one the changes were prepared for, so one this table does " - "not carry would publish this table's metadata as the recreation's", - kind, context_table_id.value(), CatalogUuid().value_or("none"))); - } - return Status::OK(); -} - std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr&& scan, const std::string& root_path) { scan->WithTablePath(root_path); @@ -354,31 +308,24 @@ std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr> FileStoreTable::NewWrite( - std::unique_ptr ctx) const { + std::unique_ptr ctx, + const std::map& dynamic_options) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); - PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("write", ctx->GetRootPath(), - ctx->GetIdentifier(), - /*context_table_id=*/std::nullopt)); - if (AsVersionManaged(ctx->GetCatalog()) != nullptr) { + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(catalog_)); + if (AsVersionManaged(catalog_)) { PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && - ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); - } std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE(Identifier identifier, CatalogUtils::BranchIdentifier( - *ctx->GetIdentifier(), CurrentBranch())); + if (catalog_) { + PAIMON_ASSIGN_OR_RAISE(Identifier identifier, + CatalogUtils::BranchIdentifier(identifier_, CurrentBranch())); catalog_identifier.emplace(std::move(identifier)); } if (ctx->GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(CurrentBranch())); } std::shared_ptr schema = std::static_pointer_cast(schema_); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(ctx->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(dynamic_options)); FileStore file_store(root_path_, options); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); @@ -396,7 +343,7 @@ Result> FileStoreTable::NewWrite( options.GetWriteFileFormat(/*level=*/0)->Identifier(), ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_manager, - file_store.NewSnapshotManager(ctx->GetCatalog(), catalog_identifier)); + file_store.NewSnapshotManager(catalog_, catalog_identifier)); std::shared_ptr io_manager; const auto& io_temp_dir = ctx->GetTempDirectory(); if (!io_temp_dir.empty()) { @@ -417,13 +364,14 @@ Result> FileStoreTable::NewWrite( } Result> FileStoreTable::NewScan( - std::unique_ptr context) const { + std::unique_ptr context, + const std::map& dynamic_options) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(CurrentBranch())); } std::shared_ptr table_schema = std::static_pointer_cast(schema_); - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, ResolveOptions(context->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, ResolveOptions(dynamic_options)); PAIMON_ASSIGN_OR_RAISE( bool read_optimized, OptionsUtils::GetValueFromMap(core_options.ToMap(), kReadOptimizedScanOption, false)); @@ -538,22 +486,23 @@ Result> FileStoreTable::CreateTableRead( } Result> FileStoreTable::NewRead( - std::unique_ptr context) const { + std::unique_ptr context, + const std::map& dynamic_options) const { std::shared_ptr read_context = std::move(context); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); if (read_context->GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(CurrentBranch())); } - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(read_context->GetOptions())); - auto fs_scheme_map = fs_scheme_map_; - for (const auto& [scheme, identifier] : read_context->GetFileSystemSchemeToIdentifierMap()) { - fs_scheme_map[scheme] = identifier; + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(dynamic_options)); + auto explicit_options = dynamic_options_; + for (const auto& [key, value] : dynamic_options) { + explicit_options[key] = value; } std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_context, - InternalReadContext::CreateForTable(read_context, loaded_schema, std::move(options), - root_path_, fs_scheme_map)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, + InternalReadContext::CreateForTable( + read_context, loaded_schema, std::move(options), root_path_, + fs_scheme_map_, catalog_, identifier_, explicit_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), @@ -566,10 +515,11 @@ Result> FileStoreTable::NewRead( return std::move(table_read); } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fallback_context, - InternalReadContext::CreateForFallbackBranch( - read_context, /*branch=*/scan_fallback_branch.value(), file_system_, - root_path_, fs_scheme_map)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr fallback_context, + InternalReadContext::CreateForFallbackBranch( + read_context, /*branch=*/scan_fallback_branch.value(), file_system_, root_path_, + fs_scheme_map_, catalog_, identifier_, explicit_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, CreateTableRead(fallback_context, read_context->GetMemoryPool(), @@ -580,30 +530,29 @@ Result> FileStoreTable::NewRead( } Result> FileStoreTable::NewCommit( - std::unique_ptr ctx) const { + std::unique_ptr ctx, + const std::map& dynamic_options) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); - PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("commit", ctx->GetRootPath(), - ctx->GetIdentifier(), ctx->GetTableId())); - if (ctx->GetCatalog() != nullptr || ctx->UseRESTCatalogCommit()) { - PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(catalog_)); + if (catalog_ && ctx->UseRESTCatalogCommit()) { + return Status::Invalid( + "a commit cannot both use a catalog and build an external REST request"); } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); + if (catalog_ || ctx->UseRESTCatalogCommit()) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); } std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE(Identifier identifier, CatalogUtils::BranchIdentifier( - *ctx->GetIdentifier(), CurrentBranch())); + if (catalog_) { + PAIMON_ASSIGN_OR_RAISE(Identifier identifier, + CatalogUtils::BranchIdentifier(identifier_, CurrentBranch())); catalog_identifier.emplace(std::move(identifier)); } FileStoreCommitImpl::SchemaIdLoader schema_id_loader; - if (ctx->GetCatalog() != nullptr) { + if (catalog_ != nullptr) { if (!catalog_identifier) { return Status::Invalid("a catalog commit requires a table identifier"); } - std::shared_ptr catalog = ctx->GetCatalog(); + std::shared_ptr catalog = catalog_; Identifier identifier = catalog_identifier.value(); schema_id_loader = [catalog, identifier]() -> Result { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, @@ -620,15 +569,14 @@ Result> FileStoreTable::NewCommit( std::shared_ptr schema = std::static_pointer_cast(schema_); std::shared_ptr arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(ctx->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(dynamic_options)); FileStore file_store(root_path_, options); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); assert(options.GetFileSystem()); assert(options.GetFileFormat()); PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); - if (is_object_store && !ctx->UseRESTCatalogCommit() && - AsVersionManaged(ctx->GetCatalog()) == nullptr && + if (is_object_store && !ctx->UseRESTCatalogCommit() && AsVersionManaged(catalog_) == nullptr && options.ToMap().count(kObjectStoreCommitInteTestOption) == 0) { return Status::NotImplemented( "commit operation does not support object store file system for now"); @@ -642,7 +590,7 @@ Result> FileStoreTable::NewCommit( options.GetFileFormat()->Identifier(), ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_manager, - file_store.NewSnapshotManager(ctx->GetCatalog(), catalog_identifier)); + file_store.NewSnapshotManager(catalog_, catalog_identifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, file_store.NewManifestList(path_factory, ctx->GetMemoryPool())); @@ -666,7 +614,8 @@ Result> FileStoreTable::NewCommit( ctx->GetExecutor(), ctx->GetMemoryPool()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_commit, - NewSnapshotCommit(*ctx, catalog_identifier, options, snapshot_manager)); + NewSnapshotCommit(*ctx, catalog_, CatalogUuid(), catalog_identifier, + options, snapshot_manager)); return std::make_unique( ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path_, ctx->GetCommitUser(), diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index f1b72bd3d..6afbf38eb 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -36,6 +36,7 @@ namespace paimon { +class Catalog; class InternalReadContext; class IOManager; class ManifestFile; @@ -56,13 +57,21 @@ class FileStoreTable : public Table { const std::shared_ptr& specified_file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::string& branch, - const std::map& fs_scheme_map = {}); + const std::map& fs_scheme_map = {}, + const std::shared_ptr& catalog = nullptr, + const std::map& dynamic_options = {}); /// Binds a resolved store without replacing the loaded schema's options. static Result> Create( std::unique_ptr store, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, - const std::map& fs_scheme_map = {}); + const std::map& fs_scheme_map = {}, + const std::shared_ptr& catalog = nullptr, + const std::map& dynamic_options = {}); + + const std::map& GetFileSystemSchemeToIdentifierMap() const { + return fs_scheme_map_; + } std::string CurrentBranch() const { return store_->CurrentBranch(); @@ -84,26 +93,24 @@ class FileStoreTable : public Table { return identifier_.GetFullName(); } - Result> NewScan(std::unique_ptr context) const override; + Result> NewScan( + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; - Result> NewRead(std::unique_ptr context) const override; + Result> NewRead( + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; Result> NewWrite( - std::unique_ptr context) const override; + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; Result> NewCommit( - std::unique_ptr context) const override; + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; private: Result ResolveOptions(const std::map& options) const; - /// Rejects paths, identifiers or UUIDs that contradict this table's identity. - /// Missing context fields are allowed. - /// @param kind Operation name used in errors. - /// @param context_table_id Catalog commit UUID; nullopt for writes. - Status CheckContextNamesThisTable(const std::string& kind, const std::string& context_root_path, - const std::optional& context_identifier, - const std::optional& context_table_id) const; - /// Creates a reader for the internal context's schema and primary keys. Result> CreateTableRead( const std::shared_ptr& internal_context, @@ -151,6 +158,8 @@ class FileStoreTable : public Table { const std::shared_ptr& pool) const = 0; std::unique_ptr store_; + std::shared_ptr catalog_; + std::map dynamic_options_; /// URI routing supplied when this table was loaded. std::map fs_scheme_map_; diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index dbd42952a..a044152d7 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -222,11 +222,6 @@ Result> FormatTable::GetArrowSchema() const { namespace { Result> NewFormatTableWrite( const std::shared_ptr& table, const WriteContext& ctx) { - if (ctx.GetCatalog() != nullptr) { - return Status::Invalid( - "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " - "format table"); - } if (ctx.IsStreamingMode()) { return Status::NotImplemented( "a format table has no snapshots, so there is nothing a streaming write could " @@ -315,15 +310,6 @@ Result> NewFormatTableCommit( "a format table commits by renaming files into place, not by sending a snapshot " "to a rest catalog"); } - if (ctx.GetTableId()) { - return Status::NotImplemented( - "a format table sends no commit table request, so there is nowhere to put a table id"); - } - if (ctx.GetCatalog() != nullptr) { - return Status::NotImplemented( - "a format table keeps no snapshot for a catalog to take, so it commits by renaming " - "files into place rather than through the catalog it was loaded from"); - } if (ctx.AppendCommitCheckConflict()) { return Status::NotImplemented( "a format table has no manifests to check a concurrent commit against"); @@ -337,29 +323,35 @@ Result> FormatTable::ForContext( const std::map& options) const { return Copy(std::make_shared(*this), options); } -Result> FormatTable::NewScan(std::unique_ptr ctx) const { +Result> FormatTable::NewScan( + std::unique_ptr ctx, + const std::map& dynamic_options) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "scan")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(dynamic_options)); std::shared_ptr context = std::move(ctx); return NewFormatTableScan(table, context); } -Result> FormatTable::NewRead(std::unique_ptr ctx) const { +Result> FormatTable::NewRead( + std::unique_ptr ctx, + const std::map& dynamic_options) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(dynamic_options)); std::shared_ptr context = std::move(ctx); return NewFormatTableRead(table, context); } Result> FormatTable::NewWrite( - std::unique_ptr ctx) const { + std::unique_ptr ctx, + const std::map& dynamic_options) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(dynamic_options)); std::shared_ptr context = std::move(ctx); return NewFormatTableWrite(table, *context); } Result> FormatTable::NewCommit( - std::unique_ptr ctx) const { + std::unique_ptr ctx, + const std::map& dynamic_options) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(dynamic_options)); std::shared_ptr context = std::move(ctx); return NewFormatTableCommit(table, *context); } diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index f1e1c7aab..75d9a17d4 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -34,6 +34,7 @@ #include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/cache/cache.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" #include "paimon/common/table/special_fields.h" @@ -656,8 +657,9 @@ TEST(FormatTableTest, TestReadRejectsSearchForDirectAndStaticEntrypoints) { for (int32_t entrypoint : {0, 1, 2}) { SCOPED_TRACE(vector_search); SCOPED_TRACE(entrypoint); - ReadContextBuilder builder = - entrypoint == 2 ? ReadContextBuilder(table) : ReadContextBuilder(dir->Str()); + ReadContextBuilder builder; + TableLoadOptions context_load_options{dir->Str()}; + std::map context_options; builder.EnablePreciseFileIndexEvaluate(true); if (vector_search) { builder.SetVectorSearch(std::make_shared( @@ -668,16 +670,18 @@ TEST(FormatTableTest, TestReadRejectsSearchForDirectAndStaticEntrypoints) { std::make_shared("name", R"({"match":{"query":"paimon"}})", 3)); } ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - auto read = entrypoint == 0 ? loaded_table->NewRead(std::move(context)) - : TableRead::Create(std::move(context)); + auto read = entrypoint != 1 ? loaded_table->NewRead(std::move(context), context_options) + : TableRead::Create(context_load_options, + std::move(context), context_options); ASSERT_NOK_WITH_MSG(read, "Precise file-index evaluation is not supported for format tables"); ASSERT_TRUE(read.status().IsNotImplemented()); } } - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder(dir->Str()).Finish()); - ASSERT_OK(loaded_table->NewRead(std::move(context))); + std::map context_builder_options; + ReadContextBuilder context_builder; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder.Finish()); + ASSERT_OK(loaded_table->NewRead(std::move(context), context_builder_options)); } TEST(FormatTableTest, TestFileCompressionComesFromCoreOptions) { @@ -808,26 +812,44 @@ TEST(FormatTableTest, TestTheGenericEntryPointsValidateOptionsGivenAtTheCall) { const std::map refused = { {Options::METASTORE_PARTITIONED_TABLE, "true"}}; - ScanContextBuilder scan_builder(dir->Str()); - scan_builder.SetOptions(refused); + TableLoadOptions scan_builder_load_options{dir->Str()}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options = refused; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG( + TableScan::Create(scan_builder_load_options, std::move(scan_context), scan_builder_options), + "metastore.partitioned-table"); + + TableLoadOptions read_builder_load_options{dir->Str()}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = refused; - ReadContextBuilder read_builder(dir->Str()); - read_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG( + TableRead::Create(read_builder_load_options, std::move(read_context), read_builder_options), + "metastore.partitioned-table"); + + TableLoadOptions write_builder_load_options{dir->Str()}; + std::map write_builder_options; + WriteContextBuilder write_builder("test-user"); + write_builder_options = refused; - WriteContextBuilder write_builder(dir->Str(), "test-user"); - write_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options), "metastore.partitioned-table"); - CommitContextBuilder commit_builder(dir->Str(), "test-user"); - commit_builder.SetOptions(refused); + TableLoadOptions commit_builder_load_options{dir->Str()}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("test-user"); + commit_builder_options = refused; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(commit_builder_load_options, + std::move(commit_context), commit_builder_options), "metastore.partitioned-table"); } @@ -1085,6 +1107,96 @@ TEST(FormatTableTest, TestExternalLocationHasNoReservedDirectories) { ASSERT_EQ(rows, (std::vector{"1|alice|schema"})); } +TEST(FormatTableTest, TestCatalogFactoriesRetainExternalLocationWithExplicitPath) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr schema, + TableSchema::Create(0, MakeSchema(), {"dt"}, {}, + {{Options::TYPE, "format-table"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE, "true"}})); + for (bool explicit_path : {false, true}) { + SCOPED_TRACE(explicit_path); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr external, + FormatTable::Create(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"), + checked_pointer_cast(schema), + /*location_carries_paimon_metadata=*/false, {})); + auto catalog = std::make_shared(); + catalog->SetTable(external); + TableLoadOptions load; + load.catalog = catalog; + load.identifier.emplace("db", "tbl"); + if (explicit_path) { + load.path = dir->Str() + "/"; + } + int32_t id = 0; + for (const std::string partition : {"schema", "branch"}) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + WriteContextBuilder("writer").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + FileStoreWrite::Create(load, std::move(write_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + MakeBatch({++id}, {partition}, partition, {{"dt", partition}})); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + writer->PrepareCommit()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + CommitContextBuilder("writer").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + FileStoreCommit::Create(load, std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(writer->Close()); + } + std::vector files; + ASSERT_OK(dir->GetFileSystem()->ListDir(dir->Str() + "/schema", &files)); + bool renamed = false; + for (const auto& file : files) { + if (!file.IsDir() && StringUtils::EndsWith(file.GetPath(), ".parquet")) { + ASSERT_FALSE(renamed); + ASSERT_OK(dir->GetFileSystem()->Rename(file.GetPath(), + dir->Str() + "/schema/schema-0.parquet")); + renamed = true; + } + } + ASSERT_TRUE(renamed); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(load, std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 2u); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + TableRead::Create(load, std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + read->CreateReader(plan->Splits())); + std::vector partitions; + while (true) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, ImportBatch(batch)); + auto values = checked_pointer_cast(rows->GetColumnByName("dt")); + for (int64_t row = 0; row < rows->num_rows(); ++row) { + partitions.push_back(values->GetString(row)); + } + } + reader->Close(); + std::sort(partitions.begin(), partitions.end()); + ASSERT_EQ(partitions, (std::vector{"branch", "schema"})); + ASSERT_TRUE(catalog->CommitCalls().empty()); + load.path = dir->Str() + "/other"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr mismatch_context, + ScanContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(load, std::move(mismatch_context)), + "does not match the catalog table location"); + } +} + TEST(FormatTableTest, TestFileSuffixIncludesCompressionWhenAsked) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -2709,10 +2821,13 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - WriteContextBuilder write_builder(dir->Str(), "test-user"); + TableLoadOptions write_builder_load_options{dir->Str()}; + std::map write_builder_options; + WriteContextBuilder write_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); ASSERT_NOK_WITH_MSG(write->GetBaseSnapshotId({{"dt", "20240101"}}, 0), "format tables do not have base snapshots"); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, @@ -2728,67 +2843,105 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { "cannot be compacted"); // A write id prefixes a postpone-bucket writer's files; a format table has no buckets. - WriteContextBuilder write_id_builder(dir->Str(), "test-user"); + TableLoadOptions write_id_builder_load_options{dir->Str()}; + std::map write_id_builder_options; + WriteContextBuilder write_id_builder("test-user"); write_id_builder.WithWriteId(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id_context, write_id_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id_context)), - "a write id would name nothing"); + ASSERT_NOK_WITH_MSG( + FileStoreWrite::Create(write_id_builder_load_options, std::move(write_id_context), + write_id_builder_options), + "a write id would name nothing"); - CommitContextBuilder empty_commit_builder(dir->Str(), "test-user"); + TableLoadOptions empty_commit_builder_load_options{dir->Str()}; + std::map empty_commit_builder_options; + CommitContextBuilder empty_commit_builder("test-user"); empty_commit_builder.IgnoreEmptyCommit(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr empty_commit_context, empty_commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(empty_commit_context)), - "cannot record an empty commit"); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(empty_commit_builder_load_options, std::move(empty_commit_context), + empty_commit_builder_options), + "cannot record an empty commit"); - CommitContextBuilder rest_builder(dir->Str(), "test-user"); + TableLoadOptions rest_builder_load_options{dir->Str()}; + std::map rest_builder_options; + CommitContextBuilder rest_builder("test-user"); rest_builder.UseRESTCatalogCommit(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_context, rest_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(rest_context)), "rest catalog"); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(rest_builder_load_options, std::move(rest_context), + rest_builder_options), + "rest catalog"); + + TableLoadOptions table_id_builder_load_options{dir->Str()}; + std::map table_id_builder_options; + CommitContextBuilder table_id_builder("test-user"); + table_id_builder_load_options.table_id = "table-uuid"; - CommitContextBuilder table_id_builder(dir->Str(), "test-user"); - table_id_builder.WithTableId("table-uuid"); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_id_context, table_id_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(table_id_context)), - "a format table sends no commit table request"); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(table_id_builder_load_options, std::move(table_id_context), + table_id_builder_options), + "a table id requires a catalog or an external REST commit request"); auto format_catalog = std::make_shared(); - format_catalog->SetTableSchema(table->GetSchema()); - CommitContextBuilder catalog_builder(dir->Str(), "test-user"); - catalog_builder.WithCatalog(format_catalog, Identifier("db", "fmt")); - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_context, catalog_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_context)), - "keeps no snapshot for a catalog to take"); + format_catalog->SetTable(table); + TableLoadOptions catalog_builder_load_options{dir->Str()}; + std::map catalog_builder_options; + CommitContextBuilder catalog_builder("test-user"); + catalog_builder_load_options.catalog = format_catalog; + catalog_builder_load_options.identifier.emplace(Identifier("db", "fmt")); - CommitContextBuilder conflict_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_context, catalog_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr catalog_commit, + FileStoreCommit::Create(catalog_builder_load_options, std::move(catalog_context), + catalog_builder_options)); + ASSERT_TRUE(format_catalog->CommitCalls().empty()); + + TableLoadOptions conflict_builder_load_options{dir->Str()}; + std::map conflict_builder_options; + CommitContextBuilder conflict_builder("test-user"); conflict_builder.AppendCommitCheckConflict(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr conflict_context, conflict_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(conflict_context)), - "no manifests to check a concurrent commit against"); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(conflict_builder_load_options, std::move(conflict_context), + conflict_builder_options), + "no manifests to check a concurrent commit against"); - CommitContextBuilder commit_builder(dir->Str(), "test-user"); + TableLoadOptions commit_builder_load_options{dir->Str()}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_OK(commit->Commit(messages)); // The same for the snapshot half of the commit interface. ASSERT_NOK_WITH_MSG(commit->Expire(), "no snapshots to expire"); // Closing after a prepared commit must not take back what the commit just published. ASSERT_OK(write->Close()); - ScanContextBuilder scan_builder(dir->Str()); + TableLoadOptions scan_builder_load_options{dir->Str()}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1u); - ReadContextBuilder read_builder(dir->Str()); + TableLoadOptions read_builder_load_options{dir->Str()}; + std::map read_builder_options; + ReadContextBuilder read_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch()); ASSERT_FALSE(BatchReader::IsEofBatch(read_batch)); @@ -2826,17 +2979,24 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { // The same location by path alone is not a table at all: there is no schema under it to say // what kind of table it is. - ScanContextBuilder path_builder(data_dir->Str()); + TableLoadOptions path_builder_load_options{data_dir->Str()}; + std::map path_builder_options; + ScanContextBuilder path_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr path_context, path_builder.Finish()); - ASSERT_NOK(TableScan::Create(std::move(path_context))); + ASSERT_NOK(TableScan::Create(path_builder_load_options, std::move(path_context), + path_builder_options)); // Options given at the call still win over the ones the schema stored, so a file is rolled // per row rather than one file holding all three. - WriteContextBuilder write_builder(external); - write_builder.SetOptions({{Options::TARGET_FILE_ROW_NUM, "1"}}); + const auto write_builder_table = external; + std::map write_builder_options; + WriteContextBuilder write_builder(""); + write_builder_options = {{Options::TARGET_FILE_ROW_NUM, "1"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write, + write_builder_table->NewWrite(std::move(write_context), write_builder_options)); for (int32_t i = 1; i <= 3; i++) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({i}, {"name"}, "20240101", {{"dt", "20240101"}})); @@ -2846,25 +3006,34 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { write->PrepareCommit()); ASSERT_EQ(messages.size(), 3u) << "target-file-row-num given at the call was ignored"; - CommitContextBuilder commit_builder(external); + const auto commit_builder_table = external; + std::map commit_builder_options; + CommitContextBuilder commit_builder(""); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + commit_builder_table->NewCommit(std::move(commit_context), commit_builder_options)); ASSERT_OK(commit->Commit(messages)); ASSERT_OK(write->Close()); - ScanContextBuilder scan_builder(external); + const auto scan_builder_table = external; + std::map scan_builder_options; + ScanContextBuilder scan_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + scan_builder_table->NewScan(std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(PartitionList partitions, scan->ListPartitions()); ASSERT_EQ(partitions, (PartitionList{{{"dt", "20240101"}}})); - ReadContextBuilder read_builder(external); + const auto read_builder_table = external; + std::map read_builder_options; + ReadContextBuilder read_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + read_builder_table->NewRead(std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); int64_t rows = 0; while (true) { @@ -2880,64 +3049,6 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { ASSERT_EQ(rows, 3); } -TEST(FormatTableTest, TestAContextBuiltFromAFormatTableRefusesASecondAnswer) { - // The table already carries its schema and the file system it was loaded through, from a - // source the context cannot see behind, so a second answer is refused rather than dropped. - std::unique_ptr dir = UniqueTestDirectory::Create(); - ASSERT_TRUE(dir); - ASSERT_OK_AND_ASSIGN(std::shared_ptr table, - CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - SchemaManager schema_manager(dir->GetFileSystem(), dir->Str()); - ASSERT_OK_AND_ASSIGN(std::optional> latest, - schema_manager.Latest()); - ASSERT_TRUE(latest.has_value()); - ASSERT_OK_AND_ASSIGN(std::string schema_json, (*latest)->GetJsonSchema()); - - ScanContextBuilder schema_builder(table); - schema_builder.SetTableSchema(schema_json); - ASSERT_NOK_WITH_MSG(schema_builder.Finish(), "carries its own schema"); - - ScanContextBuilder fs_builder(table); - fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(fs_builder.Finish(), "carries the file system"); - - ReadContextBuilder branch_builder(table); - branch_builder.WithBranch("other"); - ASSERT_NOK_WITH_MSG(branch_builder.Finish(), "has no branches"); - - WriteContextBuilder write_fs_builder(table); - write_fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(write_fs_builder.Finish(), "carries the file system"); - - auto catalog = std::make_shared(); - WriteContextBuilder write_catalog_builder(table); - ASSERT_NOK_WITH_MSG( - write_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() requires a native table"); - ReadContextBuilder read_catalog_builder(table); - ASSERT_NOK_WITH_MSG(read_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() cannot be used with one"); - ScanContextBuilder scan_catalog_builder(table); - ASSERT_NOK_WITH_MSG(scan_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() cannot be used with one"); - catalog->SetTableSchema(latest.value()); - WriteContextBuilder path_catalog_builder(dir->Str(), "test-user"); - ASSERT_OK_AND_ASSIGN( - auto path_context, - path_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(path_context)), - "use WriteContextBuilder(FormatTable)"); - - CommitContextBuilder commit_fs_builder(table); - commit_fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(commit_fs_builder.Finish(), "carries the file system"); - - // A null table is the caller's mistake, and says so rather than complaining about a path it - // was never given. - ScanContextBuilder null_builder{std::shared_ptr()}; - ASSERT_NOK_WITH_MSG(null_builder.Finish(), "null format table"); -} - TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) { // `WithBranch()` is refused on a context built from a format table, and the `branch` option // says the same thing by another name: a format table keeps no metadata to branch, and a read @@ -2958,27 +3069,45 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) FormatTable::Create(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"), branch), "has no branches"); - ScanContextBuilder scan_builder(table); - scan_builder.SetOptions(branch); + const auto scan_builder_table = table; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options = branch; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(scan_builder_table->NewScan(std::move(scan_context), scan_builder_options), + "has no branches"); + + const auto read_builder_table = table; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = branch; - ReadContextBuilder read_builder(table); - read_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(read_builder_table->NewRead(std::move(read_context), read_builder_options), + "has no branches"); + + const auto write_builder_table = table; + std::map write_builder_options; + WriteContextBuilder write_builder(""); + write_builder_options = branch; - WriteContextBuilder write_builder(table); - write_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "has no branches"); + ASSERT_NOK_WITH_MSG( + write_builder_table->NewWrite(std::move(write_context), write_builder_options), + "has no branches"); // The commit is the one that matters most: an overwrite clears what it replaces before it // publishes anything, so the refusal has to come before the deleting does. - CommitContextBuilder commit_builder(table); - commit_builder.SetOptions(branch); + const auto commit_builder_table = table; + std::map commit_builder_options; + CommitContextBuilder commit_builder(""); + commit_builder_options = branch; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "has no branches"); + ASSERT_NOK_WITH_MSG( + commit_builder_table->NewCommit(std::move(commit_context), commit_builder_options), + "has no branches"); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, @@ -3014,23 +3143,36 @@ TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFr MakeSchema(), /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {{Options::TYPE, "format-table"}, {Options::FILE_FORMAT, "parquet"}})); - ReadContextBuilder read_builder(dir->Str()); - read_builder.WithBranch("dev"); + TableLoadOptions read_builder_load_options{dir->Str()}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options["branch"] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "has no branches"); + ASSERT_NOK_WITH_MSG( + TableRead::Create(read_builder_load_options, std::move(read_context), read_builder_options), + "has no branches"); // A scan takes its branch from the option rather than from a `WithBranch()` of its own, which // is the same branch by another name and gets the same answer. - ScanContextBuilder scan_builder(dir->Str()); - scan_builder.SetOptions({{Options::BRANCH, "dev"}}); + TableLoadOptions scan_builder_load_options{dir->Str()}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options = {{Options::BRANCH, "dev"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "has no branches"); + ASSERT_NOK_WITH_MSG( + TableScan::Create(scan_builder_load_options, std::move(scan_context), scan_builder_options), + "has no branches"); // The main branch is untouched by any of it. - ScanContextBuilder main_builder(dir->Str()); + TableLoadOptions main_builder_load_options{dir->Str()}; + std::map main_builder_options; + ScanContextBuilder main_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr main_context, main_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan, - TableScan::Create(std::move(main_context))); + TableScan::Create(main_builder_load_options, std::move(main_context), + main_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, main_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::vector rows, ReadAll(table, plan->Splits())); ASSERT_EQ(rows, (std::vector{"1|alice|20240101"})); @@ -3047,11 +3189,15 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { // `target-file-row-num` is stored nowhere in this table's schema, so a file would hold every // row. Given at the call it has to roll a file per row, as it does for a managed table. - WriteContextBuilder write_builder(dir->Str(), "test-user"); - write_builder.SetOptions({{Options::TARGET_FILE_ROW_NUM, "1"}}); + TableLoadOptions write_builder_load_options{dir->Str()}; + std::map write_builder_options; + WriteContextBuilder write_builder("test-user"); + write_builder_options = {{Options::TARGET_FILE_ROW_NUM, "1"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); for (int32_t i = 0; i < 3; i++) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({i}, {"name"}, "20240101", {})); @@ -3063,20 +3209,28 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { // Published through the generic commit, or nothing below would see the rows: until the commit // renames them the files sit in the hidden `_temporary` directory a scan skips. - CommitContextBuilder generic_commit_builder(dir->Str(), "test-user"); + TableLoadOptions generic_commit_builder_load_options{dir->Str()}; + std::map generic_commit_builder_options; + CommitContextBuilder generic_commit_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr generic_commit_context, generic_commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr generic_commit, - FileStoreCommit::Create(std::move(generic_commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr generic_commit, + FileStoreCommit::Create(generic_commit_builder_load_options, + std::move(generic_commit_context), generic_commit_builder_options)); ASSERT_OK(generic_commit->Commit(messages)); ASSERT_OK(write->Close()); // A branch this table has no schema on is a managed table as far as dispatch is concerned, so // the format branch must not answer for it. - ReadContextBuilder branch_builder(dir->Str()); - branch_builder.WithBranch("nosuchbranch"); + TableLoadOptions branch_builder_load_options{dir->Str()}; + std::map branch_builder_options; + ReadContextBuilder branch_builder; + branch_builder_options["branch"] = "nosuchbranch"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_context, branch_builder.Finish()); - ASSERT_NOK(TableRead::Create(std::move(branch_context))); + ASSERT_NOK(TableRead::Create(branch_builder_load_options, std::move(branch_context), + branch_builder_options)); // A caller-held schema is used instead of reading one from under the path. SchemaManager schema_manager(dir->GetFileSystem(), dir->Str()); @@ -3084,11 +3238,15 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { schema_manager.Latest()); ASSERT_TRUE(latest.has_value()); ASSERT_OK_AND_ASSIGN(std::string schema_json, (*latest)->GetJsonSchema()); - ScanContextBuilder seeded_builder(dir->Str()); - seeded_builder.SetTableSchema(schema_json); + TableLoadOptions seeded_builder_load_options{dir->Str()}; + std::map seeded_builder_options; + ScanContextBuilder seeded_builder; + seeded_builder_load_options.table_schema_json = schema_json; + ASSERT_OK_AND_ASSIGN(std::unique_ptr seeded_context, seeded_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr seeded_scan, - TableScan::Create(std::move(seeded_context))); + TableScan::Create(seeded_builder_load_options, std::move(seeded_context), + seeded_builder_options)); // Planning, not just dispatch: handing the schema over must not turn the `schema` directory // the table keeps under its own path into data, which a plan would then read as a data file. ASSERT_OK_AND_ASSIGN(std::shared_ptr seeded_plan, seeded_scan->CreatePlan()); @@ -3099,11 +3257,15 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { // `type` is structural, so an option given at the call must not decide what kind of table // this is. Checked by reading the rows back rather than by counting splits: three small files // are packed into one split, so a split count says nothing about how many there are. - ScanContextBuilder retyped_builder(dir->Str()); - retyped_builder.SetOptions({{Options::TYPE, "table"}}); + TableLoadOptions retyped_builder_load_options{dir->Str()}; + std::map retyped_builder_options; + ScanContextBuilder retyped_builder; + retyped_builder_options = {{Options::TYPE, "table"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_context, retyped_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_scan, - TableScan::Create(std::move(retyped_context))); + TableScan::Create(retyped_builder_load_options, std::move(retyped_context), + retyped_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr retyped_plan, retyped_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::vector retyped_rows, ReadAll(table, retyped_plan->Splits())); @@ -3150,15 +3312,17 @@ TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { MakeBatch({1, 2, 3}, {"alice", "bob", "carol"}, "20240101", {{"dt", "20240101"}})); ASSERT_OK(WriteAndCommit(table, std::move(batch))); - ScanContextBuilder scan_builder(dir->Str()); + TableLoadOptions scan_builder_load_options{dir->Str()}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - auto read_rows = [&plan](std::unique_ptr context) -> Result { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - TableRead::Create(std::move(context))); + auto read_rows = [&plan, &table](std::unique_ptr context) -> Result { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, table->NewRead(std::move(context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); int64_t rows = 0; @@ -3177,7 +3341,7 @@ TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { }; // Prefetch is honoured rather than refused, and reads back what was written. - ReadContextBuilder prefetch_builder(dir->Str()); + ReadContextBuilder prefetch_builder; prefetch_builder.EnablePrefetch(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr prefetch_context, prefetch_builder.Finish()); ASSERT_OK_AND_ASSIGN(int64_t prefetched_rows, read_rows(std::move(prefetch_context))); @@ -3185,7 +3349,7 @@ TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { // And the cache the context carries reaches the format reader. auto cache = std::make_shared(); - ReadContextBuilder cache_builder(dir->Str()); + ReadContextBuilder cache_builder; cache_builder.WithCache(cache); ASSERT_OK_AND_ASSIGN(std::unique_ptr cache_context, cache_builder.Finish()); ASSERT_OK_AND_ASSIGN(int64_t cached_rows, read_rows(std::move(cache_context))); @@ -3203,33 +3367,47 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { // A projected read schema can rename a column, prune a nested one and give it metadata of its // own; a format table's projection is a list of top-level names. - ReadContextBuilder read_schema_builder(dir->Str()); + TableLoadOptions read_schema_builder_load_options{dir->Str()}; + std::map read_schema_builder_options; + ReadContextBuilder read_schema_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr<::ArrowSchema> projected_schema, table->GetArrowSchema()); read_schema_builder.SetReadSchema(std::move(projected_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_schema_context, read_schema_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_schema_context)), - "does not take a projected read schema"); + ASSERT_NOK_WITH_MSG( + TableRead::Create(read_schema_builder_load_options, std::move(read_schema_context), + read_schema_builder_options), + "does not take a projected read schema"); - ScanContextBuilder streaming_builder(dir->Str()); + TableLoadOptions streaming_builder_load_options{dir->Str()}; + std::map streaming_builder_options; + ScanContextBuilder streaming_builder; streaming_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context, streaming_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(streaming_builder_load_options, + std::move(streaming_context), streaming_builder_options), "nothing for a streaming scan to follow"); - ScanContextBuilder predicate_builder(dir->Str()); + TableLoadOptions predicate_builder_load_options{dir->Str()}; + std::map predicate_builder_options; + ScanContextBuilder predicate_builder; predicate_builder.SetPredicate(PredicateBuilder::GreaterThan( /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(1))); ASSERT_OK_AND_ASSIGN(std::unique_ptr predicate_context, predicate_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(predicate_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(predicate_builder_load_options, + std::move(predicate_context), predicate_builder_options), "does not take a predicate"); - CommitContextBuilder commit_builder(dir->Str(), "test-user"); + TableLoadOptions commit_builder_load_options{dir->Str()}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); // A commit takes batch writes only. The rest of the snapshot half is // `TestTheGenericCommitRefusesEveryCallAboutSnapshots`. ASSERT_NOK_WITH_MSG(commit->Commit({}, /*commit_identifier=*/7), "commit identifier"); @@ -3252,10 +3430,14 @@ TEST(FormatTableTest, TestTheGenericCommitRefusesEveryCallAboutSnapshots) { ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - CommitContextBuilder commit_builder(dir->Str(), "test-user"); + TableLoadOptions commit_builder_load_options{dir->Str()}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress({}, BATCH_WRITE_COMMIT_IDENTIFIER, /*watermark=*/std::nullopt), @@ -3296,10 +3478,13 @@ TEST(FormatTableTest, TestTheGenericCommitOverwritesOnePartition) { MakeBatch({9}, {"zoe"}, "20240102", {{"dt", "20240102"}})); ASSERT_OK(WriteAndCommit(table, std::move(other))); - WriteContextBuilder write_builder(dir->Str(), "test-user"); + TableLoadOptions write_builder_load_options{dir->Str()}; + std::map write_builder_options; + WriteContextBuilder write_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr replacement, MakeBatch({3}, {"carol"}, "20240101", {{"dt", "20240101"}})); ASSERT_OK(write->Write(std::move(replacement))); @@ -3309,10 +3494,14 @@ TEST(FormatTableTest, TestTheGenericCommitOverwritesOnePartition) { write->PrepareCommit()); ASSERT_EQ(messages.size(), 1u); - CommitContextBuilder commit_builder(dir->Str(), "test-user"); + TableLoadOptions commit_builder_load_options{dir->Str()}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); // Batch writes only, and refused before anything is cleared: an overwrite that failed on its // arguments must not have deleted the partition it named. ASSERT_NOK_WITH_MSG(commit->Overwrite({{"dt", "20240101"}}, messages, /*commit_identifier=*/7), diff --git a/src/paimon/core/table/sink/commit_message_test.cpp b/src/paimon/core/table/sink/commit_message_test.cpp index 8fd31c4c8..7ce13b431 100644 --- a/src/paimon/core/table/sink/commit_message_test.cpp +++ b/src/paimon/core/table/sink/commit_message_test.cpp @@ -33,6 +33,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" #include "paimon/common/utils/linked_hash_map.h" @@ -1230,16 +1231,17 @@ TEST(CommitMessageTest, TestSerialize) { /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - "commit_user1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::FILE_FORMAT, "mock_format") - .AddOption(Options::TARGET_FILE_SIZE, "1024") - .Finish()); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user1"); + context_builder_options[Options::FILE_FORMAT] = "mock_format"; + context_builder_options[Options::TARGET_FILE_SIZE] = "1024"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - std::string root_path = write_context->GetRootPath(); + std::string root_path = context_builder_load_options.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); for (size_t i = 0; i < 10240; i++) { auto f0_array = std::make_shared(); diff --git a/src/paimon/core/table/source/append_count_reader_test.cpp b/src/paimon/core/table/source/append_count_reader_test.cpp index 3b191a3d0..1814f2b39 100644 --- a/src/paimon/core/table/source/append_count_reader_test.cpp +++ b/src/paimon/core/table/source/append_count_reader_test.cpp @@ -25,6 +25,7 @@ #include #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/defs.h" #include "paimon/fs/local/local_file_system.h" @@ -51,14 +52,18 @@ class AppendCountReaderTest : public testing::Test { protected: Result>> CreateSplits( const std::string& table_path, const std::optional& snapshot_id) { - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; if (snapshot_id.has_value()) { - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, - std::to_string(snapshot_id.value())); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = + std::to_string(snapshot_id.value()); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); if (snapshot_id.has_value() && (!plan->SnapshotId().has_value() || plan->SnapshotId().value() != snapshot_id)) { @@ -133,10 +138,14 @@ TEST_F(AppendCountReaderTest, TestAppendOnlyTableReadCreateCountReaderPredicateN auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(13.0)); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/5)); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader(splits), diff --git a/src/paimon/core/table/source/pk_count_reader_test.cpp b/src/paimon/core/table/source/pk_count_reader_test.cpp index 39c0b6b50..70d27cf4f 100644 --- a/src/paimon/core/table/source/pk_count_reader_test.cpp +++ b/src/paimon/core/table/source/pk_count_reader_test.cpp @@ -25,6 +25,7 @@ #include #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/types/data_field.h" #include "paimon/core/operation/internal_read_context.h" #include "paimon/core/schema/schema_manager.h" @@ -51,14 +52,18 @@ class PKCountReaderTest : public testing::Test { protected: Result>> CreateSplits( const std::string& table_path, const std::optional& snapshot_id) { - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; if (snapshot_id.has_value()) { - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, - std::to_string(snapshot_id.value())); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = + std::to_string(snapshot_id.value()); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); if (snapshot_id.has_value()) { if (!plan->SnapshotId().has_value() || plan->SnapshotId().value() != snapshot_id) { @@ -70,7 +75,9 @@ class PKCountReaderTest : public testing::Test { Result> CreateInternalContext( const std::string& table_path) { - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, read_context_builder.Finish()); @@ -78,7 +85,13 @@ class PKCountReaderTest : public testing::Test { PAIMON_ASSIGN_OR_RAISE(auto table_schema, schema_manager.ReadSchema(0)); PAIMON_ASSIGN_OR_RAISE( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + read_context_builder_load_options.file_system, + read_context_builder_load_options.path, + read_context_builder_load_options.fs_scheme_to_identifier_map, + read_context_builder_load_options.catalog, + read_context_builder_load_options.identifier, read_context_builder_options)); return std::shared_ptr(std::move(internal_context)); } diff --git a/src/paimon/core/table/source/scan_exception_safety_test.cpp b/src/paimon/core/table/source/scan_exception_safety_test.cpp index 246ab10f7..c453a6933 100644 --- a/src/paimon/core/table/source/scan_exception_safety_test.cpp +++ b/src/paimon/core/table/source/scan_exception_safety_test.cpp @@ -30,6 +30,7 @@ #include "arrow/memory_pool.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -117,18 +118,21 @@ Result> OrdinaryScan( std::ifstream schema_file(input.path + "/schema/schema-0"); std::string schema{std::istreambuf_iterator(schema_file), std::istreambuf_iterator()}; - ScanContextBuilder builder(input.path); - builder.SetTableSchema(schema).WithMemoryPool(pool).WithExecutor(executor).AddOption( - "scan.snapshot-id", input.snapshot); + TableLoadOptions builder_load_options{input.path}; + std::map builder_options; + ScanContextBuilder builder; + builder_load_options.table_schema_json = schema; + builder_options["scan.snapshot-id"] = input.snapshot; + builder.WithMemoryPool(pool).WithExecutor(executor); if (input.cache) { - builder.WithCache(input.cache) - .AddOption("scan.manifest-entry-cache.max-snapshots", "8") - .AddOption("scan.manifest-entry.lazy-decode.enabled", "true"); + builder_options["scan.manifest-entry-cache.max-snapshots"] = "8"; + builder_options["scan.manifest-entry.lazy-decode.enabled"] = "true"; + builder.WithCache(input.cache); } if (input.bucket) builder.SetBucketFilter(*input.bucket); - if (input.file_system) builder.WithFileSystem(input.file_system); + builder_load_options.file_system = input.file_system; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return TableScan::Create(std::move(context)); + return TableScan::Create(builder_load_options, std::move(context), builder_options); } class RejectingPlanningExecutor final : public Executor { diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 3433cd670..e28b83cea 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -32,6 +32,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" #include "paimon/catalog/table.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/catalog/catalog_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/core/core_options.h" @@ -65,71 +66,92 @@ TEST(TableReadTest, TestReadWithInvalidContext) { std::string path = paimon::test::GetDataDir() + "/orc/append_10.db/append_10/"; { // read with non-exist field - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f0", "f1", "non-exist"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options), "Get field non-exist failed: not exist in table schema"); } { // field type and literal type mismatch auto predicate = PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(int64_t{15})); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableRead::Create(std::move(read_context)), + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options), "field f3 has field type BIGINT in literal, mismatch field type DOUBLE in predicate"); } { // field type in predicate mismatch schema auto predicate = PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::BIGINT, Literal(int64_t{15})); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options), "schema type double mismatches predicate field type BIGINT"); } { // field idx in predicate mismatch in schema auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f3", FieldType::DOUBLE, Literal(15.0)); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableRead::Create(std::move(read_context)), + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options), "field f3 has field idx 0 in input schema, mismatch field idx 2 in predicate"); } { // literal cannot be null auto predicate = PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(FieldType::DOUBLE)); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options), "literal cannot be null in predicate, field name f3"); } { // schema with duplicate field f3 - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f1", "f3"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options), "validate schema failed: read schema has duplicate field f3"); } } TEST(TableReadTest, TestReadWithSpecifiedInvalidSchema) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"field_no_exist"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options), "Get field field_no_exist failed: not exist in table schema"); } @@ -147,20 +169,38 @@ TEST(TableReadTest, TestFullTextMatchReadEntryAndScoreValidation) { FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"), "", "main")); for (bool table_entry : {false, true}) { - ReadContextBuilder builder(table_path); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"f0", "f1"}) .SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK(table_entry ? table->NewRead(std::move(context)) - : TableRead::Create(std::move(context))); + ASSERT_OK(table_entry ? table->NewRead(std::move(context), context_options) + : TableRead::Create(context_load_options, std::move(context), + context_options)); builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}) .SetPredicate(predicate); + context_load_options.path = builder_load_options.path; + context_load_options.file_system = builder_load_options.file_system; + context_load_options.fs_scheme_to_identifier_map = + builder_load_options.fs_scheme_to_identifier_map; + context_load_options.table_schema_json = builder_load_options.table_schema_json; + context_load_options.catalog = builder_load_options.catalog; + context_load_options.table_id = builder_load_options.table_id; + context_load_options.identifier.reset(); + if (builder_load_options.identifier) { + context_load_options.identifier.emplace(*builder_load_options.identifier); + } + context_options = builder_options; ASSERT_OK_AND_ASSIGN(context, builder.Finish()); - ASSERT_NOK_WITH_MSG(table_entry ? table->NewRead(std::move(context)) - : TableRead::Create(std::move(context)), + ASSERT_NOK_WITH_MSG(table_entry ? table->NewRead(std::move(context), context_options) + : TableRead::Create(context_load_options, + std::move(context), context_options), "Score-producing FullTextMatch cannot appear under OR"); } } @@ -177,23 +217,30 @@ TEST(TableReadTest, TestPerFieldScoreReadEntry) { FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"), "", "main")); for (bool table_entry : {false, true}) { - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true).SetFullTextSearch(search).SetReadFieldNames( {"f1", "_f0_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK(table_entry ? table->NewRead(std::move(context)) - : TableRead::Create(std::move(context))); + ASSERT_OK(table_entry ? table->NewRead(std::move(context), builder_options) + : TableRead::Create(builder_load_options, std::move(context), + builder_options)); } } TEST(TableReadTest, TestMixedFilePredicateProducesScoresThroughPublicReadEntry) { const std::string path = GetDataDir() + "/orc/multi_partition_append_table.db/multi_partition_append_table"; - ScanContextBuilder scan_builder(path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); + TableLoadOptions scan_builder_load_options{path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); std::shared_ptr selected_split; for (const auto& split : plan->Splits()) { @@ -222,13 +269,17 @@ TEST(TableReadTest, TestMixedFilePredicateProducesScoresThroughPublicReadEntry) auto scalar = PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.0)); ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::And({text, scalar})); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_options[Options::READ_BATCH_SIZE] = "1"; builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"_f0_INDEX_SCORE", "f3", "f0"}) - .SetPredicate(predicate) - .AddOption(Options::READ_BATCH_SIZE, "1"); + .SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(selected_split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -247,12 +298,18 @@ TEST(TableReadTest, TestMixedFilePredicateProducesScoresThroughPublicReadEntry) TEST(TableReadTest, TestCreateKeyValueTableRead) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["orc.read.enable-lazy-decoding"] = "true"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); auto key_value_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(key_value_table_read); } @@ -262,12 +319,16 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; auto search = std::make_shared("f0", R"({"match":{"query":"Alice"}})", /*limit=*/1); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"f0"}) .SetFullTextSearch(search); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader({}), "CreateCountReader does not support precise file-index evaluation"); } @@ -277,12 +338,16 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { auto search = std::make_shared("f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"f0"}) .SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader({}), "CreateCountReader does not support precise file-index evaluation"); } @@ -290,10 +355,13 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { TEST(TableReadTest, TestPreciseEvaluationRejectsSystemAndFormatTables) { std::string path = GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder system_builder(path + "$snapshots"); + TableLoadOptions system_builder_load_options{path + "$snapshots"}; + std::map system_builder_options; + ReadContextBuilder system_builder; system_builder.EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(auto system_context, system_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(system_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(system_builder_load_options, std::move(system_context), + system_builder_options), "Precise file-index evaluation is not supported for system tables"); auto directory = UniqueTestDirectory::Create(); @@ -306,11 +374,14 @@ TEST(TableReadTest, TestPreciseEvaluationRejectsSystemAndFormatTables) { FormatTable::Create(directory->GetFileSystem(), directory->Str(), Identifier("db", "table"), {})); for (bool loaded_table : {false, true}) { - ReadContextBuilder builder = - loaded_table ? ReadContextBuilder(format_table) : ReadContextBuilder(directory->Str()); + ReadContextBuilder builder; + TableLoadOptions builder_load_options{directory->Str()}; + std::map builder_options; builder.EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(context)), + ASSERT_NOK_WITH_MSG(loaded_table + ? format_table->NewRead(std::move(context)) + : TableRead::Create(builder_load_options, std::move(context)), "Precise file-index evaluation is not supported for format tables"); } } @@ -321,13 +392,16 @@ TEST(TableReadTest, TestPreciseScalarEvaluationRejectsUnsupportedTableModes) { GetDataDir() + (primary_key ? "/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality" : "/orc/append_09.db/append_09"); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true).SetReadFieldNames({"f3"}).SetPredicate( PredicateBuilder::GreaterThan(0, "f3", FieldType::DOUBLE, Literal(0.0))); if (primary_key) { - builder.AddOption(Options::DELETION_VECTORS_ENABLED, "false"); + builder_options[Options::DELETION_VECTORS_ENABLED] = "false"; + } else { - builder.AddOption(Options::DATA_EVOLUTION_ENABLED, "true"); + builder_options[Options::DATA_EVOLUTION_ENABLED] = "true"; } ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); const std::string expected_error = @@ -335,7 +409,9 @@ TEST(TableReadTest, TestPreciseScalarEvaluationRejectsUnsupportedTableModes) { "deletion vectors" : "Precise file-index evaluation does not support data-evolution or " "real-time reads"; - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(context)), expected_error); + ASSERT_NOK_WITH_MSG( + TableRead::Create(builder_load_options, std::move(context), builder_options), + expected_error); } } @@ -345,11 +421,15 @@ TEST(TableReadTest, TestPrimaryKeyFileIndexSearchDoesNotFallBackToMergeReader) { auto search = std::make_shared("f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.EnablePreciseFileIndexEvaluate(true).SetReadFieldNames({"f0"}).SetVectorSearch( search); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + TableRead::Create(context_builder_load_options, std::move(context), + context_builder_options)); BinaryRow partition = BinaryRowGenerator::GenerateRow({10}, GetDefaultPool().get()); DataSplitImpl::Builder builder(partition, /*bucket=*/0, /*bucket_path=*/"", /*data_files=*/{}); @@ -369,26 +449,40 @@ TEST(TableReadTest, TestPrimaryKeyFileIndexSearchDoesNotFallBackToMergeReader) { TEST(TableReadTest, TestCreateAppendOnlyTableRead) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["orc.read.enable-lazy-decoding"] = "true"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); auto append_only_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(append_only_table_read); } TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); - context_builder.AddOption("bucket", "10"); - context_builder.AddOption("bucket-key", "f3"); + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["orc.read.enable-lazy-decoding"] = "true"; + + context_builder_options["bucket"] = "10"; + + context_builder_options["bucket-key"] = "f3"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); auto append_only_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(append_only_table_read); auto* typed_split_read = @@ -413,15 +507,19 @@ TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { ASSERT_OK_AND_ASSIGN(std::string schema_json, table_schema->ToJsonString()); for (bool explicit_header : {false, true}) { - ReadContextBuilder builder("warehouse/db.db/table"); - builder.SetTableSchema(schema_json); + TableLoadOptions builder_load_options{"warehouse/db.db/table"}; + std::map builder_options; + ReadContextBuilder builder; + builder_load_options.table_schema_json = schema_json; + if (explicit_header) { - builder.AddOption(read_via_option, "read-view"); + builder_options[read_via_option] = "read-view"; } ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, builder.Finish()); // Exercise the actual table/read merge before creating the dependency context. - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(builder_load_options, std::move(read_context), builder_options)); auto* append_only_table_read = dynamic_cast(table_read.get()); ASSERT_NE(append_only_table_read, nullptr); const auto& internal_context = append_only_table_read->context_; diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp index 435e2254f..4e4b8f062 100644 --- a/src/paimon/core/table/source/table_scan_test.cpp +++ b/src/paimon/core/table/source/table_scan_test.cpp @@ -25,6 +25,7 @@ #include #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/defs.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/metrics.h" @@ -60,10 +61,14 @@ TEST(TableScanTest, TestDefaultMetricsSnapshot) { TEST(TableScanTest, TestNoSnapshot) { std::string path = paimon::test::GetDataDir() + "/orc/append_table_with_nested_type.db/append_table_with_nested_type/"; - ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(builder_load_options, + std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_FALSE(plan->SnapshotId()); ASSERT_TRUE(plan->Splits().empty()); @@ -71,23 +76,31 @@ TEST(TableScanTest, TestNoSnapshot) { TEST(TableScanTest, TestNonExistTable) { std::string path = paimon::test::GetDataDir() + "/non-exist.db/non-exist/"; - ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), "not found latest schema"); + ASSERT_NOK_WITH_MSG( + TableScan::Create(builder_load_options, std::move(context), builder_options), + "not found latest schema"); } TEST(TableScanTest, TestStreamingSnapshotsRemoved) { auto dir = UniqueTestDirectory::Create("local"); ASSERT_TRUE(dir); ASSERT_TRUE(TestUtil::CopyDirectory(GetDataDir() + "/orc/append_09.db/append_09", dir->Str())); - ScanContextBuilder builder(dir->Str()); - builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_MODE, "latest") - .WithStreamingMode(true); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::FILE_FORMAT] = "orc"; + builder_options[Options::SCAN_MODE] = "latest"; + builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_plan, table_scan->CreatePlan()); ASSERT_TRUE(initial_plan->Splits().empty()); ASSERT_OK_AND_ASSIGN(std::shared_ptr waiting_plan, table_scan->CreatePlan()); @@ -103,14 +116,19 @@ TEST(TableScanTest, TestStreamingNextSnapshotBoundary) { std::string path = GetDataDir() + "/orc/append_09.db/append_09"; for (const std::string next_snapshot_id : {"6", "7"}) { SCOPED_TRACE(next_snapshot_id); - ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .AddOption(Options::SCAN_SNAPSHOT_ID, next_snapshot_id) + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::FILE_FORMAT] = "orc"; + builder_options[Options::SCAN_MODE] = "from-snapshot"; + builder_options[Options::SCAN_SNAPSHOT_ID] = next_snapshot_id; + builder + .WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_plan, table_scan->CreatePlan()); ASSERT_TRUE(initial_plan->Splits().empty()); @@ -131,10 +149,14 @@ TEST(TableScanTest, TestStreamingNextSnapshotBoundary) { TEST(TableScanTest, TestPkSchemaEvolutionScan) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table/"; - ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(builder_load_options, + std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_TRUE(plan->SnapshotId()); ASSERT_FALSE(plan->Splits().empty()); @@ -164,14 +186,18 @@ TEST(TableScanTest, TestPkSchemaEvolutionScan) { TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table$ro"; - ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::FILE_FORMAT] = "orc"; + builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), - "read-optimized system table does not support streaming scan for primary " - "key table"); + ASSERT_NOK_WITH_MSG( + TableScan::Create(builder_load_options, std::move(context), builder_options), + "read-optimized system table does not support streaming scan for primary " + "key table"); } } // namespace paimon::test diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 0967aefc2..3f1f43b47 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -390,19 +390,18 @@ Result> AuditLogSystemTable::ArrowSchema() const } Result> AuditLogSystemTable::NewScan( - std::unique_ptr ctx) const { + std::unique_ptr ctx, + const std::map& dynamic_options) const { const std::shared_ptr context = std::move(ctx); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { return Status::NotImplemented("audit_log system table predicate pushdown is not supported"); } std::shared_ptr scan_filter = context->GetScanFilters(); - ScanContextBuilder builder(table_path_); + ScanContextBuilder builder; builder.WithStreamingMode(context->IsStreamingMode()) - .SetOptions(context->GetOptions()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(GetFileSystem()) .WithCache(context->GetCache()); if (scan_filter) { if (scan_filter->GetBucketFilter()) { @@ -418,20 +417,22 @@ Result> AuditLogSystemTable::NewScan( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, CreateDataTable(table_schema_, options_)); - return data_table->NewScan(std::move(data_context)); + return data_table->NewScan(std::move(data_context), dynamic_options); } Result> AuditLogSystemTable::NewRead( - std::unique_ptr ctx) const { + std::unique_ptr ctx, + const std::map& dynamic_options) const { const std::shared_ptr context = std::move(ctx); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); - return NewChangelogRead(context, std::make_shared()); + return NewChangelogRead(context, std::make_shared(), dynamic_options); } Result> AuditLogSystemTable::NewChangelogRead( const std::shared_ptr& context, - std::shared_ptr converter) const { + std::shared_ptr converter, + const std::map& dynamic_options) const { if (table_schema_->PrimaryKeys().empty()) { return Status::NotImplemented(Name(), " system table only supports primary key table"); } @@ -445,10 +446,13 @@ Result> AuditLogSystemTable::NewChangelogRead( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, CreateDataReadContext(context, read_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, - CreateDataTable(table_schema_, read_options, - context->GetFileSystemSchemeToIdentifierMap())); + CreateDataTable(table_schema_, read_options)); + auto options = dynamic_options; + if (core_options.TableReadSequenceNumberEnabled()) { + options[Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED] = "true"; + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_read, - data_table->NewRead(std::move(data_context))); + data_table->NewRead(std::move(data_context), options)); auto* key_value_read = dynamic_cast(data_read.get()); if (!key_value_read) { return Status::Invalid("audit_log system table requires key-value table read"); @@ -466,16 +470,10 @@ Result> AuditLogSystemTable::CreateDataReadContext( const std::map& read_options) const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr base_read_schema, BaseReadSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); - auto options = context->GetOptions(); - if (core_options.TableReadSequenceNumberEnabled()) { - options[Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED] = "true"; - } - ReadContextBuilder builder(table_path_); + ReadContextBuilder builder; builder.SetReadFieldNames(base_read_schema->field_names()) - .SetOptions(options) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(GetFileSystem()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) .SetPrefetchMaxParallelNum(context->GetPrefetchMaxParallelNum()) diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index bc15c385f..40089a998 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -60,8 +60,12 @@ class AuditLogSystemTable : public SystemTable { std::map options); Result> ArrowSchema() const override; - Result> NewScan(std::unique_ptr context) const override; - Result> NewRead(std::unique_ptr context) const override; + Result> NewScan( + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; + Result> NewRead( + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; protected: /// Constructor for derived changelog views with a different system-table name. @@ -71,7 +75,8 @@ class AuditLogSystemTable : public SystemTable { Result> NewChangelogRead( const std::shared_ptr& context, - std::shared_ptr converter) const; + std::shared_ptr converter, + const std::map& dynamic_options) const; /// Build the context for the data table underneath this changelog view. /// /// The builder starts from the defaults, so every setting the caller configured has to be diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index 7b73ce1dc..e5827a7d6 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -96,11 +96,12 @@ Result> BinlogSystemTable::ArrowSchema() const { } Result> BinlogSystemTable::NewRead( - std::unique_ptr ctx) const { + std::unique_ptr ctx, + const std::map& dynamic_options) const { const std::shared_ptr context = std::move(ctx); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); - return NewChangelogRead(context, CreateBinlogBatchConverter()); + return NewChangelogRead(context, CreateBinlogBatchConverter(), dynamic_options); } } // namespace paimon diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index fcdcb84b6..010d5b3f5 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -42,7 +42,9 @@ class BinlogSystemTable : public AuditLogSystemTable { std::map options); Result> ArrowSchema() const override; - Result> NewRead(std::unique_ptr context) const override; + Result> NewRead( + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; }; } // namespace paimon diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index 08c7d7726..66649928d 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -111,17 +111,19 @@ InMemorySystemTable::InMemorySystemTable(const std::shared_ptr& fs, : SystemTable(fs, table_path, std::move(name)) {} Result> InMemorySystemTable::NewScan( - std::unique_ptr /*context*/) const { + std::unique_ptr /*context*/, + const std::map& dynamic_options) const { return std::make_unique(GetRootPath()); } Result> InMemorySystemTable::NewRead( - std::unique_ptr ctx) const { + std::unique_ptr ctx, + const std::map& dynamic_options) const { const std::shared_ptr context = std::move(ctx); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - ForRead(context->GetOptions())); + ForRead(dynamic_options)); return std::make_unique(std::move(table), context->GetMemoryPool()); } diff --git a/src/paimon/core/table/system/in_memory_system_table.h b/src/paimon/core/table/system/in_memory_system_table.h index d0909f360..28a7eb3be 100644 --- a/src/paimon/core/table/system/in_memory_system_table.h +++ b/src/paimon/core/table/system/in_memory_system_table.h @@ -41,8 +41,11 @@ class InMemorySystemTable : public SystemTable { std::string name); Result> NewScan( - std::unique_ptr /*context*/) const override; - Result> NewRead(std::unique_ptr context) const override; + std::unique_ptr /*context*/, + const std::map& dynamic_options = {}) const override; + Result> NewRead( + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; virtual Result> BuildRows() const = 0; protected: diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index b2d4bf99e..86d47216d 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -58,7 +58,8 @@ std::map ReadOptimizedSystemTable::ReadOptimizedOption } Result> ReadOptimizedSystemTable::NewScan( - std::unique_ptr ctx) const { + std::unique_ptr ctx, + const std::map& dynamic_options) const { const std::shared_ptr context = std::move(ctx); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetRealtimeContext() && !table_schema_->PrimaryKeys().empty()) { @@ -66,12 +67,10 @@ Result> ReadOptimizedSystemTable::NewScan( "PK real-time union read does not support read-optimized scans"); } auto options = ReadOptimizedOptions(); - ScanContextBuilder builder(table_path_); + ScanContextBuilder builder; builder.WithStreamingMode(context->IsStreamingMode()) - .SetOptions(context->GetOptions()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(GetFileSystem()) .WithCache(context->GetCache()); if (context->GetLimit().has_value()) { builder.SetLimit(context->GetLimit().value()); @@ -89,12 +88,12 @@ Result> ReadOptimizedSystemTable::NewScan( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, CreateDataTable(table_schema_, options)); - return data_table->NewScan(std::move(base_context)); + return data_table->NewScan(std::move(base_context), dynamic_options); } Result> ReadOptimizedSystemTable::CreateDataReadContext( const std::shared_ptr& context) const { - ReadContextBuilder builder(table_path_); + ReadContextBuilder builder; builder.SetPredicate(context->GetPredicate()) .EnablePredicateFilter(context->EnablePredicateFilter()) .EnablePrefetch(context->EnablePrefetch()) @@ -102,10 +101,8 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon .SetPrefetchMaxParallelNum(context->GetPrefetchMaxParallelNum()) .EnableMultiThreadRowToBatch(context->EnableMultiThreadRowToBatch()) .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber()) - .SetOptions(context->GetOptions()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(GetFileSystem()) .SetReadAheadCacheEnabled(context->ReadAheadCacheEnabled()) .WithCacheConfig(context->GetCacheConfig()) .SetWarmupLevel(context->GetWarmupLevel()) @@ -123,16 +120,16 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon } Result> ReadOptimizedSystemTable::NewRead( - std::unique_ptr ctx) const { + std::unique_ptr ctx, + const std::map& dynamic_options) const { const std::shared_ptr context = std::move(ctx); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, CreateDataReadContext(context)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr data_table, - CreateDataTable(table_schema_, options_, context->GetFileSystemSchemeToIdentifierMap())); - return data_table->NewRead(std::move(base_context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, options_)); + return data_table->NewRead(std::move(base_context), dynamic_options); } } // namespace paimon diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h index 164b2c79c..3d0985552 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.h +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -40,8 +40,12 @@ class ReadOptimizedSystemTable : public SystemTable { std::map options); Result> ArrowSchema() const override; - Result> NewScan(std::unique_ptr context) const override; - Result> NewRead(std::unique_ptr context) const override; + Result> NewScan( + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; + Result> NewRead( + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; private: /// Build the context for the data table underneath this read-optimized view. diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 34c06489c..fc9c0f786 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -217,27 +217,24 @@ Status SystemTable::AdoptOwnSchema() { Result> SystemTable::CreateDataTable( const std::shared_ptr& table_schema, - const std::map& options, - const std::map& fs_scheme_map_overrides) const { - auto fs_scheme_map = fs_scheme_map_; - for (const auto& [scheme, identifier] : fs_scheme_map_overrides) { - fs_scheme_map[scheme] = identifier; - } + const std::map& options) const { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, table_schema->MergeOptions(options)); const std::string branch = BranchManager::NormalizeBranch(LoadBranch(merged_schema->Options())); return FileStoreTable::Create( GetFileSystem(), GetRootPath(), std::shared_ptr(std::move(merged_schema)), - CatalogUtils::IdentifierFromPath(GetRootPath(), branch), "", branch, fs_scheme_map); + CatalogUtils::IdentifierFromPath(GetRootPath(), branch), "", branch, fs_scheme_map_); } Result> SystemTable::NewWrite( - std::unique_ptr /*context*/) const { + std::unique_ptr /*context*/, + const std::map& dynamic_options) const { return Status::NotImplemented(Name(), " system table is read-only: writing is not supported"); } Result> SystemTable::NewCommit( - std::unique_ptr /*context*/) const { + std::unique_ptr /*context*/, + const std::map& dynamic_options) const { return Status::NotImplemented(Name(), " system table is read-only: committing for it is not supported"); } diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 73420559b..49aa7393f 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -68,10 +68,12 @@ class SystemTable : public Table, public std::enable_shared_from_this> ArrowSchema() const = 0; Result> NewWrite( - std::unique_ptr context) const override; + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; Result> NewCommit( - std::unique_ptr context) const override; + std::unique_ptr context, + const std::map& dynamic_options = {}) const override; protected: /// @param fs Data table file system. @@ -92,8 +94,7 @@ class SystemTable : public Table, public std::enable_shared_from_this> CreateDataTable( const std::shared_ptr& table_schema, - const std::map& options, - const std::map& fs_scheme_map_overrides = {}) const; + const std::map& options) const; friend class SystemTableLoader; friend class GlobalSystemTableLoader; diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 9f837e4a1..0e0c43cf4 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -31,6 +31,7 @@ #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/generic_row.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/core_options.h" @@ -253,8 +254,8 @@ TEST(SystemTableTest, TestNewReadPropagatesWarmupLevel) { options); for (WarmupLevel level : {WarmupLevel::NONE, WarmupLevel::RAW, WarmupLevel::DECODED}) { - ReadContextBuilder builder("/tmp/table"); - builder.SetOptions(options).SetWarmupLevel(level); + ReadContextBuilder builder; + builder.SetWarmupLevel(level); ASSERT_OK_AND_ASSIGN(std::unique_ptr caller_unique_context, builder.Finish()); std::shared_ptr caller_context(std::move(caller_unique_context)); ASSERT_EQ(level, caller_context->GetWarmupLevel()); diff --git a/src/paimon/core/table/table_factory.cpp b/src/paimon/core/table/table_factory.cpp index 2f12551b1..648fb4b7d 100644 --- a/src/paimon/core/table/table_factory.cpp +++ b/src/paimon/core/table/table_factory.cpp @@ -28,6 +28,7 @@ #include "paimon/catalog/catalog.h" #include "paimon/catalog/table.h" #include "paimon/commit_context.h" +#include "paimon/common/utils/path_util.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/core_options.h" @@ -50,329 +51,239 @@ namespace paimon { namespace { -Result> ManagedTable( - const std::string& path, const std::string& branch, const std::shared_ptr& schema, - CoreOptions options, const std::optional& identifier, - const std::optional& table_id, - const std::map& fs_scheme_map) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr table, - FileStoreTable::Create(std::make_unique(path, std::move(options)), schema, - identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)), - table_id.value_or(""), fs_scheme_map)); - return std::shared_ptr
(std::move(table)); -} -} // namespace -Result> TableFactory::Create(const WriteContext* ctx) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return std::shared_ptr
(given_table); +Status CheckCatalogTable(const TableLoadOptions& load, const std::shared_ptr
& table) { + if (load.table_id && load.table_id != table->CatalogUuid()) { + return Status::Invalid("the expected table id does not match the catalog table id"); } - - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && - ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); + if (!load.path.empty() && !table->GetRootPath().empty()) { + PAIMON_ASSIGN_OR_RAISE(std::string supplied_path, PathUtil::NormalizePath(load.path)); + PAIMON_ASSIGN_OR_RAISE(std::string catalog_path, + PathUtil::NormalizePath(table->GetRootPath())); + if (supplied_path != catalog_path) { + return Status::Invalid("the table path does not match the catalog table location"); } - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs, - ctx->GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx->GetBranch(); - if (ctx->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); - } - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); + if (std::dynamic_pointer_cast(table) == nullptr && + (load.table_schema_json || load.file_system || !load.fs_scheme_to_identifier_map.empty())) { + return Status::Invalid( + "a catalog format or system table carries its schema and file system"); } - // Only the main branch may have a current schema the catalog alone holds. A branch publishes - // its own under `branch/branch-`, which is where the load below and a read of the branch - // both read it, so a catalog answering for no branch schema does not fail a write to one. - std::optional catalog_table_schema; - if (ctx->GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog write requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); - } + return Status::OK(); +} + +} // namespace + +Result> TableFactory::Create( + const TableLoadOptions& load, const std::map& overrides, + const char* operation) { + if (load.catalog == nullptr && load.identifier) { + return Status::Invalid("a table identifier requires a non-null catalog"); } - // A format table writes plain data files into a directory, so it never reaches the manifest - // path below. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch, - ctx->GetOptions(), catalog_table_schema, schema_manager.get(), - &latest_schema)); - if (format_table != nullptr) { - return std::shared_ptr
(format_table); + if (load.catalog != nullptr && !load.identifier) { + return Status::Invalid("a catalog load requires a table identifier"); } - // The schema the dispatch above already read through `schema_manager`, rather than a second - // read of the same file. - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); + if (load.file_system && !load.fs_scheme_to_identifier_map.empty()) { + return Status::Invalid("a file system and URI scheme routing cannot be specified together"); } - const std::shared_ptr& schema = latest_schema; - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; + auto fs_scheme_map = load.fs_scheme_to_identifier_map; + const bool writing = std::string(operation) == "write" || std::string(operation) == "commit"; + if (writing && load.identifier) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(*load.identifier, operation)); } - // The metadata paths above are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; PAIMON_ASSIGN_OR_RAISE( - CoreOptions options, - CoreOptions::FromMap(opts, specific_fs, ctx->GetFileSystemSchemeToIdentifierMap())); - return ManagedTable(ctx->GetRootPath(), branch, schema, std::move(options), - ctx->GetIdentifier(), std::nullopt, - ctx->GetFileSystemSchemeToIdentifierMap()); -} - -Result> TableFactory::Create(const CommitContext* ctx) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return std::shared_ptr
(given_table); + std::string branch, + BranchManager::ResolveBranch(load.identifier, overrides, std::nullopt, operation)); + auto dynamic_options = overrides; + if (load.identifier || dynamic_options.count(Options::BRANCH)) { + dynamic_options[Options::BRANCH] = branch; } - - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - // Use catalog credentials unless the caller supplied a file system. - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); + if (writing && load.catalog) { + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(load.catalog)); + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); + PAIMON_ASSIGN_OR_RAISE(std::optional named_branch, + load.identifier->GetBranchName()); + if (!BranchManager::IsMainBranch(branch) && !named_branch) { + return Status::Invalid( + "a catalog write or commit must name its branch in the table identifier"); } - // Use the credentials of this table, which a catalog issuing per-table temporary - // ones only hands out through GetTableFileSystem. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); - const std::string& root_path = ctx->GetRootPath(); - // The branch the context was built for, already validated by `CommitContextBuilder`. - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - ctx->GetIdentifier(), ctx->GetOptions(), - /*explicit_branch=*/std::nullopt, "commit")); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); } - // Catalog-managed schemas may be absent from the table directory, but only on the main branch: - // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail - // a commit to one. - std::optional catalog_table_schema; - if (ctx->GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); + std::string path = load.path; + std::optional table_id = load.table_id; + std::optional schema_json = load.table_schema_json; + std::shared_ptr file_system = load.file_system; + // A catalog-only load obtains identity, schema and location from one table generation. + if (load.catalog && (path.empty() || load.table_id)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + load.catalog->GetTable(*load.identifier)); + PAIMON_RETURN_NOT_OK(CheckCatalogTable(load, table)); + if (std::dynamic_pointer_cast(table) == nullptr) { + return table; } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); - catalog_table_schema = std::move(schema_json); + if (path.empty()) { + path = table->GetRootPath(); + if (!schema_json) { + PAIMON_ASSIGN_OR_RAISE(schema_json, table->GetSchema()->GetJsonSchema()); + } } + if (!file_system && fs_scheme_map.empty()) { + file_system = table->GetFileSystem(); + } + if (auto managed = std::dynamic_pointer_cast(table)) { + auto inherited_map = managed->GetFileSystemSchemeToIdentifierMap(); + for (const auto& [scheme, identifier] : fs_scheme_map) { + inherited_map[scheme] = identifier; + } + fs_scheme_map = std::move(inherited_map); + } + table_id = table->CatalogUuid(); } - - // A format table commits by renaming files into place, so it never reaches the snapshot path - // below. The managed path here reads the commit's branch, so this reads the same one: the two - // must not dispatch on different schemas. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), root_path, branch); - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTableLoader::TryLoad( - tmp_options.GetFileSystem(), root_path, branch, ctx->GetOptions(), - catalog_table_schema, schema_manager.get(), &latest_schema)); - if (format_table != nullptr) { - return std::shared_ptr
(format_table); + PAIMON_ASSIGN_OR_RAISE(path, PathUtil::NormalizePath(path)); + if (path.empty()) { + return Status::Invalid("table path is empty"); } - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); + if (load.catalog && !file_system && fs_scheme_map.empty()) { + PAIMON_ASSIGN_OR_RAISE(file_system, load.catalog->GetTableFileSystem(*load.identifier)); } - const std::shared_ptr& schema = latest_schema; - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; + if (writing && load.catalog && BranchManager::IsMainBranch(branch) && !schema_json) { + PAIMON_ASSIGN_OR_RAISE(Identifier identifier, + CatalogUtils::BranchIdentifier(*load.identifier, branch)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + load.catalog->LoadTableSchema(identifier)); + PAIMON_ASSIGN_OR_RAISE(schema_json, schema->GetJsonSchema()); } - // The metadata paths below are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, specific_fs)); - return ManagedTable(ctx->GetRootPath(), branch, schema, std::move(options), - ctx->GetIdentifier(), ctx->GetTableId(), {}); -} - -Result> TableFactory::Create(const ScanContext* context) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (context->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); - return std::shared_ptr
(format_table); + PAIMON_ASSIGN_OR_RAISE(CoreOptions temporary, + CoreOptions::FromMap(dynamic_options, file_system, fs_scheme_map)); + PAIMON_ASSIGN_OR_RAISE(std::optional system_path, + SystemTableLoader::TryParsePath(path)); + if (system_path) { + if (writing) { + return Status::Invalid("system tables are read-only"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + SystemTableLoader::LoadFromPath(temporary.GetFileSystem(), path, + dynamic_options, fs_scheme_map)); + return std::shared_ptr
(std::move(table)); } - // load schema - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); - if (context->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(tmp_options.GetBranch())); + std::shared_ptr schema; + Result> format_result = FormatTableLoader::TryLoad( + temporary.GetFileSystem(), path, branch, dynamic_options, schema_json, nullptr, &schema); + if (load.catalog && BranchManager::IsMainBranch(branch) && + (!format_result.ok() || format_result.value() != nullptr || !schema)) { + // An external format table may have real data under `schema`, including file names + // that look like schema versions. Its catalog determines the directory semantics. + Result> catalog_result = load.catalog->GetTable(*load.identifier); + if (catalog_result.ok() && + std::dynamic_pointer_cast(catalog_result.value()) != nullptr) { + PAIMON_RETURN_NOT_OK(CheckCatalogTable(load, catalog_result.value())); + return catalog_result; + } + PAIMON_RETURN_NOT_OK(format_result.status()); + if (format_result.value() != nullptr) { + PAIMON_RETURN_NOT_OK(catalog_result.status()); + return Status::Invalid("the catalog did not return a format table"); + } } - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(context->GetPath())); - if (system_table_path) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), context->GetPath(), - context->GetOptions())); - return std::shared_ptr
(system_table); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, std::move(format_result)); + if (format_table) { + if (load.table_id) { + return Status::Invalid("a format table has no catalog commit table id"); + } + return std::shared_ptr
(std::move(format_table)); } - // A format table is planned by listing directories, so it never reaches the manifest path - // below. One `TableScan` interface serves both, as Java Paimon serves both through one - // `ReadBuilder`. - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), context->GetPath(), - BranchManager::NormalizeBranch(tmp_options.GetBranch()), - context->GetOptions(), context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - return std::shared_ptr
(format_table); + if (!schema) { + return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); } - if (!latest_schema) { - return Status::Invalid("not found latest schema"); + auto merged = schema->Options(); + for (const auto& [key, value] : dynamic_options) { + merged[key] = value; } - auto options_map = latest_schema->Options(); - for (const auto& [key, value] : context->GetOptions()) { - options_map[key] = value; + if (!writing) { + PAIMON_ASSIGN_OR_RAISE( + branch, BranchManager::ResolveBranch(load.identifier, merged, std::nullopt, operation)); } - PAIMON_ASSIGN_OR_RAISE( - std::string branch, - BranchManager::ResolveBranch(std::nullopt, options_map, std::nullopt, "scan")); - if (options_map.count(Options::BRANCH) != 0) { - options_map[Options::BRANCH] = branch; + if (writing || merged.count(Options::BRANCH) != 0) { + merged[Options::BRANCH] = branch; } PAIMON_ASSIGN_OR_RAISE(CoreOptions options, - CoreOptions::FromMap(options_map, context->GetSpecificFileSystem(), {})); - return ManagedTable(context->GetPath(), branch, latest_schema, std::move(options), - context->GetIdentifier(), std::nullopt, {}); + CoreOptions::FromMap(merged, file_system, fs_scheme_map)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + FileStoreTable::Create( + std::make_unique(path, std::move(options)), schema, + load.identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)), + table_id.value_or(""), fs_scheme_map, load.catalog, dynamic_options)); + return std::shared_ptr
(std::move(table)); } -Result> TableFactory::Create(const ReadContext* context) { +Result> TableRead::Create( + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (context->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); - return std::shared_ptr
(format_table); - } - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_core_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(context->GetPath())); - if (system_table_path) { - if (context->EnablePreciseFileIndexEvaluate()) { - return Status::NotImplemented( - "Precise file-index evaluation is not supported for system tables"); - } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetOptions(), - context->GetFileSystemSchemeToIdentifierMap())); - return std::shared_ptr
(system_table); - } - - // A format table has no manifest and a file another engine wrote carries no field ids, so - // which files there are and how a row is put back together is its own. How a file is opened - // is not: both paths go through `DataFileReaderFactory`. - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetBranch(), context->GetOptions(), - context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - return std::shared_ptr
(format_table); - } - - if (!latest_schema) { - return Status::Invalid("not found latest schema"); - } - auto options_map = latest_schema->Options(); - for (const auto& [key, value] : context->GetOptions()) { - options_map[key] = value; + if (load_options.table_id && !load_options.catalog) { + return Status::Invalid("a table id requires a catalog or an external REST commit request"); } - PAIMON_ASSIGN_OR_RAISE( - std::string branch, - BranchManager::ResolveBranch(std::nullopt, options_map, std::nullopt, "read")); - if (options_map.count(Options::BRANCH) != 0) { - options_map[Options::BRANCH] = branch; + if (context->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::string branch, + BranchManager::ResolveBranch(load_options.identifier, + dynamic_options, std::nullopt, "read")); + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); } - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, - CoreOptions::FromMap(options_map, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - return ManagedTable(context->GetPath(), branch, latest_schema, std::move(options), - context->GetIdentifier(), std::nullopt, - context->GetFileSystemSchemeToIdentifierMap()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + TableFactory::Create(load_options, dynamic_options, "read")); + return table->NewRead(std::move(context), dynamic_options); } -Result> TableScan::Create(std::unique_ptr context) { +Result> TableScan::Create( + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); - return table->NewScan(std::move(context)); -} - -Result> TableRead::Create(std::unique_ptr context) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); - return table->NewRead(std::move(context)); + if (load_options.table_id && !load_options.catalog) { + return Status::Invalid("a table id requires a catalog or an external REST commit request"); + } + if (context->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::string branch, + BranchManager::ResolveBranch(load_options.identifier, + dynamic_options, std::nullopt, "scan")); + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + TableFactory::Create(load_options, dynamic_options, "scan")); + return table->NewScan(std::move(context), dynamic_options); } Result> FileStoreWrite::Create( - std::unique_ptr context) { + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "write")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); - return table->NewWrite(std::move(context)); + if (load_options.table_id && !load_options.catalog) { + return Status::Invalid("a table id requires a catalog or an external REST commit request"); + } + if (context->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( + load_options.identifier, dynamic_options, + std::nullopt, "write")); + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + TableFactory::Create(load_options, dynamic_options, "write")); + return table->NewWrite(std::move(context), dynamic_options); } Result> FileStoreCommit::Create( - std::unique_ptr context) { + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "commit")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); - return table->NewCommit(std::move(context)); + if (load_options.catalog && context->UseRESTCatalogCommit()) { + return Status::Invalid( + "a commit cannot both use a catalog and build an external REST request"); + } + if (load_options.table_id && !load_options.catalog && !context->UseRESTCatalogCommit()) { + return Status::Invalid("a table id requires a catalog or an external REST commit request"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + TableFactory::Create(load_options, dynamic_options, "commit")); + return table->NewCommit(std::move(context), dynamic_options); } + } // namespace paimon diff --git a/src/paimon/core/table/table_factory.h b/src/paimon/core/table/table_factory.h index 7c560b752..9f55fcda9 100644 --- a/src/paimon/core/table/table_factory.h +++ b/src/paimon/core/table/table_factory.h @@ -17,21 +17,22 @@ */ #pragma once + +#include #include +#include +#include "paimon/catalog/table_load_options.h" #include "paimon/result.h" + namespace paimon { class Table; -class ScanContext; -class ReadContext; -class WriteContext; -class CommitContext; -/// Compatibility loading for the existing static operator factories. + +/// Loads the table used by a static operator factory. class TableFactory { public: - static Result> Create(const ScanContext* context); - static Result> Create(const ReadContext* context); - static Result> Create(const WriteContext* context); - static Result> Create(const CommitContext* context); + static Result> Create( + const TableLoadOptions& load_options, + const std::map& dynamic_options, const char* operation); }; } // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index a02f72d3f..39403f05f 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -27,6 +27,7 @@ #include "arrow/api.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/catalog/catalog_context.h" #include "paimon/common/utils/checked_cast.h" @@ -55,6 +56,7 @@ #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" +#include "paimon/testing/mock/mock_file_system.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -193,6 +195,45 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { } } +TEST(TableTest, TestScanWithMainBranchAlias) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string table_path = dir->Str() + "/db.db/tbl"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_table, + CreateManagedTable(fs, table_path, Identifier("db", "tbl"))); + auto catalog = + std::make_shared(fs, dir->Str(), std::map()); + for (const std::string alias : {"main", "MAIN", "Main", "mAiN"}) { + SCOPED_TRACE(alias); + const Identifier identifier("db", "tbl$branch_" + alias); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(identifier)); + for (bool explicit_main : {false, true}) { + SCOPED_TRACE(explicit_main); + TableLoadOptions load_options{table_path}; + load_options.catalog = catalog; + load_options.identifier.emplace(identifier); + std::map dynamic_options; + if (explicit_main) { + dynamic_options[Options::BRANCH] = "main"; + } + ScanContextBuilder builder; + ASSERT_OK_AND_ASSIGN(std::unique_ptr static_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr static_scan, + TableScan::Create(load_options, std::move(static_context), dynamic_options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr static_plan, static_scan->CreatePlan()); + ASSERT_TRUE(static_plan->Splits().empty()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr loaded_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr loaded_scan, + table->NewScan(std::move(loaded_context), dynamic_options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr loaded_plan, loaded_scan->CreatePlan()); + ASSERT_TRUE(loaded_plan->Splits().empty()); + } + } +} + TEST(TableTest, TestCurrentBranch) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -264,12 +305,14 @@ TEST(TableTest, TestGetTableDispatchesByTableType) { ASSERT_EQ((std::vector{"key", "value"}), system_table->GetSchema()->FieldNames()); ASSERT_EQ(nullptr, std::dynamic_pointer_cast(system_table->GetSchema())); for (bool direct : {false, true}) { + TableLoadOptions context_builder_load_options{managed_dir + "$options"}; + std::map context_builder_options; + ReadContextBuilder context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder(managed_dir + "$options") - .EnablePreciseFileIndexEvaluate(true) - .Finish()); - auto read = direct ? system_table->NewRead(std::move(context)) - : TableRead::Create(std::move(context)); + context_builder.EnablePreciseFileIndexEvaluate(true).Finish()); + auto read = direct ? system_table->NewRead(std::move(context), context_builder_options) + : TableRead::Create(context_builder_load_options, std::move(context), + context_builder_options); ASSERT_NOK_WITH_MSG(read, "Precise file-index evaluation is not supported for system tables"); } @@ -323,10 +366,11 @@ TEST(TableTest, TestNewScanCreatesATableScan) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, - ScanContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + std::map ctx_builder_options; + ScanContextBuilder ctx_builder; + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ctx_builder.Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, - table->NewScan(std::move(ctx))); + table->NewScan(std::move(ctx), ctx_builder_options)); } TEST(TableTest, TestNewReadCreatesATableRead) { @@ -339,10 +383,11 @@ TEST(TableTest, TestNewReadCreatesATableRead) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, - ReadContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + std::map ctx_builder_18211_options; + ReadContextBuilder ctx_builder_18211; + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ctx_builder_18211.Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, - table->NewRead(std::move(ctx))); + table->NewRead(std::move(ctx), ctx_builder_18211_options)); } TEST(TableTest, TestNewWriteCreatesAFileStoreWrite) { @@ -357,11 +402,11 @@ TEST(TableTest, TestNewWriteCreatesAFileStoreWrite) { // A catalog write reads the current schema from the catalog on the main branch. catalog->SetTableSchema(table->GetSchema()); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr ctx, - WriteContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); + std::map ctx_builder_18974_options; + WriteContextBuilder ctx_builder_18974("commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ctx_builder_18974.Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr write, - table->NewWrite(std::move(ctx))); + table->NewWrite(std::move(ctx), ctx_builder_18974_options)); } TEST(TableTest, TestNewCommitCreatesAFileStoreCommit) { @@ -376,11 +421,11 @@ TEST(TableTest, TestNewCommitCreatesAFileStoreCommit) { // Like a write, a catalog commit reads the current schema from the catalog. catalog->SetTableSchema(table->GetSchema()); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr ctx, - CommitContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); + std::map ctx_builder_19886_options; + CommitContextBuilder ctx_builder_19886("commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ctx_builder_19886.Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, - table->NewCommit(std::move(ctx))); + table->NewCommit(std::move(ctx), ctx_builder_19886_options)); } TEST(TableTest, TestNewScanReadWriteCommitRejectANullContext) { @@ -412,27 +457,46 @@ TEST(TableTest, TestCreateBuildsTheOperator) { // `TableScan::Create()` and friends build the operator straight from the context, which // carries the catalog and identifier it resolves the table through. - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, - ScanContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + TableLoadOptions scan_ctx_builder_load_options{dir->Str()}; + std::map scan_ctx_builder_options; + ScanContextBuilder scan_ctx_builder; + scan_ctx_builder_load_options.catalog = catalog; + scan_ctx_builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_ctx_builder.Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, - TableScan::Create(std::move(scan_ctx))); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, - ReadContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + TableScan::Create(scan_ctx_builder_load_options, std::move(scan_ctx), + scan_ctx_builder_options)); + + TableLoadOptions read_ctx_builder_load_options{dir->Str()}; + std::map read_ctx_builder_options; + ReadContextBuilder read_ctx_builder; + read_ctx_builder_load_options.catalog = catalog; + read_ctx_builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, read_ctx_builder.Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, - TableRead::Create(std::move(read_ctx))); - - ASSERT_OK_AND_ASSIGN( - std::unique_ptr write_ctx, - WriteContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); + TableRead::Create(read_ctx_builder_load_options, std::move(read_ctx), + read_ctx_builder_options)); + + TableLoadOptions write_ctx_builder_load_options{dir->Str()}; + std::map write_ctx_builder_options; + WriteContextBuilder write_ctx_builder("commit-user"); + write_ctx_builder_load_options.catalog = catalog; + write_ctx_builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_ctx, write_ctx_builder.Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr write, - FileStoreWrite::Create(std::move(write_ctx))); - + FileStoreWrite::Create(write_ctx_builder_load_options, + std::move(write_ctx), write_ctx_builder_options)); + + TableLoadOptions commit_ctx_builder_load_options{dir->Str()}; + std::map commit_ctx_builder_options; + CommitContextBuilder commit_ctx_builder("commit-user"); + commit_ctx_builder_load_options.catalog = catalog; + commit_ctx_builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_ctx, commit_ctx_builder.Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_ctx, - CommitContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_ctx))); + [[maybe_unused]] std::unique_ptr commit, + FileStoreCommit::Create(commit_ctx_builder_load_options, std::move(commit_ctx), + commit_ctx_builder_options)); } TEST(TableTest, TestLoadedSchemaAndPathAreUsedWithoutReloading) { @@ -446,14 +510,17 @@ TEST(TableTest, TestLoadedSchemaAndPathAreUsedWithoutReloading) { ASSERT_EQ(schema, table->GetSchema()); ASSERT_EQ((std::vector{"id"}), schema->FieldNames()); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr read_context, - ReadContextBuilder(dir->Str() + "/other").SetReadFieldNames({"id"}).Finish()); + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + read_context_builder.SetReadFieldNames({"id"}).Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, - table->NewRead(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(dir->Str() + "/other").Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + table->NewRead(std::move(read_context), read_context_builder_options)); + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + table->NewScan(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_TRUE(plan->Splits().empty()); } @@ -466,22 +533,38 @@ TEST(TableTest, TestContextOptionsCannotChangeLoadedTypeOrBranch) { CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); for (const auto& options : {std::map{{Options::TYPE, "format-table"}}, std::map{{Options::BRANCH, "dev"}}}) { + std::map scan_context_builder_25016_options; + ScanContextBuilder scan_context_builder_25016; + scan_context_builder_25016_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(dir->Str()).SetOptions(options).Finish()); - ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), - "contradicts the loaded table"); + scan_context_builder_25016.Finish()); + ASSERT_NOK_WITH_MSG( + table->NewScan(std::move(scan_context), scan_context_builder_25016_options), + "contradicts the loaded table"); + std::map read_context_builder_25926_options; + ReadContextBuilder read_context_builder_25926; + read_context_builder_25926_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(dir->Str()).SetOptions(options).Finish()); - ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), - "contradicts the loaded table"); + read_context_builder_25926.Finish()); + ASSERT_NOK_WITH_MSG( + table->NewRead(std::move(read_context), read_context_builder_25926_options), + "contradicts the loaded table"); + std::map write_context_builder_options; + WriteContextBuilder write_context_builder("user"); + write_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - WriteContextBuilder(dir->Str(), "user").SetOptions(options).Finish()); - ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), - "contradicts the loaded table"); + write_context_builder.Finish()); + ASSERT_NOK_WITH_MSG( + table->NewWrite(std::move(write_context), write_context_builder_options), + "contradicts the loaded table"); + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("user"); + commit_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - CommitContextBuilder(dir->Str(), "user").SetOptions(options).Finish()); - ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), - "contradicts the loaded table"); + commit_context_builder.Finish()); + ASSERT_NOK_WITH_MSG( + table->NewCommit(std::move(commit_context), commit_context_builder_options), + "contradicts the loaded table"); } } @@ -494,15 +577,19 @@ TEST(TableTest, TestOperationOptionsDoNotMutateTheLoadedTable) { auto schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, schema); auto options = schema->Options(); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr invalid_context, - ScanContextBuilder(dir->Str()).SetOptions({{Options::BUCKET, "0"}}).Finish()); - ASSERT_NOK_WITH_MSG(table->NewScan(std::move(invalid_context)), "bucket=0"); + std::map invalid_context_builder_options; + ScanContextBuilder invalid_context_builder; + invalid_context_builder_options = {{Options::BUCKET, "0"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_context, + invalid_context_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(invalid_context), invalid_context_builder_options), + "bucket=0"); ASSERT_EQ(options, schema->Options()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ScanContextBuilder(dir->Str()).Finish()); + std::map context_builder_27781_options; + ScanContextBuilder context_builder_27781; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder_27781.Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, - table->NewScan(std::move(context))); + table->NewScan(std::move(context), context_builder_27781_options)); } TEST(TableTest, TestPathIdentifierPreservesBranch) { @@ -527,13 +614,21 @@ TEST(TableTest, TestGlobalSystemTableRetainsCatalog) { catalog->GetTable(Identifier("sys", "tables"))); ASSERT_NE(nullptr, std::dynamic_pointer_cast(table)); ASSERT_EQ(nullptr, std::dynamic_pointer_cast(table->GetSchema())); + std::map scan_context_builder_28121_options; + ScanContextBuilder scan_context_builder_28121; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(table->GetRootPath()).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + scan_context_builder_28121.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + table->NewScan(std::move(scan_context), scan_context_builder_28121_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + std::map read_context_builder_29694_options; + ReadContextBuilder read_context_builder_29694; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(table->GetRootPath()).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + read_context_builder_29694.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + table->NewRead(std::move(read_context), read_context_builder_29694_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); catalog.reset(); table.reset(); @@ -558,17 +653,20 @@ TEST(TableTest, TestExplicitContextBranchMustMatchLoadedTable) { {std::optional(), std::optional("main"), std::optional("dev"), std::optional(" ")}) { SCOPED_TRACE(table_branch + ":" + requested.value_or("unset")); - WriteContextBuilder write_builder(table->GetRootPath(), "user"); - ReadContextBuilder read_builder(table->GetRootPath()); + std::map write_builder_options; + WriteContextBuilder write_builder("user"); + std::map read_builder_options; + ReadContextBuilder read_builder; if (requested) { - write_builder.WithBranch(*requested); - read_builder.WithBranch(*requested); + write_builder_options["branch"] = *requested; + + read_builder_options["branch"] = *requested; } ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - auto write = table->NewWrite(std::move(write_context)); - auto read = table->NewRead(std::move(read_context)); + auto write = table->NewWrite(std::move(write_context), write_builder_options); + auto read = table->NewRead(std::move(read_context), read_builder_options); if (requested && BranchManager::NormalizeBranch(*requested) != table_branch) { ASSERT_NOK_WITH_MSG(write, "context branch contradicts the loaded table"); ASSERT_NOK_WITH_MSG(read, "context branch contradicts the loaded table"); @@ -611,18 +709,29 @@ TEST(TableTest, TestPreloadedSchemaBranchMatchesScanAndRead) { std::map{{Options::BRANCH, "main"}}}) { const std::string branch = overrides.empty() ? "rt" : overrides.at(Options::BRANCH); SCOPED_TRACE(branch); + TableLoadOptions scan_context_builder_30072_load_options{path}; + std::map scan_context_builder_30072_options; + ScanContextBuilder scan_context_builder_30072; + scan_context_builder_30072_load_options.table_schema_json = schema_json; + scan_context_builder_30072_options = overrides; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + scan_context_builder_30072.Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr scan_context, - ScanContextBuilder(path).SetTableSchema(schema_json).SetOptions(overrides).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + std::unique_ptr scan, + TableScan::Create(scan_context_builder_30072_load_options, std::move(scan_context), + scan_context_builder_30072_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(branch == "rt" ? 1 : 2, plan->SnapshotId()); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr read_context, - ReadContextBuilder(path).SetTableSchema(schema_json).SetOptions(overrides).Finish()); + TableLoadOptions read_context_builder_34363_load_options{path}; + std::map read_context_builder_34363_options; + ReadContextBuilder read_context_builder_34363; + read_context_builder_34363_load_options.table_schema_json = schema_json; + read_context_builder_34363_options = overrides; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + read_context_builder_34363.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, - TableFactory::Create(read_context.get())); + TableFactory::Create(read_context_builder_34363_load_options, + read_context_builder_34363_options, "read")); ASSERT_EQ(branch, std::dynamic_pointer_cast(loaded)->CurrentBranch()); // Explicit non-main loads keep using that branch's persisted schema. const auto& expected_options = overrides.count(Options::BRANCH) != 0 && branch == "rt" @@ -630,8 +739,9 @@ TEST(TableTest, TestPreloadedSchemaBranchMatchesScanAndRead) { : preloaded->Options(); ASSERT_EQ(expected_options, std::dynamic_pointer_cast(loaded->GetSchema())->Options()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - loaded->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + loaded->NewRead(std::move(read_context), read_context_builder_34363_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, @@ -641,10 +751,16 @@ TEST(TableTest, TestPreloadedSchemaBranchMatchesScanAndRead) { ASSERT_TRUE(snapshot); ASSERT_EQ(snapshot->TotalRecordCount(), rows->length()); } - ASSERT_OK_AND_ASSIGN( - std::unique_ptr explicit_main, - ReadContextBuilder(path).SetTableSchema(schema_json).WithBranch("main").Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, TableFactory::Create(explicit_main.get())); + TableLoadOptions explicit_main_builder_load_options{path}; + std::map explicit_main_builder_options; + ReadContextBuilder explicit_main_builder; + explicit_main_builder_load_options.table_schema_json = schema_json; + explicit_main_builder_options["branch"] = "main"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr explicit_main, + explicit_main_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + TableFactory::Create(explicit_main_builder_load_options, + explicit_main_builder_options, "read")); ASSERT_EQ("main", std::dynamic_pointer_cast(loaded)->CurrentBranch()); } @@ -666,23 +782,27 @@ TEST(TableTest, TestResolvedStoreUsesOptionsBranch) { ASSERT_EQ("dev", loaded->GetSnapshotManager()->Branch()); } -TEST(TableTest, TestLoadedTableReadRetainsOperationSchemeMap) { +TEST(TableTest, TestLoadedTableReadRetainsLoadSchemeMap) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); const std::string path = dir->Str() + "/db.db/tbl"; ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
created, CreateManagedTable(dir->GetFileSystem(), path, Identifier("db", "tbl"))); - FileSystemCatalog catalog(dir->GetFileSystem(), dir->Str(), {}); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db", "tbl"))); const std::map scheme_map = {{"custom", "local"}}; + FileSystemCatalog catalog(dir->GetFileSystem(), dir->Str(), {}, scheme_map); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db", "tbl"))); for (bool direct : {false, true}) { SCOPED_TRACE(direct); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr context, - ReadContextBuilder(path).WithFileSystemSchemeToIdentifierMap(scheme_map).Finish()); + TableLoadOptions context_builder_36442_load_options{path}; + std::map context_builder_36442_options; + ReadContextBuilder context_builder_36442; + context_builder_36442_load_options.fs_scheme_to_identifier_map = scheme_map; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder_36442.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr read, - direct ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context))); + direct ? table->NewRead(std::move(context), context_builder_36442_options) + : TableRead::Create(context_builder_36442_load_options, std::move(context), + context_builder_36442_options)); auto* append_read = dynamic_cast(read.get()); ASSERT_NE(nullptr, append_read); ASSERT_EQ( @@ -695,7 +815,7 @@ TEST(TableTest, TestLoadedTableReadRetainsOperationSchemeMap) { } } -TEST(TableTest, TestReadMergesSchemeMapsForMainAndFallback) { +TEST(TableTest, TestReadRetainsLoadSchemeMapForMainAndFallback) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); @@ -704,9 +824,7 @@ TEST(TableTest, TestReadMergesSchemeMapsForMainAndFallback) { ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
dev, CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl", "dev"), "dev")); const std::map table_map = {{"base", "local"}, {"shared", "mock_fs"}}; - const std::map read_map = {{"custom", "local"}, {"shared", "local"}}; - const std::map expected = { - {"base", "local"}, {"custom", "local"}, {"shared", "local"}}; + ASSERT_OK_AND_ASSIGN( std::shared_ptr table, FileStoreTable::Create(fs, dir->Str(), @@ -714,13 +832,15 @@ TEST(TableTest, TestReadMergesSchemeMapsForMainAndFallback) { Identifier("db", "tbl"), "", "main", table_map)); for (bool fallback : {false, true}) { SCOPED_TRACE(fallback); - ReadContextBuilder builder("unused_path"); - builder.WithFileSystemSchemeToIdentifierMap(read_map); + std::map builder_options; + ReadContextBuilder builder; + if (fallback) { - builder.AddOption(Options::SCAN_FALLBACK_BRANCH, "dev"); + builder_options[Options::SCAN_FALLBACK_BRANCH] = "dev"; } ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + table->NewRead(std::move(context), builder_options)); std::vector reads; if (fallback) { auto* fallback_read = dynamic_cast(read.get()); @@ -733,15 +853,17 @@ TEST(TableTest, TestReadMergesSchemeMapsForMainAndFallback) { auto* append_read = dynamic_cast(data_read); ASSERT_NE(nullptr, append_read); const auto& internal = append_read->context_; - ASSERT_EQ(expected, + ASSERT_EQ(table_map, internal->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); ASSERT_EQ(fs, internal->GetCoreOptions().GetFileSystem()); ASSERT_EQ(dir->Str(), internal->GetPath()); } } - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder("unused_path").Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(context))); + std::map context_builder_38424_options; + ReadContextBuilder context_builder_38424; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder_38424.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + table->NewRead(std::move(context), context_builder_38424_options)); auto* append_read = dynamic_cast(read.get()); ASSERT_NE(nullptr, append_read); ASSERT_EQ(table_map, @@ -757,17 +879,42 @@ TEST(TableTest, TestSystemTablePathRetainsSchemeMapForDependencyReads) { const std::map scheme_map = {{"file", "local"}}; for (bool reload_context : {false, true}) { SCOPED_TRACE(reload_context); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder(dir->Str() + "$ro") - .WithFileSystemSchemeToIdentifierMap(scheme_map) - .Finish()); + TableLoadOptions context_builder_41670_load_options{dir->Str() + "$ro"}; + std::map context_builder_41670_options; + ReadContextBuilder context_builder_41670; + context_builder_41670_load_options.fs_scheme_to_identifier_map = scheme_map; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder_41670.Finish()); std::unique_ptr read; if (reload_context) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, TableFactory::Create(context.get())); - ASSERT_OK_AND_ASSIGN(context, ReadContextBuilder(dir->Str()).Finish()); - ASSERT_OK_AND_ASSIGN(read, table->NewRead(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + TableFactory::Create(context_builder_41670_load_options, + context_builder_41670_options, "read")); + TableLoadOptions context_builder_42682_load_options{dir->Str()}; + std::map context_builder_42682_options; + ReadContextBuilder context_builder_42682; + context_builder_41670_load_options.path = context_builder_42682_load_options.path; + context_builder_41670_load_options.file_system = + context_builder_42682_load_options.file_system; + context_builder_41670_load_options.fs_scheme_to_identifier_map = + context_builder_42682_load_options.fs_scheme_to_identifier_map; + context_builder_41670_load_options.table_schema_json = + context_builder_42682_load_options.table_schema_json; + context_builder_41670_load_options.catalog = context_builder_42682_load_options.catalog; + context_builder_41670_load_options.table_id = + context_builder_42682_load_options.table_id; + context_builder_41670_load_options.identifier.reset(); + if (context_builder_42682_load_options.identifier) { + context_builder_41670_load_options.identifier.emplace( + *context_builder_42682_load_options.identifier); + } + context_builder_41670_options = context_builder_42682_options; + ASSERT_OK_AND_ASSIGN(context, context_builder_42682.Finish()); + ASSERT_OK_AND_ASSIGN(read, + table->NewRead(std::move(context), context_builder_41670_options)); } else { - ASSERT_OK_AND_ASSIGN(read, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + read, TableRead::Create(context_builder_41670_load_options, std::move(context), + context_builder_41670_options)); } auto* append_read = dynamic_cast(read.get()); ASSERT_NE(nullptr, append_read); @@ -778,30 +925,29 @@ TEST(TableTest, TestSystemTablePathRetainsSchemeMapForDependencyReads) { } } -TEST(TableTest, TestSystemTableReadMergesSchemeMapsWithoutReplacingFileSystem) { +TEST(TableTest, TestSystemTableReadRetainsLoadSchemeMapWithoutReplacingFileSystem) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); ASSERT_OK_AND_ASSIGN(std::shared_ptr
data_table, CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"))); const std::map table_map = {{"base", "local"}}; - const std::map read_map = {{"file", "local"}}; - const std::map expected = {{"base", "local"}, {"file", "local"}}; ASSERT_OK_AND_ASSIGN( std::shared_ptr table, SystemTableLoader::Load("ro", fs, dir->Str(), std::dynamic_pointer_cast(data_table->GetSchema()), {}, "tbl", table_map)); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr context, - ReadContextBuilder("unused_path").WithFileSystemSchemeToIdentifierMap(read_map).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(context))); + std::map context_builder_43111_options; + ReadContextBuilder context_builder_43111; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder_43111.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + table->NewRead(std::move(context), context_builder_43111_options)); auto* append_read = dynamic_cast(read.get()); ASSERT_NE(nullptr, append_read); const auto& internal = append_read->context_; ASSERT_EQ(fs, internal->GetCoreOptions().GetFileSystem()); ASSERT_EQ(dir->Str(), internal->GetPath()); - ASSERT_EQ(expected, internal->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); + ASSERT_EQ(table_map, internal->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); } TEST(TableTest, TestRealtimeFallbackOnlyAllowsMainBranch) { @@ -825,13 +971,16 @@ TEST(TableTest, TestRealtimeFallbackOnlyAllowsMainBranch) { for (bool realtime_enabled : {false, true}) { for (const std::string fallback : {"main", "dev"}) { SCOPED_TRACE(fallback + (realtime_enabled ? ":realtime" : ":disk")); - ReadContextBuilder builder(dir->Str()); - builder.SetOptions({{Options::SCAN_FALLBACK_BRANCH, fallback}}); + std::map builder_options; + ReadContextBuilder builder; + builder_options = {{Options::SCAN_FALLBACK_BRANCH, fallback}}; + if (realtime_enabled) { - builder.WithRealtimeContext(realtime).AddOption(Options::REALTIME_ENABLED, "true"); + builder_options[Options::REALTIME_ENABLED] = "true"; + builder.WithRealtimeContext(realtime); } ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - auto read = table->NewRead(std::move(context)); + auto read = table->NewRead(std::move(context), builder_options); if (realtime_enabled && fallback != "main") { ASSERT_NOK_WITH_MSG(read, "real-time operations only support the main branch"); } else { @@ -842,3 +991,155 @@ TEST(TableTest, TestRealtimeFallbackOnlyAllowsMainBranch) { } } // namespace paimon::test + +namespace paimon::test { + +TEST(TableTest, TestLoadOptionsValidateIdentityBeforeMetadataAccess) { + auto catalog = std::make_shared(); + TableLoadOptions load{"unused/table"}; + load.identifier.emplace("db", "tbl"); + ASSERT_NOK_WITH_MSG(TableFactory::Create(load, {}, "read"), "non-null catalog"); + load.catalog = catalog; + load.identifier.reset(); + ASSERT_NOK_WITH_MSG(TableFactory::Create(load, {}, "write"), "table identifier"); + load.identifier.emplace("db", "tbl", "dev"); + ASSERT_NOK(TableFactory::Create(load, {{Options::BRANCH, "other"}}, "read")); + ASSERT_NOK_WITH_MSG(TableFactory::Create(load, {{Options::BRANCH, "bad/branch"}}, "write"), + "branch"); + ASSERT_EQ(catalog->GetTableCalls(), 0u); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); +} + +TEST(TableTest, TestLoadOptionsRejectConflictingFileSystemResolution) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + TableLoadOptions load{dir->Str()}; + load.file_system = dir->GetFileSystem(); + load.fs_scheme_to_identifier_map = {{"file", "local"}}; + ASSERT_NOK_WITH_MSG(TableFactory::Create(load, {}, "read"), "cannot be specified together"); +} + +TEST(TableTest, TestPreloadedSchemaIsIndependentOfReadProjection) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + TableSchema::Create(0, + arrow::schema({arrow::field("id", arrow::int32()), + arrow::field("value", arrow::utf8())}), + {}, {}, {})); + TableLoadOptions load{dir->Str() + "/without-metadata"}; + load.file_system = dir->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(load.table_schema_json, schema->GetJsonSchema()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ReadContextBuilder().SetReadFieldNames({"value"}).Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, TableFactory::Create(load, {}, "read")); + ASSERT_EQ(table->GetSchema()->FieldNames(), (std::vector{"id", "value"})); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + table->NewRead(std::move(context))); +} + +TEST(TableTest, TestLoadOptionsValidateExpectedCatalogUuid) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr schema, + TableSchema::Create(0, arrow::schema({arrow::field("id", arrow::int32())}), {}, {}, {})); + auto catalog = std::make_shared(true, "current-uuid"); + catalog->SetTableSchema(schema); + catalog->SetFileSystem(dir->GetFileSystem()); + TableLoadOptions load{dir->Str()}; + load.catalog = catalog; + load.identifier.emplace("db", "tbl"); + load.table_id = "previous-uuid"; + ASSERT_NOK_WITH_MSG(TableFactory::Create(load, {}, "commit"), "does not match"); + load.table_id = "current-uuid"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, TableFactory::Create(load, {}, "commit")); + ASSERT_EQ(table->CatalogUuid(), load.table_id); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + CommitContextBuilder("writer").Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(context), {{Options::BRANCH, "dev"}}), + "branch contradicts"); +} + +TEST(TableTest, TestExplicitPathAndUuidRetainCatalogFileSystemWithRouting) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr schema, + TableSchema::Create(0, arrow::schema({arrow::field("id", arrow::int32())}), {}, {}, {})); + ASSERT_OK_AND_ASSIGN(std::string schema_json, schema->GetJsonSchema()); + for (bool with_routing : {false, true}) { + std::map routes; + if (with_routing) { + routes["dependency"] = "local"; + } + auto scoped_fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr catalog_table, + FileStoreTable::Create(scoped_fs, dir->Str(), schema, Identifier("db", "tbl"), + "table-uuid", "main", routes)); + auto catalog = std::make_shared(); + catalog->SetTable(catalog_table); + for (bool override_file_system : {false, true}) { + auto supplied_fs = std::make_shared(); + TableLoadOptions load{dir->Str()}; + load.catalog = catalog; + load.identifier.emplace("db", "tbl"); + load.table_id = "table-uuid"; + load.table_schema_json = schema_json; + if (override_file_system) { + load.file_system = supplied_fs; + } + for (const std::string operation : {"scan", "read", "write", "commit"}) { + SCOPED_TRACE(operation + (with_routing ? ":routing" : ":no-routing") + + (override_file_system ? ":override" : ":catalog")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + TableFactory::Create(load, {}, operation.c_str())); + ASSERT_EQ(override_file_system ? supplied_fs : scoped_fs, table->GetFileSystem()); + auto managed = std::dynamic_pointer_cast(table); + ASSERT_NE(managed, nullptr); + ASSERT_EQ(routes, managed->GetFileSystemSchemeToIdentifierMap()); + } + } + } +} + +TEST(TableTest, TestExplicitPathRetainsMissingSchemaErrorWithCatalog) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + TableLoadOptions load{dir->Str() + "/missing-table"}; + load.catalog = std::make_shared(dir->GetFileSystem(), dir->Str(), + std::map{}); + load.identifier.emplace("db", "missing-table"); + for (const std::string operation : {"scan", "read"}) { + SCOPED_TRACE(operation); + auto result = TableFactory::Create(load, {}, operation.c_str()); + ASSERT_TRUE(result.status().IsInvalid()) << result.status().ToString(); + ASSERT_NOK_WITH_MSG(result, "not found latest schema in branch main"); + } +} + +} // namespace paimon::test + +namespace paimon::test { +TEST(TableTest, TestLoadedTableRetainsCatalogAfterStackCatalogIsDestroyed) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::string path = dir->Str() + "/db.db/tbl"; + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
created, + CreateManagedTable(dir->GetFileSystem(), path, Identifier("db", "tbl"))); + std::shared_ptr
table; + const std::map catalog_options = {{"dependency.setting", "retained"}}; + { + FileSystemCatalog catalog(dir->GetFileSystem(), dir->Str(), catalog_options); + ASSERT_OK_AND_ASSIGN(table, catalog.GetTable(Identifier("db", "tbl"))); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(context))); + auto* append_read = dynamic_cast(read.get()); + ASSERT_NE(append_read, nullptr); + auto dependency = append_read->context_->CreateDependencyCatalogContext(); + ASSERT_EQ(dependency->root_path, dir->Str()); + ASSERT_EQ(dependency->options.at("dependency.setting"), "retained"); +} +} // namespace paimon::test diff --git a/src/paimon/core/utils/blob_view_lookup.cpp b/src/paimon/core/utils/blob_view_lookup.cpp index 30999fe62..0e02439b8 100644 --- a/src/paimon/core/utils/blob_view_lookup.cpp +++ b/src/paimon/core/utils/blob_view_lookup.cpp @@ -28,6 +28,7 @@ #include "arrow/c/bridge.h" #include "fmt/format.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/executor/future.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -154,27 +155,32 @@ Result BlobViewLookup::LoadTableDescriptorChu Catalog::Create(catalog_context->root_path, catalog_context->options, /*file_system=*/nullptr, catalog_context->fs_scheme_to_identifier_map)); PAIMON_ASSIGN_OR_RAISE(std::string table_path, catalog->GetTableLocation(identifier)); - ScanContextBuilder scan_builder(table_path); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); - scan_builder.SetGlobalIndexResult(global_index_result) - .WithMemoryPool(pool) - .WithCatalog(catalog, identifier); + scan_builder_load_options.catalog = catalog; + scan_builder_load_options.identifier.emplace(identifier); + scan_builder.SetGlobalIndexResult(global_index_result).WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); - ReadContextBuilder read_builder(table_path); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; std::vector read_field_ids = field_ids; read_field_ids.push_back(SpecialFieldIds::ROW_ID); - read_builder.SetReadFieldIds(read_field_ids) - .AddOption(Options::BLOB_AS_DESCRIPTOR, "true") - .EnablePrefetch(true) - .WithMemoryPool(pool) - .WithCatalog(catalog, identifier); + read_builder_options[Options::BLOB_AS_DESCRIPTOR] = "true"; + read_builder_load_options.catalog = catalog; + read_builder_load_options.identifier.emplace(identifier); + read_builder.SetReadFieldIds(read_field_ids).EnablePrefetch(true).WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); diff --git a/src/paimon/core/utils/branch_manager.cpp b/src/paimon/core/utils/branch_manager.cpp index 6a9a46b3d..49b3ebc3d 100644 --- a/src/paimon/core/utils/branch_manager.cpp +++ b/src/paimon/core/utils/branch_manager.cpp @@ -41,6 +41,9 @@ Result BranchManager::ResolveBranch(const std::optional std::optional identifier_branch; if (identifier) { PAIMON_ASSIGN_OR_RAISE(identifier_branch, identifier.value().GetBranchName()); + if (identifier_branch) { + PAIMON_ASSIGN_OR_RAISE(identifier_branch, identifier.value().GetBranchNameOrDefault()); + } } auto option_iter = options.find(Options::BRANCH); std::optional option_branch = diff --git a/src/paimon/core/utils/branch_manager.h b/src/paimon/core/utils/branch_manager.h index eebb45647..747f23cbf 100644 --- a/src/paimon/core/utils/branch_manager.h +++ b/src/paimon/core/utils/branch_manager.h @@ -78,7 +78,9 @@ class BranchManager { /// option and `explicit_branch` may each name. /// /// An absent name and a name which `NormalizeBranch` maps to `main` both mean the main - /// branch. Names which disagree are refused rather than silently chosen between, with + /// branch. Identifier names also fold main branch aliases as the catalog does through + /// `GetBranchNameOrDefault`; option and explicit branch names keep their case. + /// Names which disagree are refused rather than silently chosen between, with /// `operation` naming the operation in that error. Every name is put through /// `CheckValidBranch` before it is compared, so neither the branch this returns nor a /// branch it names in an error is one that would leave the table root. diff --git a/src/paimon/core/utils/branch_manager_test.cpp b/src/paimon/core/utils/branch_manager_test.cpp index c73459241..ea114bef9 100644 --- a/src/paimon/core/utils/branch_manager_test.cpp +++ b/src/paimon/core/utils/branch_manager_test.cpp @@ -101,6 +101,35 @@ TEST(BranchManagerTest, TestResolveBranch) { "branch name cannot contain control characters"); } +TEST(BranchManagerTest, TestResolveMainBranchAliases) { + for (const std::string alias : {"main", "MAIN", "Main", "mAiN"}) { + SCOPED_TRACE(alias); + const Identifier identifier("db", "tbl$branch_" + alias); + ASSERT_OK_AND_ASSIGN(std::string branch, + BranchManager::ResolveBranch(identifier, {}, std::nullopt, "scan")); + ASSERT_EQ("main", branch); + ASSERT_OK_AND_ASSIGN( + std::string agreed, + BranchManager::ResolveBranch(identifier, {{Options::BRANCH, "main"}}, "main", "scan")); + ASSERT_EQ("main", agreed); + ASSERT_NOK_WITH_MSG(BranchManager::ResolveBranch(identifier, {{Options::BRANCH, "dev"}}, + std::nullopt, "scan"), + "a scan is aimed at one branch, but both 'main' and 'dev' were named"); + } + ASSERT_OK_AND_ASSIGN(std::string option_branch, + BranchManager::ResolveBranch(std::nullopt, {{Options::BRANCH, "MAIN"}}, + std::nullopt, "scan")); + ASSERT_EQ("MAIN", option_branch); + ASSERT_OK_AND_ASSIGN(std::string explicit_branch, + BranchManager::ResolveBranch(std::nullopt, {}, "MAIN", "scan")); + ASSERT_EQ("MAIN", explicit_branch); + ASSERT_OK_AND_ASSIGN( + std::string unnamed, + BranchManager::ResolveBranch(Identifier("db", "tbl"), {{Options::BRANCH, "dev"}}, + std::nullopt, "scan")); + ASSERT_EQ("dev", unnamed); +} + TEST(BranchManagerTest, TestCheckCatalogAddressableBranch) { ASSERT_OK(BranchManager::CheckCatalogAddressableBranch(BranchManager::DEFAULT_MAIN_BRANCH)); ASSERT_OK(BranchManager::CheckCatalogAddressableBranch("")); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 8f23150f1..203796131 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -529,9 +529,13 @@ Result> RestCatalog::GetTable(const Identifier& identifie // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); + std::shared_ptr catalog = std::const_pointer_cast(weak_from_this().lock()); + if (!catalog) { + catalog = std::make_shared(*this); + } return FileStoreTable::Create(table_file_system, table_path, schema, identifier, uuid, branch.value_or(Identifier::kDefaultMainBranch), - fs_scheme_to_identifier_map_); + fs_scheme_to_identifier_map_, catalog); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 0f710bb47..f0a4c6c68 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -38,6 +38,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/table.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/catalog_options.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" @@ -823,13 +824,17 @@ TEST_F(RestCatalogTest, GlobalSystemTableRetainsFactoryCatalog) { std::weak_ptr weak_catalog = catalog; ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("sys", "tables"))); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(table->GetRootPath()).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + table->NewScan(std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(table->GetRootPath()).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + table->NewRead(std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); catalog.reset(); table.reset(); @@ -1686,10 +1691,13 @@ TEST_F(RestCatalogTest, FileStoreCommitIsBuiltFromTheCatalog) { requests_before = state_->get_table_requests; } std::shared_ptr shared_catalog(std::move(rest_catalog)); - CommitContextBuilder builder(dir->Str(), "commit-user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.WithCatalog(shared_catalog, Identifier("db1", "t1")).Finish()); - ASSERT_OK(FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + CommitContextBuilder builder("commit-user"); + builder_load_options.catalog = shared_catalog; + builder_load_options.identifier.emplace(Identifier("db1", "t1")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK(FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); { std::lock_guard lock(state_->mutex); ASSERT_GT(state_->get_table_requests, requests_before); @@ -1919,10 +1927,13 @@ TEST_P(RestCatalogCommitRecoveryTest, CommitAcceptedThenLostRecoversAfterRestart std::shared_ptr catalog(std::move(rest_catalog)); auto create_commit = [&]() -> Result> { - CommitContextBuilder builder(dir->Str(), "commit-user"); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.WithCatalog(catalog, identifier).Finish()); - return FileStoreCommit::Create(std::move(context)); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + CommitContextBuilder builder("commit-user"); + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(identifier); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + return FileStoreCommit::Create(builder_load_options, std::move(context), builder_options); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, create_commit()); @@ -2094,16 +2105,20 @@ class RestCatalogCommitLeaseTest : public RestCatalogTest { Result> CreateCommit( const std::shared_ptr& catalog, const std::optional& table_id, const std::map& options) { - CommitContextBuilder builder(dir_->Str(), "commit-user"); - builder.WithCatalog(catalog, Identifier("db1", "t1")); + TableLoadOptions builder_load_options{dir_->Str()}; + std::map builder_options; + CommitContextBuilder builder("commit-user"); + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(Identifier("db1", "t1")); + if (table_id) { - builder.WithTableId(table_id.value()); + builder_load_options.table_id = table_id.value(); } for (const auto& [key, value] : options) { - builder.AddOption(key, value); + builder_options[key] = value; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreCommit::Create(std::move(context)); + return FileStoreCommit::Create(builder_load_options, std::move(context), builder_options); } std::vector TableEvents() { diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 5f8a8d52f..862b19e06 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -114,6 +114,9 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { Result> GetTable(const Identifier& identifier) const override { ++get_table_calls_; + if (table_ != nullptr) { + return table_; + } // Serves the per-table file system when one was set, matching `GetTableFileSystem` // without recording the request, so a test can still tell the two apart. No location: // this mock's `GetTableLocation` is the unsupported stub. @@ -136,6 +139,13 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { table_schema_ = table_schema; } + void SetTable(const std::shared_ptr
& table) { + table_ = table; + table_uuid_ = table->CatalogUuid().value_or(""); + table_schema_ = table->GetSchema(); + table_file_system_ = table->GetFileSystem(); + } + Result> LoadTableSchema(const Identifier& identifier) const override { load_table_schema_identifiers_.push_back(identifier); if (table_schema_ == nullptr) { @@ -179,6 +189,10 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { return load_snapshot_identifiers_.size(); } + void SetTableUuid(const std::string& table_uuid) { + table_uuid_ = table_uuid; + } + void CheckTableUuid() { check_table_uuid_ = true; } @@ -305,6 +319,7 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { Result answer_; std::string table_uuid_; + std::shared_ptr
table_; std::shared_ptr table_schema_; std::shared_ptr file_system_; std::shared_ptr table_file_system_; diff --git a/src/paimon/testing/utils/test_helper.h b/src/paimon/testing/utils/test_helper.h index 2bc1e9bbc..59cc5db15 100644 --- a/src/paimon/testing/utils/test_helper.h +++ b/src/paimon/testing/utils/test_helper.h @@ -32,6 +32,7 @@ #include "fmt/format.h" #include "paimon/api.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/types/data_field.h" @@ -90,23 +91,30 @@ class TestHelper { PAIMON_ASSIGN_OR_RAISE(auto file_system, FileSystemFactory::Get(file_system_identifier, table_path, options)); std::string commit_user = "commit_user"; - WriteContextBuilder context_builder(table_path, commit_user); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder(commit_user); if (!temp_directory.empty()) { context_builder.WithTempDirectory(temp_directory); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, - context_builder.SetOptions(options) - .WithStreamingMode(is_streaming_mode) - .WithWriteId(12345) - .Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); - CommitContextBuilder commit_context_builder(table_path, commit_user); + context_builder_options = options; PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).IgnoreEmptyCommit(false).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + std::unique_ptr write_context, + context_builder.WithStreamingMode(is_streaming_mode).WithWriteId(12345).Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder(commit_user); + commit_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); return std::unique_ptr(new TestHelper(std::move(file_system), std::move(write), std::move(commit), commit_user, table_path, options)); @@ -156,16 +164,20 @@ class TestHelper { Result>> NewScan(StartupMode startup_mode, std::optional snapshot_id, bool is_streaming = true) { - ScanContextBuilder scan_context_builder(table_path_); - scan_context_builder.WithStreamingMode(is_streaming) - .SetOptions(options_) - .AddOption(Options::SCAN_MODE, startup_mode.ToString()); + TableLoadOptions scan_context_builder_load_options{table_path_}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options_; + scan_context_builder_options[Options::SCAN_MODE] = startup_mode.ToString(); + scan_context_builder.WithStreamingMode(is_streaming); if (snapshot_id) { - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, - std::to_string(snapshot_id.value())); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = + std::to_string(snapshot_id.value()); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(scan_, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + scan_, TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); return Scan(); } @@ -276,14 +288,19 @@ class TestHelper { Result> ReadResult( const std::vector>& splits, std::unique_ptr<::ArrowSchema> read_schema) { - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options_); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options_; + if (read_schema != nullptr) { read_context_builder.SetReadSchema(std::move(read_schema)); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr collected, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -302,11 +319,16 @@ class TestHelper { Result ReadAndCheckResult(const std::shared_ptr& data_type, const std::vector>& splits, const std::string& expected_result) { - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options_); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index 1424533de..3c92b1d83 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -24,6 +24,7 @@ #include "arrow/api.h" #include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" @@ -412,10 +413,16 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithMap auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar")); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{ + PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1035,10 +1042,16 @@ TEST_P(AppendCompactionInteTest, TestAppendTableCompactionDictionaryPassthrough) auto predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"s", FieldType::STRING, Literal(FieldType::STRING, indexed_value.data(), indexed_value.size())); - ReadContextBuilder read_context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar")); - read_context_builder.SetOptions(options).SetPredicate(predicate); + TableLoadOptions read_context_builder_load_options{ + PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto filtered, ReadResultCollector::CollectResult(batch_reader.get())); auto expected = std::make_shared( diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index c721658f5..781a11f82 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -40,6 +40,7 @@ #include "arrow/ipc/json_simple.h" #include "arrow/type.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_array_writer.h" #include "paimon/common/data/binary_row.h" @@ -141,11 +142,15 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter const std::vector& write_cols, const std::vector>& write_arrays) const { // write - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithWriteSchema(write_cols); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); for (const auto& write_array : write_arrays) { ArrowArray c_array; @@ -176,11 +181,15 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { // commit - CommitContextBuilder commit_builder(table_path, "commit_user_1"); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); return file_store_commit->Commit(commit_msgs); } @@ -267,14 +276,18 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter Result> ScanTable(const std::string& table_path, const std::shared_ptr& predicate = nullptr, const std::vector& row_ranges = {}) const { - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetPredicate(predicate); if (!row_ranges.empty()) { auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); scan_context_builder.SetGlobalIndexResult(global_index_result); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); return result_plan; } @@ -285,15 +298,19 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter const std::shared_ptr& plan, const std::shared_ptr& predicate = nullptr, const std::map& options = {}) const { auto splits = plan->Splits(); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetReadFieldNames(read_schema).SetPredicate(predicate); read_context_builder.EnableLateMaterializing(false); if (!options.empty()) { - read_context_builder.SetOptions(options); + read_context_builder_options = options; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4885,10 +4902,15 @@ TEST_P(BlobTableInteTest, TestReadArrayBlobTableFromJava) { const std::vector read_fields = {"id", "array_payloads"}; for (int64_t snapshot_id : {1, 2, 3}) { - ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); size_t array_layer_count = 0; @@ -4946,10 +4968,15 @@ TEST_P(BlobTableInteTest, TestReadMapBlobTableFromJava) { "large_decimal_payloads"}; for (int64_t snapshot_id : {1, 3}) { - ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); size_t string_layer_count = 0; diff --git a/test/inte/clean_inte_test.cpp b/test/inte/clean_inte_test.cpp index ec418d2c0..4a32b7d48 100644 --- a/test/inte/clean_inte_test.cpp +++ b/test/inte/clean_inte_test.cpp @@ -42,6 +42,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -204,11 +205,16 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { auto dir = UniqueTestDirectory::Create(); std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 4); } @@ -219,11 +225,16 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath( table_path, "manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1"))); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 4); } @@ -234,11 +245,16 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath( table_path, "f1=10/bucket-1/data-7a912f84-04b7-4bbb-8dc6-53f4a292ea25-0.parquet"))); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 4); } @@ -248,11 +264,16 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath(table_path, "snapshot/snapshot-3"))); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 1); } @@ -282,12 +303,16 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { CreateTestTable(dir->Str(), /*db_name=*/"foo", /*table_name=*/"bar", &arrow_schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); - WriteContextBuilder context_builder(table_path, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); @@ -303,18 +328,24 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_load_options.file_system = file_system_; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "2"; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + commit_context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; + commit_context_builder_options[Options::MANIFEST_MERGE_MIN_COUNT] = "2"; + commit_context_builder_options[Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "2") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .AddOption(Options::MANIFEST_MERGE_MIN_COUNT, "2") - .AddOption(Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true") + commit_context_builder + .IgnoreEmptyCommit(false) .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0, 10)); ASSERT_OK_AND_ASSIGN( @@ -430,12 +461,17 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { /*table_name=*/"bar", &arrow_schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); - WriteContextBuilder context_builder(table_path, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + context_builder.WithStreamingMode(true).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); @@ -451,19 +487,25 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_load_options.file_system = file_system_; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "2"; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + commit_context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; + commit_context_builder_options[Options::MANIFEST_MERGE_MIN_COUNT] = "2"; + commit_context_builder_options[Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder + + .IgnoreEmptyCommit(false) + .Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "2") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .AddOption(Options::MANIFEST_MERGE_MIN_COUNT, "2") - .AddOption(Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true") - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( @@ -588,12 +630,16 @@ TEST_F(CleanInteTest, TestOrphanFilesClean) { /*partition_keys=*/{"f1", "f2"}, /*primary_keys=*/{}, options)); - WriteContextBuilder context_builder(table_path, commit_user); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder(commit_user); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}, {"f2", "1"}}, 0)); @@ -609,13 +655,16 @@ TEST_F(CleanInteTest, TestOrphanFilesClean) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); - CommitContextBuilder commit_context_builder(table_path, commit_user); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder(commit_user); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_load_options.file_system = file_system_; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( @@ -702,12 +751,16 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithFileRetainCondition) { /*partition_keys=*/{"f1", "f2"}, /*primary_keys=*/{}, options)); - WriteContextBuilder context_builder(table_path, commit_user); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder(commit_user); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}, {"f2", "1"}}, 0)); @@ -723,13 +776,16 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithFileRetainCondition) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); - CommitContextBuilder commit_context_builder(table_path, commit_user); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder(commit_user); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_load_options.file_system = file_system_; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( @@ -812,12 +868,16 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithIOException) { /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); - WriteContextBuilder context_builder(table_path, commit_user); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder(commit_user); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); - std::string root_path = write_context->GetRootPath(); + context_builder.WithStreamingMode(true).Finish()); + std::string root_path = context_builder_load_options.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); @@ -833,14 +893,20 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithIOException) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); - CommitContextBuilder commit_context_builder(root_path, commit_user); + TableLoadOptions commit_context_builder_load_options{root_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder(commit_user); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_load_options.file_system = file_system_; + commit_context_builder_options[Options::BUCKET] = "2"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .AddOption(Options::BUCKET, "2") + commit_context_builder + .IgnoreEmptyCommit(false) .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp index 79e16a6f2..5b8f333f0 100644 --- a/test/inte/data_evolution_table_test.cpp +++ b/test/inte/data_evolution_table_test.cpp @@ -24,6 +24,7 @@ #include "arrow/type.h" #include "fmt/format.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" @@ -111,14 +112,18 @@ class DataEvolutionTableTest : public ::testing::Test, const std::vector>& write_arrays, const std::map& write_options = {}) const { // write - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithWriteSchema(write_cols); for (const auto& [key, value] : write_options) { - write_builder.AddOption(key, value); + write_builder_options[key] = value; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); for (const auto& write_array : write_arrays) { ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); @@ -165,11 +170,15 @@ class DataEvolutionTableTest : public ::testing::Test, Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { // commit - CommitContextBuilder commit_builder(table_path, "commit_user_1"); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); return file_store_commit->Commit(commit_msgs); } @@ -308,29 +317,38 @@ class DataEvolutionTableTest : public ::testing::Test, const std::string& table_path, const std::vector& read_schema, int32_t limit, const std::shared_ptr& predicate = nullptr, const std::vector& row_ranges = {}, bool enable_predicate_filter = false) const { - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetLimit(limit); scan_context_builder.SetPredicate(predicate); if (!row_ranges.empty()) { scan_context_builder.SetGlobalIndexResult( BitmapGlobalIndexResult::FromRanges(row_ranges)); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - FinishScanContext(scan_context_builder)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result_plan, table_scan->CreatePlan()); LimitScanResult result; result.splits = result_plan->Splits(); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetReadFieldNames(read_schema) .SetPredicate(predicate) .EnablePredicateFilter(enable_predicate_filter); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(result.splits)); PAIMON_ASSIGN_OR_RAISE(result.rows, @@ -343,11 +361,16 @@ class DataEvolutionTableTest : public ::testing::Test, /// the row count derived from them. Reading the splits back is ScanAndReadWithLimit's job, /// which verifies the returned rows after their reader has been destroyed. Result>> PlanSplits(const std::string& table_path) const { - ScanContextBuilder scan_context_builder(table_path); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - FinishScanContext(scan_context_builder)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); return plan->Splits(); } @@ -416,11 +439,15 @@ class DataEvolutionTableTest : public ::testing::Test, const std::string& table_path, const std::vector>& commit_msgs, std::optional row_id_check_from_snapshot) const { - CommitContextBuilder commit_builder(table_path, "commit_user_1"); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); file_store_commit->RowIdCheckConflict(row_id_check_from_snapshot); return file_store_commit->Commit(commit_msgs); } @@ -478,11 +505,16 @@ class DataEvolutionTableTest : public ::testing::Test, ASSERT_OK_AND_ASSIGN(std::string anchor, PlannedAnchorFileName(table_path)); ASSERT_OK(CommitDeletionVectors(table_path, base_msgs[0], {{anchor, deleted}})); } - ScanContextBuilder scan_builder(table_path); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; scan_builder.SetGlobalIndexResult(std::make_shared( RoaringBitmap64::From(candidates), std::vector(scores))); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_builder, scan_builder_options)); + ASSERT_OK_AND_ASSIGN( + auto scan, TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); auto scored_split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -513,12 +545,16 @@ class DataEvolutionTableTest : public ::testing::Test, std::shared_ptr bound_predicate, PredicateUtils::CreatePickedFieldFilter(filter_case.predicate, {{"f0", project_row_id ? 3 : 2}})); - ReadContextBuilder read_builder(table_path); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; read_builder.SetReadFieldNames(read_fields) .SetPredicate(bound_predicate) .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(read_builder_load_options, + std::move(read_context), + read_builder_options)); // Reuse the same TableRead with and without scores: the internal row-id // projection must not affect a subsequent split's output schema. for (bool with_scores : {true, false}) { @@ -568,14 +604,19 @@ class DataEvolutionTableTest : public ::testing::Test, bool check_scan_plan_when_empty_result = true, bool apply_predicate_to_scan = true) const { // scan - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetPredicate(apply_predicate_to_scan ? predicate : nullptr); if (!row_ranges.empty()) { auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); scan_context_builder.SetGlobalIndexResult(global_index_result); } - PAIMON_ASSIGN_OR_RAISE(auto scan_context, FinishScanContext(scan_context_builder)); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, FinishScanContext(scan_context_builder, + scan_context_builder_options)); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); if (!expected_array && check_scan_plan_when_empty_result) { if (!result_plan->Splits().empty()) { @@ -585,11 +626,15 @@ class DataEvolutionTableTest : public ::testing::Test, // read auto splits = result_plan->Splits(); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetReadFieldNames(read_schema).SetPredicate(predicate); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -634,14 +679,19 @@ class DataEvolutionTableTest : public ::testing::Test, const std::vector>& expected_first_row_ids, const std::vector& expected_row_counts) { ASSERT_EQ(expected_first_row_ids.size(), expected_row_counts.size()); - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetPredicate(predicate); if (!row_ranges.empty()) { auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); scan_context_builder.SetGlobalIndexResult(global_index_result); } - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); const auto& result_splits = result_plan->Splits(); if (expected_first_row_ids.empty()) { @@ -666,14 +716,15 @@ class DataEvolutionTableTest : public ::testing::Test, ASSERT_EQ(result_row_counts, expected_row_counts); } - Result> FinishScanContext(ScanContextBuilder& builder) const { + Result> FinishScanContext( + ScanContextBuilder& builder, std::map& builder_options) const { if (EnableSnapshotLiveManifestCache()) { if (!snapshot_live_manifest_cache_) { snapshot_live_manifest_cache_ = std::make_shared(/*max_weight=*/64 * 1024 * 1024); } - builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .WithCache(snapshot_live_manifest_cache_); + builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder.WithCache(snapshot_live_manifest_cache_); } return builder.Finish(); } @@ -1098,16 +1149,25 @@ TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite) auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2019,11 +2079,14 @@ TEST_P(DataEvolutionTableTest, TestReadJavaIndependentlyRolledVectorStore) { } auto selected_gap_expected = checked_pointer_cast( arrow::Concatenate(selected_gap_chunks).ValueOrDie()); - ReadContextBuilder read_builder(table_path); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; read_builder.SetReadFieldNames({"embedding"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); const std::vector, std::shared_ptr>> gap_cases = {{gap_split, gap_expected}, {std::make_shared(gap_split, selected_ranges), @@ -2084,12 +2147,16 @@ TEST_P(DataEvolutionTableTest, TestReadJavaIndependentlyRolledVectorStore) { {std::vector{"id", "embedding", "_ROW_ID", "_SEQUENCE_NUMBER"}, std::vector{"id", "_ROW_ID", "_SEQUENCE_NUMBER"}, std::vector{"_ROW_ID", "_SEQUENCE_NUMBER"}}) { - ReadContextBuilder sequence_read_builder(table_path); + TableLoadOptions sequence_read_builder_load_options{table_path}; + std::map sequence_read_builder_options; + ReadContextBuilder sequence_read_builder; sequence_read_builder.SetReadFieldNames(projection); ASSERT_OK_AND_ASSIGN(std::unique_ptr sequence_context, sequence_read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr sequence_read, - TableRead::Create(std::move(sequence_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr sequence_read, + TableRead::Create(sequence_read_builder_load_options, std::move(sequence_context), + sequence_read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr sequence_reader, sequence_read->CreateReader(split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr sequence_result, @@ -2197,16 +2264,25 @@ TEST_P(DedicatedVectorTableTest, TestDedicatedVectorStore) { ASSERT_EQ(group[0]->row_count, group[1]->row_count); } - WriteContextBuilder partial_builder(table_path, "partial"); + TableLoadOptions partial_builder_load_options{table_path}; + std::map partial_builder_options; + WriteContextBuilder partial_builder("partial"); partial_builder.WithWriteSchema({"embedding"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr partial_context, partial_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(partial_context)).status(), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(partial_builder_load_options, + std::move(partial_context), partial_builder_options) + .status(), "partial writes"); - WriteContextBuilder invalid_builder(table_path, "invalid"); - invalid_builder.AddOption(Options::DATA_EVOLUTION_ENABLED, "false"); + TableLoadOptions invalid_builder_load_options{table_path}; + std::map invalid_builder_options; + WriteContextBuilder invalid_builder("invalid"); + invalid_builder_options[Options::DATA_EVOLUTION_ENABLED] = "false"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_context, invalid_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(invalid_context)).status(), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(invalid_builder_load_options, + std::move(invalid_context), invalid_builder_options) + .status(), "requires row-tracking.enabled and data-evolution.enabled"); } @@ -3972,12 +4048,16 @@ TEST_P(DataEvolutionDvTableTest, TestReadWithDeletionVectorsOnEveryRowRangeGroup // a count query answers from that metadata alone, never reading a row, so the deletion // vectors have to reach it too: without them it reports the 8 rows the files hold - ReadContextBuilder count_context_builder(table_path); + TableLoadOptions count_context_builder_load_options{table_path}; + std::map count_context_builder_options; + ReadContextBuilder count_context_builder; count_context_builder.SetReadFieldNames(schema->field_names()); ASSERT_OK_AND_ASSIGN(std::unique_ptr count_context, count_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr count_table_read, - TableRead::Create(std::move(count_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr count_table_read, + TableRead::Create(count_context_builder_load_options, std::move(count_context), + count_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr count_reader, count_table_read->CreateCountReader(planned_splits)); ASSERT_OK_AND_ASSIGN(int64_t counted_rows, count_reader->CountRows()); diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index 3c24f83f8..3777be290 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -25,6 +25,7 @@ #include "fmt/format.h" #include "fmt/ranges.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/global_index/bitmap/bitmap_global_index_factory.h" #include "paimon/common/global_index/bitmap/bitmap_global_index_options.h" @@ -217,12 +218,17 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter const std::vector& write_cols, const std::shared_ptr& write_array) const { // write - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithWriteSchema(write_cols); - write_builder.WithFileSystem(fs_); + write_builder_load_options.file_system = fs_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); ArrowArray c_array; EXPECT_TRUE(arrow::ExportArray(*write_array, &c_array).ok()); auto record_batch = std::make_unique( @@ -245,23 +251,33 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { // commit - CommitContextBuilder commit_builder(table_path, "commit_user_1"); - commit_builder.WithFileSystem(fs_); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); + commit_builder_load_options.file_system = fs_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); return file_store_commit->Commit(commit_msgs); } Result> ScanData( const std::string& table_path, const std::vector>& partition_filters) const { - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetPartitionFilter(partition_filters); - scan_context_builder.WithFileSystem(fs_); + scan_context_builder_load_options.file_system = fs_; + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); EXPECT_EQ(result_plan->Splits().size(), 1); return std::dynamic_pointer_cast(result_plan->Splits()[0]); @@ -789,16 +805,19 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter const std::map& options = {}, const std::shared_ptr& index_result = nullptr, const std::shared_ptr& cache = nullptr) const { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetPredicate(predicate) - .SetOptions(options) - .SetGlobalIndexResult(index_result) - .WithFileSystem(fs_); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_load_options.file_system = fs_; + scan_context_builder.SetPredicate(predicate).SetGlobalIndexResult(index_result); if (cache) { scan_context_builder.WithCache(cache); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); return result_plan; } @@ -830,14 +849,18 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter const std::shared_ptr& predicate, const std::shared_ptr& result_plan) const { auto splits = result_plan->Splits(); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_load_options.file_system = fs_; read_context_builder.SetReadFieldNames(read_schema) .SetPredicate(predicate) - .WithFileSystem(fs_) .EnableLateMaterializing(false); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1221,13 +1244,17 @@ TEST_P(GlobalIndexTest, TestWriteIndexDecodesNestedOrcDictionaries) { ScanData(table_path, /*partition_filters=*/{})); std::shared_ptr indexed_split = std::make_shared(split, std::vector{Range(0, 3)}); - ReadContextBuilder read_builder(table_path); - read_builder.SetOptions(options).WithFileSystem(fs_).SetReadFieldNames( - {"key", "details", "attributes", "plain_details", "plain_attributes", - SpecialFields::RowId().Name()}); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options; + read_builder_load_options.file_system = fs_; + read_builder.SetReadFieldNames({"key", "details", "attributes", "plain_details", + "plain_attributes", SpecialFields::RowId().Name()}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(indexed_split)); ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); diff --git a/test/inte/nested_column_pruning_inte_test.cpp b/test/inte/nested_column_pruning_inte_test.cpp index b9b1b99ca..6adf94267 100644 --- a/test/inte/nested_column_pruning_inte_test.cpp +++ b/test/inte/nested_column_pruning_inte_test.cpp @@ -29,6 +29,7 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/path_util.h" @@ -92,25 +93,34 @@ class NestedColumnPruningInteTest : public ::testing::Test, const std::shared_ptr& expected_schema, const std::string& expected_json, const std::shared_ptr& predicate = nullptr) const { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + if (predicate) { scan_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*expected_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetReadSchema(std::move(c_schema)); if (predicate) { read_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -350,10 +360,14 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsAllNonExistent) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options), "does not support schema evolution inside struct"); } @@ -412,10 +426,14 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithNonExistentField) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options), "does not support schema evolution inside struct"); } @@ -491,10 +509,15 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithTypeMismatchShouldFa helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); ASSERT_FALSE(data_splits.empty()); - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateReader(data_splits), "PruneDataType nested field type mismatch for 'a': read string vs " @@ -581,10 +604,15 @@ TEST_P(NestedColumnPruningInteTest, auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); ASSERT_FALSE(data_splits.empty()); @@ -766,10 +794,15 @@ TEST_P(NestedColumnPruningInteTest, PruneNestedStructWithSpecialFields) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1178,10 +1211,16 @@ TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) { ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); - WriteContextBuilder write_context_builder(table_path_, "commit_user_1"); + TableLoadOptions write_context_builder_load_options{table_path_}; + std::map write_context_builder_options; + WriteContextBuilder write_context_builder("commit_user_1"); + write_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + write_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(write_context_builder_load_options, std::move(write_context), + write_context_builder_options)); auto write_one_row = [&](const std::string& data) -> Status { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, @@ -1201,10 +1240,15 @@ TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) { /*commit_identifier=*/0)); ASSERT_OK(file_store_write->Close()); - CommitContextBuilder commit_context_builder(table_path_, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path_}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(commit_msgs, /*commit_identifier=*/0)); // Read selected MAP keys together with nested STRUCT sub-fields. @@ -1309,10 +1353,15 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysWithOrcDictionaryEncodedMap) auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1464,10 +1513,15 @@ TEST_P(NestedColumnPruningInteTest, PruneListStructSubFields) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); auto create_reader_result = table_read->CreateReader(data_splits); ASSERT_NOK_WITH_MSG(create_reader_result, "partial projection inside list"); } diff --git a/test/inte/paimon_read_compat_inte_test.cpp b/test/inte/paimon_read_compat_inte_test.cpp index 5761720d4..f0053fcbd 100644 --- a/test/inte/paimon_read_compat_inte_test.cpp +++ b/test/inte/paimon_read_compat_inte_test.cpp @@ -29,6 +29,7 @@ #include "arrow/api.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/variant/generic_variant.h" #include "paimon/data/blob_descriptor.h" #include "paimon/defs.h" @@ -71,20 +72,30 @@ Result> ReadTable( std::string table_path = GetDataDir() + "/" + file_format + "/" + kDatabaseName + ".db/" + table_name; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames(field_names); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadFieldNames(field_names); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); return ReadResultCollector::CollectResult(std::move(batch_reader)); diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp index 153a7926c..390f7317b 100644 --- a/test/inte/pk_compaction_inte_test.cpp +++ b/test/inte/pk_compaction_inte_test.cpp @@ -31,6 +31,7 @@ #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/table/special_fields.h" @@ -351,15 +352,19 @@ class PkCompactionInteTest : public ::testing::Test { int32_t bucket, const std::shared_ptr& write_array, int64_t commit_identifier, bool is_streaming = true, const std::vector& row_kinds = {}, bool write_only = true) const { - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(is_streaming) .WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp")); if (write_only) { - write_builder.AddOption(Options::WRITE_ONLY, "true"); + write_builder_options[Options::WRITE_ONLY] = "true"; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); auto record_batch = std::make_unique(partition, bucket, row_kinds, &c_array); @@ -372,11 +377,15 @@ class PkCompactionInteTest : public ::testing::Test { Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { - CommitContextBuilder commit_builder(table_path, "commit_user_1"); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); return file_store_commit->Commit(commit_msgs); } @@ -397,13 +406,18 @@ class PkCompactionInteTest : public ::testing::Test { const std::string& table_path, const std::map& partition, int32_t bucket, bool full_compaction, int64_t commit_identifier, const std::map& options = {}) { - WriteContextBuilder write_builder(table_path, "commit_user_1"); - write_builder.SetOptions(options); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); + write_builder_options = options; + write_builder.WithStreamingMode(true).WithTempDirectory( PathUtil::JoinPath(dir_->Str(), "tmp")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); PAIMON_RETURN_NOT_OK(file_store_write->Compact(partition, bucket, full_compaction)); PAIMON_ASSIGN_OR_RAISE( std::vector> commit_messages, @@ -422,16 +436,21 @@ class PkCompactionInteTest : public ::testing::Test { std::map options = {{Options::FILE_SYSTEM, "local"}}; ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(table_path, options, /*is_streaming_mode=*/false)); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false).SetOptions(options).AddOption( - Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); if (predicate) { scan_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_plan, table_scan->CreatePlan()); std::vector> data_splits = result_plan->Splits(); @@ -456,15 +475,20 @@ class PkCompactionInteTest : public ::testing::Test { auto iter = expected_data_per_partition_bucket.find(key); ASSERT_TRUE(iter != expected_data_per_partition_bucket.end()) << "Unexpected partition=" << key.first << " bucket=" << key.second; - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + if (predicate) { read_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_result, @@ -490,20 +514,30 @@ class PkCompactionInteTest : public ::testing::Test { template void ScanAllRows(const std::string& table_path, Fn consume) { std::map options = {{Options::FILE_SYSTEM, "local"}}; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false).SetOptions(options).AddOption( - Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_plan, table_scan->CreatePlan()); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -617,13 +651,16 @@ TEST_F(PkCompactionInteTest, TestMetadataOnlyLevelUpgradeKeepsValueStats) { ASSERT_FALSE(before->value_stats == SimpleStats::EmptyStats()); ASSERT_EQ(before->value_stats, after->value_stats); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false) - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::FILE_SYSTEM] = "local"; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_EQ(1u, plan->Splits().size()); auto split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -997,9 +1034,13 @@ TEST_F(PkCompactionInteTest, DeduplicateWithBitmap64HighPositions) { ASSERT_TRUE(HasDeletionVectorIndexFiles(compact_msgs)); // Reload committed metadata and DV bytes; no test-created DV or lookup result is injected. - ScanContextBuilder scan_builder(table_path); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); int32_t dv_count = 0; for (const auto& split : plan->Splits()) { @@ -1746,10 +1787,15 @@ TEST_F(PkCompactionInteTest, CompactWithSchemaEvolution) { // Step 5: Read and verify using DataSplits from CompactAfter in write CommitMessages. { - ReadContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "parquet"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = "parquet"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); // Use the latest snapshot id for both splits. // The original table has 6 snapshots. Each write with force-compact produces @@ -2011,12 +2057,16 @@ TEST_F(PkCompactionInteTest, ContinuousWriteWithBackgroundCompact) { // Step 1-4: Streaming writes with background compact in one FileStoreWrite. { - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory( PathUtil::JoinPath(dir_->Str(), "tmp")); ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); auto write_batch = [&](const std::string& json_data) -> Status { auto array = @@ -2220,12 +2270,15 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { auto data_type = arrow::struct_(fields); // Step 2: Write data to the branch-rt and compact. - WriteContextBuilder write_builder(table_path, "commit_user_1"); - write_builder.WithStreamingMode(true) - .WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp")) - .WithBranch("rt"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); + write_builder_options["branch"] = "rt"; + write_builder.WithStreamingMode(true).WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp")); ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); // dt=20240726, bucket-0 only has ["20240726", "cherry", 3] one row auto write_array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ @@ -2276,11 +2329,17 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { ASSERT_EQ(compact_after[0]->row_count, 2); // Step 4: Commit to branch-rt. - CommitContextBuilder commit_builder(table_path, "commit_user_1"); - commit_builder.AddOption(Options::FILE_SYSTEM, "local").AddOption(Options::BRANCH, "rt"); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); + commit_builder_options[Options::FILE_SYSTEM] = "local"; + commit_builder_options[Options::BRANCH] = "rt"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); auto fs = std::make_shared(); @@ -2299,13 +2358,16 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { // Step 5: Scan and read branch-rt. std::map branch_options = {{Options::FILE_SYSTEM, "local"}, {Options::BRANCH, "rt"}}; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false) - .SetOptions(branch_options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = branch_options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); std::vector> splits = plan->Splits(); @@ -2316,11 +2378,15 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { ASSERT_EQ(split_impl->SnapshotId(), 3); } - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(branch_options); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = branch_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, diff --git a/test/inte/primary_key_sorted_index_inte_test.cpp b/test/inte/primary_key_sorted_index_inte_test.cpp index f2811a699..d72fc77d3 100644 --- a/test/inte/primary_key_sorted_index_inte_test.cpp +++ b/test/inte/primary_key_sorted_index_inte_test.cpp @@ -31,6 +31,7 @@ #include "arrow/api.h" #include "fmt/format.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/table/source/data_split_impl.h" @@ -199,24 +200,27 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, const std::optional& snapshot_id, bool index_enabled, const std::vector>& partition_filters, const std::string& branch, const std::shared_ptr& file_system) const { - ScanContextBuilder builder(TablePath(partitioned)); - builder.SetPredicate(predicate).AddOption(Options::GLOBAL_INDEX_ENABLED, - index_enabled ? "true" : "false"); + TableLoadOptions builder_load_options{TablePath(partitioned)}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::GLOBAL_INDEX_ENABLED] = index_enabled ? "true" : "false"; + builder.SetPredicate(predicate); if (snapshot_id != std::nullopt) { - builder.AddOption(Options::SCAN_SNAPSHOT_ID, fmt::format("{}", snapshot_id.value())); + builder_options[Options::SCAN_SNAPSHOT_ID] = fmt::format("{}", snapshot_id.value()); } if (!partition_filters.empty()) { builder.SetPartitionFilter(partition_filters); } if (!branch.empty()) { - builder.AddOption(Options::BRANCH, branch); + builder_options[Options::BRANCH] = branch; } if (file_system != nullptr) { - builder.WithFileSystem(file_system); + builder_load_options.file_system = file_system; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(builder_load_options, std::move(context), builder_options)); return table_scan->CreatePlan(); } @@ -224,7 +228,9 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, const std::vector>& splits, const std::map& options, const std::shared_ptr& file_system) const { - ReadContextBuilder builder(TablePath(partitioned)); + TableLoadOptions builder_load_options{TablePath(partitioned)}; + std::map builder_options; + ReadContextBuilder builder; if (partitioned) { builder.SetReadFieldNames({"pt", "id", "score", "tag"}); } else { @@ -232,14 +238,15 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, } builder.SetPredicate(predicate).EnablePredicateFilter(true); for (const auto& [key, value] : options) { - builder.AddOption(key, value); + builder_options[key] = value; } if (file_system != nullptr) { - builder.WithFileSystem(file_system); + builder_load_options.file_system = file_system; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -541,13 +548,16 @@ TEST_P(PrimaryKeySortedIndexInteTest, ScoredSplitFailsBeforeForceKeepDeleteFallb inner_split, std::vector{Range(first_range.from, first_range.from)}, std::vector{0.5F}); - ReadContextBuilder builder(TablePath(/*partitioned=*/false)); + TableLoadOptions builder_load_options{TablePath(/*partitioned=*/false)}; + std::map builder_options; + ReadContextBuilder builder; builder.SetReadFieldNames({"id", "score", "tag"}) .SetPredicate(predicate) .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); auto* key_value_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(key_value_table_read != nullptr); key_value_table_read->ForceKeepDelete(true); diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 43642fd92..eecf7a9b3 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -39,6 +39,7 @@ #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" #include "paimon/catalog/table.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/factories/io_hook.h" @@ -280,22 +281,30 @@ Result> ReadSystemTable( const std::string& system_table_path, const std::map& options, bool streaming_mode = false, const std::shared_ptr& predicate = nullptr, const std::vector& read_field_names = {}, bool read_next_plan = false) { - ScanContextBuilder scan_context_builder(system_table_path); - scan_context_builder.SetOptions(options).WithStreamingMode(streaming_mode); + TableLoadOptions scan_context_builder_load_options{system_table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder.WithStreamingMode(streaming_mode); if (predicate) { scan_context_builder.SetPredicate(predicate); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); if (read_next_plan) { PAIMON_ASSIGN_OR_RAISE(plan, table_scan->CreatePlan()); } - ReadContextBuilder read_context_builder(system_table_path); - read_context_builder.SetOptions(options); + TableLoadOptions read_context_builder_load_options{system_table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + if (predicate) { read_context_builder.SetPredicate(predicate); } @@ -304,8 +313,10 @@ Result> ReadSystemTable( } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -377,18 +388,23 @@ TEST_P(ReadInteTest, TestAppendSimple) { DataField(2, arrow::field("f2", arrow::int32())), DataField(3, arrow::field("f3", arrow::float64()))}; - ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption("orc.read.enable-metrics", "true"); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = param.file_format; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder.EnablePrefetch(param.enable_prefetch); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); if (specific_table_schema) { - context_builder.SetTableSchema(specific_table_schema.value()); + context_builder_load_options.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -460,18 +476,24 @@ TEST_P(ReadInteTest, TestReadWithLimits) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption(Options::READ_BATCH_SIZE, "1"); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options[Options::READ_BATCH_SIZE] = "1"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options["orc.read.enable-metrics"] = "true"; context_builder.EnablePrefetch(param.enable_prefetch) .SetReadAheadCacheEnabled(param.read_ahead_cache_enabled) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .AddOption("orc.read.enable-metrics", "true") + .SetPrefetchBatchCount(10); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -523,14 +545,19 @@ TEST_P(ReadInteTest, TestReadAheadCacheMetrics) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = param.file_format; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", "false") .SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -597,16 +624,21 @@ TEST_P(ReadInteTest, TestReadOnlyPartitionField) { DataField(0, arrow::field("dt", arrow::utf8())), }; - ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder.SetReadFieldNames({"dt"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector> data_splits; data_splits.reserve(3); @@ -656,17 +688,27 @@ TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) { ArrowSchemaRelease(&schema); std::string system_table_path = PathUtil::JoinPath(dir->Str(), "warehouse/db1.db/tbl1$options"); - ScanContextBuilder scan_context_builder(system_table_path); - scan_context_builder.SetOptions(options); + TableLoadOptions scan_context_builder_load_options{system_table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); - ReadContextBuilder read_context_builder(system_table_path); - read_context_builder.SetOptions(options); + TableLoadOptions read_context_builder_load_options{system_table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); ASSERT_TRUE(result); @@ -688,17 +730,27 @@ TEST(SystemTableReadInteTest, TestReadBranchOptionsSystemTable) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}}; std::string system_table_path = table_path + "$branch_rt$options"; - ScanContextBuilder scan_context_builder(system_table_path); - scan_context_builder.SetOptions(options); + TableLoadOptions scan_context_builder_load_options{system_table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); - ReadContextBuilder read_context_builder(system_table_path); - read_context_builder.SetOptions(options); + TableLoadOptions read_context_builder_load_options{system_table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -935,17 +987,23 @@ TEST(SystemTableReadInteTest, TestReadMetadataSystemTablesAtPinnedSnapshot) { const auto bound_schema = table->GetSchema(); auto create_reader = [&table](const std::map& read_options) -> Result> { + std::map scan_context_builder_0_options; + ScanContextBuilder scan_context_builder_0; + scan_context_builder_0_options = read_options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, + scan_context_builder_0.Finish()); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan_context, - ScanContextBuilder("unused_path").SetOptions(read_options).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - table->NewScan(std::move(scan_context))); + std::unique_ptr scan, + table->NewScan(std::move(scan_context), scan_context_builder_0_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); + std::map read_context_builder_48503_options; + ReadContextBuilder read_context_builder_48503; + read_context_builder_48503_options = read_options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, + read_context_builder_48503.Finish()); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr read_context, - ReadContextBuilder("unused_path").SetOptions(read_options).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - table->NewRead(std::move(read_context))); + std::unique_ptr read, + table->NewRead(std::move(read_context), read_context_builder_48503_options)); return read->CreateReader(plan->Splits()); }; @@ -1106,23 +1164,32 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP std::shared_ptr predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k", FieldType::INT, Literal(2)); - ScanContextBuilder ro_scan_context_builder(table_path + "$ro"); - ro_scan_context_builder.SetOptions(options).SetPredicate(predicate); + TableLoadOptions ro_scan_context_builder_load_options{table_path + "$ro"}; + std::map ro_scan_context_builder_options; + ScanContextBuilder ro_scan_context_builder; + ro_scan_context_builder_options = options; + ro_scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_scan_context, ro_scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_table_scan, - TableScan::Create(std::move(ro_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr ro_table_scan, + TableScan::Create(ro_scan_context_builder_load_options, std::move(ro_scan_context), + ro_scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr ro_plan, ro_table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(int64_t ro_file_count, CountDataFiles(ro_plan->Splits())); ASSERT_EQ(ro_file_count, 1); - ReadContextBuilder read_context_builder(table_path + "$ro"); + TableLoadOptions read_context_builder_load_options{table_path + "$ro"}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; // Do not enable row-level predicate filtering: the result must come from the file reader's // row-group/page predicate pushdown. - read_context_builder.SetOptions(options).SetPredicate(predicate).SetReadFieldNames({"k", "v"}); + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate).SetReadFieldNames({"k", "v"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(ro_plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -1166,11 +1233,15 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(WriteAndFullCompact(std::move(batch), /*commit_identifier=*/0, helper.get())); - ScanContextBuilder scan_context_builder(table_path + "$ro"); - scan_context_builder.SetOptions(options); + TableLoadOptions scan_context_builder_load_options{table_path + "$ro"}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); auto projected_schema = arrow::schema({ @@ -1179,11 +1250,15 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { }); auto c_projected_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_projected_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path + "$ro"); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_projected_schema)); + TableLoadOptions read_context_builder_load_options{table_path + "$ro"}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_projected_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -1375,12 +1450,16 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite /*is_streaming_mode=*/true)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - WriteContextBuilder write_context_builder(table_path, "partial-write"); + TableLoadOptions write_context_builder_load_options{table_path}; + std::map write_context_builder_options; + WriteContextBuilder write_context_builder("partial-write"); write_context_builder.WithWriteSchema({"score"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write, + FileStoreWrite::Create(write_context_builder_load_options, std::move(write_context), + write_context_builder_options)); ASSERT_OK_AND_ASSIGN( std::unique_ptr batch, TestHelper::MakeRecordBatch(arrow::struct_({fields[2]}), R"([[10], [20]])", @@ -1390,11 +1469,15 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite write->PrepareCommit()); ASSERT_OK(write->Close()); - CommitContextBuilder commit_context_builder(table_path, "partial-write"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("partial-write"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(commit_messages)); ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); @@ -1760,17 +1843,31 @@ TEST(SystemTableReadInteTest, TestChangelogSequenceNumberCannotBeDisabledByReadO for (bool direct : {false, true}) { SCOPED_TRACE(direct); const std::string path = table->GetRootPath() + "$" + kind; + TableLoadOptions scan_context_builder_48980_load_options{path}; + std::map scan_context_builder_48980_options; + ScanContextBuilder scan_context_builder_48980; + scan_context_builder_48980_options = read_options; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(path).SetOptions(read_options).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - direct ? table->NewScan(std::move(scan_context)) - : TableScan::Create(std::move(scan_context))); + scan_context_builder_48980.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + direct ? table->NewScan(std::move(scan_context), scan_context_builder_48980_options) + : TableScan::Create(scan_context_builder_48980_load_options, + std::move(scan_context), + scan_context_builder_48980_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + TableLoadOptions read_context_builder_93839_load_options{path}; + std::map read_context_builder_93839_options; + ReadContextBuilder read_context_builder_93839; + read_context_builder_93839_options = read_options; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(path).SetOptions(read_options).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - direct ? table->NewRead(std::move(read_context)) - : TableRead::Create(std::move(read_context))); + read_context_builder_93839.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + direct ? table->NewRead(std::move(read_context), read_context_builder_93839_options) + : TableRead::Create(read_context_builder_93839_load_options, + std::move(read_context), + read_context_builder_93839_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -2065,17 +2162,23 @@ TEST_P(ReadInteTest, TestAppendReadWithMultipleBuckets) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder + .EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2146,19 +2249,22 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicate) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .SetPredicate(predicate) - .EnablePredicateFilter(true) - .EnablePrefetch(param.enable_prefetch) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .AddOption("orc.read.enable-metrics", "true"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder.SetPredicate(predicate).EnablePredicateFilter(true).EnablePrefetch( + param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2248,18 +2354,23 @@ TEST_P(ReadInteTest, TestAppendReadWithComplexTypePredicate) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_complex_data.db/append_complex_data"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f6", "f2", "f4", "f3", "f5"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2322,19 +2433,25 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateOnlyPushdown) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder + .SetPredicate(predicate) .EnableLateMaterializing(false) .EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2406,18 +2523,24 @@ TEST_P(ReadInteTest, TestAppendReadWithLateMaterializing) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder + .SetPredicate(predicate) .EnableLateMaterializing(true) .EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2484,19 +2607,25 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateAllFiltered) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder + .SetPredicate(predicate) .EnablePredicateFilter(true) .EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2570,18 +2699,22 @@ TEST_P(ReadInteTest, TestAppendReadIOException) { for (size_t i = 0; i < 200; i++) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - ReadContextBuilder context_builder(paimon::test::GetDataDir() + "/" + param.file_format + - "/append_09.db/append_09/"); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = + paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09/"; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = TableRead::Create( + context_builder_load_options, std::move(read_context), context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(data_splits); @@ -2616,16 +2749,21 @@ TEST_P(ReadInteTest, TestPkTableWithDeletionVectorSimple) { data_file = "data-ed5d7184-cf42-4f4a-bf83-1e3080d9012d-0.parquet"; dv_file = "index/index-62372b7b-d1cf-4dde-a162-c9d9d35006b4-1"; } - ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); DataSplitsDv input_data_splits = { {path + "f1=10/bucket-1", @@ -2665,16 +2803,20 @@ TEST_P(ReadInteTest, TestPkTableWithDeletionVector) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list; std::string deletion_file; @@ -2732,19 +2874,24 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot6) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(15.0)); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2818,17 +2965,22 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot8) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f0", "f3", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2897,20 +3049,25 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolution) { DataField(5, arrow::field("a", arrow::int32())), DataField(8, arrow::field("e", arrow::int32()))}; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { - context_builder.SetTableSchema(specific_table_schema.value()); + context_builder_load_options.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3004,15 +3161,20 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithReaderBuildParallelism /*schema ids*/ {1, 0}}}; auto data_splits = CreateDataSplits(input_data_splits, /*snapshot_id=*/2); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3069,20 +3231,25 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateFilter) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_with_alter_table.db/append_table_with_alter_table/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"a", "k", "key1", "d", "key0", "c"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3149,18 +3316,23 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateOnlyPushDown) std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_with_alter_table.db/" "append_table_with_alter_table/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"a", "k", "key1", "d", "key0", "c"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3223,17 +3395,22 @@ TEST_P(ReadInteTest, TestPkReadSnapshot5WithSchemaEvolution) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3309,17 +3486,22 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolution) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3395,18 +3577,22 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateOnlyPush FieldType::INT, Literal(510)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetPredicate(predicate); - context_builder.EnablePrefetch(param.enable_prefetch) - .EnableLateMaterializing(false) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch).EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3481,18 +3667,22 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithLateMaterializing FieldType::INT, Literal(510)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetPredicate(predicate); - context_builder.EnableLateMaterializing(true) - .EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnableLateMaterializing(true).EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3565,19 +3755,24 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateFilter) FieldType::INT, Literal(510)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3658,17 +3853,22 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithBuildInFieldId) { "data-11b12094-192f-4ad8-92a8-ae8cba5e25ef-0.parquet"}; } - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key0", "key1", "k", "c", "d", "a", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); DataSplitsSchema input_data_splits = { {path + "key0=0/key1=1/bucket-0", BinaryRowGenerator::GenerateRow({0, 1}, pool_.get()), @@ -3722,16 +3922,21 @@ TEST_P(ReadInteTest, TestAppendReadNestedType) { DataField(7, arrow::field("f6", arrow::decimal128(2, 2)))}; std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -3778,17 +3983,22 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCast) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_alter_table_with_cast.db/" "append_table_alter_table_with_cast/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3861,18 +4071,23 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCastWithPredicatePushD std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_alter_table_with_cast.db/" "append_table_alter_table_with_cast/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3941,17 +4156,21 @@ TEST_P(ReadInteTest, TestReadWithPKFallBackBranch) { DataField(2, arrow::field("amount", arrow::int32())), }; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { - context_builder.SetTableSchema(specific_table_schema.value()); + context_builder_load_options.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3997,11 +4216,15 @@ TEST_P(ReadInteTest, TestReadWithAppendFallBackBranch) { data_splits.push_back(split); } - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.EnablePrefetch(param.enable_prefetch); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4041,13 +4264,17 @@ TEST_P(ReadInteTest, TestFallBackBranchStreamRead) { std::vector read_fields = {DataField(0, arrow::field("dt", arrow::utf8())), DataField(1, arrow::field("name", arrow::utf8())), DataField(2, arrow::field("amount", arrow::int32()))}; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_split)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4087,17 +4314,21 @@ TEST_P(ReadInteTest, TestReadWithPKRtBranch) { DataField(2, arrow::field("amount", arrow::int32())), }; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .WithBranch("rt"); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options["branch"] = "rt"; + context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { - context_builder.SetTableSchema(specific_table_schema.value()); + context_builder_load_options.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4144,17 +4375,21 @@ TEST_P(ReadInteTest, TestReadWithAppendPtBranch) { DataField(2, arrow::field("value2", arrow::int32())), }; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .WithBranch("test"); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options["branch"] = "test"; + context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { - context_builder.SetTableSchema(specific_table_schema.value()); + context_builder_load_options.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4275,15 +4510,20 @@ TEST_P(ReadInteTest, TestSpecificFs) { auto countable_fs = std::make_shared(std::make_shared(), &io_count); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption("orc.read.enable-metrics", "true") - .WithFileSystem(countable_fs); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder_load_options.file_system = countable_fs; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -4343,20 +4583,26 @@ Result> ReadGlobalSystemTable( std::string sys_path = PathUtil::JoinPath(PathUtil::JoinPath(warehouse, "sys"), table_name); - ScanContextBuilder scan_context_builder(sys_path); - scan_context_builder.SetOptions(options); + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - system_table->NewScan(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + system_table->NewScan(std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); - ReadContextBuilder read_context_builder(sys_path); - read_context_builder.SetOptions(options); + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - system_table->NewRead(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + system_table->NewRead(std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 84741f657..170dcd5db 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -45,6 +45,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/predicate/equal.h" @@ -541,13 +542,16 @@ class RealtimeWriteInteTest : public ::testing::Test { Result> CreateRealtimeWriter( const std::shared_ptr& realtime_context, const std::string& temp_directory) const { - WriteContextBuilder builder(table_path_, commit_user_); - builder.SetOptions(options_).WithStreamingMode(true).WithRealtimeContext(realtime_context); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options_; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); if (!temp_directory.empty()) { builder.WithTempDirectory(temp_directory); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreWrite::Create(std::move(context)); + return FileStoreWrite::Create(builder_load_options, std::move(context), builder_options); } Result> CreateRealtimeWriter() const { @@ -721,11 +725,14 @@ class RealtimeWriteInteTest : public ::testing::Test { Result Commit(const std::vector& realtime_commits, int64_t commit_identifier) const { - CommitContextBuilder builder(table_path_, commit_user_); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(commit_user_); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); return commit->CommitWithProgress(realtime_commits, commit_identifier, /*watermark=*/std::nullopt); } @@ -746,11 +753,14 @@ class RealtimeWriteInteTest : public ::testing::Test { Result DropPartition(const std::map& partition, int64_t commit_identifier) const { - CommitContextBuilder builder(table_path_, kDropPartitionCommitUser); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(kDropPartitionCommitUser); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); PAIMON_RETURN_NOT_OK(commit->DropPartition({partition}, commit_identifier)); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_)); SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_); @@ -763,11 +773,14 @@ class RealtimeWriteInteTest : public ::testing::Test { } Result TruncateTable(int64_t commit_identifier) const { - CommitContextBuilder builder(table_path_, kTruncateCommitUser); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(kTruncateCommitUser); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); PAIMON_RETURN_NOT_OK(commit->TruncateTable(commit_identifier)); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_)); SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_); @@ -780,11 +793,14 @@ class RealtimeWriteInteTest : public ::testing::Test { } Result RollbackToAsLatest(int64_t target_snapshot_id) const { - CommitContextBuilder builder(table_path_, kRollbackCommitUser); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(kRollbackCommitUser); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(bool rolled_back, commit->RollbackToAsLatest(target_snapshot_id)); if (!rolled_back) { return Status::Invalid("failed to commit rollback snapshot"); @@ -802,11 +818,15 @@ class RealtimeWriteInteTest : public ::testing::Test { Result>> CompactAndPrepareCommit( const std::vector& partition_buckets, int64_t commit_identifier, bool full_compaction, std::unique_ptr* compactor) const { - WriteContextBuilder builder(table_path_, "regression-compactor"); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + WriteContextBuilder builder("regression-compactor"); + builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).WithStreamingMode(true).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, - FileStoreWrite::Create(std::move(context))); + builder.WithStreamingMode(true).Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(context), builder_options)); for (const RealtimePartitionBucket& scope : partition_buckets) { PAIMON_RETURN_NOT_OK(writer->Compact(scope.partition, scope.bucket, full_compaction)); } @@ -818,11 +838,14 @@ class RealtimeWriteInteTest : public ::testing::Test { Result CommitCompaction(const std::vector>& messages, int64_t commit_identifier) const { - CommitContextBuilder builder(table_path_, "regression-compactor"); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("regression-compactor"); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); PAIMON_RETURN_NOT_OK(commit->Commit(messages, commit_identifier)); SnapshotManager snapshot_manager(dir_->GetFileSystem(), table_path_); PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, snapshot_manager.LatestSnapshot()); @@ -835,12 +858,17 @@ class RealtimeWriteInteTest : public ::testing::Test { Result CompactAndCommit(const std::map& partition, int32_t bucket, int64_t commit_identifier, bool full_compaction = true) const { - WriteContextBuilder write_builder(table_path_, commit_user_); - write_builder.SetOptions(options_).WithStreamingMode(true).WithTempDirectory( + TableLoadOptions write_builder_load_options{table_path_}; + std::map write_builder_options; + WriteContextBuilder write_builder(commit_user_); + write_builder_options = options_; + write_builder.WithStreamingMode(true).WithTempDirectory( PathUtil::JoinPath(dir_->Str(), "compact-tmp")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr compaction_writer, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr compaction_writer, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); PAIMON_RETURN_NOT_OK(compaction_writer->Compact(partition, bucket, full_compaction)); PAIMON_ASSIGN_OR_RAISE( std::vector> compaction_messages, @@ -849,11 +877,16 @@ class RealtimeWriteInteTest : public ::testing::Test { return Status::Invalid("compaction did not produce a commit message"); } - CommitContextBuilder commit_builder(table_path_, commit_user_); + TableLoadOptions commit_builder_load_options{table_path_}; + std::map commit_builder_options; + CommitContextBuilder commit_builder(commit_user_); + commit_builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr compaction_commit, - FileStoreCommit::Create(std::move(commit_context))); + commit_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr compaction_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); PAIMON_RETURN_NOT_OK(compaction_commit->Commit(compaction_messages, commit_identifier)); PAIMON_RETURN_NOT_OK(compaction_writer->Close()); @@ -878,40 +911,49 @@ class RealtimeWriteInteTest : public ::testing::Test { } Result ExpireSnapshots() const { - CommitContextBuilder builder(table_path_, commit_user_); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(commit_user_); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); return commit->Expire(); } Result> CreatePlan( const std::shared_ptr& realtime_context, const std::shared_ptr& predicate) const { - ScanContextBuilder scan_builder(table_path_); + TableLoadOptions scan_builder_load_options{table_path_}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; if (realtime_context) { scan_builder.WithRealtimeContext(realtime_context); } scan_builder.SetPredicate(predicate).WithMemoryPool(pool_); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - scan_builder.SetOptions(options_).Finish()); + scan_builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); return scan->CreatePlan(); } Result> CreateQueryReader( const std::shared_ptr& plan, const std::shared_ptr& realtime_context) const { - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); return table_read->CreateReader(plan->Splits()); } @@ -926,16 +968,19 @@ class RealtimeWriteInteTest : public ::testing::Test { const std::shared_ptr& plan, const std::shared_ptr& realtime_context, const std::vector& read_fields, const std::shared_ptr& predicate, bool enable_predicate_filter) const { - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames(read_fields) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadFieldNames(read_fields) .SetPredicate(predicate) .EnablePredicateFilter(enable_predicate_filter) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -949,14 +994,17 @@ class RealtimeWriteInteTest : public ::testing::Test { const std::string& expected_json) const { std::unique_ptr c_read_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_read_schema.get()).ok()); - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadSchema(std::move(c_read_schema)) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadSchema(std::move(c_read_schema)) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -1038,13 +1086,15 @@ class RealtimeWriteInteTest : public ::testing::Test { Result CountRows(const std::shared_ptr& plan, const std::shared_ptr& realtime_context) const { - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .WithRealtimeContext(realtime_context) - .WithMemoryPool(pool_); + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.WithRealtimeContext(realtime_context).WithMemoryPool(pool_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr count_reader, table_read->CreateCountReader(plan->Splits())); return count_reader->CountRows(); @@ -1334,31 +1384,45 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeOperationsRequireEnabledOption) { ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - WriteContextBuilder write_builder(table_path_, commit_user_); - write_builder.SetOptions(disabled_options) - .WithStreamingMode(true) - .WithRealtimeContext(realtime_context); + TableLoadOptions write_builder_load_options{table_path_}; + std::map write_builder_options; + WriteContextBuilder write_builder(commit_user_); + write_builder_options = disabled_options; + write_builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options), "real-time write requires realtime.enabled=true"); - ScanContextBuilder scan_builder(table_path_); - scan_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); + TableLoadOptions scan_builder_load_options{table_path_}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options = disabled_options; + scan_builder.WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), - "real-time scan requires realtime.enabled=true"); - - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); + ASSERT_NOK_WITH_MSG( + TableScan::Create(scan_builder_load_options, std::move(scan_context), scan_builder_options), + "real-time scan requires realtime.enabled=true"); + + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = disabled_options; + read_builder.WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), - "real-time read requires realtime.enabled=true"); - - CommitContextBuilder commit_builder(table_path_, commit_user_); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_builder.SetOptions(disabled_options).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_NOK_WITH_MSG( + TableRead::Create(read_builder_load_options, std::move(read_context), read_builder_options), + "real-time read requires realtime.enabled=true"); + + TableLoadOptions commit_builder_load_options{table_path_}; + std::map commit_builder_options; + CommitContextBuilder commit_builder(commit_user_); + commit_builder_options = disabled_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(/*realtime_commits=*/{}, /*commit_identifier=*/0, /*watermark=*/std::nullopt), @@ -1458,14 +1522,16 @@ TEST_F(RealtimeWriteInteTest, TestRefreshPreparationAllowsScanWriteAndStoreRegis ConfigureRealtimeDeduplicateTable(); auto fs = std::make_shared(dir_->GetFileSystem()); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, RealtimeContext::Create()); - WriteContextBuilder builder(table_path_, commit_user_); - builder.SetOptions(options_) - .WithStreamingMode(true) - .WithRealtimeContext(context) - .WithFileSystem(fs); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options_; + builder_load_options.file_system = fs; + builder.WithStreamingMode(true).WithRealtimeContext(context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); const std::vector initial = {{1, "old", "p0"}}; ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(initial, /*partitioned=*/false)); @@ -1569,14 +1635,16 @@ TEST_F(RealtimeWriteInteTest, TestRefreshPreparationFailureKeepsProgressAndCanRe ConfigureRealtimeDeduplicateTable(); auto fs = std::make_shared(dir_->GetFileSystem()); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, RealtimeContext::Create()); - WriteContextBuilder builder(table_path_, commit_user_); - builder.SetOptions(options_) - .WithStreamingMode(true) - .WithRealtimeContext(context) - .WithFileSystem(fs); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options_; + builder_load_options.file_system = fs; + builder.WithStreamingMode(true).WithRealtimeContext(context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); const std::vector initial = {{1, "old", "p0"}}; ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(initial, /*partitioned=*/false)); @@ -1827,11 +1895,16 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryMergesLatestDv) auto concurrent_message = std::make_shared( initial_message->Partition(), /*bucket=*/0, /*total_buckets=*/1, DataIncrement({}, {}, {}, {concurrent_dv}, {}), CompactIncrement({}, {}, {}, {}, {})); - CommitContextBuilder concurrent_builder(table_path_, "concurrent-dv-writer"); + TableLoadOptions concurrent_builder_load_options{table_path_}; + std::map concurrent_builder_options; + CommitContextBuilder concurrent_builder("concurrent-dv-writer"); + concurrent_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_context, - concurrent_builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_commit, - FileStoreCommit::Create(std::move(concurrent_context))); + concurrent_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr concurrent_commit, + FileStoreCommit::Create(concurrent_builder_load_options, std::move(concurrent_context), + concurrent_builder_options)); bool interleaved = false; auto fs = std::make_shared( PathUtil::JoinPath(table_path_, "snapshot/snapshot-2"), [&]() -> Status { @@ -1839,11 +1912,15 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryMergesLatestDv) return concurrent_commit->Commit({concurrent_message}, /*commit_identifier=*/0, /*watermark=*/std::nullopt); }); - CommitContextBuilder builder(table_path_, commit_user_); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).WithFileSystem(fs).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(commit_user_); + builder_options = options_; + builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN( int64_t snapshot_id, commit->CommitWithProgress(progress, /*commit_identifier=*/1, /*watermark=*/std::nullopt)); @@ -1905,11 +1982,15 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryRemapsCompactedO interleaved = true; return Status::OK(); }); - CommitContextBuilder builder(table_path_, commit_user_); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).WithFileSystem(fs).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(commit_user_); + builder_options = options_; + builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN( int64_t snapshot_id, commit->CommitWithProgress(progress, /*commit_identifier=*/2, /*watermark=*/std::nullopt)); @@ -1956,11 +2037,15 @@ TEST_P(RealtimeDeduplicateCompactionInteTest, TestCommitTransfersLatestDvToCompa ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot)); } - WriteContextBuilder write_builder(table_path_, "compactor"); + TableLoadOptions write_builder_load_options{table_path_}; + std::map write_builder_options; + WriteContextBuilder write_builder("compactor"); + write_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_builder.SetOptions(options_).WithStreamingMode(true).Finish()); + write_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr compactor, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr forbidden_batch, MakeBatch({{99, "not-written", "p0"}}, /*partitioned=*/false)); ASSERT_NOK_WITH_MSG(compactor->Write(std::move(forbidden_batch)), "only supports compaction"); @@ -2033,11 +2118,16 @@ TEST_P(RealtimeDeduplicateCompactionInteTest, TestCommitTransfersLatestDvToCompa } else { ASSERT_OK(publish_build()); } - CommitContextBuilder commit_builder(table_path_, "compactor"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_builder.SetOptions(options_).WithFileSystem(fs).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions commit_builder_load_options{table_path_}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("compactor"); + commit_builder_options = options_; + commit_builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_OK(commit->Commit(messages, /*commit_identifier=*/0)); ASSERT_TRUE(interleaved); // Only unpublished intermediate outputs and their sidecars are reclaimed after publication. @@ -2163,11 +2253,15 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCompactionRejectsReplacedIn ASSERT_EQ(1, initial_snapshot); ASSERT_OK(writer->Close()); - WriteContextBuilder write_builder(table_path_, "stale-compactor"); + TableLoadOptions write_builder_load_options{table_path_}; + std::map write_builder_options; + WriteContextBuilder write_builder("stale-compactor"); + write_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_builder.SetOptions(options_).WithStreamingMode(true).Finish()); + write_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr compactor, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); ASSERT_OK(compactor->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN( std::vector> messages, @@ -2180,11 +2274,15 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCompactionRejectsReplacedIn ASSERT_OK_AND_ASSIGN(Snapshot replacement, CompactAndCommit(/*partition=*/{}, /*bucket=*/0, /*commit_identifier=*/1)); ASSERT_EQ(2, replacement.Id()); - CommitContextBuilder commit_builder(table_path_, "stale-compactor"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions commit_builder_load_options{table_path_}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("stale-compactor"); + commit_builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_NOK_WITH_MSG(commit->Commit(messages, /*commit_identifier=*/0), "input is no longer active"); ASSERT_OK_AND_ASSIGN(std::vector rows, ReadRows()); @@ -2726,11 +2824,14 @@ TEST_F(RealtimeWriteInteTest, TestNoHitPreservesBaselineDv) { auto no_hit = std::make_shared(); no_hit->Add(0); progress[0].deleted_offsets = no_hit; - CommitContextBuilder builder(table_path_, commit_user_); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(commit_user_); + builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN( int64_t snapshot, commit->CommitWithProgress(progress, /*commit_identifier=*/4, /*watermark=*/std::nullopt)); @@ -2854,11 +2955,15 @@ TEST_F(RealtimeWriteInteTest, TestPublicationConflictCleansOnlyAttemptDv) { } return Status::OK(); }); - CommitContextBuilder builder(table_path_, "regression-compactor"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).WithFileSystem(fs).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("regression-compactor"); + builder_options = options_; + builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK(commit->Commit(messages, /*commit_identifier=*/0)); ASSERT_OK_AND_ASSIGN(uint64_t attempts, commit->GetCommitMetrics()->GetCounter( CommitMetrics::LAST_COMMIT_ATTEMPTS)); @@ -3791,18 +3896,26 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesSnapshotConflict) { const std::string first_snapshot_path = PathUtil::JoinPath(table_path_, "snapshot/snapshot-1"); auto conflict_file_system = std::make_shared(first_snapshot_path); - CommitContextBuilder bucket0_builder(table_path_, "bucket0_realtime_commit_user"); + TableLoadOptions bucket0_builder_load_options{table_path_}; + std::map bucket0_builder_options; + CommitContextBuilder bucket0_builder("bucket0_realtime_commit_user"); + bucket0_builder_options = options_; + bucket0_builder_load_options.file_system = conflict_file_system; + ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket0_context, bucket0_builder.Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr bucket0_context, - bucket0_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket0_commit, - FileStoreCommit::Create(std::move(bucket0_context))); - CommitContextBuilder bucket1_builder(table_path_, "bucket1_realtime_commit_user"); + std::unique_ptr bucket0_commit, + FileStoreCommit::Create(bucket0_builder_load_options, std::move(bucket0_context), + bucket0_builder_options)); + TableLoadOptions bucket1_builder_load_options{table_path_}; + std::map bucket1_builder_options; + CommitContextBuilder bucket1_builder("bucket1_realtime_commit_user"); + bucket1_builder_options = options_; + bucket1_builder_load_options.file_system = conflict_file_system; + ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket1_context, bucket1_builder.Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr bucket1_context, - bucket1_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket1_commit, - FileStoreCommit::Create(std::move(bucket1_context))); + std::unique_ptr bucket1_commit, + FileStoreCommit::Create(bucket1_builder_load_options, std::move(bucket1_context), + bucket1_builder_options)); std::optional> bucket0_result; std::optional> bucket1_result; @@ -3870,13 +3983,18 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesConflictWithCompaction) { ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id)); } - WriteContextBuilder compact_write_builder(table_path_, "compaction_write_user"); - compact_write_builder.SetOptions(options_).WithStreamingMode(true).WithTempDirectory( + TableLoadOptions compact_write_builder_load_options{table_path_}; + std::map compact_write_builder_options; + WriteContextBuilder compact_write_builder("compaction_write_user"); + compact_write_builder_options = options_; + compact_write_builder.WithStreamingMode(true).WithTempDirectory( PathUtil::JoinPath(dir_->Str(), "conflict-compact-tmp")); ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_write_context, compact_write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_writer, - FileStoreWrite::Create(std::move(compact_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr compact_writer, + FileStoreWrite::Create(compact_write_builder_load_options, std::move(compact_write_context), + compact_write_builder_options)); ASSERT_OK(compact_writer->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN( @@ -3900,18 +4018,28 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesConflictWithCompaction) { PathUtil::JoinPath(table_path_, "snapshot/snapshot-3"); auto conflict_file_system = std::make_shared(conflict_snapshot_path); - CommitContextBuilder append_commit_builder(table_path_, commit_user_); + TableLoadOptions append_commit_builder_load_options{table_path_}; + std::map append_commit_builder_options; + CommitContextBuilder append_commit_builder(commit_user_); + append_commit_builder_options = options_; + append_commit_builder_load_options.file_system = conflict_file_system; + ASSERT_OK_AND_ASSIGN(std::unique_ptr append_commit_context, + append_commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr append_commit_context, - append_commit_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr append_commit, - FileStoreCommit::Create(std::move(append_commit_context))); - CommitContextBuilder compact_commit_builder(table_path_, "compaction_commit_user"); + std::unique_ptr append_commit, + FileStoreCommit::Create(append_commit_builder_load_options, + std::move(append_commit_context), append_commit_builder_options)); + TableLoadOptions compact_commit_builder_load_options{table_path_}; + std::map compact_commit_builder_options; + CommitContextBuilder compact_commit_builder("compaction_commit_user"); + compact_commit_builder_options = options_; + compact_commit_builder_load_options.file_system = conflict_file_system; + ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_commit_context, + compact_commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr compact_commit_context, - compact_commit_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_commit, - FileStoreCommit::Create(std::move(compact_commit_context))); + std::unique_ptr compact_commit, + FileStoreCommit::Create(compact_commit_builder_load_options, + std::move(compact_commit_context), compact_commit_builder_options)); std::optional> append_result; std::optional compact_status; @@ -4175,14 +4303,17 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) { ASSERT_OK_AND_ASSIGN(std::shared_ptr lifetime_plan, CreatePlan(realtime_context, /*predicate=*/nullptr)); - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(lifetime_plan->Splits())); ASSERT_FALSE(query_view->expired()); @@ -4250,10 +4381,14 @@ TEST_F(RealtimeWriteInteTest, TestPkRealtimeReadOptimizedScanUnsupported) { ASSERT_OK_AND_ASSIGN(std::vector disk_rows, ReadRows()); ASSERT_EQ(rows, disk_rows); - ScanContextBuilder scan_builder(table_path_ + "$ro"); - scan_builder.SetOptions(options_).WithRealtimeContext(realtime_context).WithMemoryPool(pool_); + TableLoadOptions scan_builder_load_options{table_path_ + "$ro"}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options = options_; + scan_builder.WithRealtimeContext(realtime_context).WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - Result> scan = TableScan::Create(std::move(scan_context)); + Result> scan = + TableScan::Create(scan_builder_load_options, std::move(scan_context), scan_builder_options); ASSERT_TRUE(scan.status().IsNotImplemented()) << scan.status().ToString(); ASSERT_NE(std::string::npos, scan.status().ToString().find( "PK real-time union read does not support read-optimized")); @@ -4283,14 +4418,17 @@ TEST_F(RealtimeWriteInteTest, TestPkDeleteInsertAndPinnedReadsAcrossRefresh) { CreatePlan(realtime_context, /*predicate=*/nullptr)); ASSERT_OK_AND_ASSIGN(std::shared_ptr reader_plan, CreatePlan(realtime_context, /*predicate=*/nullptr)); - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr pinned_reader, table_read->CreateReader(reader_plan->Splits())); @@ -4481,14 +4619,17 @@ TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) { }); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadSchema(std::move(c_schema)) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadSchema(std::move(c_schema)) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -4776,22 +4917,31 @@ TEST_F(RealtimeWriteInteTest, TestPkPartitionBucketRecovery) { TEST_F(RealtimeWriteInteTest, TestPkRecovery) { CreatePkTable(); - WriteContextBuilder seed_builder(table_path_, commit_user_); - seed_builder.SetOptions(options_).WithStreamingMode(true); + TableLoadOptions seed_builder_load_options{table_path_}; + std::map seed_builder_options; + WriteContextBuilder seed_builder(commit_user_); + seed_builder_options = options_; + seed_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_context, seed_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_writer, - FileStoreWrite::Create(std::move(seed_context))); + FileStoreWrite::Create(seed_builder_load_options, std::move(seed_context), + seed_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_batch, MakeLogicalBatch({Row{99, "seed", "p0"}}, /*bucket=*/0)); ASSERT_OK(seed_writer->Write(std::move(seed_batch))); ASSERT_OK_AND_ASSIGN(std::vector> seed_messages, seed_writer->PrepareCommit(/*wait_compaction=*/false, /*commit_identifier=*/0)); - CommitContextBuilder seed_commit_builder(table_path_, commit_user_); + TableLoadOptions seed_commit_builder_load_options{table_path_}; + std::map seed_commit_builder_options; + CommitContextBuilder seed_commit_builder(commit_user_); + seed_commit_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_commit_context, - seed_commit_builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_commit, - FileStoreCommit::Create(std::move(seed_commit_context))); + seed_commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr seed_commit, + FileStoreCommit::Create(seed_commit_builder_load_options, std::move(seed_commit_context), + seed_commit_builder_options)); ASSERT_OK(seed_commit->Commit(seed_messages, /*commit_identifier=*/0)); ASSERT_OK(seed_writer->Close()); const std::vector mutations = { @@ -4878,11 +5028,16 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) { ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context)); ASSERT_EQ(0, memory_usage); } - WriteContextBuilder compact_builder(table_path_, commit_user_); - compact_builder.SetOptions(options_).WithStreamingMode(true); + TableLoadOptions compact_builder_load_options{table_path_}; + std::map compact_builder_options; + WriteContextBuilder compact_builder(commit_user_); + compact_builder_options = options_; + compact_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_context, compact_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_writer, - FileStoreWrite::Create(std::move(compact_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr compact_writer, + FileStoreWrite::Create(compact_builder_load_options, std::move(compact_context), + compact_builder_options)); ASSERT_OK(compact_writer->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN( @@ -4910,11 +5065,15 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) { std::max(compacted_live_max_sequence_number, file->max_sequence_number); } ASSERT_LT(compacted_live_max_sequence_number, kHistoricalMaxSequenceNumber); - CommitContextBuilder commit_builder(table_path_, commit_user_); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions commit_builder_load_options{table_path_}; + std::map commit_builder_options; + CommitContextBuilder commit_builder(commit_user_); + commit_builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_OK(commit->Commit(compact_messages, /*commit_identifier=*/4)); ASSERT_OK(compact_writer->Close()); @@ -5234,11 +5393,14 @@ TEST_F(RealtimeWriteInteTest, TestCommitWithProgressRejectsCoveredRangesFromAnot writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); ASSERT_OK(Commit(commits, /*commit_identifier=*/0)); - CommitContextBuilder builder(table_path_, "another_realtime_commit_user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("another_realtime_commit_user"); + builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(commits, /*commit_identifier=*/0, /*watermark=*/std::nullopt), "another commit user or identifier"); @@ -6054,14 +6216,17 @@ TEST_F(RealtimeWriteInteTest, TestSuccessfulReaderCreationConsumesRealtimeSplitT ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, CreatePlan(realtime_context, /*predicate=*/nullptr)); - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr first_reader, table_read->CreateReader(plan->Splits())); first_reader->Close(); @@ -6086,16 +6251,19 @@ TEST_F(RealtimeWriteInteTest, TestFailedReaderCreationPreservesRealtimeSplitTick CreatePlan(realtime_context, /*predicate=*/nullptr)); ASSERT_EQ(1, plan->Splits().size()); - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .SetPredicate(std::make_shared()) .EnablePredicateFilter(true) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateReader(plan->Splits()), "cannot cast predicate unsupported"); @@ -6949,14 +7117,17 @@ TEST_F(RealtimeWriteInteTest, TestReaderPinsMemoryAcrossRefresh) { std::dynamic_pointer_cast(plan->Splits()[0]); ASSERT_NE(nullptr, realtime_split); - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context_impl, diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 17807e254..019aa567f 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -36,6 +36,7 @@ #include "fmt/ranges.h" #include "gtest/gtest.h" #include "paimon/bucket/bucket_id_calculator.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" @@ -144,21 +145,23 @@ class ScanAndReadInteTest : public testing::Test, } } - void AddReadOptionsForPrefetch(ReadContextBuilder* read_context_builder) { - read_context_builder->AddOption("test.enable-adaptive-prefetch-strategy", "false"); + void AddReadOptionsForPrefetch(ReadContextBuilder* read_context_builder, + std::map& read_options) { + read_options["test.enable-adaptive-prefetch-strategy"] = "false"; if (EnablePrefetch()) { read_context_builder->EnablePrefetch(true).SetPrefetchBatchCount(3); } } - Result> FinishScanContext(ScanContextBuilder& builder) { + Result> FinishScanContext( + ScanContextBuilder& builder, std::map& builder_options) { if (EnableSnapshotLiveManifestCache()) { if (!snapshot_live_manifest_cache_) { snapshot_live_manifest_cache_ = std::make_shared(/*max_weight=*/64 * 1024 * 1024); } - builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .WithCache(snapshot_live_manifest_cache_); + builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder.WithCache(snapshot_live_manifest_cache_); } return builder.Finish(); } @@ -240,12 +243,17 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - Result> scan_context = FinishScanContext(scan_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + Result> scan_context = + FinishScanContext(scan_context_builder, scan_context_builder_options); CHECK_HOOK_STATUS(scan_context.status(), i); Result> table_scan = - TableScan::Create(std::move(scan_context).value()); + TableScan::Create(scan_context_builder_load_options, std::move(scan_context).value(), + scan_context_builder_options); CHECK_HOOK_STATUS(table_scan.status(), i); Result> result_plan = table_scan.value()->CreatePlan(); CHECK_HOOK_STATUS(result_plan.status(), i); @@ -254,12 +262,16 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { auto splits = result_plan.value()->Splits(); ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(splits); @@ -297,12 +309,17 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - Result> scan_context = FinishScanContext(scan_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + + Result> scan_context = + FinishScanContext(scan_context_builder, scan_context_builder_options); CHECK_HOOK_STATUS(scan_context.status(), i); Result> table_scan = - TableScan::Create(std::move(scan_context).value()); + TableScan::Create(scan_context_builder_load_options, std::move(scan_context).value(), + scan_context_builder_options); CHECK_HOOK_STATUS(table_scan.status(), i); Result> result_plan = table_scan.value()->CreatePlan(); CHECK_HOOK_STATUS(result_plan.status(), i); @@ -310,12 +327,16 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { auto splits = result_plan.value()->Splits(); ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(splits); @@ -350,21 +371,31 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot1) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -392,10 +423,16 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -403,10 +440,14 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -505,21 +546,22 @@ TEST(SelectiveManifestDecodeInteTest, TestPartitionedPointLookup) { bool entry_cache) -> Result { std::shared_ptr pool = GetMemoryPool(); scan_pools.push_back(pool); - ScanContextBuilder builder(table_path); - builder.WithMemoryPool(pool) - .WithExecutor(executor) - .WithCache(cache) - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, entry_cache ? "3" : "0") - .AddOption(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED, - lazy_decode ? "true" : "false") - .AddOption(Options::READ_BATCH_SIZE, "128"); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = entry_cache ? "3" : "0"; + builder_options[Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED] = + lazy_decode ? "true" : "false"; + builder_options[Options::READ_BATCH_SIZE] = "128"; + builder.WithMemoryPool(pool).WithExecutor(executor).WithCache(cache); if (entry_cache) { // No explicit bucket or partition filter: the predicate must select the bucket. builder.SetPredicate(predicate); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(builder_load_options, std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); PAIMON_ASSIGN_OR_RAISE(uint64_t enabled, table_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED)); @@ -537,11 +579,14 @@ TEST(SelectiveManifestDecodeInteTest, TestPartitionedPointLookup) { (historical && data_split->Bucket() == kBuckets / 2 - 1)); ASSERT_EQ(data_split->GetFileList().size(), 1); } - ReadContextBuilder builder(table_path); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ReadContextBuilder builder; builder.SetPredicate(predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(result.plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, @@ -690,15 +735,17 @@ TEST(SelectiveManifestDecodeInteTest, TestSchemaEvolutionPreservesBucketPruning) } auto cache = std::make_shared(capacities); for (int32_t attempt = 0; attempt < 2; ++attempt) { - ScanContextBuilder builder(table_path); - builder.SetPredicate(predicate) - .WithCache(cache) - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .AddOption(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED, - lazy_decode ? "true" : "false"); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder_options[Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED] = + lazy_decode ? "true" : "false"; + builder.SetPredicate(predicate).WithCache(cache); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); auto split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -707,12 +754,16 @@ TEST(SelectiveManifestDecodeInteTest, TestSchemaEvolutionPreservesBucketPruning) ASSERT_OK_AND_ASSIGN(uint64_t hit, scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); ASSERT_EQ(hit, attempt); - ReadContextBuilder read_builder(table_path); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; read_builder.SetPredicate(predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, @@ -779,10 +830,13 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { // All buckets have overlapping stats for this key. An explicit filter bypasses // inference, proving that ordinary statistics cannot account for the pruning. for (int32_t bucket = 0; bucket < kNumBuckets; ++bucket) { - ScanContextBuilder builder(table_path); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; builder.SetPredicate(predicate).SetBucketFilter(bucket); - ASSERT_OK_AND_ASSIGN(auto context, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto context, FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(builder_load_options, + std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_FALSE(plan->Splits().empty()); @@ -804,23 +858,33 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { auto lookup_predicate = PredicateBuilder::Equal( 0, "rowkey", field_type, Literal(field_type, lookup_key.data(), lookup_key.size())); - ScanContextBuilder scan_builder(table_path); - scan_builder.SetPredicate(lookup_predicate) - .AddOption(Options::BUCKET, current_bucket_count); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::BUCKET] = current_bucket_count; + scan_builder.SetPredicate(lookup_predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_builder, scan_builder_options)); + ASSERT_OK_AND_ASSIGN( + auto scan, TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); if (EnableSnapshotLiveManifestCache()) { ASSERT_OK_AND_ASSIGN( uint64_t enabled, scan->GetMetrics()->GetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED)); ASSERT_EQ(enabled, 1); - ScanContextBuilder cached_builder(table_path); - cached_builder.SetPredicate(lookup_predicate) - .AddOption(Options::BUCKET, current_bucket_count); - ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); - ASSERT_OK_AND_ASSIGN(auto cached_scan, - TableScan::Create(std::move(cached_context))); + TableLoadOptions cached_builder_load_options{table_path}; + std::map cached_builder_options; + ScanContextBuilder cached_builder; + cached_builder_options[Options::BUCKET] = current_bucket_count; + cached_builder.SetPredicate(lookup_predicate); + ASSERT_OK_AND_ASSIGN(auto cached_context, + FinishScanContext(cached_builder, cached_builder_options)); + ASSERT_OK_AND_ASSIGN( + auto cached_scan, + TableScan::Create(cached_builder_load_options, std::move(cached_context), + cached_builder_options)); ASSERT_OK_AND_ASSIGN(plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); @@ -833,11 +897,15 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { ASSERT_TRUE(data_split); ASSERT_EQ(data_split->Bucket(), bucket_ids[lookup_index]); } - ReadContextBuilder read_builder(table_path); - AddReadOptionsForPrefetch(&read_builder); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; + AddReadOptionsForPrefetch(&read_builder, read_builder_options); read_builder.SetPredicate(lookup_predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + auto read, TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(reader))); @@ -861,10 +929,16 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -872,10 +946,14 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -911,18 +989,28 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5WithReaderBuildParallelism) { auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -953,15 +1041,25 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotWithStreamWithDefaultMode) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 1, 2, 3, 4}; auto expected_snapshot1 = std::make_shared( @@ -1002,10 +1100,16 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshot1) { std::string table_path = GetDataDir() + "/" + file_format + "/append_10.db/append_10"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1013,10 +1117,14 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshot1) { ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1041,10 +1149,16 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshotOfNestedType) { "/append_complex_build_in_fieldid.db/" "append_complex_build_in_fieldid/"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1069,10 +1183,14 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshotOfNestedType) { std::shared_ptr arrow_data_type = DataField::ConvertDataFieldsToArrowStructType(read_fields); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1096,14 +1214,24 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6) { "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; // normal batch scan case for pk+dv, all data in level 0 is filtered out - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1137,15 +1265,25 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot1) { auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1160,16 +1298,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPartitionAndBu std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1196,8 +1344,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; // predicate: f0 != "Alice" (key predicate) and f3 > 18 (value predicate) and all data in level // 0 - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; std::string literal_str = "Alice"; auto not_equal = PredicateBuilder::NotEqual( @@ -1207,14 +1357,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { FieldType::DOUBLE, Literal(18.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, greater_than})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1237,8 +1394,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLateMaterializ auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; std::string literal_str = "Alice"; auto not_equal = PredicateBuilder::NotEqual( @@ -1248,14 +1407,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLateMaterializ FieldType::DOUBLE, Literal(18.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, greater_than})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1278,15 +1444,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot4WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + scan_context_builder.SetBucketFilter(0).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will be applied, as in kv mode value filter is enabled auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -1300,15 +1472,24 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLimit) { // normal batch scan case for pk+dv, all data in level 0 is filtered out, set row limits to 6, // data in partition 20 is truncated - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetLimit(6); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1336,17 +1517,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot4) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // from an compact snapshot std::vector> expected_snapshot_ids = {4, 5, 6}; @@ -1388,17 +1578,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot5) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // from an append snapshot, first plan is snapshot5 with merge, second plan is snapshot6 // (isStreaming=true) @@ -1434,15 +1633,25 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot6) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, "latest-full").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_MODE] = "latest-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // from an append snapshot, first plan is snapshot6 with merge, level 0 data in base manifest // list is contained @@ -1471,17 +1680,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot1) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // from the first snapshot std::vector> expected_snapshot_ids = {1, 3, 5, 6}; @@ -1530,17 +1748,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot2) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // use from-snapshot mode, first plan is empty, second plan is snapshot 3 with delta files // (snapshot 2 is compact snapshot, so skipped) @@ -1575,18 +1802,27 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithNestedType) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_nested_type.db/pk_table_nested_type/"; - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN( auto read_context, read_context_builder.SetReadFieldNames({"shopId", "dt", "hr", "col0", "col1", "col2"}) .Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -1629,14 +1865,23 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanLatestSnapshot) { "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // normal batch scan case for pk+mor, use latest snapshot if not specified - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1676,15 +1921,25 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { // normal batch scan case for pk+mor, read snapshot 2 (append snapshot), all data is in level 0 // with merge read - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -1720,16 +1975,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPartitionAndB "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // all data not in partition 10, bucket 1 is filtered out - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1758,8 +2023,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; // predicate: f0 != "Alice" and f0 < "Lucy" and f3 <= 30.0 // as f3 <= 30.0 is a value filter, it does not take effect @@ -1780,14 +2047,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { PredicateBuilder::And({not_equal, less_than, less_or_equal})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1821,8 +2095,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLateMateriali std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; std::string literal_str = "Alice"; auto not_equal = PredicateBuilder::NotEqual( @@ -1837,14 +2113,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLateMateriali ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, less_than, less_or_equal})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1869,16 +2152,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot3WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will be applied as all data files in bucket 1 in compact snapshot 3 are // all filtered auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -1890,20 +2179,31 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvWithInvalidAggregateBatchScanSnapsho std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + scan_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + read_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -1930,20 +2230,31 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorWithInvalidAggregateBatchScanSnapsh std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + scan_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + read_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -1957,24 +2268,35 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithAggregateBatchScanSnapshot3WithPredica std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "sum"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + scan_context_builder_options["fields.f3.aggregate-function"] = "sum"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will not be applied, as merge engine is aggregation auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + read_context_builder_options["fields.f3.aggregate-function"] = "sum"; - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "sum"); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -2001,24 +2323,35 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithPartialUpdateBatchScanSnapshot3WithPre std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "partial-update") - .AddOption(Options::IGNORE_DELETE, "true"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "partial-update"; + scan_context_builder_options[Options::IGNORE_DELETE] = "true"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will not be applied, as merge engine is partial-update auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "partial-update"; + read_context_builder_options[Options::IGNORE_DELETE] = "true"; - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "partial-update") - .AddOption(Options::IGNORE_DELETE, "true"); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -2047,15 +2380,24 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLimit) { // in pk mor mode, limit does not take effect, as we do not know the number of records after // merging - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetLimit(6); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -2087,17 +2429,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot4) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // first plan is snapshot4 (isStreaming=false), second plan is snapshot5 std::vector> expected_snapshot_ids = {4, 5}; @@ -2132,17 +2483,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot1) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {1, 2, 4, 5}; auto expected_snapshot1_batch = std::make_shared( @@ -2193,17 +2553,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot2) { // use from-snapshot mode, first plan is empty, snapshot 2 in second plan only includes delta // files - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 2, 4, 5}; auto expected_snapshot2_stream = std::make_shared( @@ -2238,21 +2607,30 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot5WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(50.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // first plan is snapshot 5 with empty plan, second plan is snapshot 5 with delta files // (predicates take no effects in delta mode) @@ -2274,24 +2652,30 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamPredicateFilter) { std::string table_path = paimon::test::GetDataDir() + FileFormat() + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); std::shared_ptr predicate = PredicateBuilder::GreaterThan( /*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(31.05)); scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - FinishScanContext(scan_context_builder)); + FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 5}; auto expected_snapshot5_stream = std::make_shared( @@ -2313,15 +2697,25 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowBatchScanSnapshot5) { "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; // normal batch scan case for pk+first row, all data in level 0 is filtered out - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -2353,17 +2747,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot3) { paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // from a compact snapshot std::vector> expected_snapshot_ids = {3, 4, 5}; @@ -2406,15 +2809,25 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot5) { paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, "latest-full").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_MODE] = "latest-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // from an append snapshot, first plan is snapshot5 with merge, all level 0 data is merged std::vector> expected_snapshot_ids = {5}; @@ -2445,15 +2858,24 @@ TEST_P(ScanAndReadInteTest, TestWithPKWith09VersionDvBatchScanLatestSnapshot) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_09.db/pk_09/"; // normal batch scan case for pk+dv (09 version) - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetLimit(2); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 8); @@ -2482,29 +2904,37 @@ TEST_P(ScanAndReadInteTest, TestWithPKWith09VersionDvBatchScanLatestSnapshot) { TEST_P(ScanAndReadInteTest, TestWithEmptyPartitionValue) { auto file_format = FileFormat(); - auto check_result = - [&](const std::string& table_path, - const std::vector>& partition_filters, - const std::shared_ptr& expected) { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetPartitionFilter(partition_filters); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - - ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); - ASSERT_EQ(result_plan->SnapshotId().value(), 1); - ASSERT_OK_AND_ASSIGN(auto batch_reader, - table_read->CreateReader(result_plan->Splits())); - ASSERT_OK_AND_ASSIGN(auto read_result, - ReadResultCollector::CollectResult(std::move(batch_reader))); - // check result - ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); - }; + auto check_result = [&](const std::string& table_path, + const std::vector>& + partition_filters, + const std::shared_ptr& expected) { + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder.SetPartitionFilter(partition_filters); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); + + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_EQ(result_plan->SnapshotId().value(), 1); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto read_result, + ReadResultCollector::CollectResult(std::move(batch_reader))); + // check result + ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); + }; auto expected_without_partition_filter = std::make_shared( arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type_, R"([ @@ -2552,28 +2982,36 @@ TEST_P(ScanAndReadInteTest, TestWithMultipleEmptyPartitionValue) { "/append_with_empty_partition_with_empty_value.db/" "append_with_empty_partition_with_empty_value/"; - auto check_result = - [&](const std::vector>& partition_filters, - const std::shared_ptr& expected) { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetPartitionFilter(partition_filters); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - - ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); - ASSERT_EQ(result_plan->SnapshotId().value(), 1); - ASSERT_OK_AND_ASSIGN(auto batch_reader, - table_read->CreateReader(result_plan->Splits())); - ASSERT_OK_AND_ASSIGN(auto read_result, - ReadResultCollector::CollectResult(std::move(batch_reader))); - // check result - ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); - }; + auto check_result = [&](const std::vector>& + partition_filters, + const std::shared_ptr& expected) { + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder.SetPartitionFilter(partition_filters); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); + + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_EQ(result_plan->SnapshotId().value(), 1); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto read_result, + ReadResultCollector::CollectResult(std::move(batch_reader))); + // check result + ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); + }; auto expected_without_partition_filter = std::make_shared( arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type_, R"([ @@ -2604,10 +3042,16 @@ TEST_P(ScanAndReadInteTest, TestMemoryUse) { std::string table_path = paimon::test::GetDataDir() + file_format + "/append_09.db/append_09/"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -2616,12 +3060,16 @@ TEST_P(ScanAndReadInteTest, TestMemoryUse) { // read std::shared_ptr read_pool = GetMemoryPool(); { - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.WithMemoryPool(read_pool); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2705,26 +3153,41 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { { // batch scan - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.WithStreamingMode(false); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_TRUE(result_plan->Splits().empty()); } { // stream scan: from snapshot 1 - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 1, 2}; auto expected_snapshot1 = std::make_shared( @@ -2749,16 +3212,26 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { } { // stream scan: from snapshot 2 - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 2}; auto expected_snapshot2 = std::make_shared( @@ -2775,17 +3248,26 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { } { // stream scan: from snapshot full - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); std::vector> expected_snapshot_ids = {1, 2}; auto expected_snapshot1 = std::make_shared( @@ -2817,22 +3299,31 @@ TEST_P(ScanAndReadInteTest, TestScanWithPredicateAndReadWithUnorderedFieldForPar } std::string table_path = paimon::test::GetDataDir() + "parquet/parquet_append_table.db/parquet_append_table"; - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; auto predicate = PredicateBuilder::LessThan( /*field_index=*/3, /*field_name=*/"f4", FieldType::INT, Literal(300006)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_EQ(result_plan->Splits().size(), 1); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetReadFieldNames({"f10", "f8", "f4", "f13"}); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2868,11 +3359,17 @@ TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { auto predicate = PredicateBuilder::GreaterThan( /*field_index=*/2, /*field_name=*/"key_2", FieldType::INT, Literal(500)); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); ASSERT_EQ(result_plan->Splits().size(), 1); @@ -2885,11 +3382,15 @@ TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { } ASSERT_EQ(data_file_count, 1); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2921,20 +3422,30 @@ TEST_P(ScanAndReadInteTest, TestAppendTableWithMultipleFileFormat) { paimon::test::GetDataDir() + "/append_table_with_multiple_file_format.db/append_table_with_multiple_file_format"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); auto splits = result_plan->Splits(); ASSERT_EQ(1, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2960,19 +3471,28 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndNoExternalPath) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_no_external.db/pk_dv_index_in_data_no_external"; // scan - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3000,19 +3520,28 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndNoExternalPath) { paimon::test::GetDataDir() + file_format + "/pk_dv_index_not_in_data_no_external.db/pk_dv_index_not_in_data_no_external"; // scan - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3042,9 +3571,14 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { std::string external_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_not_in_data_with_external.db/external"; // scan - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -3053,10 +3587,14 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { AdjustSplitWithExternalPath("FILE:/tmp/external", external_path, /*adjust_index=*/false, &splits); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3087,22 +3625,30 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { Literal(FieldType::STRING, "Alice0", 6)); // scan - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN( - auto scan_context, - scan_context_builder.AddOption("file-index.read.enabled", "false").Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options["file-index.read.enabled"] = "false"; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); auto splits = result_plan->Splits(); ASSERT_EQ(1, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption("file-index.read.enabled", "false"); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options["file-index.read.enabled"] = "false"; + + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3162,24 +3708,33 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { // file. Disabling file-index reads verifies that baseline explicitly. const std::shared_ptr absent_predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"indexed_value", FieldType::INT, Literal(15)); - ScanContextBuilder stats_only_scan_builder(table_path); - stats_only_scan_builder.SetOptions(options) - .AddOption(Options::FILE_INDEX_READ_ENABLED, "false") - .SetPredicate(absent_predicate); + TableLoadOptions stats_only_scan_builder_load_options{table_path}; + std::map stats_only_scan_builder_options; + ScanContextBuilder stats_only_scan_builder; + stats_only_scan_builder_options = options; + stats_only_scan_builder_options[Options::FILE_INDEX_READ_ENABLED] = "false"; + stats_only_scan_builder.SetPredicate(absent_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - TableScan::Create(std::move(stats_only_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr stats_only_scan, + TableScan::Create(stats_only_scan_builder_load_options, std::move(stats_only_scan_context), + stats_only_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); // The embedded bitmap knows that 15 is absent and eliminates the file during scan planning. - ScanContextBuilder indexed_scan_builder(table_path); - indexed_scan_builder.SetOptions(options).SetPredicate(absent_predicate); + TableLoadOptions indexed_scan_builder_load_options{table_path}; + std::map indexed_scan_builder_options; + ScanContextBuilder indexed_scan_builder; + indexed_scan_builder_options = options; + indexed_scan_builder.SetPredicate(absent_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan_context, indexed_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan, - TableScan::Create(std::move(indexed_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr indexed_scan, + TableScan::Create(indexed_scan_builder_load_options, std::move(indexed_scan_context), + indexed_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr indexed_plan, indexed_scan->CreatePlan()); ASSERT_TRUE(indexed_plan->Splits().empty()); @@ -3188,22 +3743,29 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { // the other rows. const std::shared_ptr present_predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"indexed_value", FieldType::INT, Literal(20)); - ScanContextBuilder retained_scan_builder(table_path); - retained_scan_builder.SetOptions(options).SetPredicate(present_predicate); + TableLoadOptions retained_scan_builder_load_options{table_path}; + std::map retained_scan_builder_options; + ScanContextBuilder retained_scan_builder; + retained_scan_builder_options = options; + retained_scan_builder.SetPredicate(present_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr retained_scan_context, retained_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr retained_scan, - TableScan::Create(std::move(retained_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr retained_scan, + TableScan::Create(retained_scan_builder_load_options, std::move(retained_scan_context), + retained_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr retained_plan, retained_scan->CreatePlan()); ASSERT_FALSE(retained_plan->Splits().empty()); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options) - .SetPredicate(present_predicate) - .EnablePredicateFilter(false); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetPredicate(present_predicate).EnablePredicateFilter(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(retained_plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3263,24 +3825,33 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc /*field_index=*/1, /*field_name=*/"renamed_value", FieldType::INT, Literal(15)); // The evolved min/max range is still [10, 30], so statistics alone retain the old file. - ScanContextBuilder stats_only_scan_builder(table_path); - stats_only_scan_builder.SetOptions(options) - .AddOption(Options::FILE_INDEX_READ_ENABLED, "false") - .SetPredicate(predicate); + TableLoadOptions stats_only_scan_builder_load_options{table_path}; + std::map stats_only_scan_builder_options; + ScanContextBuilder stats_only_scan_builder; + stats_only_scan_builder_options = options; + stats_only_scan_builder_options[Options::FILE_INDEX_READ_ENABLED] = "false"; + stats_only_scan_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - TableScan::Create(std::move(stats_only_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr stats_only_scan, + TableScan::Create(stats_only_scan_builder_load_options, std::move(stats_only_scan_context), + stats_only_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); // Reconstructing the predicate to schema-0 lets its embedded bitmap prove that 15 is absent. - ScanContextBuilder indexed_scan_builder(table_path); - indexed_scan_builder.SetOptions(options).SetPredicate(predicate); + TableLoadOptions indexed_scan_builder_load_options{table_path}; + std::map indexed_scan_builder_options; + ScanContextBuilder indexed_scan_builder; + indexed_scan_builder_options = options; + indexed_scan_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan_context, indexed_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan, - TableScan::Create(std::move(indexed_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr indexed_scan, + TableScan::Create(indexed_scan_builder_load_options, std::move(indexed_scan_context), + indexed_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr indexed_plan, indexed_scan->CreatePlan()); ASSERT_TRUE(indexed_plan->Splits().empty()); } @@ -3293,9 +3864,14 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { std::string external_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_with_external.db/external"; // scan - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -3304,10 +3880,14 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { AdjustSplitWithExternalPath("FILE:/tmp/external", external_path, /*adjust_index=*/true, &splits); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3334,9 +3914,14 @@ TEST_P(ScanAndReadInteTest, TestTimestampType) { std::string table_path = paimon::test::GetDataDir() + file_format + "/append_with_multiple_ts_precision_and_timezone.db" "/append_with_multiple_ts_precision_and_timezone/"; - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3344,10 +3929,14 @@ TEST_P(ScanAndReadInteTest, TestTimestampType) { ASSERT_EQ(1, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3382,9 +3971,14 @@ TEST_P(ScanAndReadInteTest, TestCastTimestampType) { "/append_with_cast_timestamp.db" "/append_with_cast_timestamp/"; // scan - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3392,10 +3986,14 @@ TEST_P(ScanAndReadInteTest, TestCastTimestampType) { ASSERT_EQ(1, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3430,18 +4028,24 @@ TEST_F(ScanAndReadInteTest, TestParquetDictionaryBinaryStoredArrowSchema) { "/parquet/append_dictionary_binary.db/" "append_dictionary_binary"; - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_EQ(1, plan->SnapshotId().value()); ASSERT_EQ(1, plan->Splits().size()); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3510,8 +4114,12 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { const std::vector& projection, const std::shared_ptr& expected_result, bool exact_filter = false) { - ScanContextBuilder scan_builder(table_path); - ReadContextBuilder read_builder(table_path); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; if (predicate) { scan_builder.SetPredicate(predicate); if (projection.empty()) { @@ -3523,14 +4131,19 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { read_builder.SetReadFieldNames(projection); } read_builder.EnablePredicateFilter(exact_filter); - read_builder.AddOption("read.batch-size", "1"); + read_builder_options["read.batch-size"] = "1"; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId(), std::optional(2)); ASSERT_FALSE(plan->Splits().empty()); ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto read, + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); ASSERT_TRUE(actual); @@ -3601,21 +4214,31 @@ TEST_F(ScanAndReadInteTest, TestLancePythonCompatibility) { auto check = [&](int64_t snapshot_id, const std::vector& projection, const std::shared_ptr& expected_result) { SCOPED_TRACE("snapshot " + std::to_string(snapshot_id)); - ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId(), std::optional(snapshot_id)); ASSERT_FALSE(plan->Splits().empty()); - ReadContextBuilder read_builder(table_path); - read_builder.AddOption("read.batch-size", "1"); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options["read.batch-size"] = "1"; + if (!projection.empty()) { read_builder.SetReadFieldNames(projection); } ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto read, + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); ASSERT_TRUE(actual); @@ -3675,21 +4298,31 @@ TEST_F(ScanAndReadInteTest, TestLanceLtzCompatibility) { const std::vector& projection, const std::shared_ptr& expected_result) { SCOPED_TRACE("snapshot " + std::to_string(snapshot_id)); - ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId(), std::optional(snapshot_id)); ASSERT_FALSE(plan->Splits().empty()); - ReadContextBuilder read_builder(table_path); - read_builder.AddOption("read.batch-size", "1"); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options["read.batch-size"] = "1"; + if (!projection.empty()) { read_builder.SetReadFieldNames(projection); } ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto read, + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); ASSERT_TRUE(actual); @@ -3757,22 +4390,30 @@ TEST_F(ScanAndReadInteTest, TestMosaicJavaAndPythonCompatibility) { auto check_compatibility = [](const std::string& table_path, const std::shared_ptr& predicate, const std::shared_ptr& expected_result) { - ScanContextBuilder scan_context_builder(table_path); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; if (predicate != nullptr) { scan_context_builder.SetPredicate(predicate); read_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3821,15 +4462,21 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibility) { const std::string table_name = "append_java_compat"; const std::string table_path = GetDataDir() + "/vortex/" + table_name + ".db/" + table_name; - ScanContextBuilder scan_context_builder(table_path); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3849,17 +4496,25 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibilityLtzAcrossTimezones) { const std::string timezones[] = {"UTC", "Asia/Shanghai"}; for (const std::string& tz : timezones) { TimezoneGuard timezone_guard(tz); - ScanContextBuilder scan_context_builder(table_path); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3924,15 +4579,21 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibilityAllTypes) { const std::string table_name = "all_types_java_compat"; const std::string table_path = GetDataDir() + "/vortex/" + table_name + ".db/" + table_name; - ScanContextBuilder scan_context_builder(table_path); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3947,22 +4608,32 @@ TEST_F(ScanAndReadInteTest, TestAvroWithAppendTable) { auto read_data = [](int64_t snapshot_id, const std::string& result_json) { std::string table_path = GetDataDir() + "/avro/append_multiple.db/append_multiple"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), snapshot_id); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption("test.enable-adaptive-prefetch-strategy", "false"); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + read_context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4022,10 +4693,15 @@ TEST_F(ScanAndReadInteTest, TestAvroWithPkTable) { std::string table_path = GetDataDir() + "/avro/pk_with_multiple_type.db/pk_with_multiple_type"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), snapshot_id); @@ -4033,10 +4709,14 @@ TEST_F(ScanAndReadInteTest, TestAvroWithPkTable) { ASSERT_EQ(1, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4095,32 +4775,47 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { // Historical files use two buckets even when the current option has changed. for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - scan_context_builder.AddOption(Options::BUCKET, current_bucket_count); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + + scan_context_builder_options[Options::BUCKET] = current_bucket_count; + scan_context_builder.SetPartitionFilter({{{"f1", "10"}}}); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); if (EnableSnapshotLiveManifestCache()) { ASSERT_OK_AND_ASSIGN(uint64_t enabled, table_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED)); ASSERT_EQ(enabled, 1); - ScanContextBuilder cached_builder(table_path); - cached_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6") - .AddOption(Options::BUCKET, current_bucket_count) - .SetPartitionFilter({{{"f1", "10"}}}) - .SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); - ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(std::move(cached_context))); + TableLoadOptions cached_builder_load_options{table_path}; + std::map cached_builder_options; + ScanContextBuilder cached_builder; + cached_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + cached_builder_options[Options::BUCKET] = current_bucket_count; + cached_builder.SetPartitionFilter({{{"f1", "10"}}}).SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto cached_context, + FinishScanContext(cached_builder, cached_builder_options)); + ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(cached_builder_load_options, + std::move(cached_context), + cached_builder_options)); ASSERT_OK_AND_ASSIGN(result_plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); @@ -4168,18 +4863,27 @@ TEST_P(ScanAndReadInteTest, TestReadNullableMapKey) { const std::string table_path = paimon::test::GetDataDir() + "orc/nullable_map_key.db/nullable_map_key"; - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->SnapshotId()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); ASSERT_EQ(result_plan->Splits().size(), 1); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::READ_BATCH_SIZE, "1"); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options[Options::READ_BATCH_SIZE] = "1"; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); const std::vector expected_rows = {R"([[0, 1, [["one", 10]]]])", @@ -4211,10 +4915,14 @@ TEST_P(ScanAndReadInteTest, TestCountRowsEmptySplits) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // CreateCountReader with empty splits should return 0 rows. std::vector> empty_splits; @@ -4229,25 +4937,39 @@ TEST_P(ScanAndReadInteTest, TestCountRowsConsistencyWithCreateReader) { "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Scan latest snapshot - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // Method 1: CreateCountReader + iterate batches - ReadContextBuilder count_context_builder(table_path); - AddReadOptionsForPrefetch(&count_context_builder); + TableLoadOptions count_context_builder_load_options{table_path}; + std::map count_context_builder_options; + ReadContextBuilder count_context_builder; + AddReadOptionsForPrefetch(&count_context_builder, count_context_builder_options); ASSERT_OK_AND_ASSIGN(auto count_read_context, count_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto count_table_read, TableRead::Create(std::move(count_read_context))); + ASSERT_OK_AND_ASSIGN( + auto count_table_read, + TableRead::Create(count_context_builder_load_options, std::move(count_read_context), + count_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto count_reader, count_table_read->CreateCountReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(int64_t count_result, count_reader->CountRows()); // Method 2: CreateReader + iterate batches - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4263,19 +4985,28 @@ TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithPredicateNotSupported) { "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Create splits from latest snapshot. - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // Set predicate in read context. CountReader currently does not support predicate pushdown. auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT, Literal(static_cast(0))); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader(result_plan->Splits()), "predicate pushdown is not supported"); @@ -4287,15 +5018,24 @@ TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithForceKeepDeleteNotSupported "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Create splits from latest snapshot. - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); auto* key_value_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(key_value_table_read != nullptr); diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 3f949484d..1a2a02200 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -32,6 +32,7 @@ #include "arrow/api.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" #include "paimon/common/utils/decimal_utils.h" @@ -64,15 +65,16 @@ enum class ManifestCacheMode { NoCache, Cache }; class ScanInteTest : public testing::TestWithParam { public: - Result> FinishScanContext(ScanContextBuilder& builder) { + Result> FinishScanContext( + ScanContextBuilder& builder, std::map& builder_options) { if (GetParam() == ManifestCacheMode::Cache) { if (!cache_) { cache_ = std::make_shared(std::map{ {CacheKind::MANIFEST, 64 * 1024 * 1024}, {CacheKind::SNAPSHOT_LIVE_MANIFEST, 64 * 1024 * 1024}}); } - builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .WithCache(cache_); + builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder.WithCache(cache_); } return builder.Finish(); } @@ -269,10 +271,15 @@ TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; auto run_scan = [&]() -> Result> { - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithCache(manifest_cache); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.WithCache(manifest_cache); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, + TableScan::Create(context_builder_load_options, + std::move(scan_context), context_builder_options)); return table_scan->CreatePlan(); }; @@ -293,19 +300,23 @@ TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) { Result>> RunBucketSnapshotScan( const std::string& table_path, int64_t snapshot_id, int32_t bucket, const std::shared_ptr& cache, int32_t max_snapshot_live_manifest_versions) { - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)) - .SetBucketFilter(bucket); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + context_builder.SetBucketFilter(bucket); if (max_snapshot_live_manifest_versions > 0) { - context_builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, - std::to_string(max_snapshot_live_manifest_versions)); + context_builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = + std::to_string(max_snapshot_live_manifest_versions); } if (cache) { context_builder.WithCache(cache); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); std::vector> data_splits; @@ -396,11 +407,16 @@ TEST(ScanInteManifestCacheTest, TestSnapshotLiveManifestCacheRetainsSnapshotsPer TEST_P(ScanInteTest, TestScanAppendWithSnapshot1) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -453,10 +469,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -514,13 +536,19 @@ TEST_P(ScanInteTest, TestScanAppendWithSpecificTableSchema) { std::string table_path = data_dir + "orc/append_09.db/append_09"; auto check_result = [&](const std::optional& specific_table_schema) { - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + if (specific_table_schema) { - context_builder.SetTableSchema(specific_table_schema.value()); + context_builder_load_options.table_schema_json = specific_table_schema.value(); } - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, + std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -578,10 +606,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSpecificTableSchema) { TEST_P(ScanInteTest, TestScanInvalidSnapshot) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "100"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "100"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_NOK_WITH_MSG( table_scan->CreatePlan(), "The specified scan snapshotId 100 is out of available snapshotId range [1, 5]."); @@ -589,10 +623,16 @@ TEST_P(ScanInteTest, TestScanInvalidSnapshot) { TEST_P(ScanInteTest, TestBatchScanMultipleTimes) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // batch scan multiple ASSERT_NOK_WITH_MSG(table_scan->CreatePlan(), "end of scan"); @@ -600,13 +640,19 @@ TEST_P(ScanInteTest, TestBatchScanMultipleTimes) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::SOURCE_SPLIT_OPEN_FILE_COST, "1024") - .AddOption(Options::SOURCE_SPLIT_TARGET_SIZE, "2048"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + context_builder_options[Options::SOURCE_SPLIT_OPEN_FILE_COST] = "1024"; + context_builder_options[Options::SOURCE_SPLIT_TARGET_SIZE] = "2048"; + // open cost = 1024, and split target size is 2048, indicates at most 2 files in a split - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -670,10 +716,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3").SetLimit(3); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + context_builder.SetLimit(3); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -716,10 +768,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.SetBucketFilter(0).AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + context_builder.SetBucketFilter(0); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -759,10 +817,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { TEST_P(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { // from snapshot is specified std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); DataSplitImpl::Builder builder1_1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -871,11 +935,16 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); DataSplitImpl::Builder builder1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -925,10 +994,16 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { TEST_P(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::Latest().ToString()); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_MODE] = StartupMode::Latest().ToString(); + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -981,11 +1056,16 @@ TEST_P(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::Latest().ToString()) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_MODE] = StartupMode::Latest().ToString(); + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); std::vector> expected_snapshot_ids = {}; CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, {}); @@ -994,12 +1074,17 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshot().ToString()) - .AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_MODE] = StartupMode::FromSnapshot().ToString(); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); DataSplitImpl::Builder builder2_1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -1066,13 +1151,18 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshotFull().ToString()) - .AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_MODE] = StartupMode::FromSnapshotFull().ToString(); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); ASSERT_TRUE(scan_context); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); DataSplitImpl::Builder builder1_1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -1151,20 +1241,30 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { TEST_P(ScanInteTest, TestScanAppendWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; { - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshot().ToString()) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_MODE] = StartupMode::FromSnapshot().ToString(); + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, + std::move(scan_context), context_builder_options)); ASSERT_NOK_WITH_MSG( table_scan->CreatePlan(), "scan.snapshot-id or scan.tag-name must be set when startup mode is FROM_SNAPSHOT"); } { - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::BUCKET, "-2").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::BUCKET] = "-2"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_NOK_WITH_MSG(TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options), "do not support bucket=-2 in scan process"); } } @@ -1177,10 +1277,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEqualPredicate) { PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, val.data(), val.size())); - ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1223,12 +1329,16 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { PredicateBuilder::And({greater_than_predicate, less_than_predicate, not_in_predicate, not_equal})); - ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate) - .AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate).WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); DataSplitImpl::Builder builder1_2( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -1284,10 +1394,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilter) { std::map partition_keys; partition_keys["f1"] = "10"; - ScanContextBuilder context_builder(table_path); - context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1330,10 +1446,15 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithInvalidPartitionFilter) { std::map partition_keys; partition_keys["invalid_partition_key"] = "10"; - ScanContextBuilder context_builder(table_path); - context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_NOK_WITH_MSG(TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options), "field invalid_partition_key does not exist in partition keys"); } @@ -1347,12 +1468,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilterAndPredicateF std::map partition_keys; partition_keys["f1"] = "10"; - ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate) - .SetPartitionFilter({partition_keys}) - .AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate).SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1399,10 +1524,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) { partition_keys["f1"] = "10"; partition_keys["f2"] = "0"; - ScanContextBuilder context_builder(table_path); - context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1455,10 +1586,16 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter) Literal(paimon::Decimal(23, 5, 0))); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); - ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "4"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan); // check snapshot id @@ -1516,10 +1653,16 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) Literal(FieldType::BINARY, "zoo", 3)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); - ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "4"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -1571,12 +1714,18 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { std::string table_path = paimon::test::GetDataDir() + "orc/append_10_stats_dense_store.db/append_10_stats_dense_store"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(13.0)); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1668,16 +1817,22 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_10_stats_dense_store.db/append_10_stats_dense_store"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto greater_than = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(13.0)); auto equal = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Emily", 5)); auto predicate = PredicateBuilder::And({greater_than, equal}).value(); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1714,10 +1869,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").SetBucketFilter(2); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetBucketFilter(2); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); ASSERT_TRUE(result_plan->Splits().empty()); @@ -1726,19 +1887,29 @@ TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::BUCKET, "-1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::BUCKET] = "-1"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_NOK_WITH_MSG(TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options), "do not support pk table bucket=-1 in scan process"); } TEST_P(ScanInteTest, TestReadWithNoSnapshot) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_nested_type.db/append_table_with_nested_type"; - ScanContextBuilder context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->SnapshotId()); ASSERT_TRUE(result_plan->Splits().empty()); @@ -1748,7 +1919,9 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_alter_table_with_cast.db/append_table_alter_table_with_cast"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto child1 = PredicateBuilder::Or( @@ -1770,9 +1943,13 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { FieldType::INT, Literal(80)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2, child3})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1813,7 +1990,9 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_alter_table.db/append_table_with_alter_table"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto child1 = PredicateBuilder::GreaterThan(/*field_index=*/2, /*field_name=*/"k", FieldType::INT, Literal(36)); @@ -1821,9 +2000,13 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { Literal(96)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1891,7 +2074,9 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_alter_table_with_dense_field.db/" "append_table_with_alter_table_with_dense_field"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto child1 = PredicateBuilder::GreaterThan(/*field_index=*/2, /*field_name=*/"f1", FieldType::INT, Literal(22)); @@ -1899,9 +2084,13 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { FieldType::DOUBLE, Literal(0.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child3})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1939,7 +2128,9 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap.db/append_with_bitmap/"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto child1 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Tony", 4)); @@ -1947,9 +2138,13 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { Literal(10)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -2016,7 +2211,9 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap.db/append_with_bitmap/"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto child1 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Lucy", 4)); @@ -2024,9 +2221,13 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { Literal(10)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->Splits().empty()); } @@ -2035,7 +2236,9 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto child1 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Lucy", 4)); @@ -2043,9 +2246,13 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { Literal(10)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -2083,14 +2290,20 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; // file0 will be removed as f5 not exists auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f5", FieldType::INT, Literal(100)); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2152,15 +2365,21 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto child1 = PredicateBuilder::IsNull(/*field_index=*/3, /*field_name=*/"f5", FieldType::INT); auto child2 = PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2227,7 +2446,9 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; // in stats filter: predicate is trimmed as type for f1 is not consist: int->bigint // in index filter: predicate is removed as type for f1 is converted bigint->int, which is not @@ -2235,9 +2456,13 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, Literal(100l)); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2304,7 +2529,9 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; // child1 will remove file1 for schema1 auto child1 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, @@ -2314,9 +2541,13 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { Literal(FieldType::STRING, "David", 5)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->Splits().empty()); } @@ -2324,11 +2555,16 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { TEST_P(ScanInteTest, TestScanAppendWithTag1) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_tag.db/append_table_with_tag"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_TAG_NAME, "1"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_TAG_NAME] = "1"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(1, result_plan->SnapshotId().value()); @@ -2337,10 +2573,16 @@ TEST_P(ScanInteTest, TestScanAppendWithTag1) { TEST_P(ScanInteTest, TestScanInvalidTag) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_tag.db/append_table_with_tag"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_TAG_NAME, "unknown"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_TAG_NAME] = "unknown"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_NOK_WITH_MSG(table_scan->CreatePlan(), "Tag 'unknown' doesn't exist."); } @@ -2353,38 +2595,62 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { // EarlierOrEqual(snap3.time) → snap-3 { - ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis())); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap3.TimeMillis()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 3); } // EarlierOrEqual(snap3.time - 1) → snap-2 { - ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis() - 1)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap3.TimeMillis() - 1); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 2); } // EarlierOrEqual(INT64_MAX) → snap-5 (latest) { - ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, - std::to_string(std::numeric_limits::max())); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = + std::to_string(std::numeric_limits::max()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 5); } // EarlierOrEqual(0) → no snapshot found, expect error { - ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, "0"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = "0"; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_NOK_WITH_MSG(scan->CreatePlan(), "There is currently no snapshot earlier than or equal to timestamp"); } @@ -2400,10 +2666,16 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { // T=0: no snapshot earlier than T=0, stream starts from snapshot 1. { - ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, "0").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = "0"; + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan1, scan->CreatePlan()); @@ -2411,11 +2683,16 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } // T=snap2.time+1: EarlierThan(snap2.time+1)=snap2, stream starts from snapshot 3. { - ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap2.TimeMillis() + 1)) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap2.TimeMillis() + 1); + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan1, scan->CreatePlan()); @@ -2423,11 +2700,16 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } // T=snap3.time: EarlierThan(snap3.time)=snap2, stream starts from snapshot 3. { - ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis())) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap3.TimeMillis()); + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan1, scan->CreatePlan()); @@ -2435,13 +2717,17 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } // T=INT64_MAX: EarlierThan(INT64_MAX)=snap5, stream starts from snapshot 6 (beyond latest). { - ScanContextBuilder builder(table_path); - builder - .AddOption(Options::SCAN_TIMESTAMP_MILLIS, - std::to_string(std::numeric_limits::max())) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = + std::to_string(std::numeric_limits::max()); + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_TRUE(plan0->Splits().empty()); diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 50772b438..76f526ff8 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -32,6 +32,7 @@ #include "arrow/type.h" #include "fmt/format.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/reader/reader_utils.h" @@ -155,13 +156,17 @@ class WriteAndReadInteTest Status CompactAndCommit(const std::string& table_path, const std::map& options, int64_t commit_identifier) const { - WriteContextBuilder write_context_builder(table_path, "commit_user"); + TableLoadOptions write_context_builder_load_options{table_path}; + std::map write_context_builder_options; + WriteContextBuilder write_context_builder("commit_user"); write_context_builder.WithTempDirectory(LookupTempDirectory()); + write_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, + write_context_builder.WithStreamingMode(true).Finish()); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr write_context, - write_context_builder.SetOptions(options).WithStreamingMode(true).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + std::unique_ptr file_store_write, + FileStoreWrite::Create(write_context_builder_load_options, std::move(write_context), + write_context_builder_options)); PAIMON_RETURN_NOT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); PAIMON_ASSIGN_OR_RAISE( @@ -171,11 +176,16 @@ class WriteAndReadInteTest if (compact_messages.empty()) { return Status::Invalid("expected compaction commit messages"); } - CommitContextBuilder commit_context_builder(table_path, "commit_user"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user"); + commit_context_builder_options = options; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); return file_store_commit->Commit(compact_messages, commit_identifier); } @@ -186,10 +196,15 @@ class WriteAndReadInteTest std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames(read_fields); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadFieldNames(read_fields); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -209,10 +224,15 @@ class WriteAndReadInteTest auto c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, c_schema.get())); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -241,11 +261,16 @@ class WriteAndReadInteTest Result> InnerScan( const std::map& options) const { std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options).AddOption(Options::SCAN_MODE, - StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); return table_scan->CreatePlan(); } @@ -443,16 +468,20 @@ TEST_P(WriteAndReadInteTest, TestAppendReadWithNestedPredicateAcrossBatches) { SCOPED_TRACE(batch_size); for (bool enable_filter : {false, true}) { SCOPED_TRACE(enable_filter); - ReadContextBuilder builder(table_path); - builder.SetOptions(options) - .SetReadFieldNames({"payload", "value", "id", "key"}) + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ReadContextBuilder builder; + builder_options = options; + builder_options[Options::READ_BATCH_SIZE] = std::to_string(batch_size); + builder.SetReadFieldNames({"payload", "value", "id", "key"}) .SetPredicate(predicate) .EnablePredicateFilter(enable_filter) .EnablePrefetch(false) - .EnableLateMaterializing(false) - .AddOption(Options::READ_BATCH_SIZE, std::to_string(batch_size)); + .EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + auto table_read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); ASSERT_TRUE(actual); @@ -618,19 +647,28 @@ TEST_P(WriteAndReadInteTest, TestAppendVectorWithPredicate) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(2)); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetPredicate(predicate); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -700,21 +738,30 @@ TEST_P(WriteAndReadInteTest, TestAppendWithExternalBitmapAndRangeBitmapIndexes) ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({name_predicate, score_predicate})); std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(1, plan->Splits().size()); // Keep precise post-read filtering disabled. The exact result therefore verifies that the // bitmap and range-bitmap indexes produced by the write path are consumed by the read path. - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetPredicate(predicate); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -812,13 +859,16 @@ TEST_P(WriteAndReadInteTest, TestLuminaSearchAfterRenameAcrossSchemaVersions) { auto search = std::make_shared("vec", 3, std::vector{0.0f, 0.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - ReadContextBuilder builder(path); - builder.SetOptions(current_options) - .SetReadFieldNames({"id", "_vec_INDEX_SCORE"}) + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_options = current_options; + builder.SetReadFieldNames({"id", "_vec_INDEX_SCORE"}) .SetVectorSearch(search) .EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + auto read, TableRead::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), @@ -910,14 +960,17 @@ TEST_P(WriteAndReadInteTest, TestLuminaVectorIndexAfterCompaction) { auto search = std::make_shared( "embedding", limit, std::vector{0.0f, 0.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - ReadContextBuilder builder(path); - builder.SetOptions(options) - .SetReadFieldNames({"id", "_embedding_INDEX_SCORE"}) + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_options = options; + builder.SetReadFieldNames({"id", "_embedding_INDEX_SCORE"}) .SetVectorSearch(search) .EnablePreciseFileIndexEvaluate(true); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - TableRead::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr actual, @@ -1032,13 +1085,16 @@ TEST_P(WriteAndReadInteTest, TestAppendWithLuminaVectorIndex) { "embedding", /*limit=*/3, std::vector{0.0f, 0.0f}, /*pre_filter=*/nullptr, /*predicate=*/nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - ReadContextBuilder builder(PathUtil::JoinPath(root, "foo.db/bar")); + TableLoadOptions builder_load_options{PathUtil::JoinPath(root, "foo.db/bar")}; + std::map builder_options; + ReadContextBuilder builder; builder.SetReadFieldNames({"id", "_embedding_INDEX_SCORE"}) .SetVectorSearch(search) .EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -3295,11 +3351,15 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, literal_str.data(), literal_str.size())); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_FALSE(result_plan->Splits().empty()); @@ -3308,10 +3368,14 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { // of the result is attributable to split/file/RG/page pruning, not to a // post-read row-level filter. This is what makes the exact assertion below // meaningful as a check that page-index filtering is wired and working. - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3405,11 +3469,15 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, literal_str.data(), literal_str.size())); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_FALSE(result_plan->Splits().empty()); @@ -3417,10 +3485,14 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { // Read with predicate but WITHOUT EnablePredicateFilter, so the narrowing // observed below is attributable to page-index filtering rather than a // post-read row-level filter. - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3491,12 +3563,16 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { {Literal(7), Literal(11), Literal(15)}); ASSERT_TRUE(predicate); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); @@ -3505,15 +3581,19 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { // therefore finds no data for it and skips ahead into the next row group it owns, whose first // selected row sits in the middle of that row group. Row level filtering stays off: the // expected rows below are exactly what page-index filtering selects. - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options) - .SetPredicate(predicate) + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + read_context_builder.SetPredicate(predicate) .EnablePrefetch(true) .SetPrefetchMaxParallelNum(2) - .SetPrefetchBatchCount(3) - .AddOption("test.enable-adaptive-prefetch-strategy", "false"); + .SetPrefetchBatchCount(3); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3580,10 +3660,15 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { auto cache = std::make_shared(CacheKind::DATA_FILE_FOOTER, 128 * 1024 * 1024); auto read_once = [&]() -> Result { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); if (result_plan->SnapshotId() != std::optional(1)) { return Status::Invalid("unexpected snapshot id"); @@ -3592,10 +3677,14 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { return Status::Invalid("no splits found"); } - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.WithCache(cache); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(result_plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3902,10 +3991,16 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { (void)helper; std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); - WriteContextBuilder write_context_builder(table_path, "commit_user_1"); + TableLoadOptions write_context_builder_load_options{table_path}; + std::map write_context_builder_options; + WriteContextBuilder write_context_builder("commit_user_1"); + write_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + write_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(write_context_builder_load_options, std::move(write_context), + write_context_builder_options)); auto write_one_row = [&](const std::string& data) -> Status { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, @@ -3922,27 +4017,41 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { /*commit_identifier=*/0)); ASSERT_OK(file_store_write->Close()); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(commit_msgs, /*commit_identifier=*/0)); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(10)); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetPredicate(predicate); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4999,10 +5108,15 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingStructValueSchemaEvolutionRea auto read_fields = [&](const std::vector& field_names) -> Status { PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames(field_names); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadFieldNames(field_names); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -6114,12 +6228,16 @@ TEST_P(LanceTimestampLtzInteTest, TestWriteAndReadInPinnedTimezone) { ASSERT_OK_AND_ASSIGN(std::vector> splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames( + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadFieldNames( {"ltz_row", "ltz_added", "ltz_9", "id", "ltz_array", "ltz_0", "ltz_to_ntz", "ntz_to_ltz"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp index 8e7ba94fa..e29984fa3 100644 --- a/test/inte/write_and_read_inte_with_index_test.cpp +++ b/test/inte/write_and_read_inte_with_index_test.cpp @@ -35,6 +35,7 @@ #include "arrow/ipc/json_simple.h" #include "arrow/util/string_builder.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/file_index/bitmap/bitmap_file_index_factory.h" @@ -142,16 +143,19 @@ class WriteAndReadInteWithIndexTest const std::vector> splits, const std::shared_ptr& predicate, const std::shared_ptr& expected_array) const { - ReadContextBuilder context_builder(table_path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .SetPredicate(predicate) - .EnableLateMaterializing(false); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder.SetPredicate(predicate).EnableLateMaterializing(false); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto result_array, @@ -1165,22 +1169,26 @@ class WriteAndReadInteWithIndexTest for (const auto& field : fields) { field_names.push_back(field->name()); } - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_options["read.batch-size"] = "1"; + builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; builder.EnablePreciseFileIndexEvaluate(true) .EnablePredicateFilter(false) .SetReadFieldNames(field_names) .SetPredicate(predicate) .SetVectorSearch(vector_search) .SetFullTextSearch(full_text_search) - .AddOption("read.batch-size", "1") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") + .EnableLateMaterializing(false); if (enable_prefetch_) { builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -1279,16 +1287,22 @@ TEST_P(WriteAndReadInteWithIndexTest, TestSimple) { &expected_array); ASSERT_TRUE(array_status.ok()); - ReadContextBuilder context_builder(path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption("orc.read.enable-metrics", "true") + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder + .SetPredicate(predicate); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(split)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(batch_reader.get())); std::shared_ptr read_metrics = batch_reader->GetReaderMetrics(); @@ -1352,17 +1366,23 @@ TEST_P(WriteAndReadInteWithIndexTest, TestReadWithLimits) { auto predicate = PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); - ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption(Options::READ_BATCH_SIZE, "1") - .AddOption("orc.read.enable-metrics", "true") + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options[Options::READ_BATCH_SIZE] = "1"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder + .SetPredicate(predicate); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(split)); // simulate read limits, only read 3 batches std::vector batches; @@ -1636,17 +1656,20 @@ TEST_P(WriteAndReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, literal_str.data(), literal_str.size())); - ReadContextBuilder context_builder(path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .SetPredicate(predicate) - .EnablePredicateFilter(true) - .EnableLateMaterializing(true); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder.SetPredicate(predicate).EnablePredicateFilter(true).EnableLateMaterializing( + true); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(std::vector>{split})); ASSERT_OK_AND_ASSIGN(auto result_array, @@ -2323,19 +2346,23 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { auto predicate = PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, {Literal(10l), Literal(int64_t{2147483648})}); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_options["read.batch-size"] = "2"; + builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; builder.EnablePreciseFileIndexEvaluate(true) .EnablePredicateFilter(false) .SetPredicate(predicate) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") + .EnableLateMaterializing(false); if (enable_prefetch_) { builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -3172,15 +3199,18 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithIOException) { for (size_t i = 0; i < 200; i++) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - ReadContextBuilder context_builder(path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .SetPredicate(predicate); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder.SetPredicate(predicate); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = TableRead::Create( + context_builder_load_options, std::move(read_context), context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(split); CHECK_HOOK_STATUS(batch_reader.status(), i); diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index 31a325ec1..d10319510 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -41,6 +41,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_array.h" #include "paimon/common/data/binary_array_writer.h" @@ -289,11 +290,16 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface const std::vector>& data_splits, const std::string& blob_field, const std::vector>& expected_blobs) const { - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames({blob_field}); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadFieldNames({blob_field}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(data_splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -349,11 +355,15 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface const std::vector>& commit_messages, bool ignore_empty_commit = true, int64_t commit_identifier = BATCH_WRITE_COMMIT_IDENTIFIER) const { - CommitContextBuilder commit_builder(table_path, "commit_user_1"); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); commit_builder.IgnoreEmptyCommit(ignore_empty_commit); PAIMON_ASSIGN_OR_RAISE(auto commit_context, commit_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); return file_store_commit->Commit(commit_messages, commit_identifier); } @@ -2015,11 +2025,14 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { ASSERT_EQ(3, batches_2.size()); // write data - WriteContextBuilder context_builder(root_path, "commit_user_1"); + TableLoadOptions context_builder_load_options{root_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); - Result> write = - FileStoreWrite::Create(std::move(write_context)); + context_builder.WithStreamingMode(true).Finish()); + Result> write = FileStoreWrite::Create( + context_builder_load_options, std::move(write_context), context_builder_options); CHECK_HOOK_STATUS(write.status(), i); auto& file_store_write = write.value(); // round 1 @@ -2451,12 +2464,15 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) { for (size_t i = 0; i < 200; i++) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - WriteContextBuilder context_builder(table_path, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + context_builder.WithStreamingMode(true).Finish()); - Result> write = - FileStoreWrite::Create(std::move(write_context)); + Result> write = FileStoreWrite::Create( + context_builder_load_options, std::move(write_context), context_builder_options); CHECK_HOOK_STATUS(write.status(), i); auto& file_store_write = write.value(); @@ -2476,14 +2492,16 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) { for (size_t i = 0; i < 400; i += paimon::test::RandomNumber(5, 15)) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(false) - .Finish()); - auto commit = FileStoreCommit::Create(std::move(commit_context)); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + auto commit = + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options); CHECK_HOOK_STATUS(commit.status(), i); Result committed = commit.value()->FilterAndCommit({{0, commit_messages}}); CHECK_HOOK_STATUS(committed.status(), i); @@ -2545,12 +2563,15 @@ TEST_P(WriteInteTest, TestWriteWithFieldId) { /*partition_keys=*/{}, /*primary_keys=*/{}, options)); - WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); ::ArrowArray c_array; ASSERT_TRUE(arrow::ExportArray(*src_array, &c_array).ok()); RecordBatchBuilder batch_builder(&c_array); @@ -2615,12 +2636,15 @@ TEST_P(WriteInteTest, TestAppendTableWriteAndReadWithExternalPath) { /*table_name=*/"append_table_with_external_path", &schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); // write - WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); - std::string root_path = write_context->GetRootPath(); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); + std::string root_path = context_builder_load_options.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ @@ -2652,15 +2676,25 @@ TEST_P(WriteInteTest, TestAppendTableWriteAndReadWithExternalPath) { ASSERT_TRUE(file_exist); // check read result - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options[Options::FILE_FORMAT] = file_format; - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -2712,12 +2746,15 @@ TEST_P(WriteInteTest, TestPKTableWriteAndReadWithExternalPath) { /*table_name=*/"pk_table_with_external_path", &schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{"f0", "f1"}, options)); // write - WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); - std::string root_path = write_context->GetRootPath(); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); + std::string root_path = context_builder_load_options.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ @@ -2932,12 +2969,15 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { /*primary_keys=*/{}, options)); // write - WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); - std::string root_path = write_context->GetRootPath(); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); + std::string root_path = context_builder_load_options.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); auto write = [&](const std::string& json_array, const std::map& partition) -> void { std::shared_ptr array = @@ -2973,15 +3013,25 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { arrow::field("_VALUE_KIND", arrow::int8())); auto data_type = arrow::struct_(fields_with_row_kind); { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options[Options::FILE_FORMAT] = file_format; - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3020,18 +3070,28 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + std::vector> partition_filters = { {{"dt", " "}, {"hr", "a=b?"}}}; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.SetPartitionFilter(partition_filters).Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options[Options::FILE_FORMAT] = file_format; - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3047,18 +3107,28 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + std::vector> partition_filters = { {{"dt", ""}, {"hr", "a=b?"}}}; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.SetPartitionFilter(partition_filters).Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options[Options::FILE_FORMAT] = file_format; - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3074,18 +3144,28 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + std::vector> partition_filters = { {{"dt", "__DEFAULT_PARTITION__"}, {"hr", "a=b?"}}}; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.SetPartitionFilter(partition_filters).Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options[Options::FILE_FORMAT] = file_format; - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3124,12 +3204,15 @@ TEST_P(WriteInteTest, TestWriteWithNestedSchema) { /*table_name=*/"bar", &schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options)); // write - WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); - std::string root_path = write_context->GetRootPath(); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); + std::string root_path = context_builder_load_options.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ @@ -3155,15 +3238,25 @@ TEST_P(WriteInteTest, TestWriteWithNestedSchema) { /*commit_identifier=*/1)); // check read result - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options[Options::FILE_FORMAT] = file_format; - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3220,12 +3313,15 @@ TEST_P(WriteInteTest, TestWriteWithIOException) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - WriteContextBuilder context_builder(table_path, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + context_builder.WithStreamingMode(true).Finish()); - Result> write = - FileStoreWrite::Create(std::move(write_context)); + Result> write = FileStoreWrite::Create( + context_builder_load_options, std::move(write_context), context_builder_options); CHECK_HOOK_STATUS(write.status(), i); auto& file_store_write = write.value(); @@ -3273,11 +3369,15 @@ TEST_P(WriteInteTest, TestCommitWithIOException) { /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); // write data - WriteContextBuilder context_builder(table_path, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); ASSERT_OK(file_store_write->Write(std::move(batch_0))); @@ -3293,14 +3393,16 @@ TEST_P(WriteInteTest, TestCommitWithIOException) { std::string tmp_table_path = tmp_dir->Str(); ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - CommitContextBuilder commit_context_builder(tmp_table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(false) - .Finish()); - auto commit = FileStoreCommit::Create(std::move(commit_context)); + TableLoadOptions commit_context_builder_load_options{tmp_table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + auto commit = + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options); CHECK_HOOK_STATUS(commit.status(), i); Result committed = commit.value()->FilterAndCommit({{0, commit_messages}}); CHECK_HOOK_STATUS(committed.status(), i); @@ -3348,14 +3450,17 @@ TEST_P(WriteInteTest, TestWriteMemoryUse) { /*primary_keys=*/{}, options)); std::shared_ptr write_pool = GetMemoryPool(); { - WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options) - .WithMemoryPool(write_pool) - .WithStreamingMode(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_context, + context_builder.WithMemoryPool(write_pool).WithStreamingMode(true).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); ASSERT_OK(file_store_write->Write(std::move(batch_0))); @@ -3532,11 +3637,15 @@ TEST_F(WriteInteTest, TestBranchWrite) { std::map options = {{Options::FILE_FORMAT, "orc"}, {Options::FILE_SYSTEM, "local"}}; - WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithBranch("rt").Finish()); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + context_builder_options["branch"] = "rt"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); // write to rt branch, pk table std::string data_0 = @@ -3577,11 +3686,15 @@ TEST_F(WriteInteTest, TestBranchWrite) { ASSERT_TRUE(expected_commit_message_0->TEST_Equal(*result_commit_msgs)); // write to main branch, append table - WriteContextBuilder context_builder1(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context1, - context_builder1.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write1, - FileStoreWrite::Create(std::move(write_context1))); + TableLoadOptions context_builder1_load_options{table_path}; + std::map context_builder1_options; + WriteContextBuilder context_builder1("commit_user_1"); + context_builder1_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context1, context_builder1.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_write1, + FileStoreWrite::Create(context_builder1_load_options, std::move(write_context1), + context_builder1_options)); std::string data_1 = R"([["20240725", "watermelon", 3], ["20240725", "orange", 2] @@ -3715,11 +3828,14 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { auto write_array = [&](const std::vector& write_cols, const std::shared_ptr& write_array) -> Result>> { - WriteContextBuilder builder(table_path, commit_user); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder(commit_user); builder.WithWriteSchema(write_cols); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(builder_load_options, + std::move(write_context), builder_options)); ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); auto record_batch = std::make_unique( @@ -3749,10 +3865,13 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { auto commit = [&](const std::vector>& commit_msgs, int64_t latest_snapshot_id, int64_t next_row_id) { - CommitContextBuilder builder(table_path, commit_user); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + CommitContextBuilder builder(commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(builder_load_options, + std::move(commit_context), builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); auto commit_impl = dynamic_cast(file_store_commit.get()); @@ -3765,10 +3884,13 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { // first_row_id, write_cols, min_sequence_number, max_sequence_number using MetaType = std::tuple>, int64_t, int64_t>; auto check_committed_meta = [&](const std::vector& expected_meta) { - ScanContextBuilder builder(table_path); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(builder_load_options, std::move(scan_context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); auto data_split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -4144,13 +4266,15 @@ TEST_P(WriteInteTest, TestPkSpillableMapSharedShreddingReadWrite) { /*primary_keys=*/{"id"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); + write_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_builder.SetOptions(options) - .WithStreamingMode(true) - .WithTempDirectory(tmp_dir) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir).Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); ASSERT_OK_AND_ASSIGN( auto batch_0, TestHelper::MakeRecordBatch( @@ -4210,10 +4334,14 @@ TEST_P(WriteInteTest, TestPkSpillableDiskQuotaExhaustedFallsBackToFlush) { /*primary_keys=*/{"pt", "f0"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); auto write_array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ ["Alice", 10, 1, 1.0], @@ -4266,10 +4394,14 @@ TEST_P(WriteInteTest, TestPkSpillableGlobalMemoryPreemptionDataCorrectness) { /*primary_keys=*/{"pt", "f0"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); // Write long strings to different partitions to trigger global memory preemption. std::string long_str_a(48, 'a'); @@ -4364,10 +4496,14 @@ TEST_P(WriteInteTest, TestPkSpillableTempFilesCleanedAfterPrepareCommit) { /*primary_keys=*/{"pt", "f0"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); auto batch1 = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([["Alice", 10, 1]])").ValueOrDie(); @@ -4428,10 +4564,14 @@ TEST_P(WriteInteTest, TestPkSpillableIntermediateMergeWithTempFileTracking) { std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); auto write_array_fn = [](FileStoreWrite* writer, const std::map& partition, int32_t bucket, @@ -4511,10 +4651,14 @@ TEST_P(WriteInteTest, TestPkSpillableVector) { /*primary_keys=*/{"pt", "f0"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); auto write_array = [](FileStoreWrite* writer, const std::shared_ptr& array) { ArrowArray c_array; @@ -4594,12 +4738,16 @@ TEST_P(WriteInteTest, TestPkSpillableMultiBucketMultiRoundDataCorrectness) { return writer->Write(std::move(batch)); }; - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true) .WithTempDirectory(tmp_dir) .SetWriteBufferSpillThreadNumber(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); // Round 1: Bucket 0 writes Alice(dup) + Bob, Bucket 1 writes Dave(dup) + Eve auto r1_b0_batch1 = @@ -4787,14 +4935,17 @@ TEST_P(WriteInteTest, TestPkSpillableWithIOException) { // Write with spill enabled (WithTempDirectory) std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - WriteContextBuilder context_builder(root_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options) - .WithStreamingMode(true) - .WithTempDirectory(tmp_dir) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + TableLoadOptions context_builder_load_options{root_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_context, + context_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir).Finish()); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); // Start IOHook only for spill write/read operations. io_hook->Reset(i, IOHook::Mode::RETURN_ERROR);