diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 0592a7b74..684a990e7 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -158,12 +158,27 @@ offsets when a real-time writer is created: PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, paimon::FileStoreWrite::Create(std::move(write_context))); -The ``Table`` handle is an equivalent entry point: ``table->NewCommit()``, -``table->NewWrite()``, ``table->NewRead()`` and ``table->NewScan()`` each take the -same finished context and return the same operator as ``FileStoreCommit::Create()``, -``FileStoreWrite::Create()``, ``TableRead::Create()`` and ``TableScan::Create()``. -Pick whichever reads better; the context still carries the catalog when one is -needed. See the :ref:`Table API `. +``Catalog::GetTable()`` returns a common ``Table`` interface for managed, format and +system tables. The managed implementation selects append-only or primary-key behavior +from the loaded schema; system tables reject writes and commits. + +``table->NewCommit()``, ``NewWrite()``, ``NewRead()`` and ``NewScan()`` use the table's +bound type, path, schema and file system. ``GetSchema()`` returns that schema without +metadata I/O; reload through the catalog to refresh it. Contexts still provide execution +settings, dynamic options and optional catalog services. Branch or type options that +contradict a loaded managed table are rejected. The legacy static ``Create()`` methods +load the table from their context before delegating to these methods. +See the :ref:`Table API `. + +An explicit ``WithBranch()`` or branch-qualified identifier participates in the same +branch check. Omitting a branch allows the loaded table to supply it. The static +scan and read factories use an explicit context branch first, then the branch in +the schema options, and finally main. + +Global system tables under ``sys`` retain their catalog for deferred row loading. +Before calling ``GetTable()`` for one, place the catalog in a ``std::shared_ptr``, +as in the example above. ``LoadTableSchema()`` can inspect the schema without +retaining the catalog. For real-time writes, also set ``WithStreamingMode(true)`` and ``WithRealtimeContext(...)`` with ``realtime.enabled=true``. After diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 492b5f0d0..5bda38fcf 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -40,8 +40,10 @@ format table is an insert. Reading and writing ------------------- -A format table is not served through :cpp:func:`Catalog::GetTable`, which describes a managed -table; use :cpp:func:`Catalog::GetFormatTable` instead. +Use :cpp:func:`Catalog::GetTable` to load a format table through the common ``Table`` +interface. Its ``NewScan``, ``NewRead``, ``NewWrite`` and ``NewCommit`` methods use the +loaded schema, location and file system. :cpp:func:`Catalog::GetFormatTable` also returns +the concrete ``FormatTable`` interface for callers using the legacy context constructors. Reading and writing go through the entry points every other table uses: ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create`` and ``FileStoreCommit::Create``, each from its diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 69eb6187c..cd83bd219 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -48,7 +48,7 @@ class Tag; class Identifier; /// This interface is responsible for reading and writing metadata such as database/table from a /// paimon catalog. -class PAIMON_EXPORT Catalog { +class PAIMON_EXPORT Catalog : public std::enable_shared_from_this { public: static const char SYSTEM_DATABASE_NAME[]; static const char SYSTEM_TABLE_SPLITTER[]; @@ -146,7 +146,7 @@ class PAIMON_EXPORT Catalog { virtual Status RenameTable(const Identifier& from_table, const Identifier& to_table, bool ignore_if_not_exists) = 0; - /// Gets a table. + /// Gets a managed, format or read-only system table. /// /// @param identifier Identifier of the table to get. /// @return A result containing the table, or an error status. @@ -249,11 +249,8 @@ class PAIMON_EXPORT Catalog { /// Gets a format table: a directory of data files laid out like a standard Hive table. /// - /// A format table carries no snapshots and no manifests, so it is loaded through its own - /// method rather than `GetTable()`. Reading and writing it go through the same `TableScan`, - /// `TableRead`, `FileStoreWrite` and `FileStoreCommit` entry points every other table uses: - /// hand the table back to `ScanContextBuilder`, `ReadContextBuilder`, `WriteContextBuilder` or - /// `CommitContextBuilder`, each of which takes one. + /// Returns the concrete format-table interface. `GetTable()` also loads format tables + /// through the common Table interface. Legacy context builders accept this typed handle. /// /// @param identifier Identifier of the table to get. /// @return A result containing the format table, or an error status if the table does not diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index e542c9aee..aed1e34cc 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -33,50 +33,37 @@ namespace paimon { -class TableScan; +class CommitContext; +class FileStoreCommit; +class FileStoreWrite; +class ReadContext; +class ScanContext; class TableRead; +class TableScan; +class WriteContext; -/// A table provides basic abstraction for table type. +/// Common interface for managed, format and system tables. +/// Create operators with NewScan/NewRead/NewWrite/NewCommit; unsupported operations return errors. +/// Operators use this table's type, path, schema and file system, with context execution settings. class PAIMON_EXPORT Table { public: - /// @param file_system File system the table's data is accessed through. - /// @param root_path Root path of the table in the file system. - /// @param uuid Metastore UUID, or empty if unavailable. - Table(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = ""); - - ~Table() = default; + virtual ~Table() = default; - /// Returns the file system the table was opened with, which serves the table's - /// files. A catalog that hands out per-table file systems opens a table with the - /// one for that table; it is null when the opener had none. + /// Returns the table file system, or null if none was supplied. std::shared_ptr GetFileSystem() const { return file_system_; } - /// Returns the root path of the table in its file system. It is empty when the - /// opener had no location for the table, which a mock in a test is one case of. + /// Returns the table root path, or an empty string if unavailable. const std::string& GetRootPath() const { return root_path_; } - /// A name to identify this table. - std::string Name() const { - return identifier_.GetObjectName(); - } - - /// Full name of the table, default is database.tableName. - std::string FullName() const { - return identifier_.GetFullName(); - } + /// Returns the object name for data tables, or the system table name. + virtual std::string Name() const = 0; - /// Returns the branch the table was opened for, which its identifier carries; `main` - /// when it names none or a spelling of it, which its managers are bound to. A - /// malformed object name that hides the branch is reported. - Result CurrentBranch() const { - return identifier_.GetBranchNameOrDefault(); - } + /// Returns database.tableName for data tables, or Name() for system tables. + virtual std::string FullName() const = 0; /// Returns the metastore UUID, falling back to `FullName()`. /// @note Use `CatalogUuid()` for commit requests to avoid sending a table name as an ID. @@ -90,42 +77,42 @@ class PAIMON_EXPORT Table { return uuid_.empty() ? std::nullopt : std::optional(uuid_); } - /// Loads the latest schema of table. - std::shared_ptr LatestSchema() const { + /// Returns the schema bound to this table instance without metadata I/O. + /// Reload through the catalog to obtain the latest schema; this accessor does not refresh it. + std::shared_ptr GetSchema() const { return schema_; } - /// Creates a scan of this table from a finished scan context, the table-first entry point - /// that `TableScan::Create()` also builds through. The scan is derived from the context. - /// @param context The finished scan context. - /// @return The scan, or the error building it reports. - Result> NewScan(std::unique_ptr context) const; - - /// Creates a read of this table from a finished read context, the table-first entry point - /// that `TableRead::Create()` also builds through. The read is derived from the context. - /// @param context The finished read context. - /// @return The read, or the error building it reports. - Result> NewRead(std::unique_ptr context) const; - - /// Creates a write of this table from a finished write context, the table-first entry point - /// that `FileStoreWrite::Create()` also builds through. The write is derived from the context. - /// @param context The finished write context. - /// @return The write, or the error building it reports. - Result> NewWrite(std::unique_ptr context) const; - - /// Creates a commit of this table from a finished commit context, the table-first entry point - /// that `FileStoreCommit::Create()` also builds through. The commit is derived from the - /// context. - /// @param context The finished commit context. - /// @return The commit, or the error building it reports. - Result> NewCommit( - std::unique_ptr context) const; + /// Creates a scan using this table's schema and options. + virtual Result> NewScan( + std::unique_ptr context) const = 0; + + /// Creates a reader using this table's schema and options. + virtual Result> NewRead( + std::unique_ptr context) const = 0; + + /// Creates a writer using this table's schema and options. + virtual Result> NewWrite( + std::unique_ptr context) const = 0; + + /// Creates a commit using this table's schema and options. + virtual Result> NewCommit( + std::unique_ptr context) const = 0; + + protected: + /// @param file_system File system the table's data is accessed through. + /// @param root_path Root path of the table in the file system. + /// @param uuid Metastore UUID, or empty if unavailable. + Table(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid); - private: std::shared_ptr file_system_; std::string root_path_; std::shared_ptr schema_; Identifier identifier_; + + private: std::string uuid_; }; diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index a4c310254..917fcb46c 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -24,6 +24,7 @@ #include #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/result.h" #include "paimon/schema/schema.h" #include "paimon/status.h" @@ -53,7 +54,7 @@ class FileSystem; /// every context builder has, which is how a table whose schema lives in a metastore is reached. /// /// See `docs/source/user_guide/format_table.rst` for what is not supported yet. -class PAIMON_EXPORT FormatTable { +class PAIMON_EXPORT FormatTable : public Table { public: /// Formats a format table's files can be in. enum class Format { @@ -115,11 +116,18 @@ class PAIMON_EXPORT FormatTable { const std::shared_ptr& table, const std::map& dynamic_options); - ~FormatTable(); + ~FormatTable() override; + + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; + Result> NewWrite( + std::unique_ptr context) const override; + Result> NewCommit( + std::unique_ptr context) const override; /// Directory the data files live in. const std::string& Location() const { - return location_; + return root_path_; } /// Format of every data file in the directory. @@ -156,16 +164,16 @@ class PAIMON_EXPORT FormatTable { } /// A name to identify this table. - std::string Name() const { + std::string Name() const override { return identifier_.GetObjectName(); } /// Full name of the table, database.tableName. - std::string FullName() const; + std::string FullName() const override; /// Schema of the table, including its partition fields. - std::shared_ptr LatestSchema() const { - return schema_; + std::shared_ptr GetSchema() const { + return std::static_pointer_cast(schema_); } /// Schema of the table as an arrow schema, including its partition fields. @@ -186,16 +194,14 @@ class PAIMON_EXPORT FormatTable { } private: + Result> ForContext( + const std::map& options) const; FormatTable(const std::shared_ptr& file_system, const std::string& location, const Identifier& identifier, const std::shared_ptr& schema, const std::map& options, Format format, const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata); - std::shared_ptr file_system_; - std::string location_; - Identifier identifier_; - std::shared_ptr schema_; std::map options_; Format format_; std::string file_compression_; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 4a5077c31..f39b80698 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -450,6 +450,10 @@ set(PAIMON_CORE_SRCS core/stats/simple_stats_converter.cpp core/stats/simple_stats.cpp core/stats/simple_stats_evolution.cpp + core/table/append_only_file_store_table.cpp + core/table/primary_key_file_store_table.cpp + core/table/file_store_table.cpp + core/table/table_factory.cpp core/table/file_store.cpp core/table/table.cpp core/table/bucket_mode.cpp diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp index 1f2a3800a..587648943 100644 --- a/src/paimon/core/catalog/catalog_utils.cpp +++ b/src/paimon/core/catalog/catalog_utils.cpp @@ -26,6 +26,7 @@ #include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/options/table_type.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/defs.h" @@ -149,4 +150,15 @@ Result> CatalogUtils::LoadFormatTableInTwoRequests( location_carries_paimon_metadata, /*dynamic_options=*/{}); } +Identifier CatalogUtils::IdentifierFromPath(const std::string& table_path, + const std::string& branch) { + const std::string name = PathUtil::GetName(table_path); + const std::string parent = PathUtil::GetName(PathUtil::GetParentDirPath(table_path)); + const std::string suffix = Catalog::DB_SUFFIX; + if (StringUtils::EndsWith(parent, suffix)) { + return Identifier(parent.substr(0, parent.size() - suffix.size()), name, branch); + } + return Identifier(Identifier::kUnknownDatabase, name, branch); +} + } // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h index 565a6aef5..d222300e2 100644 --- a/src/paimon/core/catalog/catalog_utils.h +++ b/src/paimon/core/catalog/catalog_utils.h @@ -38,6 +38,8 @@ class CatalogUtils { CatalogUtils() = delete; ~CatalogUtils() = delete; + /// Derives an identifier for a table loaded directly from a path. + static Identifier IdentifierFromPath(const std::string& path, const std::string& branch); /// Returns whether `db_name` is the reserved system database "sys". static bool IsSystemDatabase(const std::string& db_name); diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 87c89de8c..4aeac4d53 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -50,6 +50,10 @@ class Schema; } // namespace arrow struct ArrowSchema; +#include "paimon/core/options/table_type.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/table/format/format_table.h" + namespace paimon { FileSystemCatalog::FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, @@ -334,16 +338,56 @@ Result> FileSystemCatalog::LoadTableSchema( return checked_pointer_cast(*latest_schema); } +Result> FileSystemCatalog::LoadSystemTable( + const Identifier& identifier) const { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + PAIMON_ASSIGN_OR_RAISE( + Identifier data_identifier, + CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + TableSchemaExists(data_identifier)); + if (!latest_schema) { + return Status::NotExist(fmt::format("{} not exist", data_identifier.ToString())); + } + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); + return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, + latest_schema.value(), dynamic_options, + data_identifier.GetObjectName(), {}); +} + Result> FileSystemCatalog::GetTable(const Identifier& identifier) const { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported( + identifier.GetObjectName(), catalog_options_)); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::shared_ptr self = weak_from_this().lock(); + if (self == nullptr) { + return Status::Invalid( + "a global system table retains its catalog, so the catalog must be owned by " + "a std::shared_ptr before it is asked for one"); + } + GlobalSystemTableContext context; + context.catalog = self.get(); + context.catalog_owner = std::move(self); + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = catalog_options_; + return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); + } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier)); - // A system table reads the files of the data table it belongs to, so its root path - // is that table's. - PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); - Identifier data_identifier(identifier.GetDatabaseName(), data_table_name); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); - return std::make_shared(fs_, table_path, schema, identifier); + return LoadSystemTable(identifier); } // The schema is read on the table's own branch: `root/branch/branch-` below the // table path, which `main` resolves to itself. `GetBranchNameOrDefault` folds any @@ -353,14 +397,20 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide if (!latest_schema) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - // Only a managed table is a `Table`; describing another type as one would hand back - // snapshots and manifests it never had, so the failure belongs to the call that opens the - // table rather than at the first read. + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); + std::shared_ptr data_schema = checked_pointer_cast(*latest_schema); + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(data_schema->Options())); + if (is_format_table) { + return FormatTable::Create(fs_, table_path, identifier, data_schema, + /*location_carries_paimon_metadata=*/true, + /*dynamic_options=*/{}); + } PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); - return std::make_shared
(fs_, table_path, checked_pointer_cast(*latest_schema), - identifier); + PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); + return FileStoreTable::Create(fs_, table_path, *latest_schema, identifier, /*uuid=*/"", branch, + {}); } Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index ccc45f666..37bb23ea6 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -33,6 +33,7 @@ struct ArrowSchema; namespace paimon { +class SystemTable; class TableSchema; class FileSystem; class Identifier; @@ -79,6 +80,7 @@ class FileSystemCatalog : public Catalog { const Identifier& identifier) const override; private: + Result> LoadSystemTable(const Identifier& identifier) const; /// Fails when `db_name` cannot be used as a single path component, so that the returned /// path always stays under `warehouse`. static Result NewDatabasePath(const std::string& warehouse, diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index f02af7be3..2409942fc 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -349,7 +349,7 @@ TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { ASSERT_TRUE(exists); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(branch_identifier)); - data_schema = std::dynamic_pointer_cast(table->LatestSchema()); + data_schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, data_schema); ASSERT_EQ((std::vector{"b1_c0"}), data_schema->FieldNames()); ASSERT_EQ(3, data_schema->Id()); @@ -679,7 +679,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { ASSERT_TRUE(typed_schema.Equals(loaded_schema)); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "tbl1"))); - ASSERT_OK_AND_ASSIGN(auto arrow_schema_from_get_table, table->LatestSchema()->GetArrowSchema()); + ASSERT_OK_AND_ASSIGN(auto arrow_schema_from_get_table, table->GetSchema()->GetArrowSchema()); auto schema_from_get_table = arrow::ImportSchema(arrow_schema_from_get_table.get()).ValueOrDie(); ASSERT_TRUE(typed_schema.Equals(schema_from_get_table)); @@ -693,7 +693,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { ArrowSchemaRelease(&schema); } -TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { +TEST(FileSystemCatalogTest, TestGetTableLoadsFormatTable) { std::map options; options[Options::FILE_SYSTEM] = "local"; options[Options::FILE_FORMAT] = "orc"; @@ -714,9 +714,8 @@ TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - // A format table has no snapshots and no manifests, so handing it back as a Table would - // describe it as something it is not. - ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db1", "fmt")), "Cannot open format table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog.GetTable(Identifier("db1", "fmt"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(opened)); ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, catalog.GetFormatTable(Identifier("db1", "fmt"))); ASSERT_EQ(format_table->GetFormat(), FormatTable::Format::PARQUET); diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 09661843e..89f2cd490 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -44,7 +44,9 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/common/utils/url_utils.h" #include "paimon/core/options/map_storage_layout.h" +#include "paimon/core/realtime/realtime_utils.h" #include "paimon/core/schema/arrow_schema_validator.h" +#include "paimon/core/schema/schema_manager.h" #include "paimon/core/utils/nested_projection_utils.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/full_text_search.h" @@ -279,6 +281,12 @@ Result> InternalReadContext::Create( PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, context->GetSpecificFileSystem(), context->GetFileSystemSchemeToIdentifierMap())); + return CreateFromCoreOptions(context, table_schema, std::move(core_options)); +} + +Result> InternalReadContext::CreateFromCoreOptions( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + CoreOptions core_options) { core_options.WithCache(context->GetCache()); if (context->EnablePreciseFileIndexEvaluate()) { if (!core_options.FileIndexReadEnabled()) { @@ -482,7 +490,8 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea table_schema_(table_schema), read_schema_(read_schema), arrow_pool_(arrow_pool), - options_(options) {} + options_(options), + fs_scheme_map_(read_context->GetFileSystemSchemeToIdentifierMap()) {} Result> InternalReadContext::CreateWithSchema( const std::shared_ptr& original, @@ -493,9 +502,44 @@ Result> InternalReadContext::CreateWithSche } // Create a new InternalReadContext sharing all properties except read_schema. // The new read_schema is the minimal column set for COUNT(*). - return std::shared_ptr( + auto result = std::shared_ptr( new InternalReadContext(original->read_context_, original->table_schema_, new_read_schema, original->options_, original->arrow_pool_)); + result->path_ = original->path_; + result->fs_scheme_map_ = original->fs_scheme_map_; + return result; +} + +Result> InternalReadContext::CreateForTable( + const std::shared_ptr& context, const std::shared_ptr& schema, + CoreOptions options, const std::string& root_path, + const std::map& fs_scheme_map) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr result, + CreateFromCoreOptions(context, schema, std::move(options))); + result->path_ = root_path; + result->fs_scheme_map_ = fs_scheme_map; + return result; +} +Result> InternalReadContext::CreateForFallbackBranch( + const std::shared_ptr& context, const std::string& branch, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& fs_scheme_map) { + if (context->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); + } + SchemaManager manager(file_system, root_path, branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> schema, manager.Latest()); + if (!schema) { + return Status::Invalid("fallback schema not found"); + } + auto options = schema.value()->Options(); + for (const auto& [key, value] : context->GetOptions()) { + options[key] = value; + } + options[Options::BRANCH] = branch; + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system)); + return CreateForTable(context, schema.value(), std::move(core_options), root_path, + fs_scheme_map); } } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 0f1c0f12b..dd13e0f13 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -61,6 +61,15 @@ class InternalReadContext { const std::shared_ptr& table_schema, const std::map& options); + static Result> CreateForTable( + const std::shared_ptr& context, const std::shared_ptr& schema, + CoreOptions options, const std::string& root_path, + const std::map& fs_scheme_map); + static Result> CreateForFallbackBranch( + const std::shared_ptr& context, const std::string& branch, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& fs_scheme_map); + /// Maps score output positions in the read schema to their logical search sources. const std::map& GetIndexScoreBindings() const { return index_score_bindings_; @@ -79,7 +88,7 @@ class InternalReadContext { return table_schema_; } const std::string& GetPath() const { - return read_context_->GetPath(); + return path_ ? *path_ : read_context_->GetPath(); } std::shared_ptr CreateDependencyCatalogContext() const; @@ -141,7 +150,7 @@ class InternalReadContext { /// The scheme map the file system of this read was resolved with, which a catalog reading a /// dependency table on its behalf needs to resolve that table's location the same way. const std::map& GetFileSystemSchemeToIdentifierMap() const { - return read_context_->GetFileSystemSchemeToIdentifierMap(); + return fs_scheme_map_; } bool ReadAheadCacheEnabled() const { @@ -163,6 +172,9 @@ class InternalReadContext { const std::shared_ptr& new_read_schema); private: + static Result> CreateFromCoreOptions( + const std::shared_ptr& context, const std::shared_ptr& schema, + CoreOptions options); InternalReadContext(const std::shared_ptr& read_context, const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, @@ -185,6 +197,8 @@ class InternalReadContext { std::shared_ptr read_schema_; std::shared_ptr arrow_pool_; CoreOptions options_; + std::optional path_; + std::map fs_scheme_map_; std::map index_score_bindings_; }; diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index f2bf754cf..d51b2eef7 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -27,6 +27,7 @@ #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/full_text_search.h" @@ -420,6 +421,14 @@ Result> ReadContextBuilder::Finish() { PAIMON_ASSIGN_OR_RAISE( std::string branch, BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, impl_->branch_, "read")); + std::map options = impl_->options_; + std::optional named_branch; + if (impl_->identifier_) { + PAIMON_ASSIGN_OR_RAISE(named_branch, impl_->identifier_->GetBranchName()); + } + if (impl_->branch_ || named_branch || options.count(Options::BRANCH) != 0) { + options[Options::BRANCH] = branch; + } if (impl_->enable_prefetch_ && impl_->prefetch_max_parallel_num_ == 0) { return Status::Invalid("prefetch max parallel num should be greater than 0"); } @@ -452,7 +461,7 @@ Result> ReadContextBuilder::Finish() { impl_->prefetch_batch_count_, impl_->prefetch_max_parallel_num_, impl_->enable_multi_thread_row_to_batch_, impl_->row_to_batch_thread_number_, impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_, - impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, impl_->options_, + impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, options, impl_->read_ahead_cache_enabled_, impl_->cache_config_, impl_->cache_, impl_->format_table_, impl_->warmup_level_, impl_->catalog_, impl_->identifier_); if (impl_->read_schema_ && impl_->read_schema_->release) { diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index 3d5040a8a..4bf1ce9da 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -40,6 +40,29 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { +TEST(ReadContextTest, TestFailedFinishDoesNotRetainImplicitBranchOption) { + for (bool explicit_option : {false, true}) { + SCOPED_TRACE(explicit_option); + ReadContextBuilder builder("table_root_path"); + builder.WithBranch("dev").EnablePrefetch(true).SetPrefetchMaxParallelNum(0).AddOption( + "custom.option", "value"); + if (explicit_option) { + builder.AddOption(Options::BRANCH, "dev"); + } + ASSERT_NOK_WITH_MSG(builder.Finish(), "prefetch max parallel num should be greater than 0"); + builder.WithBranch("main").SetPrefetchMaxParallelNum(1); + auto retried = builder.Finish(); + if (explicit_option) { + ASSERT_NOK_WITH_MSG(retried, "but both 'dev' and 'main' were named"); + } else { + ASSERT_OK(retried); + ASSERT_EQ("main", retried.value()->GetBranch()); + ASSERT_EQ("main", retried.value()->GetOptions().at(Options::BRANCH)); + ASSERT_EQ("value", retried.value()->GetOptions().at("custom.option")); + } + } +} + TEST(ReadContextTest, TestWithCatalogResolvesTableFileSystem) { // WithCatalog is a shorthand for WithFileSystem(catalog->GetTableFileSystem(identifier)); it // resolves the per-table file system when Finish() builds the context, which also keeps the @@ -197,7 +220,8 @@ TEST(ReadContextTest, TestSetContent) { ASSERT_EQ(128U, ctx->GetCacheConfig().GetHoleSizeLimit()); ASSERT_EQ(2048U, ctx->GetCacheConfig().GetPreBufferLimit()); ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); - std::map expected_options = {{"key", "value"}}; + std::map expected_options = {{"key", "value"}, + {Options::BRANCH, "rt"}}; ASSERT_EQ(expected_options, ctx->GetOptions()); ASSERT_EQ(ctx->GetSpecificFileSystem(), fs); ASSERT_TRUE(ctx->GetCache()); diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 89cccaa69..339538bb7 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -18,10 +18,13 @@ #include "paimon/scan_context.h" +#include #include #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" @@ -240,6 +243,17 @@ Result> ScanContextBuilder::Finish() { PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); } + PAIMON_ASSIGN_OR_RAISE( + std::string branch, + BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, std::nullopt, "scan")); + std::map options = impl_->options_; + std::optional named_branch; + if (impl_->identifier_) { + PAIMON_ASSIGN_OR_RAISE(named_branch, impl_->identifier_->GetBranchName()); + } + if (named_branch || options.count(Options::BRANCH) != 0) { + options[Options::BRANCH] = branch; + } std::shared_ptr executor = impl_->executor_ ? impl_->executor_ : CreateDefaultExecutor(); auto ctx = std::make_unique( @@ -247,7 +261,7 @@ Result> ScanContextBuilder::Finish() { std::make_shared(impl_->predicates_, impl_->partition_filters_, impl_->bucket_filter_), impl_->global_index_result_, impl_->realtime_context_, impl_->memory_pool_, executor, - impl_->specific_file_system_, impl_->table_schema_, impl_->options_, impl_->cache_, + impl_->specific_file_system_, impl_->table_schema_, options, impl_->cache_, impl_->format_table_, impl_->catalog_, impl_->identifier_); impl_->Reset(); return ctx; diff --git a/src/paimon/core/operation/write_context.cpp b/src/paimon/core/operation/write_context.cpp index e31d64ac4..2fdf0dab2 100644 --- a/src/paimon/core/operation/write_context.cpp +++ b/src/paimon/core/operation/write_context.cpp @@ -18,6 +18,7 @@ #include "paimon/write_context.h" +#include #include #include "arrow/util/thread_pool.h" @@ -26,6 +27,7 @@ #include "paimon/common/utils/path_util.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" @@ -263,6 +265,14 @@ Result> WriteContextBuilder::Finish() { PAIMON_ASSIGN_OR_RAISE( std::string branch, BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, impl_->branch_, "write")); + std::map options = impl_->options_; + std::optional named_branch; + if (impl_->identifier_) { + PAIMON_ASSIGN_OR_RAISE(named_branch, impl_->identifier_->GetBranchName()); + } + if (impl_->branch_ || named_branch || options.count(Options::BRANCH) != 0) { + options[Options::BRANCH] = branch; + } if (impl_->catalog_ != nullptr && !BranchManager::IsMainBranch(branch)) { // Refused here rather than at the commit of what is written to the branch. PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); @@ -286,7 +296,7 @@ Result> WriteContextBuilder::Finish() { impl_->ignore_num_bucket_check_, impl_->ignore_previous_files_, enable_multi_thread_spill, impl_->write_id_, branch, impl_->write_schema_, impl_->memory_pool_, impl_->executor_, impl_->temp_directory_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, - impl_->realtime_context_, impl_->options_, impl_->format_table_, impl_->catalog_, + impl_->realtime_context_, options, impl_->format_table_, impl_->catalog_, impl_->identifier_); impl_->Reset(); return ctx; diff --git a/src/paimon/core/operation/write_context_test.cpp b/src/paimon/core/operation/write_context_test.cpp index 5e777c351..337254482 100644 --- a/src/paimon/core/operation/write_context_test.cpp +++ b/src/paimon/core/operation/write_context_test.cpp @@ -33,6 +33,20 @@ namespace paimon::test { +TEST(WriteContextTest, TestFailedFinishDoesNotRetainImplicitBranchOption) { + auto catalog = std::make_shared(); + WriteContextBuilder builder("table_root_path", "user"); + builder.WithCatalog(catalog, Identifier("db", "tbl")) + .WithBranch("dev") + .AddOption("custom.option", "value"); + ASSERT_NOK_WITH_MSG(builder.Finish(), "addresses a branch by the table identifier"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + builder.WithBranch("main").Finish()); + ASSERT_EQ("main", context->GetBranch()); + ASSERT_EQ("main", context->GetOptions().at(Options::BRANCH)); + ASSERT_EQ("value", context->GetOptions().at("custom.option")); +} + TEST(WriteContextTest, TestDefaultValue) { WriteContextBuilder builder("table_root_path", "commit_user_1"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); @@ -187,7 +201,8 @@ TEST(WriteContextTest, TestSetContent) { ASSERT_EQ(ctx->GetWriteSchema(), write_schema); ASSERT_EQ(ctx->GetFileSystemSchemeToIdentifierMap(), fs_scheme_to_identifier_map); ASSERT_EQ(ctx->GetSpecificFileSystem(), file_system); - std::map expected_options = {{"key", "value"}}; + std::map expected_options = {{"key", "value"}, + {Options::BRANCH, "test_branch"}}; ASSERT_EQ(expected_options, ctx->GetOptions()); } diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index 1d69c033a..1c0cfd333 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -194,6 +194,36 @@ TableSchema::TableSchema(int32_t version, int64_t id, const std::vector> TableSchema::MergeOptions( + const std::map& dynamic_options) const { + std::map options = options_; + for (const auto& [key, value] : dynamic_options) { + options[key] = value; + } + return Copy(options); +} + +Result> TableSchema::Copy( + const std::map& options) const { + auto merged = std::unique_ptr( + new TableSchema(version_, id_, fields_, highest_field_id_, partition_keys_, primary_keys_, + options, comment_, time_millis_)); + PAIMON_RETURN_NOT_OK(merged->DeriveBucketInfo()); + return merged; +} + +Status TableSchema::DeriveBucketInfo() { + PAIMON_ASSIGN_OR_RAISE(std::vector keys, TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, OriginalBucketKeys()); + if (bucket_keys.empty()) { + bucket_keys = keys; + } + bucket_keys_ = bucket_keys; + PAIMON_ASSIGN_OR_RAISE(num_bucket_, + OptionsUtils::GetValueFromMap(Options(), Options::BUCKET, -1)); + return Status::OK(); +} + bool TableSchema::operator==(const TableSchema& other) const { return version_ == other.version_ && fields_ == other.fields_ && partition_keys_ == other.partition_keys_ && primary_keys_ == other.primary_keys_ && @@ -332,18 +362,7 @@ Result> TableSchema::InitSchema( auto table_schema = std::unique_ptr( new TableSchema(TableSchema::CURRENT_VERSION, schema_id, fields, highest_field_id, partition_keys, primary_keys, options, comment, time_millis)); - PAIMON_ASSIGN_OR_RAISE(std::vector keys, table_schema->TrimmedPrimaryKeys()); - - // Try to validate bucket keys - PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, - table_schema->OriginalBucketKeys()); - if (bucket_keys.empty()) { - bucket_keys = keys; - } - table_schema->bucket_keys_ = bucket_keys; - PAIMON_ASSIGN_OR_RAISE( - table_schema->num_bucket_, - OptionsUtils::GetValueFromMap(table_schema->Options(), Options::BUCKET, -1)); + PAIMON_RETURN_NOT_OK(table_schema->DeriveBucketInfo()); return table_schema; } diff --git a/src/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h index eab109cf1..9b6717446 100644 --- a/src/paimon/core/schema/table_schema.h +++ b/src/paimon/core/schema/table_schema.h @@ -52,6 +52,11 @@ class TableSchema : public DataSchema, public Jsonizable { const std::vector& primary_keys, const std::map& options); + Result> Copy( + const std::map& options) const; + Result> MergeOptions( + const std::map& options) const; + static Result> CreateFromJson(const std::string& json_str); /// Computes the highest non-system field id across all nesting levels of a "fields" JSON @@ -149,6 +154,8 @@ class TableSchema : public DataSchema, public Jsonizable { const std::shared_ptr& field, int32_t field_id); private: + Status DeriveBucketInfo(); + // version of schema for paimon int32_t version_ = -1; int64_t id_ = -1; diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp new file mode 100644 index 000000000..2a2110ef7 --- /dev/null +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/append_only_file_store_table.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/operation/append_only_file_store_scan.h" +#include "paimon/core/operation/append_only_file_store_write.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/data_evolution_file_store_scan.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/source/append_only_split_generator.h" +#include "paimon/core/table/source/data_evolution_split_generator.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { + +AppendOnlyFileStoreTable::AppendOnlyFileStoreTable(std::unique_ptr store, + const std::shared_ptr& schema, + const Identifier& identifier, + const std::string& uuid) + : FileStoreTable(std::move(store), schema, identifier, uuid) {} + +Result> AppendOnlyFileStoreTable::CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const { + const std::shared_ptr& file_store_path_factory = path_factory; + auto schema = std::static_pointer_cast(schema_); + auto schema_manager = GetSchemaManager(); + const std::string& branch = CurrentBranch(); + FileStore file_store(root_path_, options); + + // append table + const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); + bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); + if (options.GetBucket() == -1) { + need_dv_maintainer_factory = false; + ignore_previous_files = true; + } else if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in append table", options.GetBucket())); + } + if (ctx.GetRealtimeContext()) { + if (!ctx.GetWriteSchema().empty()) { + return Status::Invalid("real-time append write does not support a custom schema"); + } + if (!realtime_deduplicate) { + PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx.GetRealtimeContext(), + snapshot_manager, options)); + } + } else if (realtime_deduplicate) { + if (!ctx.GetWriteSchema().empty() || ignore_previous_files || options.WriteOnly()) { + return Status::Invalid( + "deduplicate compaction requires the full schema, restored files and " + "write-only=false"); + } + } + std::shared_ptr write_schema = arrow_schema; + const auto& write_field_names = ctx.GetWriteSchema(); + if (!write_field_names.empty()) { + arrow::FieldVector write_fields; + write_fields.reserve(write_field_names.size()); + for (const auto& field_name : write_field_names) { + auto field = arrow_schema->GetFieldByName(field_name); + if (!field) { + // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER + return Status::Invalid( + fmt::format("write field {} does not exist in table schema", field_name)); + } + write_fields.push_back(field); + } + write_schema = arrow::schema(write_fields); + } + + std::shared_ptr realtime_schema_layout; + if (ctx.GetRealtimeContext()) { + const RealtimeStoreMode mode = + realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE : RealtimeStoreMode::APPEND_ONLY; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, + RealtimeSchemaLayout::Create(mode, write_schema)); + realtime_schema_layout = std::move(layout); + } + + std::shared_ptr dv_maintainer_factory; + if (need_dv_maintainer_factory) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr index_file_handler, + file_store.NewIndexFileHandler(ResolveBucketMode(options.GetBucket(), schema), + file_store_path_factory, ctx.GetMemoryPool())); + dv_maintainer_factory = std::make_shared(index_file_handler); + } + + auto file_store_write = std::make_unique( + file_store_path_factory, snapshot_manager, schema_manager, ctx.GetCommitUser(), root_path_, + schema, arrow_schema, write_schema, realtime_schema_layout, partition_schema, + dv_maintainer_factory, io_manager, options, ignore_previous_files, ctx.IsStreamingMode(), + ctx.IgnoreNumBucketCheck(), ctx.GetRealtimeContext(), ctx.GetExecutor(), + ctx.GetMemoryPool()); + if (ctx.GetRealtimeContext() && realtime_deduplicate) { + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_RETURN_NOT_OK(file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); + } + } + return std::unique_ptr(std::move(file_store_write)); +} + +Result> AppendOnlyFileStoreTable::NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const { + // TODO(liancheng.lsz): support fallback branch in scan + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + std::unique_ptr scan; + if (core_options.DataEvolutionEnabled()) { + PAIMON_ASSIGN_OR_RAISE( + scan, DataEvolutionFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), + manifest_list, manifest_file, table_schema, + arrow_schema, context->GetScanFilters(), + core_options, executor, memory_pool)); + } else { + PAIMON_ASSIGN_OR_RAISE( + scan, AppendOnlyFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), + manifest_list, manifest_file, table_schema, + arrow_schema, context->GetScanFilters(), + core_options, executor, memory_pool)); + } + return WithTablePath(std::move(scan), root_path_); +} + +Result> AppendOnlyFileStoreTable::NewSplitGenerator( + const CoreOptions& core_options) const { + auto source_split_target_size = core_options.GetSourceSplitTargetSize(); + auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); + if (core_options.DataEvolutionEnabled()) { + return std::make_unique(source_split_target_size, + source_split_open_file_cost, + core_options.BlobSplitByFileSize()); + } + BucketMode bucket_mode = + (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE : BucketMode::HASH_FIXED); + return std::make_unique(source_split_target_size, + source_split_open_file_cost, bucket_mode); +} + +CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) const { + const std::shared_ptr& schema_manager = GetSchemaManager(); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h new file mode 100644 index 000000000..a83632dfd --- /dev/null +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class FileStoreScan; +class IOManager; +class SplitGenerator; +class TableSchema; + +/// Managed table without primary keys; uses the shared FileStoreTable operator flows. +class AppendOnlyFileStoreTable : public FileStoreTable { + public: + AppendOnlyFileStoreTable(std::unique_ptr store, + const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid); + + private: + Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const override; + + Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const override; + + Result> NewSplitGenerator( + const CoreOptions& core_options) const override; + + CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index 8f1c6d7a8..68bf717de 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -34,6 +34,7 @@ #include "paimon/core/manifest/index_manifest_file.h" #include "paimon/core/manifest/manifest_file.h" #include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/schema/schema_manager.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/index_file_path_factories.h" @@ -46,6 +47,17 @@ namespace paimon { FileStore::FileStore(std::string root_path, CoreOptions options) : root_path_(std::move(root_path)), options_(std::move(options)) {} +std::string FileStore::CurrentBranch() const { + return BranchManager::NormalizeBranch(options_.GetBranch()); +} + +std::shared_ptr FileStore::GetSchemaManager() const { + return std::make_shared(options_.GetFileSystem(), root_path_, CurrentBranch()); +} +std::shared_ptr FileStore::GetSnapshotManager() const { + return std::make_shared(options_.GetFileSystem(), root_path_, CurrentBranch()); +} + Result> FileStore::NewPathFactory( const std::shared_ptr& arrow_schema, const std::vector& partition_keys, const std::string& file_format_identifier, diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 521a7bc39..0c536132d 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -43,12 +43,26 @@ class IndexManifestFile; class ManifestFile; class ManifestList; class SnapshotManager; +class SchemaManager; -/// Component factories for one managed-table operation using already resolved options. +/// Managed-table storage configuration and component factories using resolved options. class FileStore { public: FileStore(std::string root_path, CoreOptions options); + const CoreOptions& GetOptions() const { + return options_; + } + const std::string& GetRootPath() const { + return root_path_; + } + std::string CurrentBranch() const; + std::shared_ptr GetFileSystem() const { + return options_.GetFileSystem(); + } + std::shared_ptr GetSchemaManager() const; + std::shared_ptr GetSnapshotManager() const; + Result> NewPathFactory( const std::shared_ptr& arrow_schema, const std::vector& partition_keys, const std::string& file_format_identifier, diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp new file mode 100644 index 000000000..d4255fd71 --- /dev/null +++ b/src/paimon/core/table/file_store_table.cpp @@ -0,0 +1,679 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/file_store_table.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" +#include "paimon/common/predicate/predicate_validator.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/binary_row_partition_computer.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/sensitive_config_utils.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_snapshot_commit.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/commit_lock.h" +#include "paimon/core/catalog/renaming_snapshot_commit.h" +#include "paimon/core/catalog/snapshot_commit.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/disk/io_manager.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/operation/commit/realtime_commit_properties.h" +#include "paimon/core/operation/expire_snapshots.h" +#include "paimon/core/operation/file_store_commit_impl.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/internal_read_context.h" +#include "paimon/core/options/table_type.h" +#include "paimon/core/realtime/realtime_context_impl.h" +#include "paimon/core/realtime/realtime_utils.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/append_only_file_store_table.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/primary_key_file_store_table.h" +#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/source/data_evolution_batch_scan.h" +#include "paimon/core/table/source/data_table_batch_scan.h" +#include "paimon/core/table/source/data_table_stream_scan.h" +#include "paimon/core/table/source/fallback_table_read.h" +#include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/table/source/primary_key_index_batch_scan.h" +#include "paimon/core/table/source/read_optimized_scan_options.h" +#include "paimon/core/table/source/realtime_table_scan.h" +#include "paimon/core/table/source/snapshot/snapshot_reader.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/primary_key_table_utils.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/format/file_format.h" +#include "paimon/fs/file_system.h" +#include "paimon/read_context.h" +#include "paimon/realtime/realtime_context.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +namespace { + +/// Allows object-store commits in integration tests. +constexpr const char kObjectStoreCommitInteTestOption[] = "enable-object-store-commit-in-inte-test"; + +Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, + const ScanContext& context, bool read_optimized) { + if (!context.GetRealtimeContext()) { + return Status::OK(); + } + if (!core_options.RealtimeEnabled()) { + return Status::Invalid("real-time scan requires realtime.enabled=true"); + } + if (core_options.GetBucket() <= 0) { + return Status::Invalid("real-time union read requires fixed bucket mode"); + } + if (core_options.DataEvolutionEnabled()) { + return Status::Invalid("real-time union read does not support data evolution"); + } + if (!table_schema.PrimaryKeys().empty()) { + if (read_optimized) { + return Status::NotImplemented( + "PK real-time union read does not support read-optimized scans"); + } + PAIMON_RETURN_NOT_OK( + PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); + } + if (context.IsStreamingMode()) { + return Status::Invalid("real-time union read currently supports batch scans only"); + } + if (context.GetLimit()) { + return Status::Invalid("real-time union read does not support scan limit pushdown"); + } + if (context.GetGlobalIndexResult()) { + return Status::Invalid("real-time union read does not support global index splits"); + } + StartupMode startup_mode = core_options.GetStartupMode(); + const bool specified_snapshot = + startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); + if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || + specified_snapshot)) { + return Status::Invalid( + "real-time union read requires the latest snapshot or an explicit snapshot id"); + } + return Status::OK(); +} + +bool IsCatalogCommit(const CommitContext& ctx) { + return AsVersionManaged(ctx.GetCatalog()) != nullptr; +} + +Result> NewSnapshotCommit( + const CommitContext& ctx, const std::optional& catalog_identifier, + const CoreOptions& options, const std::shared_ptr& snapshot_manager) { + if (IsCatalogCommit(ctx)) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr lock, + AsVersionManaged(ctx.GetCatalog()) + ->CreateCommitLock(catalog_identifier.value(), ctx.GetTableId(), + ctx.GetCommitUser(), options)); + // Reuse the caller's UUID so table recreation cannot redirect this commit. + return std::shared_ptr(std::make_shared( + ctx.GetCatalog(), catalog_identifier.value(), ctx.GetTableId(), std::move(lock))); + } + if (ctx.UseRESTCatalogCommit()) { + if (options.RestCommitAcquireLock()) { + return Status::Invalid(fmt::format( + "'{}' needs a commit through a REST catalog with WithCatalog(); a commit table " + "request built for the caller to send cannot hold a commit lease", + Options::REST_COMMIT_ACQUIRE_LOCK)); + } + return std::shared_ptr( + std::make_shared(ctx.GetTableId())); + } + // Catalogs without version management may still supply a table ID. + if (ctx.GetTableId() && ctx.GetCatalog() == nullptr) { + return Status::Invalid( + "a commit which publishes the snapshot by writing it to the table directory sends no " + "commit table request, so there is nowhere to put a table id; commit through a " + "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); + } + return std::shared_ptr( + std::make_shared(options.GetFileSystem(), snapshot_manager)); +} + +} // namespace + +class Executor; +class MemoryPool; + +FileStoreTable::FileStoreTable(std::unique_ptr store, + const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid) + : Table(store->GetFileSystem(), store->GetRootPath(), schema, identifier, uuid), + store_(std::move(store)) {} + +Result> FileStoreTable::Create( + const std::shared_ptr& specified_file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid, const std::string& branch, + const std::map& fs_scheme_map) { + if (schema == nullptr) { + return Status::Invalid( + fmt::format("not found latest schema of {}", identifier.GetFullName())); + } + PAIMON_RETURN_NOT_OK( + CatalogUtils::CheckManagedTableType(identifier, schema, "FileStoreTable::Create")); + PAIMON_RETURN_NOT_OK(BranchManager::CheckValidBranch(branch)); + std::map options = schema->Options(); + const std::string normalized_branch = BranchManager::NormalizeBranch(branch); + if (BranchManager::IsMainBranch(normalized_branch)) { + options.erase(Options::BRANCH); + } else { + options[Options::BRANCH] = normalized_branch; + } + std::shared_ptr effective_schema = schema; + if (options != schema->Options()) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr normalized_schema, + schema->Copy(options)); + effective_schema = std::shared_ptr(std::move(normalized_schema)); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(options, specified_file_system, fs_scheme_map)); + std::unique_ptr store = + std::make_unique(root_path, std::move(core_options)); + return Create(std::move(store), effective_schema, identifier, uuid, fs_scheme_map); +} + +Result> FileStoreTable::Create( + std::unique_ptr store, const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid, + const std::map& fs_scheme_map) { + if (!store || !schema) { + return Status::Invalid("managed table requires a store and schema"); + } + PAIMON_RETURN_NOT_OK( + CatalogUtils::CheckManagedTableType(identifier, schema, "FileStoreTable::Create")); + PAIMON_RETURN_NOT_OK(BranchManager::CheckValidBranch(store->CurrentBranch())); + std::shared_ptr table; + if (!schema->PrimaryKeys().empty()) { + table = + std::make_shared(std::move(store), schema, identifier, uuid); + } else { + table = + std::make_shared(std::move(store), schema, identifier, uuid); + } + table->fs_scheme_map_ = fs_scheme_map; + return table; +} + +Result FileStoreTable::ResolveOptions( + const std::map& overrides) const { + if (overrides.empty()) { + return store_->GetOptions(); + } + auto options = store_->GetOptions().ToMap(); + for (const auto& [key, value] : overrides) { + options[key] = value; + } + auto branch = overrides.find(Options::BRANCH); + if (branch != overrides.end() && + BranchManager::NormalizeBranch(branch->second) != CurrentBranch()) { + return Status::Invalid("context branch contradicts the loaded table"); + } + PAIMON_ASSIGN_OR_RAISE(TableType type, TableTypeDefine::FromOptions(options)); + PAIMON_ASSIGN_OR_RAISE(TableType bound_type, + TableTypeDefine::FromOptions(store_->GetOptions().ToMap())); + if (type != bound_type) { + return Status::Invalid("context type contradicts the loaded table"); + } + return CoreOptions::FromMap(options, file_system_, fs_scheme_map_); +} + +Status FileStoreTable::RestoreRealtimeCommittedProgress( + const std::shared_ptr& realtime_context, + const std::shared_ptr& snapshot_manager, const CoreOptions& options) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, + RealtimeContextImpl::Cast(realtime_context)); + PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_ASSIGN_OR_RAISE( + RealtimeOffsetMap realtime_committed_offsets, + RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); + PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( + latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); + } + return Status::OK(); +} + +Status FileStoreTable::CheckContextNamesThisTable( + const std::string& kind, const std::string& context_root_path, + const std::optional& context_identifier, + const std::optional& context_table_id) const { + if (!root_path_.empty()) { + PAIMON_ASSIGN_OR_RAISE(std::string context_path, + PathUtil::NormalizePath(context_root_path)); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, PathUtil::NormalizePath(root_path_)); + if (context_path != table_path) { + return Status::Invalid(fmt::format( + "the {} context names root path '{}' but this table serves '{}': the store below " + "reads and writes this table's files, so a context naming another path would " + "leave this table's metadata there and publish it as the other's", + kind, context_root_path, root_path_)); + } + } + if (context_identifier) { + PAIMON_ASSIGN_OR_RAISE(std::optional context_branch, + context_identifier.value().GetBranchName()); + if (context_branch && context_branch.value() != CurrentBranch()) { + return Status::Invalid(fmt::format( + "the {} context names branch '{}' of '{}' but this table serves branch '{}', " + "which is the one a {} of this table writes and publishes", + kind, context_branch.value(), context_identifier.value().ToString(), + CurrentBranch(), kind)); + } + PAIMON_ASSIGN_OR_RAISE( + Identifier context_table, + CatalogUtils::BranchIdentifier(context_identifier.value(), CurrentBranch())); + PAIMON_ASSIGN_OR_RAISE(Identifier served_table, + CatalogUtils::BranchIdentifier(identifier_, CurrentBranch())); + const bool names_this_table = + (served_table.GetDatabaseName() == Identifier::kUnknownDatabase || + context_table.GetDatabaseName() == served_table.GetDatabaseName()) && + context_table.GetObjectName() == served_table.GetObjectName(); + if (!names_this_table) { + return Status::Invalid(fmt::format( + "the {} context names table '{}' but this table is '{}': the store below serves " + "this table's files, so a context naming another table would write this table's " + "metadata and publish it as the other's", + kind, context_identifier.value().ToString(), identifier_.ToString())); + } + } + if (context_table_id && context_table_id != CatalogUuid()) { + return Status::Invalid(fmt::format( + "the {} context carries table id '{}' but this table's is '{}': the id tells a " + "recreated table from the one the changes were prepared for, so one this table does " + "not carry would publish this table's metadata as the recreation's", + kind, context_table_id.value(), CatalogUuid().value_or("none"))); + } + return Status::OK(); +} + +std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr&& scan, + const std::string& root_path) { + scan->WithTablePath(root_path); + return std::move(scan); +} + +Result> FileStoreTable::NewWrite( + std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("write", ctx->GetRootPath(), + ctx->GetIdentifier(), + /*context_table_id=*/std::nullopt)); + if (AsVersionManaged(ctx->GetCatalog()) != nullptr) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && + ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + std::optional catalog_identifier; + if (ctx->GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE(Identifier identifier, CatalogUtils::BranchIdentifier( + *ctx->GetIdentifier(), CurrentBranch())); + catalog_identifier.emplace(std::move(identifier)); + } + if (ctx->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(CurrentBranch())); + } + std::shared_ptr schema = std::static_pointer_cast(schema_); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(ctx->GetOptions())); + FileStore file_store(root_path_, options); + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); + if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) { + return Status::NotImplemented( + "Dedicated VECTOR storage does not yet support custom write schemas or partial writes"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr file_store_path_factory, + file_store.NewPathFactory(arrow_schema, schema->PartitionKeys(), + options.GetWriteFileFormat(/*level=*/0)->Identifier(), + ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_manager, + file_store.NewSnapshotManager(ctx->GetCatalog(), catalog_identifier)); + std::shared_ptr io_manager; + const auto& io_temp_dir = ctx->GetTempDirectory(); + if (!io_temp_dir.empty()) { + io_manager = std::make_shared(io_temp_dir, options.GetFileSystem()); + } + + bool ignore_previous_files = ctx->IgnorePreviousFiles(); + if (ctx->GetRealtimeContext() && !options.RealtimeEnabled()) { + return Status::Invalid("real-time write requires realtime.enabled=true"); + } + if (ctx->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, + RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); + PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); + } + return CreateWrite(*ctx, arrow_schema, partition_schema, options, file_store_path_factory, + snapshot_manager, io_manager, ignore_previous_files); +} + +Result> FileStoreTable::NewScan( + std::unique_ptr context) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + if (context->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(CurrentBranch())); + } + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, ResolveOptions(context->GetOptions())); + PAIMON_ASSIGN_OR_RAISE( + bool read_optimized, + OptionsUtils::GetValueFromMap(core_options.ToMap(), kReadOptimizedScanOption, false)); + core_options.WithCache(context->GetCache()); + FileStore file_store(root_path_, core_options); + + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); + + PAIMON_RETURN_NOT_OK( + ValidateRealtimeScan(*table_schema, core_options, *context, read_optimized)); + if (core_options.GetBucket() == -1) { + if (!table_schema->PrimaryKeys().empty()) { + return Status::NotImplemented(fmt::format( + "do not support pk table bucket={} in scan process", core_options.GetBucket())); + } + } else if (core_options.GetBucket() < 1 && + !SchemaValidation::IsPostponeBucketTable(*table_schema, core_options.GetBucket())) { + return Status::Invalid( + fmt::format("do not support bucket={} in scan process", core_options.GetBucket())); + } + + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { + PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( + *arrow_schema, context->GetScanFilters()->GetPredicate(), + /*validate_field_idx=*/false)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, + file_store.NewPathFactory(arrow_schema, table_schema->PartitionKeys(), + core_options.GetFileFormat()->Identifier(), + context->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, + file_store.NewManifestList(path_factory, context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_file, + file_store.NewManifestFile(path_factory, partition_schema, context->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr file_store_scan, + NewFileStoreScan(manifest_list, manifest_file, arrow_schema, core_options, + context->GetExecutor(), context->GetMemoryPool(), context.get())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr split_generator, + NewSplitGenerator(core_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_file_handler, + file_store.NewIndexFileHandler(ResolveBucketMode(core_options.GetBucket(), table_schema), + path_factory, context->GetMemoryPool())); + auto snapshot_reader = std::make_shared( + file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler)); + const bool pk_table = !table_schema->PrimaryKeys().empty(); + if (read_optimized && pk_table && context->IsStreamingMode()) { + return Status::NotImplemented( + "read-optimized system table does not support streaming scan for primary key table"); + } + if (context->IsStreamingMode()) { + return std::make_unique(core_options, snapshot_reader); + } + auto batch_scan = std::make_unique( + /*pk_table=*/pk_table, core_options, snapshot_reader, read_optimized, context->GetLimit(), + /*realtime_pk_scan=*/pk_table && context->GetRealtimeContext() != nullptr); + if (context->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, + RealtimeContextImpl::Cast(context->GetRealtimeContext())); + return std::make_unique( + std::move(batch_scan), pk_table, realtime_context, path_factory, + snapshot_reader->GetSnapshotManager(), core_options.GetFileSystem(), + context->GetScanFilters(), core_options.GetRealtimeReadViewTtlMillis()); + } + if (core_options.DataEvolutionEnabled()) { + return std::make_unique( + root_path_, snapshot_reader, std::move(batch_scan), table_schema, + context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), + context->GetExecutor()); + } + if (pk_table && !read_optimized && core_options.GlobalIndexEnabled()) { + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*table_schema)); + if (!definitions.ScalarDefinitions().empty()) { + return PrimaryKeyIndexBatchScan::Create(snapshot_reader, std::move(batch_scan), + table_schema, path_factory, core_options, + context->GetMemoryPool()); + } + } + return batch_scan; +} + +Result> FileStoreTable::CreateTableRead( + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const { + const auto& core_options = internal_context->GetCoreOptions(); + FileStore file_store(root_path_, core_options); + const auto& table_schema = internal_context->GetTableSchema(); + if (internal_context->GetRealtimeContext() && !core_options.RealtimeEnabled()) { + return Status::Invalid("real-time read requires realtime.enabled=true"); + } + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + file_store.NewPathFactory(arrow_schema, table_schema->PartitionKeys(), + core_options.GetFileFormat()->Identifier(), memory_pool)); + if (internal_context->GetPrimaryKeys().empty()) { + return std::make_unique(path_factory, internal_context, memory_pool, + executor); + } + return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); +} + +Result> FileStoreTable::NewRead( + std::unique_ptr context) const { + std::shared_ptr read_context = std::move(context); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); + if (read_context->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(CurrentBranch())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(read_context->GetOptions())); + auto fs_scheme_map = fs_scheme_map_; + for (const auto& [scheme, identifier] : read_context->GetFileSystemSchemeToIdentifierMap()) { + fs_scheme_map[scheme] = identifier; + } + std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr internal_context, + InternalReadContext::CreateForTable(read_context, loaded_schema, std::move(options), + root_path_, fs_scheme_map)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + CreateTableRead(internal_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); + + std::optional scan_fallback_branch = + internal_context->GetCoreOptions().GetScanFallbackBranch(); + if (!scan_fallback_branch || + StringUtils::IsNullOrWhitespaceOnly(scan_fallback_branch.value())) { + return std::move(table_read); + } + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fallback_context, + InternalReadContext::CreateForFallbackBranch( + read_context, /*branch=*/scan_fallback_branch.value(), file_system_, + root_path_, fs_scheme_map)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, + CreateTableRead(fallback_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); + const std::shared_ptr& arrow_pool = internal_context->GetArrowMemoryPool(); + return std::make_unique(std::move(table_read), + std::move(fallback_table_read), arrow_pool); +} + +Result> FileStoreTable::NewCommit( + std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("commit", ctx->GetRootPath(), + ctx->GetIdentifier(), ctx->GetTableId())); + if (ctx->GetCatalog() != nullptr || ctx->UseRESTCatalogCommit()) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + std::optional catalog_identifier; + if (ctx->GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE(Identifier identifier, CatalogUtils::BranchIdentifier( + *ctx->GetIdentifier(), CurrentBranch())); + catalog_identifier.emplace(std::move(identifier)); + } + FileStoreCommitImpl::SchemaIdLoader schema_id_loader; + if (ctx->GetCatalog() != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + std::shared_ptr catalog = ctx->GetCatalog(); + Identifier identifier = catalog_identifier.value(); + schema_id_loader = [catalog, identifier]() -> Result { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + catalog->LoadTableSchema(identifier)); + std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); + if (data_schema == nullptr) { + return Status::Invalid( + "the catalog did not hand back a data table schema, which has the id a " + "snapshot records"); + } + return data_schema->Id(); + }; + } + std::shared_ptr schema = std::static_pointer_cast(schema_); + std::shared_ptr arrow_schema = + DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(ctx->GetOptions())); + FileStore file_store(root_path_, options); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); + assert(options.GetFileSystem()); + assert(options.GetFileFormat()); + PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); + PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); + if (is_object_store && !ctx->UseRESTCatalogCommit() && + AsVersionManaged(ctx->GetCatalog()) == nullptr && + options.ToMap().count(kObjectStoreCommitInteTestOption) == 0) { + return Status::NotImplemented( + "commit operation does not support object store file system for now"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr partition_computer, + file_store.NewPartitionComputer(arrow_schema, schema->PartitionKeys(), + ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + file_store.NewPathFactory(arrow_schema, schema->PartitionKeys(), + options.GetFileFormat()->Identifier(), ctx->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_manager, + file_store.NewSnapshotManager(ctx->GetCatalog(), catalog_identifier)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, + file_store.NewManifestList(path_factory, ctx->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_file, + file_store.NewManifestFile(path_factory, partition_schema, ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr index_manifest_file, + file_store.NewIndexManifestFile(ResolveBucketMode(options.GetBucket(), schema), + path_factory, ctx->GetMemoryPool())); + + auto expire_snapshots = std::make_shared( + snapshot_manager, path_factory, manifest_list, manifest_file, index_manifest_file, + options.GetFileSystem(), options.GetExpireConfig(), options.RealtimeEnabled(), + ctx->GetExecutor()); + + CommitScanner::ScanSupplier scan_supplier = + NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, arrow_schema, options, + ctx->GetExecutor(), ctx->GetMemoryPool()); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_commit, + NewSnapshotCommit(*ctx, catalog_identifier, options, snapshot_manager)); + + return std::make_unique( + ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path_, ctx->GetCommitUser(), + options, path_factory, std::move(partition_computer), snapshot_manager, snapshot_commit, + ctx->IgnoreEmptyCommit(), ctx->AppendCommitCheckConflict(), schema, + std::move(schema_id_loader), manifest_file, manifest_list, index_manifest_file, + expire_snapshots, GetSchemaManager(), std::move(scan_supplier)); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h new file mode 100644 index 000000000..f1b72bd3d --- /dev/null +++ b/src/paimon/core/table/file_store_table.h @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class InternalReadContext; +class IOManager; +class ManifestFile; +class ManifestList; +class RealtimeContext; +class SplitGenerator; + +/// Managed table backed by snapshots and manifests. +/// Shares operator setup across append-only and primary-key tables, selected from the schema. +class FileStoreTable : public Table { + public: + /// Creates an append-only or primary-key table according to the schema's primary keys. + /// @param specified_file_system Explicit file system, overriding option-based resolution. + /// @param schema Loaded table schema; must not be null. + /// @param branch Branch used by all operators and metadata managers. + /// @param fs_scheme_map URI scheme routing for option-based file system resolution. + static Result> Create( + const std::shared_ptr& specified_file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid, const std::string& branch, + const std::map& fs_scheme_map = {}); + + /// Binds a resolved store without replacing the loaded schema's options. + static Result> Create( + std::unique_ptr store, const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid, + const std::map& fs_scheme_map = {}); + + std::string CurrentBranch() const { + return store_->CurrentBranch(); + } + + std::shared_ptr GetSchemaManager() const { + return store_->GetSchemaManager(); + } + + std::shared_ptr GetSnapshotManager() const { + return store_->GetSnapshotManager(); + } + + std::string Name() const override { + return identifier_.GetObjectName(); + } + + std::string FullName() const override { + return identifier_.GetFullName(); + } + + Result> NewScan(std::unique_ptr context) const override; + + Result> NewRead(std::unique_ptr context) const override; + + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; + + private: + Result ResolveOptions(const std::map& options) const; + /// Rejects paths, identifiers or UUIDs that contradict this table's identity. + /// Missing context fields are allowed. + /// @param kind Operation name used in errors. + /// @param context_table_id Catalog commit UUID; nullopt for writes. + Status CheckContextNamesThisTable(const std::string& kind, const std::string& context_root_path, + const std::optional& context_identifier, + const std::optional& context_table_id) const; + + /// Creates a reader for the internal context's schema and primary keys. + Result> CreateTableRead( + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const; + + protected: + FileStoreTable(std::unique_ptr store, const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid); + + /// Restores real-time write progress from the latest snapshot. + static Status RestoreRealtimeCommittedProgress( + const std::shared_ptr& realtime_context, + const std::shared_ptr& snapshot_manager, const CoreOptions& options); + + static std::unique_ptr WithTablePath(std::unique_ptr&& scan, + const std::string& root_path); + + /// Creates the table-specific writer. + /// @param io_manager Optional spill manager. + /// @param ignore_previous_files Whether to start over instead of appending. + virtual Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; + + virtual Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const = 0; + + virtual Result> NewSplitGenerator( + const CoreOptions& core_options) const = 0; + + virtual CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const = 0; + + std::unique_ptr store_; + + /// URI routing supplied when this table was loaded. + std::map fs_scheme_map_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index a039baa4b..dbd42952a 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -21,6 +21,7 @@ #include #include "fmt/format.h" +#include "paimon/commit_context.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/string_utils.h" @@ -29,8 +30,21 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/format/format_table_file_store_commit.h" +#include "paimon/core/table/format/format_table_file_store_write.h" +#include "paimon/core/table/format/format_table_read.h" +#include "paimon/core/table/format/format_table_scan.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" namespace paimon { @@ -45,10 +59,7 @@ FormatTable::FormatTable(const std::shared_ptr& file_system, const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata) - : file_system_(file_system), - location_(location), - identifier_(identifier), - schema_(schema), + : Table(file_system, location, schema, identifier, ""), options_(options), format_(format), file_compression_(file_compression), @@ -192,12 +203,12 @@ Result> FormatTable::Copy( for (const auto& [key, value] : dynamic_options) { options[key] = value; } - return Create(table->file_system_, table->location_, table->identifier_, table->schema_, + return Create(table->file_system_, table->root_path_, table->identifier_, table->GetSchema(), table->location_carries_paimon_metadata_, options); } const std::vector& FormatTable::PartitionKeys() const { - return schema_->PartitionKeys(); + return GetSchema()->PartitionKeys(); } std::string FormatTable::FullName() const { @@ -205,7 +216,152 @@ std::string FormatTable::FullName() const { } Result> FormatTable::GetArrowSchema() const { - return schema_->GetArrowSchema(); + return GetSchema()->GetArrowSchema(); +} + +namespace { +Result> NewFormatTableWrite( + const std::shared_ptr& table, const WriteContext& ctx) { + if (ctx.GetCatalog() != nullptr) { + return Status::Invalid( + "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " + "format table"); + } + if (ctx.IsStreamingMode()) { + return Status::NotImplemented( + "a format table has no snapshots, so there is nothing a streaming write could " + "commit against"); + } + if (ctx.GetRealtimeContext() != nullptr) { + return Status::NotImplemented("a format table has no real-time store to write into"); + } + if (!ctx.GetWriteSchema().empty()) { + return Status::NotImplemented( + "a format table write takes the table's own columns; a write schema naming a " + "subset of them is not supported yet"); + } + // A write id prefixes a postpone-bucket writer's files so that one compaction reader can + // put them back in order. A format table has no buckets, so it would identify nothing. + if (ctx.GetWriteId().has_value()) { + return Status::NotImplemented( + "a format table has no buckets, so a write id would name nothing"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, + FormatTableFileStoreWrite::Create(table, ctx.GetMemoryPool())); + return std::unique_ptr(std::move(format_write)); +} + +Result> NewFormatTableScan(const std::shared_ptr& table, + const std::shared_ptr& context) { + // Anything a `ScanContext` carries that a format table cannot honour is refused rather than + // silently dropped. + if (context->IsStreamingMode()) { + return Status::NotImplemented( + "a format table has no snapshots, so there is nothing for a streaming scan to follow"); + } + if (context->GetRealtimeContext() != nullptr) { + return Status::NotImplemented( + "a format table has no real-time store to union with what is on disk"); + } + if (context->GetGlobalIndexResult() != nullptr) { + return Status::NotImplemented("a format table carries no global index"); + } + std::map partition_filter; + std::shared_ptr filters = context->GetScanFilters(); + if (filters != nullptr) { + if (filters->GetPredicate() != nullptr) { + return Status::NotImplemented( + "a format table scan does not take a predicate: its files carry no statistics to " + "skip by, and pruning by the partition columns a predicate names is not " + "implemented yet; give the partition values as a partition filter instead"); + } + if (filters->GetBucketFilter()) { + return Status::NotImplemented("a format table has no buckets to filter by"); + } + const std::vector>& partition_filters = + filters->GetPartitionFilters(); + if (partition_filters.size() > 1) { + return Status::NotImplemented("a format table scan takes at most one partition filter"); + } + if (partition_filters.size() == 1) { + partition_filter = partition_filters.front(); + } + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + FormatTableScan::Create(table, partition_filter, context->GetLimit())); + return std::unique_ptr(std::move(scan)); +} + +Result> NewFormatTableRead(const std::shared_ptr& table, + const std::shared_ptr& context) { + if (context->EnablePreciseFileIndexEvaluate()) { + return Status::NotImplemented( + "Precise file-index evaluation is not supported for format tables"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + FormatTableRead::Create(table, context)); + return std::unique_ptr(std::move(read)); +} + +Result> NewFormatTableCommit( + const std::shared_ptr& table, const CommitContext& ctx) { + if (!ctx.IgnoreEmptyCommit()) { + return Status::NotImplemented( + "a format table cannot record an empty commit: keeping one means writing a " + "snapshot that adds no files, and there are no snapshots here"); + } + if (ctx.UseRESTCatalogCommit()) { + return Status::NotImplemented( + "a format table commits by renaming files into place, not by sending a snapshot " + "to a rest catalog"); + } + if (ctx.GetTableId()) { + return Status::NotImplemented( + "a format table sends no commit table request, so there is nowhere to put a table id"); + } + if (ctx.GetCatalog() != nullptr) { + return Status::NotImplemented( + "a format table keeps no snapshot for a catalog to take, so it commits by renaming " + "files into place rather than through the catalog it was loaded from"); + } + if (ctx.AppendCommitCheckConflict()) { + return Status::NotImplemented( + "a format table has no manifests to check a concurrent commit against"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, + FormatTableFileStoreCommit::Create(table)); + return std::unique_ptr(std::move(format_commit)); +} +} // namespace +Result> FormatTable::ForContext( + const std::map& options) const { + return Copy(std::make_shared(*this), options); +} +Result> FormatTable::NewScan(std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "scan")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + std::shared_ptr context = std::move(ctx); + return NewFormatTableScan(table, context); +} +Result> FormatTable::NewRead(std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + std::shared_ptr context = std::move(ctx); + return NewFormatTableRead(table, context); +} +Result> FormatTable::NewWrite( + std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + std::shared_ptr context = std::move(ctx); + return NewFormatTableWrite(table, *context); +} +Result> FormatTable::NewCommit( + std::unique_ptr ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + std::shared_ptr context = std::move(ctx); + return NewFormatTableCommit(table, *context); } } // namespace paimon diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index 40e4c3ebd..f1e1c7aab 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -56,8 +56,10 @@ #include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/predicate/full_text_search.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/record_batch.h" #include "paimon/scan_context.h" @@ -644,6 +646,40 @@ TEST(FormatTableTest, TestCreateReadsOptions) { ASSERT_EQ(table->FullName(), "db.tbl"); } +TEST(FormatTableTest, TestReadRejectsSearchForDirectAndStaticEntrypoints) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + CreateTable(dir->GetFileSystem(), dir->Str(), {})); + const std::shared_ptr
loaded_table = table; + for (bool vector_search : {false, true}) { + for (int32_t entrypoint : {0, 1, 2}) { + SCOPED_TRACE(vector_search); + SCOPED_TRACE(entrypoint); + ReadContextBuilder builder = + entrypoint == 2 ? ReadContextBuilder(table) : ReadContextBuilder(dir->Str()); + builder.EnablePreciseFileIndexEvaluate(true); + if (vector_search) { + builder.SetVectorSearch(std::make_shared( + "embedding", 3, std::vector{1.0f, 2.0f}, nullptr, nullptr, + VectorSearch::DistanceType::COSINE, std::map{})); + } else { + builder.SetFullTextSearch( + std::make_shared("name", R"({"match":{"query":"paimon"}})", 3)); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + auto read = entrypoint == 0 ? loaded_table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); + ASSERT_NOK_WITH_MSG(read, + "Precise file-index evaluation is not supported for format tables"); + ASSERT_TRUE(read.status().IsNotImplemented()); + } + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ReadContextBuilder(dir->Str()).Finish()); + ASSERT_OK(loaded_table->NewRead(std::move(context))); +} + TEST(FormatTableTest, TestFileCompressionComesFromCoreOptions) { // The resolution order itself is `CoreOptionsTest.TestFormatTableFileCompression`'s business; // what matters here is that the table asks for it rather than resolving compression again. @@ -752,11 +788,10 @@ TEST(FormatTableTest, TestCatalogManagedPartitionsAreRejected) { // And the same refusal when it arrives at the call rather than in the schema. Validating the // schema's own options alone would let this one through and then drop it, leaving a caller // who asked for catalog-managed partitions with a scan that read the directories anyway. - Result> dynamic_table = - FormatTable::Create(off_dir->GetFileSystem(), off_dir->Str(), Identifier("db", "tbl"), - off_table->LatestSchema(), - /*location_carries_paimon_metadata=*/true, - {{Options::METASTORE_PARTITIONED_TABLE, "true"}}); + Result> dynamic_table = FormatTable::Create( + off_dir->GetFileSystem(), off_dir->Str(), Identifier("db", "tbl"), off_table->GetSchema(), + /*location_carries_paimon_metadata=*/true, + {{Options::METASTORE_PARTITIONED_TABLE, "true"}}); ASSERT_FALSE(dynamic_table.ok()); ASSERT_TRUE(dynamic_table.status().IsNotImplemented()) << dynamic_table.status().ToString(); } @@ -2719,7 +2754,7 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { "a format table sends no commit table request"); auto format_catalog = std::make_shared(); - format_catalog->SetTableSchema(table->LatestSchema()); + format_catalog->SetTableSchema(table->GetSchema()); CommitContextBuilder catalog_builder(dir->Str(), "test-user"); catalog_builder.WithCatalog(format_catalog, Identifier("db", "fmt")); ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_context, catalog_builder.Finish()); diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp new file mode 100644 index 000000000..936fc29c2 --- /dev/null +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/primary_key_file_store_table.h" + +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/global_index/btree/btree_defs.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/mergetree/compact/lookup_merge_function.h" +#include "paimon/core/mergetree/compact/merge_function.h" +#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_write.h" +#include "paimon/core/options/merge_engine.h" +#include "paimon/core/postpone/postpone_bucket_file_store_write.h" +#include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/source/merge_tree_split_generator.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/utils/primary_key_table_utils.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { +namespace { +Result HasHistoricalPrimaryKeyBTreeDefinition( + const std::shared_ptr& schema_manager, int64_t current_schema_id) { + if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); + for (int64_t schema_id : schema_ids) { + if (schema_id >= current_schema_id) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, + schema_manager->ReadSchema(schema_id)); + const auto& historical_options = historical_schema->Options(); + auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); + if (option != historical_options.end() && + !StringUtils::IsNullOrWhitespaceOnly(option->second)) { + return true; + } + } + return false; +} +} // namespace + +struct KeyValue; +template +class MergeFunctionWrapper; + +PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable(std::unique_ptr store, + const std::shared_ptr& schema, + const Identifier& identifier, + const std::string& uuid) + : FileStoreTable(std::move(store), schema, identifier, uuid) {} + +Result> PrimaryKeyFileStoreTable::CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const { + const std::shared_ptr& file_store_path_factory = path_factory; + auto schema = std::static_pointer_cast(schema_); + auto schema_manager = GetSchemaManager(); + const std::string& branch = CurrentBranch(); + FileStore file_store(root_path_, options); + + // pk table + if (ctx.GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); + if (ignore_previous_files) { + return Status::NotImplemented("PK realtime requires restore from the latest snapshot"); + } + if (!ctx.GetWriteSchema().empty()) { + return Status::NotImplemented("PK realtime does not support a custom write schema"); + } + PAIMON_RETURN_NOT_OK( + RestoreRealtimeCommittedProgress(ctx.GetRealtimeContext(), snapshot_manager, options)); + } + if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { + return PostponeBucketFileStoreWrite::Create( + snapshot_manager, schema_manager, ctx.GetCommitUser(), root_path_, schema, arrow_schema, + partition_schema, io_manager, options, ctx.IsStreamingMode(), + ctx.IgnoreNumBucketCheck(), ctx.GetWriteId(), fs_scheme_map_, ctx.GetExecutor(), + ctx.GetMemoryPool(), file_system_); + } + if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in key value table", options.GetBucket())); + } + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, + schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_primary_key_fields, + options.SequenceFieldSortOrderIsAscending())); + auto primary_keys = schema->PrimaryKeys(); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, + PrimaryKeyTableUtils::CreateMergeFunction(arrow_schema, primary_keys, + options, ctx.GetMemoryPool())); + if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { + // don't wrap first row, it is already OK + merge_function = std::make_unique(std::move(merge_function)); + } + std::shared_ptr> merge_function_wrapper = + std::make_shared(std::move(merge_function)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr sequence_fields_comparator, + PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); + + std::shared_ptr dv_maintainer_factory; + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + bool has_btree_index = false; + for (const PrimaryKeyIndexDefinition& definition : + primary_key_index_definitions.Definitions()) { + if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { + has_btree_index = true; + break; + } + } + std::optional latest_snapshot; + bool has_existing_index_manifest = false; + if (!has_btree_index) { + PAIMON_ASSIGN_OR_RAISE( + bool has_historical_btree_index, + HasHistoricalPrimaryKeyBTreeDefinition(schema_manager, schema->Id())); + if (has_historical_btree_index) { + PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); + has_existing_index_manifest = + latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); + } + } + std::shared_ptr index_file_handler; + if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE( + index_file_handler, + file_store.NewIndexFileHandler(ResolveBucketMode(options.GetBucket(), schema), + file_store_path_factory, ctx.GetMemoryPool())); + } + bool has_restored_btree_index = false; + if (!has_btree_index && has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE( + std::vector restored_btree_entries, + index_file_handler->Scan( + latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { + return entry.index_file->IndexType() == BtreeDefs::kIdentifier && + IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); + })); + has_restored_btree_index = !restored_btree_entries.empty(); + } + if (options.DeletionVectorsEnabled()) { + dv_maintainer_factory = std::make_shared(index_file_handler); + } + std::shared_ptr + primary_key_index_maintainer_factory; + if (has_btree_index || has_restored_btree_index) { + primary_key_index_maintainer_factory = + std::make_shared( + root_path_, branch, schema, primary_key_index_definitions.Definitions(), + file_store_path_factory, index_file_handler, options, io_manager, + ctx.EnableMultiThreadSpill(), ctx.GetExecutor(), ctx.GetMemoryPool()); + } + + std::shared_ptr realtime_schema_layout; + if (ctx.GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr layout, + RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); + realtime_schema_layout = std::move(layout); + } + + return std::make_unique( + file_store_path_factory, snapshot_manager, schema_manager, ctx.GetCommitUser(), root_path_, + schema, arrow_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, + primary_key_index_maintainer_factory, io_manager, key_comparator, + sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, + ctx.IsStreamingMode(), ctx.IgnoreNumBucketCheck(), ctx.EnableMultiThreadSpill(), + ctx.GetRealtimeContext(), ctx.GetExecutor(), ctx.GetMemoryPool()); +} + +Result> PrimaryKeyFileStoreTable::NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const { + // TODO(liancheng.lsz): support fallback branch in scan + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + std::unique_ptr scan; + PAIMON_ASSIGN_OR_RAISE( + scan, KeyValueFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), manifest_list, + manifest_file, table_schema, arrow_schema, + context->GetScanFilters(), core_options, executor, + memory_pool)); + return WithTablePath(std::move(scan), root_path_); +} + +Result> PrimaryKeyFileStoreTable::NewSplitGenerator( + const CoreOptions& core_options) const { + auto source_split_target_size = core_options.GetSourceSplitTargetSize(); + auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); + // TODO(liancheng.lsz): support evolution + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + table_schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, + table_schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); + return std::make_unique( + source_split_target_size, source_split_open_file_cost, + core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), key_comparator); +} + +CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) const { + const std::shared_ptr& schema_manager = GetSchemaManager(); + std::shared_ptr table_schema = std::static_pointer_cast(schema_); + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h new file mode 100644 index 000000000..c27335e91 --- /dev/null +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class FileStoreScan; +class IOManager; +class SplitGenerator; +class TableSchema; + +/// Managed table with primary keys; uses the shared FileStoreTable operator flows. +class PrimaryKeyFileStoreTable : public FileStoreTable { + public: + PrimaryKeyFileStoreTable(std::unique_ptr store, + const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid); + + private: + Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const override; + + Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const override; + + Result> NewSplitGenerator( + const CoreOptions& core_options) const override; + + CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 2e38731c6..3433cd670 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -39,10 +39,12 @@ #include "paimon/core/operation/abstract_split_read.h" #include "paimon/core/operation/split_read.h" #include "paimon/core/schema/schema_manager.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" #include "paimon/defs.h" +#include "paimon/fs/local/local_file_system.h" #include "paimon/predicate/full_text_match.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/literal.h" @@ -137,30 +139,51 @@ TEST(TableReadTest, TestFullTextMatchReadEntryAndScoreValidation) { auto scalar = PredicateBuilder::Equal(/*field_index=*/1, "f1", FieldType::INT, Literal(1)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({scalar, full_text})); const std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder builder(table_path); - builder.EnablePreciseFileIndexEvaluate(true) - .SetReadFieldNames({"f0", "f1"}) - .SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK(TableRead::Create(std::move(context))); + auto fs = std::make_shared(); + SchemaManager manager(fs, table_path); + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, manager.ReadSchema(0)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"), "", + "main")); + for (bool table_entry : {false, true}) { + ReadContextBuilder builder(table_path); + builder.EnablePreciseFileIndexEvaluate(true) + .SetReadFieldNames({"f0", "f1"}) + .SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); + ASSERT_OK(table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context))); - builder.EnablePreciseFileIndexEvaluate(true) - .SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}) - .SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(context)), - "Score-producing FullTextMatch cannot appear under OR"); + builder.EnablePreciseFileIndexEvaluate(true) + .SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}) + .SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(context, builder.Finish()); + ASSERT_NOK_WITH_MSG(table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)), + "Score-producing FullTextMatch cannot appear under OR"); + } } TEST(TableReadTest, TestPerFieldScoreReadEntry) { const std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; auto search = std::make_shared("f0", R"({"match":{"query":"apple"}})", /*limit=*/1); - ReadContextBuilder builder(path); - builder.EnablePreciseFileIndexEvaluate(true).SetFullTextSearch(search).SetReadFieldNames( - {"f1", "_f0_INDEX_SCORE"}); - ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK(TableRead::Create(std::move(context))); + auto fs = std::make_shared(); + SchemaManager manager(fs, path); + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, manager.ReadSchema(0)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"), "", + "main")); + for (bool table_entry : {false, true}) { + ReadContextBuilder builder(path); + builder.EnablePreciseFileIndexEvaluate(true).SetFullTextSearch(search).SetReadFieldNames( + {"f1", "_f0_INDEX_SCORE"}); + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); + ASSERT_OK(table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context))); + } } TEST(TableReadTest, TestMixedFilePredicateProducesScoresThroughPublicReadEntry) { diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 8de9742e1..0967aefc2 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -42,8 +42,10 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -357,16 +359,21 @@ std::unique_ptr CreateChangelogBatchReader( AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, + std::string data_table_name, std::map options) - : fs_(std::move(fs)), + : AuditLogSystemTable(std::move(fs), std::move(table_path), std::move(table_schema), + std::move(options), ComposeName(data_table_name, kName)) {} + +AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, + std::shared_ptr table_schema, + std::map options, + std::string name) + : SystemTable(fs, table_path, std::move(name)), + fs_(std::move(fs)), table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} -std::string AuditLogSystemTable::Name() const { - return kName; -} - Result> AuditLogSystemTable::ArrowSchema() const { std::shared_ptr row_kind_field = DataField::ConvertDataFieldToArrowField(SpecialFields::RowKind()); @@ -383,17 +390,19 @@ Result> AuditLogSystemTable::ArrowSchema() const } Result> AuditLogSystemTable::NewScan( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { return Status::NotImplemented("audit_log system table predicate pushdown is not supported"); } std::shared_ptr scan_filter = context->GetScanFilters(); ScanContextBuilder builder(table_path_); - builder.SetOptions(options_) - .WithStreamingMode(context->IsStreamingMode()) + builder.WithStreamingMode(context->IsStreamingMode()) + .SetOptions(context->GetOptions()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + .WithFileSystem(GetFileSystem()) .WithCache(context->GetCache()); if (scan_filter) { if (scan_filter->GetBucketFilter()) { @@ -407,11 +416,16 @@ Result> AuditLogSystemTable::NewScan( builder.SetLimit(context->GetLimit().value()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, builder.Finish()); - return TableScan::Create(std::move(data_context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, options_)); + return data_table->NewScan(std::move(data_context)); } Result> AuditLogSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return NewChangelogRead(context, std::make_shared()); } @@ -430,8 +444,11 @@ Result> AuditLogSystemTable::NewChangelogRead( PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, CreateDataReadContext(context, read_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, read_options, + context->GetFileSystemSchemeToIdentifierMap())); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_read, - TableRead::Create(std::move(data_context))); + data_table->NewRead(std::move(data_context))); auto* key_value_read = dynamic_cast(data_read.get()); if (!key_value_read) { return Status::Invalid("audit_log system table requires key-value table read"); @@ -449,14 +466,16 @@ Result> AuditLogSystemTable::CreateDataReadContext( const std::map& read_options) const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr base_read_schema, BaseReadSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); + auto options = context->GetOptions(); + if (core_options.TableReadSequenceNumberEnabled()) { + options[Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED] = "true"; + } ReadContextBuilder builder(table_path_); - builder.SetOptions(read_options) - .SetReadFieldNames(base_read_schema->field_names()) - .WithBranch(core_options.GetBranch()) + builder.SetReadFieldNames(base_read_schema->field_names()) + .SetOptions(options) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) - .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) + .WithFileSystem(GetFileSystem()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) .SetPrefetchMaxParallelNum(context->GetPrefetchMaxParallelNum()) diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index 780f5fa72..bc15c385f 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -54,18 +54,21 @@ class AuditLogSystemTable : public SystemTable { public: static constexpr const char* kName = "audit_log"; + /// @param data_table_name Logical table name, including any branch suffix. AuditLogSystemTable(std::shared_ptr fs, std::string table_path, - std::shared_ptr table_schema, + std::shared_ptr table_schema, std::string data_table_name, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; - Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; protected: + /// Constructor for derived changelog views with a different system-table name. + AuditLogSystemTable(std::shared_ptr fs, std::string table_path, + std::shared_ptr table_schema, + std::map options, std::string name); + Result> NewChangelogRead( const std::shared_ptr& context, std::shared_ptr converter) const; diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index 88541194d..7b73ce1dc 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -31,6 +31,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/table/source/table_read.h" @@ -72,13 +73,10 @@ std::shared_ptr CreateBinlogBatchConverter() { BinlogSystemTable::BinlogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, + std::string data_table_name, std::map options) : AuditLogSystemTable(std::move(fs), std::move(table_path), std::move(table_schema), - std::move(options)) {} - -std::string BinlogSystemTable::Name() const { - return kName; -} + std::move(options), ComposeName(data_table_name, kName)) {} Result> BinlogSystemTable::ArrowSchema() const { std::shared_ptr rowkind_field = @@ -98,7 +96,10 @@ Result> BinlogSystemTable::ArrowSchema() const { } Result> BinlogSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return NewChangelogRead(context, CreateBinlogBatchConverter()); } diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index 3b4f45bd0..fcdcb84b6 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -36,14 +36,13 @@ class BinlogSystemTable : public AuditLogSystemTable { public: static constexpr const char* kName = "binlog"; + /// @param data_table_name Logical table name, including any branch suffix. BinlogSystemTable(std::shared_ptr fs, std::string table_path, - std::shared_ptr table_schema, + std::shared_ptr table_schema, std::string data_table_name, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewRead(std::unique_ptr context) const override; }; } // namespace paimon diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 647da76c3..29b71768f 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -134,11 +134,15 @@ struct CatalogTableInfo { // Match Java CatalogUtils::listAllTables: tolerate databases or tables removed concurrently, but // propagate all other catalog and schema errors instead of returning incomplete system-table rows. -Result> LoadAllDataTables(const Catalog& catalog) { +Result> LoadAllDataTables(const Catalog* catalog, + const std::string& table_name) { + if (catalog == nullptr) { + return Status::NotImplemented("global system table requires catalog context: ", table_name); + } std::vector result; - PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog.ListDatabases()); + PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog->ListDatabases()); for (const std::string& database : databases) { - Result> tables_result = catalog.ListTables(database); + Result> tables_result = catalog->ListTables(database); if (!tables_result.ok()) { if (tables_result.status().IsNotExist()) { continue; @@ -147,7 +151,7 @@ Result> LoadAllDataTables(const Catalog& catalog) } for (const std::string& table : tables_result.value()) { Identifier identifier(database, table); - Result> schema_result = catalog.LoadTableSchema(identifier); + Result> schema_result = catalog->LoadTableSchema(identifier); if (!schema_result.ok()) { if (schema_result.status().IsNotExist()) { continue; @@ -300,7 +304,9 @@ Result> GlobalSystemTableLoader::Load( return Status::NotImplemented("global system table requires catalog context: ", table_name); } - return entry.factory(context); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, entry.factory(context)); + PAIMON_RETURN_NOT_OK(table->AdoptOwnSchema()); + return table; } } return Status::NotImplemented("unsupported global system table: ", table_name); @@ -324,11 +330,7 @@ Result> GlobalSystemTableLoader::GetSupportedTableNames // ============================================================================= CatalogOptionsSystemTable::CatalogOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/catalog_options"), context_(std::move(context)) {} - -std::string CatalogOptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/catalog_options", kName), context_(std::move(context)) {} Result> CatalogOptionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -360,11 +362,8 @@ Result> CatalogOptionsSystemTable::BuildRows() const { // ============================================================================= AllTableOptionsSystemTable::AllTableOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/all_table_options"), context_(std::move(context)) {} - -std::string AllTableOptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/all_table_options", kName), + context_(std::move(context)) {} Result> AllTableOptionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -380,7 +379,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { for (const auto& [key, value] : table.schema->Options()) { GenericRow row(schema->num_fields()); @@ -399,11 +398,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { // ============================================================================= TablesSystemTable::TablesSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/tables"), context_(std::move(context)) {} - -std::string TablesSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/tables", kName), context_(std::move(context)) {} Result> TablesSystemTable::ArrowSchema() const { return arrow::schema({ @@ -429,7 +424,7 @@ Result> TablesSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { const std::shared_ptr& data_schema = table.schema; @@ -471,11 +466,7 @@ Result> TablesSystemTable::BuildRows() const { // ============================================================================= PartitionsSystemTable::PartitionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/partitions"), context_(std::move(context)) {} - -std::string PartitionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/partitions", kName), context_(std::move(context)) {} Result> PartitionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -495,7 +486,7 @@ Result> PartitionsSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { const std::shared_ptr& data_schema = table.schema; Identifier id(table.database_name, table.table_name); diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h index 3d6936262..22d8a0262 100644 --- a/src/paimon/core/table/system/global_system_tables.h +++ b/src/paimon/core/table/system/global_system_tables.h @@ -33,7 +33,10 @@ class FileSystem; /// Context passed to global system table constructors, providing catalog-level /// access for enumerating databases, tables, and reading metadata. struct GlobalSystemTableContext { - const Catalog* catalog = nullptr; // non-owning pointer + /// Catalog used when loading rows; borrowed for synchronous schema loading. + const Catalog* catalog = nullptr; + /// Keeps the catalog alive for tables returned by Catalog::GetTable(). + std::shared_ptr catalog_owner; std::shared_ptr fs; std::string warehouse; std::map catalog_options; @@ -48,7 +51,6 @@ class CatalogOptionsSystemTable : public InMemorySystemTable { explicit CatalogOptionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -64,7 +66,6 @@ class AllTableOptionsSystemTable : public InMemorySystemTable { explicit AllTableOptionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -80,7 +81,6 @@ class TablesSystemTable : public InMemorySystemTable { explicit TablesSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -96,7 +96,6 @@ class PartitionsSystemTable : public InMemorySystemTable { explicit PartitionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -113,6 +112,7 @@ class GlobalSystemTableLoader { static Result IsSupported(const std::string& table_name, const std::map& catalog_options); + /// Loads a global system table. Views that enumerate tables require a catalog in context. static Result> Load(const std::string& table_name, const GlobalSystemTableContext& context); diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index 8d6111d79..08c7d7726 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -29,8 +29,10 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/core/io/generic_row_to_arrow_array_converter.h" #include "paimon/core/table/system/system_table_scan.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/memory/memory_pool.h" #include "paimon/read_context.h" +#include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/source/table_read.h" @@ -104,19 +106,28 @@ class InMemorySystemTableRead : public TableRead { } // namespace -InMemorySystemTable::InMemorySystemTable(std::string table_path) - : table_path_(std::move(table_path)) {} +InMemorySystemTable::InMemorySystemTable(const std::shared_ptr& fs, + std::string table_path, std::string name) + : SystemTable(fs, table_path, std::move(name)) {} Result> InMemorySystemTable::NewScan( - const std::shared_ptr& /*context*/) const { - return std::make_unique(table_path_); + std::unique_ptr /*context*/) const { + return std::make_unique(GetRootPath()); } Result> InMemorySystemTable::NewRead( - const std::shared_ptr& context) const { - return std::make_unique( - checked_pointer_cast(shared_from_this()), - context->GetMemoryPool()); + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + ForRead(context->GetOptions())); + return std::make_unique(std::move(table), context->GetMemoryPool()); +} + +Result> InMemorySystemTable::ForRead( + const std::map& /*options*/) const { + return checked_pointer_cast(shared_from_this()); } } // namespace paimon diff --git a/src/paimon/core/table/system/in_memory_system_table.h b/src/paimon/core/table/system/in_memory_system_table.h index f9fcebdb3..d0909f360 100644 --- a/src/paimon/core/table/system/in_memory_system_table.h +++ b/src/paimon/core/table/system/in_memory_system_table.h @@ -19,6 +19,7 @@ #pragma once +#include #include #include #include @@ -28,27 +29,30 @@ namespace paimon { +class FileSystem; + /// Base class for system tables whose result can be materialized as a single in-memory /// RecordBatch. /// /// It provides the common singleton split scan and one-shot batch reader. class InMemorySystemTable : public SystemTable { public: - explicit InMemorySystemTable(std::string table_path); + InMemorySystemTable(const std::shared_ptr& fs, std::string table_path, + std::string name); Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + std::unique_ptr /*context*/) const override; + Result> NewRead(std::unique_ptr context) const override; virtual Result> BuildRows() const = 0; protected: + /// Creates an independent read view when dynamic options affect materialized rows. + virtual Result> ForRead( + const std::map& options) const; + const std::string& TablePath() const { - return table_path_; + return GetRootPath(); } - - private: - std::string table_path_; }; } // namespace paimon diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index 8db1c41cf..19c650ad7 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -55,6 +55,7 @@ #include "paimon/core/manifest/manifest_file.h" #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/snapshot.h" @@ -199,6 +200,27 @@ Result CreateCoreOptions(const MetadataSystemTableContext& context) return CoreOptions::FromMap(context.options, context.fs); } +Result WithReadOptions( + const MetadataSystemTableContext& context, + const std::map& overrides) { + auto branch = overrides.find(Options::BRANCH); + if (branch != overrides.end() && + BranchManager::NormalizeBranch(branch->second) != context.branch) { + return Status::Invalid("context branch contradicts the loaded table"); + } + MetadataSystemTableContext merged = context; + for (const auto& [key, value] : overrides) { + merged.options[key] = value; + } + PAIMON_ASSIGN_OR_RAISE(TableType type, TableTypeDefine::FromOptions(merged.options)); + PAIMON_ASSIGN_OR_RAISE(TableType bound_type, TableTypeDefine::FromOptions(context.options)); + if (type != bound_type) { + return Status::Invalid("context type contradicts the loaded table"); + } + merged.options[Options::BRANCH] = context.branch; + return merged; +} + Result> CreatePathFactory( const MetadataSystemTableContext& context, const CoreOptions& core_options, const std::shared_ptr& pool) { @@ -517,13 +539,11 @@ Result> WriteColsValue( } // namespace -OptionsSystemTable::OptionsSystemTable(std::string table_path, +OptionsSystemTable::OptionsSystemTable(const std::shared_ptr& fs, + std::string table_path, std::shared_ptr table_schema) - : InMemorySystemTable(std::move(table_path)), table_schema_(std::move(table_schema)) {} - -std::string OptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), + table_schema_(std::move(table_schema)) {} Result> OptionsSystemTable::ArrowSchema() const { return arrow::schema({arrow::field("key", arrow::utf8(), /*nullable=*/false), @@ -545,13 +565,9 @@ Result> OptionsSystemTable::BuildRows() const { SnapshotsSystemTable::SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string SnapshotsSystemTable::Name() const { - return kName; -} - Result> SnapshotsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("snapshot_id", arrow::int64(), /*nullable=*/false), @@ -607,13 +623,9 @@ Result> SnapshotsSystemTable::BuildRows() const { SchemasSystemTable::SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string SchemasSystemTable::Name() const { - return kName; -} - Result> SchemasSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("schema_id", arrow::int64(), /*nullable=*/false), @@ -663,13 +675,9 @@ Result> SchemasSystemTable::BuildRows() const { TagsSystemTable::TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string TagsSystemTable::Name() const { - return kName; -} - Result> TagsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("tag_name", arrow::utf8(), /*nullable=*/false), @@ -713,13 +721,9 @@ Result> TagsSystemTable::BuildRows() const { BranchesSystemTable::BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string BranchesSystemTable::Name() const { - return kName; -} - Result> BranchesSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("branch_name", arrow::utf8(), /*nullable=*/false), @@ -752,13 +756,9 @@ Result> BranchesSystemTable::BuildRows() const { ConsumersSystemTable::ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string ConsumersSystemTable::Name() const { - return kName; -} - Result> ConsumersSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("consumer_id", arrow::utf8(), /*nullable=*/false), @@ -787,14 +787,10 @@ ManifestsSystemTable::ManifestsSystemTable(std::shared_ptr fs, std:: std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} -std::string ManifestsSystemTable::Name() const { - return kName; -} - Result> ManifestsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("file_name", arrow::utf8(), /*nullable=*/false), @@ -809,6 +805,14 @@ Result> ManifestsSystemTable::ArrowSchema() const }); } +Result> ManifestsSystemTable::ForRead( + const std::map& options) const { + PAIMON_ASSIGN_OR_RAISE(MetadataSystemTableContext context, WithReadOptions(context_, options)); + auto table = std::make_shared(*this); + table->context_ = std::move(context); + return std::shared_ptr(std::move(table)); +} + Result> ManifestsSystemTable::BuildRows() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CreateCoreOptions(context_)); @@ -857,14 +861,10 @@ Result> ManifestsSystemTable::BuildRows() const { FilesSystemTable::FilesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path, ComposeName(PathUtil::GetName(table_path), kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} -std::string FilesSystemTable::Name() const { - return kName; -} - Result> FilesSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("partition", arrow::utf8(), /*nullable=*/true), @@ -891,6 +891,14 @@ Result> FilesSystemTable::ArrowSchema() const { }); } +Result> FilesSystemTable::ForRead( + const std::map& options) const { + PAIMON_ASSIGN_OR_RAISE(MetadataSystemTableContext context, WithReadOptions(context_, options)); + auto table = std::make_shared(*this); + table->context_ = std::move(context); + return std::shared_ptr(std::move(table)); +} + Result> FilesSystemTable::BuildRows() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); std::shared_ptr pool = GetDefaultPool(); diff --git a/src/paimon/core/table/system/metadata_system_tables.h b/src/paimon/core/table/system/metadata_system_tables.h index e3961e476..cfb8c6faf 100644 --- a/src/paimon/core/table/system/metadata_system_tables.h +++ b/src/paimon/core/table/system/metadata_system_tables.h @@ -35,9 +35,9 @@ class OptionsSystemTable : public InMemorySystemTable { public: static constexpr const char* kName = "options"; - OptionsSystemTable(std::string table_path, std::shared_ptr table_schema); + OptionsSystemTable(const std::shared_ptr& fs, std::string table_path, + std::shared_ptr table_schema); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -62,7 +62,6 @@ class SnapshotsSystemTable : public InMemorySystemTable { SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -77,7 +76,6 @@ class SchemasSystemTable : public InMemorySystemTable { SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -92,7 +90,6 @@ class TagsSystemTable : public InMemorySystemTable { TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -107,7 +104,6 @@ class BranchesSystemTable : public InMemorySystemTable { BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -123,7 +119,6 @@ class ConsumersSystemTable : public InMemorySystemTable { ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -140,11 +135,13 @@ class ManifestsSystemTable : public InMemorySystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; private: + Result> ForRead( + const std::map& options) const override; + MetadataSystemTableContext context_; }; @@ -157,11 +154,13 @@ class FilesSystemTable : public InMemorySystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; private: + Result> ForRead( + const std::map& options) const override; + MetadataSystemTableContext context_; }; diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 8d5696447..b2d4bf99e 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -26,7 +26,9 @@ #include "arrow/c/bridge.h" #include "paimon/common/types/data_field.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/read_optimized_scan_options.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -35,17 +37,16 @@ namespace paimon { -ReadOptimizedSystemTable::ReadOptimizedSystemTable(std::string table_path, +ReadOptimizedSystemTable::ReadOptimizedSystemTable(const std::shared_ptr& fs, + std::string table_path, std::shared_ptr table_schema, + std::string data_table_name, std::map options) - : table_path_(std::move(table_path)), + : SystemTable(fs, table_path, ComposeName(data_table_name, kName)), + table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} -std::string ReadOptimizedSystemTable::Name() const { - return kName; -} - Result> ReadOptimizedSystemTable::ArrowSchema() const { return DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields()); } @@ -57,18 +58,20 @@ std::map ReadOptimizedSystemTable::ReadOptimizedOption } Result> ReadOptimizedSystemTable::NewScan( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetRealtimeContext() && !table_schema_->PrimaryKeys().empty()) { return Status::NotImplemented( "PK real-time union read does not support read-optimized scans"); } auto options = ReadOptimizedOptions(); ScanContextBuilder builder(table_path_); - builder.SetOptions(options) - .WithStreamingMode(context->IsStreamingMode()) + builder.WithStreamingMode(context->IsStreamingMode()) + .SetOptions(context->GetOptions()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + .WithFileSystem(GetFileSystem()) .WithCache(context->GetCache()); if (context->GetLimit().has_value()) { builder.SetLimit(context->GetLimit().value()); @@ -83,36 +86,26 @@ Result> ReadOptimizedSystemTable::NewScan( if (context->GetGlobalIndexResult()) { builder.SetGlobalIndexResult(context->GetGlobalIndexResult()); } - if (context->GetSpecificTableSchema().has_value()) { - builder.SetTableSchema(context->GetSpecificTableSchema().value()); - } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, builder.Finish()); - return TableScan::Create(std::move(base_context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, options)); + return data_table->NewScan(std::move(base_context)); } Result> ReadOptimizedSystemTable::CreateDataReadContext( const std::shared_ptr& context) const { - auto options = options_; - std::string branch = context->GetBranch(); - // SystemTableLoader injects Options::BRANCH when parsing paths such as `T$branch_dev$ro`. - auto branch_iter = options.find(Options::BRANCH); - if (branch_iter != options.end()) { - branch = branch_iter->second; - } ReadContextBuilder builder(table_path_); - builder.SetOptions(options) - .WithBranch(branch) - .SetPredicate(context->GetPredicate()) + builder.SetPredicate(context->GetPredicate()) .EnablePredicateFilter(context->EnablePredicateFilter()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) .SetPrefetchMaxParallelNum(context->GetPrefetchMaxParallelNum()) .EnableMultiThreadRowToBatch(context->EnableMultiThreadRowToBatch()) .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber()) + .SetOptions(context->GetOptions()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) - .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) + .WithFileSystem(GetFileSystem()) .SetReadAheadCacheEnabled(context->ReadAheadCacheEnabled()) .WithCacheConfig(context->GetCacheConfig()) .SetWarmupLevel(context->GetWarmupLevel()) @@ -126,17 +119,20 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, c_read_schema.get())); builder.SetReadSchema(std::move(c_read_schema)); } - if (context->GetSpecificTableSchema().has_value()) { - builder.SetTableSchema(context->GetSpecificTableSchema().value()); - } return builder.Finish(); } Result> ReadOptimizedSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, CreateDataReadContext(context)); - return TableRead::Create(std::move(base_context)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr data_table, + CreateDataTable(table_schema_, options_, context->GetFileSystemSchemeToIdentifierMap())); + return data_table->NewRead(std::move(base_context)); } } // namespace paimon diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h index 3123fe03b..164b2c79c 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.h +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -26,6 +26,7 @@ #include "paimon/core/table/system/system_table.h" namespace paimon { +class FileSystem; class TableSchema; /// System table for `T$ro`, exposing read-optimized data. @@ -33,15 +34,14 @@ class ReadOptimizedSystemTable : public SystemTable { public: static constexpr const char* kName = "ro"; - ReadOptimizedSystemTable(std::string table_path, std::shared_ptr table_schema, + /// @param data_table_name Logical table name, including any branch suffix. + ReadOptimizedSystemTable(const std::shared_ptr& fs, std::string table_path, + std::shared_ptr table_schema, std::string data_table_name, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; - Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; private: /// Build the context for the data table underneath this read-optimized view. diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 5a1a1de00..34c06489c 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -26,26 +26,38 @@ #include #include +#include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" +#include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/metadata_system_tables.h" #include "paimon/core/table/system/read_optimized_system_table.h" +#include "paimon/core/table/system/system_table_schema.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/read_context.h" #include "paimon/status.h" +#include "paimon/write_context.h" namespace paimon { namespace { using SystemTableFactory = std::function>( const std::shared_ptr&, const std::string&, const std::shared_ptr&, - const std::map&)>; + const std::map&, const std::string&)>; struct SystemTableRegistryEntry { std::string name; @@ -70,81 +82,84 @@ std::string LoadBranch(const std::map& options) { const std::vector& SystemTableRegistry() { static const std::vector registry = { {OptionsSystemTable::kName, - [](const std::shared_ptr& /*fs*/, const std::string& table_path, + [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& /*dynamic_options*/) - -> Result> { - return std::make_shared(table_path, table_schema); + const std::map& /*dynamic_options*/, + const std::string& /*data_table_name*/) -> Result> { + return std::make_shared(fs, table_path, table_schema); }}, {AuditLogSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& data_table_name) -> Result> { return std::make_shared( - fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, data_table_name, + MergeOptions(table_schema, dynamic_options)); }}, {BinlogSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& data_table_name) -> Result> { return std::make_shared( - fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, data_table_name, + MergeOptions(table_schema, dynamic_options)); }}, {ReadOptimizedSystemTable::kName, - [](const std::shared_ptr& /*fs*/, const std::string& table_path, + [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& data_table_name) -> Result> { return std::make_shared( - table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, data_table_name, + MergeOptions(table_schema, dynamic_options)); }}, {SnapshotsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {SchemasSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {TagsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {BranchesSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {ConsumersSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, {ManifestsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); auto branch = LoadBranch(options); return std::make_shared(fs, table_path, std::move(branch), @@ -153,8 +168,8 @@ const std::vector& SystemTableRegistry() { {FilesSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) - -> Result> { + const std::map& dynamic_options, + const std::string& /*data_table_name*/) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); auto branch = LoadBranch(options); return std::make_shared(fs, table_path, std::move(branch), @@ -176,6 +191,57 @@ std::optional FindSystemTableFactory(const std::string& syst } // namespace +SystemTable::SystemTable(const std::shared_ptr& fs, const std::string& table_path, + std::string name) + : Table(fs, table_path, /*schema=*/nullptr, + Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/""), + name_(std::move(name)) {} + +std::string SystemTable::ComposeName(const std::string& data_table_name, const std::string& kind) { + return data_table_name + Catalog::SYSTEM_TABLE_SPLITTER + kind; +} + +Status SystemTable::CheckNoSearch(const std::shared_ptr& context) { + if (context->EnablePreciseFileIndexEvaluate()) { + return Status::NotImplemented( + "Precise file-index evaluation is not supported for system tables"); + } + return Status::OK(); +} + +Status SystemTable::AdoptOwnSchema() { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, ArrowSchema()); + schema_ = std::make_shared(std::move(arrow_schema)); + return Status::OK(); +} + +Result> SystemTable::CreateDataTable( + const std::shared_ptr& table_schema, + const std::map& options, + const std::map& fs_scheme_map_overrides) const { + auto fs_scheme_map = fs_scheme_map_; + for (const auto& [scheme, identifier] : fs_scheme_map_overrides) { + fs_scheme_map[scheme] = identifier; + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, + table_schema->MergeOptions(options)); + const std::string branch = BranchManager::NormalizeBranch(LoadBranch(merged_schema->Options())); + return FileStoreTable::Create( + GetFileSystem(), GetRootPath(), std::shared_ptr(std::move(merged_schema)), + CatalogUtils::IdentifierFromPath(GetRootPath(), branch), "", branch, fs_scheme_map); +} + +Result> SystemTable::NewWrite( + std::unique_ptr /*context*/) const { + return Status::NotImplemented(Name(), " system table is read-only: writing is not supported"); +} + +Result> SystemTable::NewCommit( + std::unique_ptr /*context*/) const { + return Status::NotImplemented(Name(), + " system table is read-only: committing for it is not supported"); +} + bool SystemTableLoader::IsSupported(const std::string& system_table_name) { return FindSystemTableFactory(system_table_name).has_value(); } @@ -183,10 +249,20 @@ bool SystemTableLoader::IsSupported(const std::string& system_table_name) { Result> SystemTableLoader::Load( const std::string& system_table_name, const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options) { + const std::map& dynamic_options, const std::string& data_table_name, + const std::map& fs_scheme_map) { std::optional factory = FindSystemTableFactory(system_table_name); if (factory) { - return factory.value()(fs, table_path, table_schema, dynamic_options); + const std::string effective_name = + data_table_name.empty() ? PathUtil::GetName(table_path) : data_table_name; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + factory.value()(fs, table_path, table_schema, dynamic_options, effective_name)); + system_table->name_ = + SystemTable::ComposeName(effective_name, StringUtils::ToLowerCase(system_table_name)); + system_table->fs_scheme_map_ = fs_scheme_map; + PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); + return system_table; } return Status::NotImplemented("unsupported system table: ", system_table_name); } @@ -223,7 +299,8 @@ Result> SystemTableLoader::TryParsePath(const std Result> SystemTableLoader::LoadFromPath( const std::shared_ptr& fs, const std::string& path, - const std::map& dynamic_options) { + const std::map& dynamic_options, + const std::map& fs_scheme_map) { PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, TryParsePath(path)); if (!system_table_path) { return Status::Invalid("path is not a system table path: ", path); @@ -237,7 +314,6 @@ Result> SystemTableLoader::LoadFromPath( // The warehouse is the grandparent of the sys/
path context.warehouse = PathUtil::GetParentDirPath(PathUtil::GetParentDirPath(path)); context.catalog_options = dynamic_options; - // Note: context.catalog is intentionally left as nullptr here. // Global tables loaded from path do not have a Catalog reference and // cannot enumerate databases/tables. Only tables that don't require // catalog enumeration (e.g. catalog_options) will work in this path. @@ -255,7 +331,12 @@ Result> SystemTableLoader::LoadFromPath( if (parsed.branch) { options[Options::BRANCH] = parsed.branch.value(); } - return Load(parsed.system_table_name, fs, parsed.table_path, latest_schema.value(), options); + // Preserve branch-qualified logical names when deriving identifiers from paths. + const Identifier data_identifier(Identifier::kUnknownDatabase, + PathUtil::GetName(parsed.table_path), + parsed.branch.value_or(std::string())); + return Load(parsed.system_table_name, fs, parsed.table_path, latest_schema.value(), options, + data_identifier.GetObjectName(), fs_scheme_map); } } // namespace paimon diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 39e1da31b..73420559b 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -25,17 +25,20 @@ #include #include "arrow/api.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/result.h" #include "paimon/status.h" #include "paimon/type_fwd.h" namespace paimon { class FileSystem; +class FileStoreTable; class ReadContext; class ScanContext; -class TableScan; -class TableRead; class TableSchema; +class SystemTableSchema; +struct GlobalSystemTableContext; /// Parsed information for a table-scoped system table path. struct SystemTablePath { @@ -49,19 +52,55 @@ struct SystemTablePath { bool is_global = false; }; -/// Base interface for table-scoped system tables such as `T$options` and `T$snapshots`. -/// -/// Implementations expose a read-only schema and create their own scan/read objects. -class SystemTable : public std::enable_shared_from_this { +/// Read-only table-scoped or global system view. +/// Table-scoped names are $; global names are . FullName() equals Name(). +class SystemTable : public Table, public std::enable_shared_from_this { public: - virtual ~SystemTable() = default; + std::string Name() const override { + return name_; + } + + std::string FullName() const override { + return name_; + } - virtual std::string Name() const = 0; + /// Returns this view's schema, rather than its data table's schema. virtual Result> ArrowSchema() const = 0; - virtual Result> NewScan( - const std::shared_ptr& context) const = 0; - virtual Result> NewRead( - const std::shared_ptr& context) const = 0; + + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; + + protected: + /// @param fs Data table file system. + /// @param table_path Data table root path. + /// @param name System table name. + SystemTable(const std::shared_ptr& fs, const std::string& table_path, + std::string name); + + /// Composes $, preserving any branch suffix. + static std::string ComposeName(const std::string& data_table_name, const std::string& kind); + + /// Rejects TopK and full-text search requests, which system-table readers cannot evaluate. + static Status CheckNoSearch(const std::shared_ptr& context); + + /// Initializes the system schema after construction, when ArrowSchema() can be called. + Status AdoptOwnSchema(); + + /// Builds the data table using this view's data schema and options. + Result> CreateDataTable( + const std::shared_ptr& table_schema, + const std::map& options, + const std::map& fs_scheme_map_overrides = {}) const; + + friend class SystemTableLoader; + friend class GlobalSystemTableLoader; + + private: + std::string name_; + std::map fs_scheme_map_; }; /// Loads system table implementations from parsed table identifiers or table paths. @@ -74,13 +113,16 @@ class SystemTableLoader { static Result> Load( const std::string& system_table_name, const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, - const std::map& dynamic_options); + const std::map& dynamic_options, + const std::string& data_table_name = "", + const std::map& fs_scheme_map = {}); static Result> TryParsePath(const std::string& path); static Result> LoadFromPath( const std::shared_ptr& fs, const std::string& path, - const std::map& dynamic_options); + const std::map& dynamic_options, + const std::map& fs_scheme_map = {}); }; } // namespace paimon diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 7591e53bf..9f837e4a1 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -77,11 +77,11 @@ TEST(SystemTableTest, TestChangelogArrowSchemaReturnsInvalidOptions) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, CreateTableSchemaForTest(options)); - AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); ASSERT_NOK_WITH_MSG(audit_log.ArrowSchema(), "Invalid Config [table-read.sequence-number.enabled: invalid]"); - BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); ASSERT_NOK_WITH_MSG(binlog.ArrowSchema(), "Invalid Config [table-read.sequence-number.enabled: invalid]"); } @@ -92,7 +92,7 @@ TEST(SystemTableTest, TestBinlogArrowSchemaWithSequenceNumber) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, CreateTableSchemaForTest(options)); - BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, binlog.ArrowSchema()); ASSERT_EQ(schema->field_names(), @@ -219,7 +219,7 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, "/tmp/table", table_schema, /*dynamic_options=*/{})); - ASSERT_EQ(system_table->Name(), ReadOptimizedSystemTable::kName); + ASSERT_EQ(system_table->Name(), "table$ro"); ASSERT_OK_AND_ASSIGN(std::shared_ptr arrow_schema, system_table->ArrowSchema()); ASSERT_EQ(arrow_schema->field_names(), (std::vector{"pk", "v"})); @@ -247,9 +247,10 @@ TEST(SystemTableTest, TestNewReadPropagatesWarmupLevel) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, CreateTableSchemaForTest(options)); - AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); - BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); - ReadOptimizedSystemTable read_optimized("/tmp/table", table_schema, options); + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, "table", options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, "table", + options); for (WarmupLevel level : {WarmupLevel::NONE, WarmupLevel::RAW, WarmupLevel::DECODED}) { ReadContextBuilder builder("/tmp/table"); diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index ec131ef9e..c51b8400b 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -18,194 +18,7 @@ #include "paimon/catalog/table.h" -#include -#include -#include -#include -#include -#include -#include -#include - -#include "arrow/c/bridge.h" -#include "fmt/format.h" -#include "paimon/catalog/catalog.h" -#include "paimon/commit_context.h" -#include "paimon/common/global_index/btree/btree_defs.h" -#include "paimon/common/predicate/predicate_validator.h" -#include "paimon/common/types/data_field.h" -#include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/common/utils/binary_row_partition_computer.h" -#include "paimon/common/utils/fields_comparator.h" -#include "paimon/common/utils/options_utils.h" -#include "paimon/common/utils/string_utils.h" -#include "paimon/core/catalog/catalog_snapshot_commit.h" -#include "paimon/core/catalog/catalog_utils.h" -#include "paimon/core/catalog/commit_lock.h" -#include "paimon/core/catalog/renaming_snapshot_commit.h" -#include "paimon/core/catalog/snapshot_commit.h" -#include "paimon/core/catalog/version_managed_catalog.h" -#include "paimon/core/core_options.h" -#include "paimon/core/disk/io_manager.h" -#include "paimon/core/index/index_file_handler.h" -#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" -#include "paimon/core/index/pk/primary_key_index_definitions.h" -#include "paimon/core/manifest/index_manifest_entry.h" -#include "paimon/core/manifest/index_manifest_file.h" -#include "paimon/core/mergetree/compact/lookup_merge_function.h" -#include "paimon/core/mergetree/compact/merge_function.h" -#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" -#include "paimon/core/operation/append_only_file_store_scan.h" -#include "paimon/core/operation/append_only_file_store_write.h" -#include "paimon/core/operation/commit/realtime_commit_properties.h" -#include "paimon/core/operation/data_evolution_file_store_scan.h" -#include "paimon/core/operation/expire_snapshots.h" -#include "paimon/core/operation/file_store_commit_impl.h" -#include "paimon/core/operation/file_store_scan.h" -#include "paimon/core/operation/internal_read_context.h" -#include "paimon/core/operation/key_value_file_store_scan.h" -#include "paimon/core/operation/key_value_file_store_write.h" -#include "paimon/core/options/merge_engine.h" -#include "paimon/core/postpone/postpone_bucket_file_store_write.h" -#include "paimon/core/realtime/realtime_context_impl.h" -#include "paimon/core/realtime/realtime_schema_layout.h" -#include "paimon/core/realtime/realtime_utils.h" -#include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/schema_validation.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/table/bucket_mode.h" -#include "paimon/core/table/file_store.h" -#include "paimon/core/table/format/format_table_file_store_commit.h" -#include "paimon/core/table/format/format_table_file_store_write.h" -#include "paimon/core/table/format/format_table_loader.h" -#include "paimon/core/table/format/format_table_read.h" -#include "paimon/core/table/format/format_table_scan.h" -#include "paimon/core/table/source/abstract_table_scan.h" -#include "paimon/core/table/source/append_only_split_generator.h" -#include "paimon/core/table/source/append_only_table_read.h" -#include "paimon/core/table/source/data_evolution_batch_scan.h" -#include "paimon/core/table/source/data_evolution_split_generator.h" -#include "paimon/core/table/source/data_table_batch_scan.h" -#include "paimon/core/table/source/data_table_stream_scan.h" -#include "paimon/core/table/source/fallback_table_read.h" -#include "paimon/core/table/source/key_value_table_read.h" -#include "paimon/core/table/source/merge_tree_split_generator.h" -#include "paimon/core/table/source/primary_key_index_batch_scan.h" -#include "paimon/core/table/source/read_optimized_scan_options.h" -#include "paimon/core/table/source/realtime_table_scan.h" -#include "paimon/core/table/source/snapshot/snapshot_reader.h" -#include "paimon/core/table/source/split_generator.h" -#include "paimon/core/table/system/system_table.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/core/utils/field_mapping.h" -#include "paimon/core/utils/file_store_path_factory.h" -#include "paimon/core/utils/primary_key_table_utils.h" -#include "paimon/core/utils/snapshot_manager.h" -#include "paimon/defs.h" -#include "paimon/file_store_commit.h" -#include "paimon/file_store_write.h" -#include "paimon/format/file_format.h" -#include "paimon/fs/file_system.h" -#include "paimon/read_context.h" -#include "paimon/realtime/realtime_context.h" -#include "paimon/result.h" -#include "paimon/scan_context.h" -#include "paimon/schema/schema.h" -#include "paimon/status.h" -#include "paimon/table/format/format_table.h" -#include "paimon/table/source/table_read.h" -#include "paimon/table/source/table_scan.h" -#include "paimon/write_context.h" - -namespace arrow { -class Schema; -} // namespace arrow - namespace paimon { -class DataSplit; -class Executor; -struct KeyValue; -class MemoryPool; -template -class MergeFunctionWrapper; - -namespace { - -Status RestoreRealtimeCommittedProgress(const std::shared_ptr& realtime_context, - const std::shared_ptr& snapshot_manager, - const CoreOptions& options) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, - RealtimeContextImpl::Cast(realtime_context)); - PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_ASSIGN_OR_RAISE( - RealtimeOffsetMap realtime_committed_offsets, - RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); - PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( - latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); - } - return Status::OK(); -} - -/// Maps a `WriteContext` onto `FormatTableFileStoreWrite`, refusing by name what a format table -/// cannot honour rather than silently dropping it. -Result> NewFormatTableWrite( - const std::shared_ptr& table, const WriteContext& ctx) { - if (ctx.GetCatalog() != nullptr) { - return Status::Invalid( - "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " - "format table"); - } - if (ctx.IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing a streaming write could " - "commit against"); - } - if (ctx.GetRealtimeContext() != nullptr) { - return Status::NotImplemented("a format table has no real-time store to write into"); - } - if (!ctx.GetWriteSchema().empty()) { - return Status::NotImplemented( - "a format table write takes the table's own columns; a write schema naming a " - "subset of them is not supported yet"); - } - // A write id prefixes a postpone-bucket writer's files so that one compaction reader can - // put them back in order. A format table has no buckets, so it would identify nothing. - if (ctx.GetWriteId().has_value()) { - return Status::NotImplemented( - "a format table has no buckets, so a write id would name nothing"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, - FormatTableFileStoreWrite::Create(table, ctx.GetMemoryPool())); - return std::unique_ptr(std::move(format_write)); -} - -Result HasHistoricalPrimaryKeyBTreeDefinition( - const std::shared_ptr& schema_manager, int64_t current_schema_id) { - if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { - return false; - } - PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); - for (int64_t schema_id : schema_ids) { - if (schema_id >= current_schema_id) { - continue; - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, - schema_manager->ReadSchema(schema_id)); - const auto& historical_options = historical_schema->Options(); - auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); - if (option != historical_options.end() && - !StringUtils::IsNullOrWhitespaceOnly(option->second)) { - return true; - } - } - return false; -} - -} // namespace - Table::Table(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid) @@ -214,1144 +27,4 @@ Table::Table(const std::shared_ptr& file_system, const std::string& schema_(schema), identifier_(identifier), uuid_(uuid) {} - -Result> Table::NewScan(std::unique_ptr context) const { - return TableScan::Create(std::move(context)); -} - -Result> Table::NewRead(std::unique_ptr context) const { - return TableRead::Create(std::move(context)); -} - -Result> Table::NewWrite( - std::unique_ptr context) const { - return FileStoreWrite::Create(std::move(context)); -} - -Result> Table::NewCommit( - std::unique_ptr context) const { - return FileStoreCommit::Create(std::move(context)); -} - -Result> FileStoreWrite::Create(std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("write context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableWrite(given_table, *ctx); - } - - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && - ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); - } - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs, - ctx->GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx->GetBranch(); - if (ctx->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); - } - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } - // Only the main branch may have a current schema the catalog alone holds. A branch publishes - // its own under `branch/branch-`, which is where the load below and a read of the branch - // both read it, so a catalog answering for no branch schema does not fail a write to one. - std::optional catalog_table_schema; - if (ctx->GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog write requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); - } - } - // A format table writes plain data files into a directory, so it never reaches the manifest - // path below. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch, - ctx->GetOptions(), catalog_table_schema, schema_manager.get(), - &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableWrite(format_table, *ctx); - } - // The schema the dispatch above already read through `schema_manager`, rather than a second - // read of the same file. - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths above are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; - PAIMON_ASSIGN_OR_RAISE( - CoreOptions options, - CoreOptions::FromMap(opts, specific_fs, ctx->GetFileSystemSchemeToIdentifierMap())); - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); - if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) { - return Status::NotImplemented( - "Dedicated VECTOR storage does not yet support custom write schemas or partial writes"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); - - FileStore file_store(ctx->GetRootPath(), options); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr file_store_path_factory, - file_store.NewPathFactory(arrow_schema, schema->PartitionKeys(), - options.GetWriteFileFormat(/*level=*/0)->Identifier(), - ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_manager, - file_store.NewSnapshotManager(ctx->GetCatalog(), catalog_identifier)); - std::shared_ptr io_manager; - const auto& io_temp_dir = ctx->GetTempDirectory(); - if (!io_temp_dir.empty()) { - io_manager = std::make_shared(io_temp_dir, options.GetFileSystem()); - } - - bool ignore_previous_files = ctx->IgnorePreviousFiles(); - if (ctx->GetRealtimeContext() && !options.RealtimeEnabled()) { - return Status::Invalid("real-time write requires realtime.enabled=true"); - } - if (ctx->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, - RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); - PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); - } - if (schema->PrimaryKeys().empty()) { - // append table - const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); - bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); - if (options.GetBucket() == -1) { - need_dv_maintainer_factory = false; - ignore_previous_files = true; - } else if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in append table", options.GetBucket())); - } - if (ctx->GetRealtimeContext()) { - if (!ctx->GetWriteSchema().empty()) { - return Status::Invalid("real-time append write does not support a custom schema"); - } - if (!realtime_deduplicate) { - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - } else if (realtime_deduplicate) { - if (!ctx->GetWriteSchema().empty() || ignore_previous_files || options.WriteOnly()) { - return Status::Invalid( - "deduplicate compaction requires the full schema, restored files and " - "write-only=false"); - } - } - std::shared_ptr write_schema = arrow_schema; - const auto& write_field_names = ctx->GetWriteSchema(); - if (!write_field_names.empty()) { - arrow::FieldVector write_fields; - write_fields.reserve(write_field_names.size()); - for (const auto& field_name : write_field_names) { - auto field = arrow_schema->GetFieldByName(field_name); - if (!field) { - // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER - return Status::Invalid( - fmt::format("write field {} does not exist in table schema", field_name)); - } - write_fields.push_back(field); - } - write_schema = arrow::schema(write_fields); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - const RealtimeStoreMode mode = realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE - : RealtimeStoreMode::APPEND_ONLY; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, - RealtimeSchemaLayout::Create(mode, write_schema)); - realtime_schema_layout = std::move(layout); - } - - std::shared_ptr dv_maintainer_factory; - if (need_dv_maintainer_factory) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr index_file_handler, - file_store.NewIndexFileHandler(ResolveBucketMode(options.GetBucket(), schema), - file_store_path_factory, ctx->GetMemoryPool())); - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - - auto file_store_write = std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, write_schema, realtime_schema_layout, - partition_schema, dv_maintainer_factory, io_manager, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetRealtimeContext(), - ctx->GetExecutor(), ctx->GetMemoryPool()); - if (ctx->GetRealtimeContext() && realtime_deduplicate) { - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_RETURN_NOT_OK( - file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); - } - } - return std::unique_ptr(std::move(file_store_write)); - } else { - // pk table - if (ctx->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); - if (ignore_previous_files) { - return Status::NotImplemented( - "PK realtime requires restore from the latest snapshot"); - } - if (!ctx->GetWriteSchema().empty()) { - return Status::NotImplemented("PK realtime does not support a custom write schema"); - } - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { - return PostponeBucketFileStoreWrite::Create( - snapshot_manager, schema_manager, ctx->GetCommitUser(), ctx->GetRootPath(), schema, - arrow_schema, partition_schema, io_manager, options, ctx->IsStreamingMode(), - ctx->IgnoreNumBucketCheck(), ctx->GetWriteId(), - ctx->GetFileSystemSchemeToIdentifierMap(), ctx->GetExecutor(), ctx->GetMemoryPool(), - specific_fs); - } - if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in key value table", options.GetBucket())); - } - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, - schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_primary_key_fields, - options.SequenceFieldSortOrderIsAscending())); - auto primary_keys = schema->PrimaryKeys(); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, - PrimaryKeyTableUtils::CreateMergeFunction( - arrow_schema, primary_keys, options, ctx->GetMemoryPool())); - if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { - // don't wrap first row, it is already OK - merge_function = std::make_unique(std::move(merge_function)); - } - std::shared_ptr> merge_function_wrapper = - std::make_shared(std::move(merge_function)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr sequence_fields_comparator, - PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); - - std::shared_ptr dv_maintainer_factory; - PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, - PrimaryKeyIndexDefinitions::Create(*schema)); - bool has_btree_index = false; - for (const PrimaryKeyIndexDefinition& definition : - primary_key_index_definitions.Definitions()) { - if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { - has_btree_index = true; - break; - } - } - std::optional latest_snapshot; - bool has_existing_index_manifest = false; - if (!has_btree_index) { - PAIMON_ASSIGN_OR_RAISE( - bool has_historical_btree_index, - HasHistoricalPrimaryKeyBTreeDefinition(schema_manager, schema->Id())); - if (has_historical_btree_index) { - PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); - has_existing_index_manifest = - latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); - } - } - std::shared_ptr index_file_handler; - if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - index_file_handler, - file_store.NewIndexFileHandler(ResolveBucketMode(options.GetBucket(), schema), - file_store_path_factory, ctx->GetMemoryPool())); - } - bool has_restored_btree_index = false; - if (!has_btree_index && has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - std::vector restored_btree_entries, - index_file_handler->Scan( - latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { - return entry.index_file->IndexType() == BtreeDefs::kIdentifier && - IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); - })); - has_restored_btree_index = !restored_btree_entries.empty(); - } - if (options.DeletionVectorsEnabled()) { - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - std::shared_ptr - primary_key_index_maintainer_factory; - if (has_btree_index || has_restored_btree_index) { - primary_key_index_maintainer_factory = - std::make_shared( - ctx->GetRootPath(), branch, schema, primary_key_index_definitions.Definitions(), - file_store_path_factory, index_file_handler, options, io_manager, - ctx->EnableMultiThreadSpill(), ctx->GetExecutor(), ctx->GetMemoryPool()); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr layout, - RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); - realtime_schema_layout = std::move(layout); - } - - return std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, realtime_schema_layout, partition_schema, - dv_maintainer_factory, primary_key_index_maintainer_factory, io_manager, key_comparator, - sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->EnableMultiThreadSpill(), - ctx->GetRealtimeContext(), ctx->GetExecutor(), ctx->GetMemoryPool()); - } -} - -namespace { - -class TableScanImpl { - public: - static Result> CreateFileStoreScan( - const std::shared_ptr& path_factory, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const std::shared_ptr& executor, const std::shared_ptr& memory_pool, - const ScanContext* context) { - auto fs = core_options.GetFileSystem(); - std::string branch = BranchManager::NormalizeBranch(core_options.GetBranch()); - FileStore file_store(context->GetPath(), core_options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_manager, - file_store.NewSnapshotManager(/*catalog=*/nullptr, - /*catalog_identifier=*/std::nullopt)); - // TODO(liancheng.lsz): support fallback branch in scan - auto schema_manager = std::make_shared(fs, context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, - file_store.NewManifestList(path_factory, memory_pool)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - file_store.NewManifestFile(path_factory, partition_schema, memory_pool)); - std::unique_ptr scan; - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - PAIMON_ASSIGN_OR_RAISE( - scan, DataEvolutionFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, AppendOnlyFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, - context->GetScanFilters(), core_options, - executor, memory_pool)); - } - return WithTablePath(std::move(scan), context); - } - - static std::unique_ptr WithTablePath(std::unique_ptr&& scan, - const ScanContext* context) { - scan->WithTablePath(context->GetPath()); - return std::move(scan); - } - - static Result> CreateSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const ScanContext* context) { - auto source_split_target_size = core_options.GetSourceSplitTargetSize(); - auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.BlobSplitByFileSize()); - } - BucketMode bucket_mode = (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE - : BucketMode::HASH_FIXED); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, bucket_mode); - } else { - // TODO(liancheng.lsz): support evolution - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - table_schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, - table_schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), - key_comparator); - } - } -}; - -/// @param loaded_schema The table's schema, when the caller already read it, or null when it has -/// to be read here. `TableScan::Create()` reads one to dispatch on its table type. -Result> NewDataTableScan( - const std::shared_ptr& context, const std::shared_ptr& loaded_schema); - -/// Maps a `ScanContext` onto `FormatTableScan`, which takes one partition filter and a limit and -/// nothing else. -/// -/// A format table has no buckets, so a bucket filter has nothing to act on. A predicate is -/// refused too, which is narrower than Java: `FormatReadBuilder.newScan()` hands the partition -/// half of one to the scan to prune directories with, which this does not do yet. -Result> NewFormatTableScan(const std::shared_ptr& table, - const std::shared_ptr& context) { - // Anything a `ScanContext` carries that a format table cannot honour is refused rather than - // silently dropped. - if (context->IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing for a streaming scan to follow"); - } - if (context->GetRealtimeContext() != nullptr) { - return Status::NotImplemented( - "a format table has no real-time store to union with what is on disk"); - } - if (context->GetGlobalIndexResult() != nullptr) { - return Status::NotImplemented("a format table carries no global index"); - } - std::map partition_filter; - std::shared_ptr filters = context->GetScanFilters(); - if (filters != nullptr) { - if (filters->GetPredicate() != nullptr) { - return Status::NotImplemented( - "a format table scan does not take a predicate: its files carry no statistics to " - "skip by, and pruning by the partition columns a predicate names is not " - "implemented yet; give the partition values as a partition filter instead"); - } - if (filters->GetBucketFilter()) { - return Status::NotImplemented("a format table has no buckets to filter by"); - } - const std::vector>& partition_filters = - filters->GetPartitionFilters(); - if (partition_filters.size() > 1) { - return Status::NotImplemented("a format table scan takes at most one partition filter"); - } - if (partition_filters.size() == 1) { - partition_filter = partition_filters.front(); - } - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - FormatTableScan::Create(table, partition_filter, context->GetLimit())); - return std::unique_ptr(std::move(scan)); -} - -} // namespace - -Result> TableScan::Create(std::unique_ptr context) { - if (context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - std::shared_ptr shared_context = std::move(context); - // A table the caller already loaded says what it is, so nothing is read to find out. - if (shared_context->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTable::Copy(shared_context->GetFormatTable(), shared_context->GetOptions())); - return NewFormatTableScan(format_table, shared_context); - } - // load schema - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(shared_context->GetOptions(), - shared_context->GetSpecificFileSystem(), {})); - if (shared_context->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(tmp_options.GetBranch())); - } - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(shared_context->GetPath())); - if (system_table_path) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), shared_context->GetPath(), - shared_context->GetOptions())); - return system_table->NewScan(shared_context); - } - // A format table is planned by listing directories, so it never reaches the manifest path - // below. One `TableScan` interface serves both, as Java Paimon serves both through one - // `ReadBuilder`. - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), shared_context->GetPath(), - BranchManager::NormalizeBranch(tmp_options.GetBranch()), - shared_context->GetOptions(), - shared_context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableScan(format_table, shared_context); - } - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableScan(shared_context, latest_schema); -} - -namespace { - -Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, - const ScanContext& context, bool read_optimized) { - if (!context.GetRealtimeContext()) { - return Status::OK(); - } - if (!core_options.RealtimeEnabled()) { - return Status::Invalid("real-time scan requires realtime.enabled=true"); - } - if (core_options.GetBucket() <= 0) { - return Status::Invalid("real-time union read requires fixed bucket mode"); - } - if (core_options.DataEvolutionEnabled()) { - return Status::Invalid("real-time union read does not support data evolution"); - } - if (!table_schema.PrimaryKeys().empty()) { - if (read_optimized) { - return Status::NotImplemented( - "PK real-time union read does not support read-optimized scans"); - } - PAIMON_RETURN_NOT_OK( - PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); - } - if (context.IsStreamingMode()) { - return Status::Invalid("real-time union read currently supports batch scans only"); - } - if (context.GetLimit()) { - return Status::Invalid("real-time union read does not support scan limit pushdown"); - } - if (context.GetGlobalIndexResult()) { - return Status::Invalid("real-time union read does not support global index splits"); - } - StartupMode startup_mode = core_options.GetStartupMode(); - const bool specified_snapshot = - startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); - if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || - specified_snapshot)) { - return Status::Invalid( - "real-time union read requires the latest snapshot or an explicit snapshot id"); - } - return Status::OK(); -} - -Result> NewDataTableScan( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); - std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema, - schema_manager.Latest()); - if (latest_table_schema == std::nullopt) { - return Status::Invalid("not found latest schema"); - } - table_schema = latest_table_schema.value(); - } - PAIMON_ASSIGN_OR_RAISE(bool read_optimized, - OptionsUtils::GetValueFromMap(context->GetOptions(), - kReadOptimizedScanOption, false)); - // merge options - auto options = table_schema->Options(); - for (const auto& [key, value] : context->GetOptions()) { - options[key] = value; - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, context->GetSpecificFileSystem(), {})); - core_options.WithCache(context->GetCache()); - - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); - - PAIMON_RETURN_NOT_OK( - ValidateRealtimeScan(*table_schema, core_options, *context, read_optimized)); - // validate options - if (core_options.GetBucket() == -1) { - if (!table_schema->PrimaryKeys().empty()) { - return Status::NotImplemented(fmt::format( - "do not support pk table bucket={} in scan process", core_options.GetBucket())); - } - } else if (core_options.GetBucket() < 1 && - !SchemaValidation::IsPostponeBucketTable(*table_schema, core_options.GetBucket())) { - return Status::Invalid( - fmt::format("do not support bucket={} in scan process", core_options.GetBucket())); - } - - // validate schema and scan filter - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { - PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( - *arrow_schema, context->GetScanFilters()->GetPredicate(), - /*validate_field_idx=*/false)); - } - FileStore file_store(context->GetPath(), core_options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, - file_store.NewPathFactory(arrow_schema, table_schema->PartitionKeys(), - core_options.GetFileFormat()->Identifier(), - context->GetMemoryPool())); - - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_store_scan, - TableScanImpl::CreateFileStoreScan( - path_factory, arrow_schema, table_schema, core_options, - context->GetExecutor(), context->GetMemoryPool(), context.get())); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr split_generator, - TableScanImpl::CreateSplitGenerator(table_schema, core_options, context.get())); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr index_file_handler, - file_store.NewIndexFileHandler(ResolveBucketMode(core_options.GetBucket(), table_schema), - path_factory, context->GetMemoryPool())); - auto snapshot_reader = std::make_shared( - file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler)); - const bool pk_table = !table_schema->PrimaryKeys().empty(); - if (read_optimized && pk_table && context->IsStreamingMode()) { - return Status::NotImplemented( - "read-optimized system table does not support streaming scan for primary key table"); - } - if (context->IsStreamingMode()) { - return std::make_unique(core_options, snapshot_reader); - } - auto batch_scan = std::make_unique( - /*pk_table=*/pk_table, core_options, snapshot_reader, read_optimized, context->GetLimit(), - /*realtime_pk_scan=*/pk_table && context->GetRealtimeContext() != nullptr); - if (context->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, - RealtimeContextImpl::Cast(context->GetRealtimeContext())); - return std::make_unique( - std::move(batch_scan), pk_table, realtime_context, path_factory, - snapshot_reader->GetSnapshotManager(), core_options.GetFileSystem(), - context->GetScanFilters(), core_options.GetRealtimeReadViewTtlMillis()); - } - if (core_options.DataEvolutionEnabled()) { - return std::make_unique( - context->GetPath(), snapshot_reader, std::move(batch_scan), table_schema, - context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), - context->GetExecutor()); - } - if (pk_table && !read_optimized && core_options.GlobalIndexEnabled()) { - PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, - PrimaryKeyIndexDefinitions::Create(*table_schema)); - if (!definitions.ScalarDefinitions().empty()) { - return PrimaryKeyIndexBatchScan::Create(snapshot_reader, std::move(batch_scan), - table_schema, path_factory, core_options, - context->GetMemoryPool()); - } - } - return batch_scan; -} - -} // namespace - -namespace { - -/// @param loaded_schema The schema of `branch`, when the caller already read it, or null when it -/// has to be read here: `TableRead::Create()` reads one to dispatch on the table type, and -/// reading it again would cost a second listing and read. -Result> CreateInternalReadContext( - const std::shared_ptr& context, const std::string& branch, - const std::shared_ptr& loaded_schema) { - if (context->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); - } - std::map tmp_options = context->GetOptions(); - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, - CoreOptions::FromMap(tmp_options, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - SchemaManager schema_manager(tmp_core_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - schema_manager.Latest()); - if (!latest_schema) { - return Status::Invalid(fmt::format("schema file not found in path {}, branch {}", - context->GetPath(), branch)); - } - table_schema = latest_schema.value(); - } - assert(table_schema); - - // merge options - auto options = table_schema->Options(); - for (const auto& [key, value] : tmp_options) { - options[key] = value; - } - if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { - options[Options::BRANCH] = branch; - } - return InternalReadContext::Create(context, table_schema, options); -} - -Result> CreateTableRead( - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor) { - const auto& core_options = internal_context->GetCoreOptions(); - const auto& table_schema = internal_context->GetTableSchema(); - if (internal_context->GetRealtimeContext() && !core_options.RealtimeEnabled()) { - return Status::Invalid("real-time read requires realtime.enabled=true"); - } - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - FileStore file_store(internal_context->GetPath(), core_options); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - file_store.NewPathFactory(arrow_schema, table_schema->PartitionKeys(), - core_options.GetFileFormat()->Identifier(), memory_pool)); - - if (internal_context->GetPrimaryKeys().empty()) { - return std::make_unique(path_factory, internal_context, memory_pool, - executor); - } - return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); -} - -Result> NewDataTableRead( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, - CreateInternalReadContext(context, context->GetBranch(), loaded_schema)); - - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr table_read, - CreateTableRead(internal_context, context->GetMemoryPool(), context->GetExecutor())); - - std::optional scan_fallback_branch = - internal_context->GetCoreOptions().GetScanFallbackBranch(); - if (!scan_fallback_branch || - StringUtils::IsNullOrWhitespaceOnly(scan_fallback_branch.value())) { - return std::move(table_read); - } - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr fallback_context, - CreateInternalReadContext(context, /*branch=*/scan_fallback_branch.value(), - /*loaded_schema=*/nullptr)); - - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr fallback_table_read, - CreateTableRead(fallback_context, context->GetMemoryPool(), context->GetExecutor())); - const std::shared_ptr& arrow_pool = internal_context->GetArrowMemoryPool(); - return std::make_unique(std::move(table_read), - std::move(fallback_table_read), arrow_pool); -} - -/// Maps a `ReadContext` onto `FormatTableRead`, which reads everything it needs out of the context -/// itself: the columns, the predicate, the pool and executor, and what a file is opened with. It -/// refuses by name what a format table cannot honour. -Result> NewFormatTableRead(const std::shared_ptr& table, - const std::shared_ptr& context) { - if (context->EnablePreciseFileIndexEvaluate()) { - return Status::NotImplemented( - "Precise file-index evaluation is not supported for format tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - FormatTableRead::Create(table, context)); - return std::unique_ptr(std::move(read)); -} - -} // namespace - -Result> TableRead::Create(std::unique_ptr ctx) { - std::shared_ptr context = std::move(ctx); - if (context == nullptr) { - return Status::Invalid("read context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - if (context->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(context->GetBranch())); - } - // A table the caller already loaded says what it is, so nothing is read to find out. - if (context->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); - return NewFormatTableRead(format_table, context); - } - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_core_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(context->GetPath())); - if (system_table_path) { - if (context->EnablePreciseFileIndexEvaluate()) { - return Status::NotImplemented( - "Precise file-index evaluation is not supported for system tables"); - } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetOptions())); - return system_table->NewRead(context); - } - - // A format table has no manifest and a file another engine wrote carries no field ids, so - // which files there are and how a row is put back together is its own. How a file is opened - // is not: both paths go through `DataFileReaderFactory`. - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetBranch(), context->GetOptions(), - context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableRead(format_table, context); - } - - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableRead(context, latest_schema); -} - -namespace { - -CommitScanner::ScanSupplier CreateAppendScanSupplier( - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - -CommitScanner::ScanSupplier CreatePkScanSupplier( - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - -bool IsCatalogCommit(const CommitContext& ctx) { - return AsVersionManaged(ctx.GetCatalog()) != nullptr; -} - -Result> NewSnapshotCommit( - const CommitContext& ctx, const std::optional& catalog_identifier, - const CoreOptions& options, const std::shared_ptr& snapshot_manager) { - if (IsCatalogCommit(ctx)) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr lock, - AsVersionManaged(ctx.GetCatalog()) - ->CreateCommitLock(catalog_identifier.value(), ctx.GetTableId(), - ctx.GetCommitUser(), options)); - // Reuse the caller's UUID so table recreation cannot redirect this commit. - return std::shared_ptr(std::make_shared( - ctx.GetCatalog(), catalog_identifier.value(), ctx.GetTableId(), std::move(lock))); - } - if (ctx.UseRESTCatalogCommit()) { - if (options.RestCommitAcquireLock()) { - return Status::Invalid(fmt::format( - "'{}' needs a commit through a REST catalog with WithCatalog(); a commit table " - "request built for the caller to send cannot hold a commit lease", - Options::REST_COMMIT_ACQUIRE_LOCK)); - } - return std::shared_ptr( - std::make_shared(ctx.GetTableId())); - } - // Catalogs without version management may still supply a table ID. - if (ctx.GetTableId() && ctx.GetCatalog() == nullptr) { - return Status::Invalid( - "a commit which publishes the snapshot by writing it to the table directory sends no " - "commit table request, so there is nowhere to put a table id; commit through a " - "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); - } - return std::shared_ptr( - std::make_shared(options.GetFileSystem(), snapshot_manager)); -} - -/// Creates a format-table commit, rejecting non-default options that require snapshots. -Result> NewFormatTableCommit( - const std::shared_ptr& table, const CommitContext& ctx) { - if (!ctx.IgnoreEmptyCommit()) { - return Status::NotImplemented( - "a format table cannot record an empty commit: keeping one means writing a " - "snapshot that adds no files, and there are no snapshots here"); - } - if (ctx.UseRESTCatalogCommit()) { - return Status::NotImplemented( - "a format table commits by renaming files into place, not by sending a snapshot " - "to a rest catalog"); - } - if (ctx.GetTableId()) { - return Status::NotImplemented( - "a format table sends no commit table request, so there is nowhere to put a table id"); - } - if (ctx.GetCatalog() != nullptr) { - return Status::NotImplemented( - "a format table keeps no snapshot for a catalog to take, so it commits by renaming " - "files into place rather than through the catalog it was loaded from"); - } - if (ctx.AppendCommitCheckConflict()) { - return Status::NotImplemented( - "a format table has no manifests to check a concurrent commit against"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, - FormatTableFileStoreCommit::Create(table)); - return std::unique_ptr(std::move(format_commit)); -} - -} // namespace - -Result> FileStoreCommit::Create( - std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("commit context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableCommit(given_table, *ctx); - } - - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - // Use catalog credentials unless the caller supplied a file system. - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - // Use the credentials of this table, which a catalog issuing per-table temporary - // ones only hands out through GetTableFileSystem. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); - const std::string& root_path = ctx->GetRootPath(); - // The branch the context was built for, already validated by `CommitContextBuilder`. - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - ctx->GetIdentifier(), ctx->GetOptions(), - /*explicit_branch=*/std::nullopt, "commit")); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } - // Catalog-managed schemas may be absent from the table directory, but only on the main branch: - // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail - // a commit to one. - std::optional catalog_table_schema; - FileStoreCommitImpl::SchemaIdLoader schema_id_loader; - if (ctx->GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); - catalog_table_schema = std::move(schema_json); - } - std::shared_ptr catalog = ctx->GetCatalog(); - Identifier identifier = catalog_identifier.value(); - schema_id_loader = [catalog, identifier]() -> Result { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - catalog->LoadTableSchema(identifier)); - std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); - if (data_schema == nullptr) { - return Status::Invalid( - "the catalog did not hand back a data table schema, which has the id a " - "snapshot records"); - } - return data_schema->Id(); - }; - } - // A format table commits by renaming files into place, so it never reaches the snapshot path - // below. The managed path here reads the commit's branch, so this reads the same one: the two - // must not dispatch on different schemas. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), root_path, branch); - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTableLoader::TryLoad( - tmp_options.GetFileSystem(), root_path, branch, ctx->GetOptions(), - catalog_table_schema, schema_manager.get(), &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableCommit(format_table, *ctx); - } - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths below are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; - std::shared_ptr arrow_schema = - DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, specific_fs)); - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); - assert(options.GetFileSystem()); - assert(options.GetFileFormat()); - PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); - PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path)); - // Catalog commits do not require atomic rename on the object store. - if (is_object_store && !ctx->UseRESTCatalogCommit() && !IsCatalogCommit(*ctx) && - opts.find("enable-object-store-commit-in-inte-test") == opts.end()) { - return Status::NotImplemented( - "commit operation does not support object store file system for now"); - } - FileStore file_store(root_path, options); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr partition_computer, - file_store.NewPartitionComputer(arrow_schema, schema->PartitionKeys(), - ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - file_store.NewPathFactory(arrow_schema, schema->PartitionKeys(), - options.GetFileFormat()->Identifier(), ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_manager, - file_store.NewSnapshotManager(ctx->GetCatalog(), catalog_identifier)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, - file_store.NewManifestList(path_factory, ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - file_store.NewManifestFile(path_factory, partition_schema, ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr index_manifest_file, - file_store.NewIndexManifestFile(ResolveBucketMode(options.GetBucket(), schema), - path_factory, ctx->GetMemoryPool())); - - auto expire_snapshots = std::make_shared( - snapshot_manager, path_factory, manifest_list, manifest_file, index_manifest_file, - options.GetFileSystem(), options.GetExpireConfig(), options.RealtimeEnabled(), - ctx->GetExecutor()); - - CommitScanner::ScanSupplier scan_supplier; - if (schema->PrimaryKeys().empty()) { - scan_supplier = CreateAppendScanSupplier(snapshot_manager, schema_manager, manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); - } else { - scan_supplier = CreatePkScanSupplier(snapshot_manager, schema_manager, manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); - } - - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_commit, - NewSnapshotCommit(*ctx, catalog_identifier, options, snapshot_manager)); - - return std::make_unique( - ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path, ctx->GetCommitUser(), - options, path_factory, std::move(partition_computer), snapshot_manager, snapshot_commit, - ctx->IgnoreEmptyCommit(), ctx->AppendCommitCheckConflict(), schema, - std::move(schema_id_loader), manifest_file, manifest_list, index_manifest_file, - expire_snapshots, schema_manager, std::move(scan_supplier)); -} - } // namespace paimon diff --git a/src/paimon/core/table/table_factory.cpp b/src/paimon/core/table/table_factory.cpp new file mode 100644 index 000000000..2f12551b1 --- /dev/null +++ b/src/paimon/core/table/table_factory.cpp @@ -0,0 +1,378 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/table_factory.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/table.h" +#include "paimon/commit_context.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/realtime/realtime_utils.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/format/format_table_loader.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/table/format/format_table.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" + +namespace paimon { +namespace { +Result> ManagedTable( + const std::string& path, const std::string& branch, const std::shared_ptr& schema, + CoreOptions options, const std::optional& identifier, + const std::optional& table_id, + const std::map& fs_scheme_map) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + FileStoreTable::Create(std::make_unique(path, std::move(options)), schema, + identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)), + table_id.value_or(""), fs_scheme_map)); + return std::shared_ptr
(std::move(table)); +} +} // namespace + +Result> TableFactory::Create(const WriteContext* ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + + // A table the caller already loaded says what it is, so nothing is read to find out. + if (ctx->GetFormatTable() != nullptr) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, + FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); + return std::shared_ptr
(given_table); + } + + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && + ctx->GetCatalog() != nullptr) { + if (!ctx->GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the write uses the credentials of this table. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx->GetOptions(), specific_fs, + ctx->GetFileSystemSchemeToIdentifierMap())); + std::string branch = ctx->GetBranch(); + if (ctx->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); + } + std::optional catalog_identifier; + if (ctx->GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE( + Identifier branch_identifier, + CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); + catalog_identifier.emplace(std::move(branch_identifier)); + } + // Only the main branch may have a current schema the catalog alone holds. A branch publishes + // its own under `branch/branch-`, which is where the load below and a read of the branch + // both read it, so a catalog answering for no branch schema does not fail a write to one. + std::optional catalog_table_schema; + if (ctx->GetCatalog() != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog write requires a table identifier"); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); + } + } + // A format table writes plain data files into a directory, so it never reaches the manifest + // path below. + auto schema_manager = + std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); + std::shared_ptr latest_schema; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr format_table, + FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch, + ctx->GetOptions(), catalog_table_schema, schema_manager.get(), + &latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr
(format_table); + } + // The schema the dispatch above already read through `schema_manager`, rather than a second + // read of the same file. + if (latest_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); + } + const std::shared_ptr& schema = latest_schema; + auto opts = schema->Options(); + for (const auto& [key, value] : ctx->GetOptions()) { + opts[key] = value; + } + // The metadata paths above are built for this branch, so what reads through these options, + // whose cache keys are scoped by it, has to read the same one. + opts[Options::BRANCH] = branch; + PAIMON_ASSIGN_OR_RAISE( + CoreOptions options, + CoreOptions::FromMap(opts, specific_fs, ctx->GetFileSystemSchemeToIdentifierMap())); + return ManagedTable(ctx->GetRootPath(), branch, schema, std::move(options), + ctx->GetIdentifier(), std::nullopt, + ctx->GetFileSystemSchemeToIdentifierMap()); +} + +Result> TableFactory::Create(const CommitContext* ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + + // A table the caller already loaded says what it is, so nothing is read to find out. + if (ctx->GetFormatTable() != nullptr) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, + FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); + return std::shared_ptr
(given_table); + } + + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + // Use catalog credentials unless the caller supplied a file system. + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetCatalog() != nullptr) { + if (!ctx->GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + // Use the credentials of this table, which a catalog issuing per-table temporary + // ones only hands out through GetTableFileSystem. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); + const std::string& root_path = ctx->GetRootPath(); + // The branch the context was built for, already validated by `CommitContextBuilder`. + PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( + ctx->GetIdentifier(), ctx->GetOptions(), + /*explicit_branch=*/std::nullopt, "commit")); + std::optional catalog_identifier; + if (ctx->GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE( + Identifier branch_identifier, + CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); + catalog_identifier.emplace(std::move(branch_identifier)); + } + // Catalog-managed schemas may be absent from the table directory, but only on the main branch: + // a branch publishes its own under `branch/branch-`, which is where the load below and a + // read of the branch both read it, so a catalog answering for no branch schema does not fail + // a commit to one. + std::optional catalog_table_schema; + if (ctx->GetCatalog() != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, + ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); + catalog_table_schema = std::move(schema_json); + } + } + + // A format table commits by renaming files into place, so it never reaches the snapshot path + // below. The managed path here reads the commit's branch, so this reads the same one: the two + // must not dispatch on different schemas. + auto schema_manager = + std::make_shared(tmp_options.GetFileSystem(), root_path, branch); + std::shared_ptr latest_schema; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + FormatTableLoader::TryLoad( + tmp_options.GetFileSystem(), root_path, branch, ctx->GetOptions(), + catalog_table_schema, schema_manager.get(), &latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr
(format_table); + } + if (latest_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); + } + const std::shared_ptr& schema = latest_schema; + auto opts = schema->Options(); + for (const auto& [key, value] : ctx->GetOptions()) { + opts[key] = value; + } + // The metadata paths below are built for this branch, so what reads through these options, + // whose cache keys are scoped by it, has to read the same one. + opts[Options::BRANCH] = branch; + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, specific_fs)); + return ManagedTable(ctx->GetRootPath(), branch, schema, std::move(options), + ctx->GetIdentifier(), ctx->GetTableId(), {}); +} + +Result> TableFactory::Create(const ScanContext* context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + + // A table the caller already loaded says what it is, so nothing is read to find out. + if (context->GetFormatTable() != nullptr) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); + return std::shared_ptr
(format_table); + } + // load schema + PAIMON_ASSIGN_OR_RAISE( + CoreOptions tmp_options, + CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); + if (context->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(tmp_options.GetBranch())); + } + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(context->GetPath())); + if (system_table_path) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), context->GetPath(), + context->GetOptions())); + return std::shared_ptr
(system_table); + } + // A format table is planned by listing directories, so it never reaches the manifest path + // below. One `TableScan` interface serves both, as Java Paimon serves both through one + // `ReadBuilder`. + std::shared_ptr latest_schema; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr format_table, + FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), context->GetPath(), + BranchManager::NormalizeBranch(tmp_options.GetBranch()), + context->GetOptions(), context->GetSpecificTableSchema(), + /*schema_manager=*/nullptr, &latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr
(format_table); + } + if (!latest_schema) { + return Status::Invalid("not found latest schema"); + } + auto options_map = latest_schema->Options(); + for (const auto& [key, value] : context->GetOptions()) { + options_map[key] = value; + } + PAIMON_ASSIGN_OR_RAISE( + std::string branch, + BranchManager::ResolveBranch(std::nullopt, options_map, std::nullopt, "scan")); + if (options_map.count(Options::BRANCH) != 0) { + options_map[Options::BRANCH] = branch; + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, + CoreOptions::FromMap(options_map, context->GetSpecificFileSystem(), {})); + return ManagedTable(context->GetPath(), branch, latest_schema, std::move(options), + context->GetIdentifier(), std::nullopt, {}); +} + +Result> TableFactory::Create(const ReadContext* context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + + // A table the caller already loaded says what it is, so nothing is read to find out. + if (context->GetFormatTable() != nullptr) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); + return std::shared_ptr
(format_table); + } + PAIMON_ASSIGN_OR_RAISE( + CoreOptions tmp_core_options, + CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), + context->GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(context->GetPath())); + if (system_table_path) { + if (context->EnablePreciseFileIndexEvaluate()) { + return Status::NotImplemented( + "Precise file-index evaluation is not supported for system tables"); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), context->GetPath(), + context->GetOptions(), + context->GetFileSystemSchemeToIdentifierMap())); + return std::shared_ptr
(system_table); + } + + // A format table has no manifest and a file another engine wrote carries no field ids, so + // which files there are and how a row is put back together is its own. How a file is opened + // is not: both paths go through `DataFileReaderFactory`. + std::shared_ptr latest_schema; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr format_table, + FormatTableLoader::TryLoad(tmp_core_options.GetFileSystem(), context->GetPath(), + context->GetBranch(), context->GetOptions(), + context->GetSpecificTableSchema(), + /*schema_manager=*/nullptr, &latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr
(format_table); + } + + if (!latest_schema) { + return Status::Invalid("not found latest schema"); + } + auto options_map = latest_schema->Options(); + for (const auto& [key, value] : context->GetOptions()) { + options_map[key] = value; + } + PAIMON_ASSIGN_OR_RAISE( + std::string branch, + BranchManager::ResolveBranch(std::nullopt, options_map, std::nullopt, "read")); + if (options_map.count(Options::BRANCH) != 0) { + options_map[Options::BRANCH] = branch; + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, + CoreOptions::FromMap(options_map, context->GetSpecificFileSystem(), + context->GetFileSystemSchemeToIdentifierMap())); + return ManagedTable(context->GetPath(), branch, latest_schema, std::move(options), + context->GetIdentifier(), std::nullopt, + context->GetFileSystemSchemeToIdentifierMap()); +} + +Result> TableScan::Create(std::unique_ptr context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); + return table->NewScan(std::move(context)); +} + +Result> TableRead::Create(std::unique_ptr context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); + return table->NewRead(std::move(context)); +} + +Result> FileStoreWrite::Create( + std::unique_ptr context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "write")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); + return table->NewWrite(std::move(context)); +} + +Result> FileStoreCommit::Create( + std::unique_ptr context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); + return table->NewCommit(std::move(context)); +} +} // namespace paimon diff --git a/src/paimon/core/table/table_factory.h b/src/paimon/core/table/table_factory.h new file mode 100644 index 000000000..7c560b752 --- /dev/null +++ b/src/paimon/core/table/table_factory.h @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +#include "paimon/result.h" +namespace paimon { +class Table; +class ScanContext; +class ReadContext; +class WriteContext; +class CommitContext; +/// Compatibility loading for the existing static operator factories. +class TableFactory { + public: + static Result> Create(const ScanContext* context); + static Result> Create(const ReadContext* context); + static Result> Create(const WriteContext* context); + static Result> Create(const CommitContext* context); +}; +} // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 4792a8461..a02f72d3f 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -18,6 +18,7 @@ #include "paimon/catalog/table.h" +#include #include #include #include @@ -27,18 +28,34 @@ #include "arrow/api.h" #include "gtest/gtest.h" #include "paimon/commit_context.h" +#include "paimon/common/catalog/catalog_context.h" #include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/catalog/file_system_catalog.h" +#include "paimon/core/core_options.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/append_only_file_store_table.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/primary_key_file_store_table.h" +#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/source/fallback_table_read.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/table/table_factory.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/realtime/realtime_context.h" #include "paimon/scan_context.h" #include "paimon/schema/schema.h" +#include "paimon/table/format/format_table.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -57,9 +74,11 @@ Result> CreateManagedTable(const std::shared_ptr created, schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{})); - return std::make_shared
( - file_system, path, - checked_pointer_cast(std::shared_ptr(std::move(created))), identifier); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + FileStoreTable::Create(file_system, path, std::shared_ptr(std::move(created)), + identifier, "", branch)); + return std::shared_ptr
(std::move(table)); } } // namespace @@ -94,7 +113,7 @@ TEST(TableTest, TestGetTableWithUnknownDatabase) { EXPECT_EQ(table->Name(), "tbl1"); EXPECT_EQ(table->FullName(), "tbl1"); - std::shared_ptr latest_schema = table->LatestSchema(); + std::shared_ptr latest_schema = table->GetSchema(); ASSERT_NE(latest_schema, nullptr); auto data_schema = std::dynamic_pointer_cast(latest_schema); ASSERT_TRUE(data_schema != nullptr); @@ -123,11 +142,11 @@ TEST(TableTest, TestGetTableWithBranch) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "tbl1$branch_b1"))); ASSERT_EQ("tbl1$branch_b1", table->Name()); - ASSERT_EQ((std::vector{"b1_c0"}), table->LatestSchema()->FieldNames()); + ASSERT_EQ((std::vector{"b1_c0"}), table->GetSchema()->FieldNames()); for (const auto& main_identifier : {Identifier("db1", "tbl1$branch_main"), Identifier("db1", "tbl1$branch_MAIN")}) { ASSERT_OK_AND_ASSIGN(table, catalog.GetTable(main_identifier)); - ASSERT_EQ((std::vector{"main_c0"}), table->LatestSchema()->FieldNames()); + ASSERT_EQ((std::vector{"main_c0"}), table->GetSchema()->FieldNames()); } Status missing_branch = catalog.GetTable(Identifier("db1", "tbl1$branch_missing")).status(); @@ -164,8 +183,12 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { SCOPED_TRACE(main_identifier.ToString()); // A catalog that loaded the schema itself calls the constructor directly, which a // rest catalog is shaped like. The alias resolves to the main branch as well. - Table table(fs, dir->Str(), schema, main_identifier); - ASSERT_OK_AND_ASSIGN(std::string branch, table.CurrentBranch()); + ASSERT_OK_AND_ASSIGN(std::string resolved_branch, main_identifier.GetBranchNameOrDefault()); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, dir->Str(), std::dynamic_pointer_cast(schema), + main_identifier, "", resolved_branch)); + std::string branch = table->CurrentBranch(); EXPECT_EQ("main", branch); } } @@ -178,14 +201,14 @@ TEST(TableTest, TestCurrentBranch) { ASSERT_OK_AND_ASSIGN( std::shared_ptr
main_table, CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); - ASSERT_OK_AND_ASSIGN(std::string branch, main_table->CurrentBranch()); + std::string branch = std::dynamic_pointer_cast(main_table)->CurrentBranch(); EXPECT_EQ(branch, "main"); // The identifier a table was opened with names the branch it is for. ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", Identifier("db", "tbl", "dev"), /*branch=*/"dev")); - ASSERT_OK_AND_ASSIGN(branch, branch_table->CurrentBranch()); + branch = std::dynamic_pointer_cast(branch_table)->CurrentBranch(); EXPECT_EQ(branch, "dev"); } @@ -197,24 +220,16 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); auto catalog = std::make_shared(); - catalog->SetTableSchema(table->LatestSchema()); - - // A catalog that hands out no per-table file system opens the table without one, and one - // that does opens it with the file system issued for that table. - ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog->GetTable(identifier)); - EXPECT_EQ(opened->GetFileSystem(), nullptr); + catalog->SetTableSchema(table->GetSchema()); catalog->SetTableFileSystem(dir->GetFileSystem()); - ASSERT_OK_AND_ASSIGN(opened, catalog->GetTable(identifier)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog->GetTable(identifier)); EXPECT_EQ(opened->GetFileSystem(), dir->GetFileSystem()); // This mock serves no table location, so the table it opens carries none. EXPECT_EQ(opened->GetRootPath(), ""); } -TEST(TableTest, TestGetTableRejectsATableTypeThatIsNotAManagedTable) { - // A `Table` describes a managed paimon table. Handing back another type as one would promise - // snapshots and manifests it never had, so the failure belongs at the call that opens the - // table rather than at the first read. +TEST(TableTest, TestGetTableDispatchesByTableType) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); @@ -230,16 +245,43 @@ TEST(TableTest, TestGetTableRejectsATableTypeThatIsNotAManagedTable) { format_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {{"type", "format-table"}, {"file.format", "parquet"}})); - ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db", "format")), "Cannot open format table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
format_table, + catalog.GetTable(Identifier("db", "format"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(format_table)); - // A managed table with no `type` option at all is what a `Table` is for, and still opens. const std::string managed_dir = dir->Str() + "/db.db/managed"; ASSERT_OK(fs->Mkdirs(managed_dir)); SchemaManager managed_schema_manager(fs, managed_dir); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr managed_schema, managed_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {})); - ASSERT_OK(catalog.GetTable(Identifier("db", "managed"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
managed_table, + catalog.GetTable(Identifier("db", "managed"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(managed_table)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
system_table, + catalog.GetTable(Identifier("db", "managed$options"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(system_table)); + ASSERT_EQ((std::vector{"key", "value"}), system_table->GetSchema()->FieldNames()); + ASSERT_EQ(nullptr, std::dynamic_pointer_cast(system_table->GetSchema())); + for (bool direct : {false, true}) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ReadContextBuilder(managed_dir + "$options") + .EnablePreciseFileIndexEvaluate(true) + .Finish()); + auto read = direct ? system_table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); + ASSERT_NOK_WITH_MSG(read, + "Precise file-index evaluation is not supported for system tables"); + } + ASSERT_NOK_WITH_MSG(system_table->NewWrite(nullptr), "read-only"); + ASSERT_NOK_WITH_MSG(system_table->NewCommit(nullptr), "read-only"); + + const std::string pk_dir = dir->Str() + "/db.db/pk"; + SchemaManager pk_schema_manager(fs, pk_dir); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr pk_schema, + pk_schema_manager.CreateTable(schema, {}, {"id"}, {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
pk_table, catalog.GetTable(Identifier("db", "pk"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(pk_table)); } TEST(TableTest, TestGetTableFailedWithNonExistSchema) { @@ -313,7 +355,7 @@ TEST(TableTest, TestNewWriteCreatesAFileStoreWrite) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); // A catalog write reads the current schema from the catalog on the main branch. - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(table->GetSchema()); ASSERT_OK_AND_ASSIGN( std::unique_ptr ctx, @@ -332,7 +374,7 @@ TEST(TableTest, TestNewCommitCreatesAFileStoreCommit) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); // Like a write, a catalog commit reads the current schema from the catalog. - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(table->GetSchema()); ASSERT_OK_AND_ASSIGN( std::unique_ptr ctx, @@ -366,7 +408,7 @@ TEST(TableTest, TestCreateBuildsTheOperator) { auto catalog = std::make_shared(); catalog->SetFileSystem(dir->GetFileSystem()); // A catalog write and commit read the current schema from the catalog on the main branch. - catalog->SetTableSchema(table->LatestSchema()); + catalog->SetTableSchema(table->GetSchema()); // `TableScan::Create()` and friends build the operator straight from the context, which // carries the catalog and identifier it resolves the table through. @@ -393,4 +435,410 @@ TEST(TableTest, TestCreateBuildsTheOperator) { FileStoreCommit::Create(std::move(commit_ctx))); } +TEST(TableTest, TestLoadedSchemaAndPathAreUsedWithoutReloading) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"))); + auto schema = table->GetSchema(); + ASSERT_OK(fs->Delete(dir->Str() + "/schema", true)); + ASSERT_EQ(schema, table->GetSchema()); + ASSERT_EQ((std::vector{"id"}), schema->FieldNames()); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read_context, + ReadContextBuilder(dir->Str() + "/other").SetReadFieldNames({"id"}).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str() + "/other").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_TRUE(plan->Splits().empty()); +} + +TEST(TableTest, TestContextOptionsCannotChangeLoadedTypeOrBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); + for (const auto& options : {std::map{{Options::TYPE, "format-table"}}, + std::map{{Options::BRANCH, "dev"}}}) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(dir->Str()).SetOptions(options).Finish()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), + "contradicts the loaded table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(dir->Str()).SetOptions(options).Finish()); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), + "contradicts the loaded table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + WriteContextBuilder(dir->Str(), "user").SetOptions(options).Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), + "contradicts the loaded table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + CommitContextBuilder(dir->Str(), "user").SetOptions(options).Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), + "contradicts the loaded table"); + } +} + +TEST(TableTest, TestOperationOptionsDoNotMutateTheLoadedTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); + auto schema = std::dynamic_pointer_cast(table->GetSchema()); + ASSERT_NE(nullptr, schema); + auto options = schema->Options(); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr invalid_context, + ScanContextBuilder(dir->Str()).SetOptions({{Options::BUCKET, "0"}}).Finish()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(invalid_context)), "bucket=0"); + ASSERT_EQ(options, schema->Options()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + table->NewScan(std::move(context))); +} + +TEST(TableTest, TestPathIdentifierPreservesBranch) { + Identifier catalog_table = CatalogUtils::IdentifierFromPath("/warehouse/db.db/tbl/", "dev"); + ASSERT_EQ("db", catalog_table.GetDatabaseName()); + ASSERT_EQ("tbl$branch_dev", catalog_table.GetObjectName()); + for (const std::string path : {"tbl", "/warehouse/db.db.extra/tbl"}) { + Identifier table = CatalogUtils::IdentifierFromPath(path, "dev"); + ASSERT_EQ(Identifier::kUnknownDatabase, table.GetDatabaseName()); + ASSERT_EQ("tbl$branch_dev", table.GetObjectName()); + } +} + +TEST(TableTest, TestGlobalSystemTableRetainsCatalog) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr unique_catalog, + Catalog::Create(dir->Str(), {}, dir->GetFileSystem())); + std::shared_ptr catalog(std::move(unique_catalog)); + std::weak_ptr weak_catalog = catalog; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("sys", "tables"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(table)); + ASSERT_EQ(nullptr, std::dynamic_pointer_cast(table->GetSchema())); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(table->GetRootPath()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(table->GetRootPath()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); + catalog.reset(); + table.reset(); + ASSERT_FALSE(weak_catalog.expired()); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + ASSERT_TRUE(BatchReader::IsEofBatch(batch)); + reader->Close(); + reader.reset(); + read.reset(); + ASSERT_TRUE(weak_catalog.expired()); +} + +TEST(TableTest, TestExplicitContextBranchMustMatchLoadedTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + for (const std::string table_branch : {"main", "dev"}) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/" + table_branch, + Identifier("db", "tbl", table_branch), table_branch)); + for (const std::optional& requested : + {std::optional(), std::optional("main"), + std::optional("dev"), std::optional(" ")}) { + SCOPED_TRACE(table_branch + ":" + requested.value_or("unset")); + WriteContextBuilder write_builder(table->GetRootPath(), "user"); + ReadContextBuilder read_builder(table->GetRootPath()); + if (requested) { + write_builder.WithBranch(*requested); + read_builder.WithBranch(*requested); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + auto write = table->NewWrite(std::move(write_context)); + auto read = table->NewRead(std::move(read_context)); + if (requested && BranchManager::NormalizeBranch(*requested) != table_branch) { + ASSERT_NOK_WITH_MSG(write, "context branch contradicts the loaded table"); + ASSERT_NOK_WITH_MSG(read, "context branch contradicts the loaded table"); + } else { + ASSERT_OK(write); + ASSERT_OK(read); + } + } + } +} + +TEST(TableTest, TestPreloadedSchemaBranchMatchesScanAndRead) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/table"; + ASSERT_TRUE(TestUtil::CopyDirectory( + std::filesystem::path( + GetDataDir() + "/parquet/append_table_with_rt_branch.db/append_table_with_rt_branch"), + std::filesystem::path(path))); + SnapshotManager main_snapshots(fs, path); + ASSERT_OK_AND_ASSIGN(Snapshot original, main_snapshots.LoadSnapshot(1)); + rapidjson::Document snapshot_json; + snapshot_json.SetObject(); + snapshot_json.CopyFrom(original.ToJson(&snapshot_json.GetAllocator()), + snapshot_json.GetAllocator()); + snapshot_json[Snapshot::FIELD_ID].SetInt64(2); + std::string json; + ASSERT_TRUE(RapidJsonUtil::ToJson(snapshot_json, &json)); + ASSERT_OK(fs->AtomicStore(main_snapshots.SnapshotPath(2), json)); + ASSERT_OK(main_snapshots.CommitLatestHint(2)); + SchemaManager schemas(fs, path, "rt"); + ASSERT_OK_AND_ASSIGN(std::optional> schema, schemas.Latest()); + ASSERT_TRUE(schema); + ASSERT_OK_AND_ASSIGN(std::unique_ptr preloaded, + schema.value()->MergeOptions({{Options::BRANCH, "rt"}})); + ASSERT_OK_AND_ASSIGN(std::string schema_json, preloaded->ToJsonString()); + for (const auto& overrides : {std::map{}, + std::map{{Options::BRANCH, "rt"}}, + std::map{{Options::BRANCH, "main"}}}) { + const std::string branch = overrides.empty() ? "rt" : overrides.at(Options::BRANCH); + SCOPED_TRACE(branch); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan_context, + ScanContextBuilder(path).SetTableSchema(schema_json).SetOptions(overrides).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(branch == "rt" ? 1 : 2, plan->SnapshotId()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read_context, + ReadContextBuilder(path).SetTableSchema(schema_json).SetOptions(overrides).Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + TableFactory::Create(read_context.get())); + ASSERT_EQ(branch, std::dynamic_pointer_cast(loaded)->CurrentBranch()); + // Explicit non-main loads keep using that branch's persisted schema. + const auto& expected_options = overrides.count(Options::BRANCH) != 0 && branch == "rt" + ? schema.value()->Options() + : preloaded->Options(); + ASSERT_EQ(expected_options, + std::dynamic_pointer_cast(loaded->GetSchema())->Options()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + loaded->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, + ReadResultCollector::CollectResult(std::move(reader))); + SnapshotManager snapshots(fs, path, branch); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, snapshots.LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_EQ(snapshot->TotalRecordCount(), rows->length()); + } + ASSERT_OK_AND_ASSIGN( + std::unique_ptr explicit_main, + ReadContextBuilder(path).SetTableSchema(schema_json).WithBranch("main").Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, TableFactory::Create(explicit_main.get())); + ASSERT_EQ("main", std::dynamic_pointer_cast(loaded)->CurrentBranch()); +} + +TEST(TableTest, TestResolvedStoreUsesOptionsBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::BRANCH, "dev"}}, dir->GetFileSystem())); + auto store = std::make_unique(dir->Str(), options); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr loaded, + FileStoreTable::Create(std::move(store), + std::dynamic_pointer_cast(table->GetSchema()), + Identifier("db", "tbl", "dev"), "")); + ASSERT_EQ("dev", loaded->CurrentBranch()); + ASSERT_EQ("dev", loaded->GetSnapshotManager()->Branch()); +} + +TEST(TableTest, TestLoadedTableReadRetainsOperationSchemeMap) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::string path = dir->Str() + "/db.db/tbl"; + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
created, + CreateManagedTable(dir->GetFileSystem(), path, Identifier("db", "tbl"))); + FileSystemCatalog catalog(dir->GetFileSystem(), dir->Str(), {}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db", "tbl"))); + const std::map scheme_map = {{"custom", "local"}}; + for (bool direct : {false, true}) { + SCOPED_TRACE(direct); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr context, + ReadContextBuilder(path).WithFileSystemSchemeToIdentifierMap(scheme_map).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + direct ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context))); + auto* append_read = dynamic_cast(read.get()); + ASSERT_NE(nullptr, append_read); + ASSERT_EQ( + scheme_map, + append_read->context_->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); + if (direct) { + ASSERT_EQ(table->GetFileSystem(), + append_read->context_->GetCoreOptions().GetFileSystem()); + } + } +} + +TEST(TableTest, TestReadMergesSchemeMapsForMainAndFallback) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main, + CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"))); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
dev, + CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl", "dev"), "dev")); + const std::map table_map = {{"base", "local"}, {"shared", "mock_fs"}}; + const std::map read_map = {{"custom", "local"}, {"shared", "local"}}; + const std::map expected = { + {"base", "local"}, {"custom", "local"}, {"shared", "local"}}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, dir->Str(), + std::dynamic_pointer_cast(main->GetSchema()), + Identifier("db", "tbl"), "", "main", table_map)); + for (bool fallback : {false, true}) { + SCOPED_TRACE(fallback); + ReadContextBuilder builder("unused_path"); + builder.WithFileSystemSchemeToIdentifierMap(read_map); + if (fallback) { + builder.AddOption(Options::SCAN_FALLBACK_BRANCH, "dev"); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(context))); + std::vector reads; + if (fallback) { + auto* fallback_read = dynamic_cast(read.get()); + ASSERT_NE(nullptr, fallback_read); + reads = {fallback_read->main_table_.get(), fallback_read->fallback_table_.get()}; + } else { + reads = {read.get()}; + } + for (TableRead* data_read : reads) { + auto* append_read = dynamic_cast(data_read); + ASSERT_NE(nullptr, append_read); + const auto& internal = append_read->context_; + ASSERT_EQ(expected, + internal->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); + ASSERT_EQ(fs, internal->GetCoreOptions().GetFileSystem()); + ASSERT_EQ(dir->Str(), internal->GetPath()); + } + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ReadContextBuilder("unused_path").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(context))); + auto* append_read = dynamic_cast(read.get()); + ASSERT_NE(nullptr, append_read); + ASSERT_EQ(table_map, + append_read->context_->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); +} + +TEST(TableTest, TestSystemTablePathRetainsSchemeMapForDependencyReads) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::shared_ptr
data_table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); + const std::map scheme_map = {{"file", "local"}}; + for (bool reload_context : {false, true}) { + SCOPED_TRACE(reload_context); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ReadContextBuilder(dir->Str() + "$ro") + .WithFileSystemSchemeToIdentifierMap(scheme_map) + .Finish()); + std::unique_ptr read; + if (reload_context) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, TableFactory::Create(context.get())); + ASSERT_OK_AND_ASSIGN(context, ReadContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(read, table->NewRead(std::move(context))); + } else { + ASSERT_OK_AND_ASSIGN(read, TableRead::Create(std::move(context))); + } + auto* append_read = dynamic_cast(read.get()); + ASSERT_NE(nullptr, append_read); + const auto& internal = append_read->context_; + ASSERT_EQ(scheme_map, internal->GetFileSystemSchemeToIdentifierMap()); + ASSERT_EQ(scheme_map, + internal->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); + } +} + +TEST(TableTest, TestSystemTableReadMergesSchemeMapsWithoutReplacingFileSystem) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
data_table, + CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"))); + const std::map table_map = {{"base", "local"}}; + const std::map read_map = {{"file", "local"}}; + const std::map expected = {{"base", "local"}, {"file", "local"}}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + SystemTableLoader::Load("ro", fs, dir->Str(), + std::dynamic_pointer_cast(data_table->GetSchema()), {}, + "tbl", table_map)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr context, + ReadContextBuilder("unused_path").WithFileSystemSchemeToIdentifierMap(read_map).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(context))); + auto* append_read = dynamic_cast(read.get()); + ASSERT_NE(nullptr, append_read); + const auto& internal = append_read->context_; + ASSERT_EQ(fs, internal->GetCoreOptions().GetFileSystem()); + ASSERT_EQ(dir->Str(), internal->GetPath()); + ASSERT_EQ(expected, internal->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); +} + +TEST(TableTest, TestRealtimeFallbackOnlyAllowsMainBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + auto arrow_schema = arrow::schema({arrow::field("id", arrow::int32(), false)}); + const std::map table_options = {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}}; + SchemaManager main_schemas(fs, dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_schema, + main_schemas.CreateTable(arrow_schema, {}, {}, table_options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, dir->Str(), std::shared_ptr(std::move(main_schema)), + Identifier("db", "tbl"), "", "main")); + SchemaManager dev_schemas(fs, dir->Str(), "dev"); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr dev_schema, + dev_schemas.CreateTable(arrow_schema, {}, {}, table_options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime, RealtimeContext::Create()); + for (bool realtime_enabled : {false, true}) { + for (const std::string fallback : {"main", "dev"}) { + SCOPED_TRACE(fallback + (realtime_enabled ? ":realtime" : ":disk")); + ReadContextBuilder builder(dir->Str()); + builder.SetOptions({{Options::SCAN_FALLBACK_BRANCH, fallback}}); + if (realtime_enabled) { + builder.WithRealtimeContext(realtime).AddOption(Options::REALTIME_ENABLED, "true"); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + auto read = table->NewRead(std::move(context)); + if (realtime_enabled && fallback != "main") { + ASSERT_NOK_WITH_MSG(read, "real-time operations only support the main branch"); + } else { + ASSERT_OK(read); + } + } + } +} + } // namespace paimon::test diff --git a/src/paimon/core/utils/context_utils.h b/src/paimon/core/utils/context_utils.h new file mode 100644 index 000000000..5e52e64a0 --- /dev/null +++ b/src/paimon/core/utils/context_utils.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "fmt/format.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// Shared operator-context checks. +class ContextUtils { + public: + ContextUtils() = delete; + ~ContextUtils() = delete; + + /// Requires a context, memory pool and executor; kind labels errors. + template + static Status CheckOperatorContext(const ContextPointer& context, const char* kind) { + if (context == nullptr) { + return Status::Invalid(fmt::format("{} context is null pointer", kind)); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + return Status::OK(); + } +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 273804d0d..8f23150f1 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -32,8 +32,10 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" @@ -448,19 +450,88 @@ Result> RestCatalog::LoadFormatTable( /*dynamic_options=*/{}); } +Result> RestCatalog::LoadSystemTable( + const Identifier& identifier, const Identifier& load_identifier, + const std::optional& branch) const { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::string data_table_path; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, + LoadDataTableSchema(load_identifier, branch, &data_table_path, + /*table_id=*/nullptr)); + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, + GetTableFileSystem(identifier)); + // Use the catalog name, including any branch, even when data lives under a UUID directory. + PAIMON_ASSIGN_OR_RAISE( + Identifier data_identifier, + CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); + return SystemTableLoader::Load(system_table_name.value(), table_file_system, data_table_path, + latest_schema, dynamic_options, data_identifier.GetObjectName(), + fs_scheme_to_identifier_map_); +} + Result> RestCatalog::GetTable(const Identifier& identifier) const { - // Read UUID, path and schema together to avoid mixing table generations after recreation. + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, + GlobalSystemTableLoader::IsSupported(identifier.GetObjectName(), + api_->GetMergedOptions())); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::shared_ptr self = weak_from_this().lock(); + if (self == nullptr) { + return Status::Invalid( + "a global system table retains its catalog, so the catalog must be owned by " + "a std::shared_ptr before it is asked for one"); + } + GlobalSystemTableContext context; + context.catalog = self.get(); + context.catalog_owner = std::move(self); + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = api_->GetMergedOptions(); + return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + if (is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + return LoadSystemTable(identifier, load_identifier, NormalizeBranch(std::move(branch))); + } + // Load UUID, location and schema together to avoid mixing table generations. + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + branch = NormalizeBranch(std::move(branch)); + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); std::string uuid; std::string table_path; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - LoadTableSchema(identifier, &uuid, &table_path)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + LoadDataTableSchema(load_identifier, branch, &table_path, &uuid)); + std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); + if (data_schema != nullptr) { + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(data_schema->Options())); + if (is_format_table) { + return FormatTable::Create(fs_, table_path, identifier, data_schema, + /*location_carries_paimon_metadata=*/false, + /*dynamic_options=*/{}); + } + } PAIMON_RETURN_NOT_OK( CatalogUtils::CheckManagedTableType(identifier, schema, "Catalog::GetTable")); // The table holds the file system its own data is served by, which a server issuing data // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - return std::make_shared
(table_file_system, table_path, schema, identifier, uuid); + return FileStoreTable::Create(table_file_system, table_path, schema, identifier, uuid, + branch.value_or(Identifier::kDefaultMainBranch), + fs_scheme_to_identifier_map_); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 2f57e912d..fc1db3d55 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -35,6 +35,7 @@ struct ArrowSchema; namespace paimon { +class SystemTable; class FileSystem; class TableSchema; @@ -124,6 +125,9 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { const Identifier& identifier) const override; private: + Result> LoadSystemTable( + const Identifier& identifier, const Identifier& load_identifier, + const std::optional& branch) const; RestCatalog(std::shared_ptr api, const std::shared_ptr& fs, const std::string& warehouse, bool data_token_enabled, bool fs_explicitly_supplied, const std::map& fs_scheme_to_identifier_map); diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 147a908ce..0f710bb47 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -49,15 +49,21 @@ #include "paimon/core/core_options.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" #include "paimon/rest/mock_rest_server.h" #include "paimon/rest/rest_api.h" +#include "paimon/scan_context.h" #include "paimon/schema/schema.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" #include "paimon/testing/utils/snapshot_test_helper.h" #include "paimon/testing/utils/testharness.h" @@ -810,6 +816,32 @@ TEST_F(RestCatalogTest, CatalogFactoryMetastoreDispatch) { ASSERT_NOK_WITH_MSG(status, "unsupported metastore"); } +TEST_F(RestCatalogTest, GlobalSystemTableRetainsFactoryCatalog) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr unique_catalog, + Catalog::Create(kWarehouse, options_)); + std::shared_ptr catalog(std::move(unique_catalog)); + std::weak_ptr weak_catalog = catalog; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("sys", "tables"))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(table->GetRootPath()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(table->GetRootPath()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); + catalog.reset(); + table.reset(); + ASSERT_FALSE(weak_catalog.expired()); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + ASSERT_TRUE(BatchReader::IsEofBatch(batch)); + reader->Close(); + reader.reset(); + read.reset(); + ASSERT_TRUE(weak_catalog.expired()); +} + TEST_F(RestCatalogTest, CreateWithWrongTokenFails) { options_[CatalogOptions::TOKEN] = "wrong-token"; ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "not authorized"); @@ -918,8 +950,7 @@ TEST_F(RestCatalogTest, TableOperations) { std::lock_guard lock(state_->mutex); ASSERT_EQ(requests_before_get_table + 1, state_->get_table_requests); } - std::shared_ptr schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, schema); ASSERT_EQ((std::vector{"f0", "f1"}), schema->FieldNames()); ASSERT_EQ((std::vector{"f0"}), schema->PrimaryKeys()); @@ -937,7 +968,7 @@ TEST_F(RestCatalogTest, TableOperations) { // timeMillis is backed by the server's audit "updatedAt" instead of the current // time, keeping the conversion deterministic std::shared_ptr table_schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, table_schema); ASSERT_EQ(123, table_schema->TimeMillis()); @@ -959,7 +990,7 @@ TEST_F(RestCatalogTest, TableOperations) { } ASSERT_OK_AND_ASSIGN(std::shared_ptr
commented_table, catalog->GetTable(Identifier("db1", "commented"))); - ASSERT_EQ("a table comment", commented_table->LatestSchema()->Comment().value_or("")); + ASSERT_EQ("a table comment", commented_table->GetSchema()->Comment().value_or("")); ASSERT_OK(catalog->RenameTable(identifier, Identifier("db1", "t2"), /*ignore_if_not_exists=*/false)); @@ -1062,8 +1093,8 @@ TEST_F(RestCatalogTest, SystemTableSchema) { } ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); ASSERT_EQ("t1$options", table->Name()); - ASSERT_EQ((std::vector{"key", "value"}), table->LatestSchema()->FieldNames()); - ASSERT_EQ("db1.t1$options", table->Uuid()); + ASSERT_EQ((std::vector{"key", "value"}), table->GetSchema()->FieldNames()); + ASSERT_EQ("t1$options", table->Uuid()); { std::lock_guard lock(state_->mutex); ASSERT_EQ(requests_before + 1, state_->get_table_requests); @@ -1105,8 +1136,7 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(branch_identifier)); ASSERT_EQ("t1$branch_b1", table->Name()); - std::shared_ptr schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, schema); // the branch's own schema and schema id, not the main table's ASSERT_EQ((std::vector{"b0"}), schema->FieldNames()); @@ -1118,7 +1148,7 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_table, catalog->GetTable(Identifier("db1", "t1$branch_main"))); std::shared_ptr main_schema = - std::dynamic_pointer_cast(main_table->LatestSchema()); + std::dynamic_pointer_cast(main_table->GetSchema()); ASSERT_NE(nullptr, main_schema); ASSERT_EQ((std::vector{"f0", "f1"}), main_schema->FieldNames()); ASSERT_EQ(0, main_schema->Options().count(Options::BRANCH)); @@ -1128,13 +1158,14 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_case_table, catalog->GetTable(Identifier("db1", "t1$branch_MAIN"))); std::shared_ptr main_case_schema = - std::dynamic_pointer_cast(main_case_table->LatestSchema()); + std::dynamic_pointer_cast(main_case_table->GetSchema()); ASSERT_NE(nullptr, main_case_schema); ASSERT_EQ((std::vector{"f0", "f1"}), main_case_schema->FieldNames()); ASSERT_EQ(0, main_case_schema->Options().count(Options::BRANCH)); // the branch the alias spells resolves to the main branch, not to a branch // directory named after the alias - ASSERT_OK_AND_ASSIGN(std::string main_case_branch, main_case_table->CurrentBranch()); + std::string main_case_branch = + std::dynamic_pointer_cast(main_case_table)->CurrentBranch(); ASSERT_EQ("main", main_case_branch); // a system table on a branch resolves against the branch's data table: the system @@ -1275,7 +1306,9 @@ TEST_F(RestCatalogTest, FormatTableIsLoadedInOneRequest) { // directory named "schema" there is a partition value, not metadata to skip. ASSERT_FALSE(table->LocationCarriesPaimonMetadata()); - ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "fmt")), "Cannot open format table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, + catalog->GetTable(Identifier("db1", "fmt"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(opened)); // A managed table stays out of this path, and so does a system table. ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); @@ -1389,8 +1422,7 @@ TEST_F(RestCatalogTest, NestedSchemaHighestFieldId) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("db1", "nested"))); - std::shared_ptr schema = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, schema); ASSERT_EQ(5, schema->Id()); // 9 lives inside the value row of the map: ROW, ARRAY element and MAP key/value @@ -1484,8 +1516,7 @@ TEST_F(RestCatalogTest, PartitionKeysRoundTrip) { /*ignore_if_exists=*/false)); // the partition keys survive both the create request and the load response conversion ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("db1", "pt"))); - std::shared_ptr loaded = - std::dynamic_pointer_cast(table->LatestSchema()); + std::shared_ptr loaded = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, loaded); ASSERT_EQ((std::vector{"f1"}), loaded->PartitionKeys()); } diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 1a2f0b7d7..5f8a8d52f 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -34,7 +34,9 @@ #include "paimon/core/catalog/commit_lock.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/partition/partition_statistics.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/result.h" #include "paimon/schema/schema.h" #include "paimon/status.h" @@ -115,9 +117,19 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { // Serves the per-table file system when one was set, matching `GetTableFileSystem` // without recording the request, so a test can still tell the two apart. No location: // this mock's `GetTableLocation` is the unsupported stub. - return std::make_shared
( - table_file_system_ != nullptr ? table_file_system_ : file_system_, - /*root_path=*/"", table_schema_, identifier, table_uuid_); + if (!table_schema_) { + return Status::NotExist("table schema not set"); + } + PAIMON_ASSIGN_OR_RAISE(std::string json, table_schema_->GetJsonSchema()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr schema, + TableSchema::CreateFromJson(json)); + PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + FileStoreTable::Create( + table_file_system_ != nullptr ? table_file_system_ : file_system_, "", + std::shared_ptr(std::move(schema)), identifier, table_uuid_, branch)); + return std::shared_ptr
(std::move(table)); } void SetTableSchema(const std::shared_ptr& table_schema) { diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index ca889342c..43642fd92 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -38,6 +38,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/factories/io_hook.h" @@ -49,6 +50,7 @@ #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/read_ahead_cache.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/snapshot.h" @@ -924,6 +926,49 @@ TEST(SystemTableReadInteTest, TestReadMetadataSystemTablesAtPinnedSnapshot) { "snapshot 3 does not exist"); ASSERT_NOK_WITH_MSG(ReadSystemTable(table_path + "$manifests", missing_options), "snapshot 3 does not exist"); + + FileSystemCatalog catalog(dir->GetFileSystem(), dir->Str(), {}); + for (const std::string kind : {"files", "manifests"}) { + SCOPED_TRACE(kind); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog.GetTable(Identifier("foo", "bar$" + kind))); + const auto bound_schema = table->GetSchema(); + auto create_reader = [&table](const std::map& read_options) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan_context, + ScanContextBuilder("unused_path").SetOptions(read_options).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + table->NewScan(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr read_context, + ReadContextBuilder("unused_path").SetOptions(read_options).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + table->NewRead(std::move(read_context))); + return read->CreateReader(plan->Splits()); + }; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr pinned_reader, + create_reader(pinned_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr latest_reader, create_reader({})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr missing_reader, + create_reader(missing_options)); + ASSERT_NOK_WITH_MSG(ReadResultCollector::CollectResult(std::move(missing_reader)), + "snapshot 3 does not exist"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr pinned, + ReadResultCollector::CollectResult(std::move(pinned_reader))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr latest, + ReadResultCollector::CollectResult(std::move(latest_reader))); + ASSERT_TRUE(pinned->Equals(kind == "files" ? pinned_files : pinned_manifests)); + ASSERT_TRUE(latest->Equals(kind == "files" ? latest_files : latest_manifests)); + ASSERT_EQ(bound_schema, table->GetSchema()); + for (const auto& overrides : + {std::map{{Options::BRANCH, "dev"}}, + std::map{{Options::TYPE, "format-table"}}}) { + ASSERT_NOK_WITH_MSG(create_reader(overrides), "contradicts the loaded table"); + } + } } TEST(SystemTableReadInteTest, TestReadOptimizedSystemTable) { @@ -1688,6 +1733,57 @@ TEST(SystemTableReadInteTest, TestReadAuditLogSystemTableWithSequenceNumber) { ])"); } +TEST(SystemTableReadInteTest, TestChangelogSequenceNumberCannotBeDisabledByReadOptions) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), + arrow::field("v", arrow::int32())}; + const std::map options = { + {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "1"}, + {Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, "true"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(dir->Str(), arrow::schema(fields), {}, {"pk"}, options, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["a", 1]])", {}, 0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), 0, std::nullopt)); + FileSystemCatalog catalog(dir->GetFileSystem(), dir->Str(), {}); + const std::map read_options = { + {Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "false"}}; + for (const std::string kind : {"audit_log", "binlog"}) { + SCOPED_TRACE(kind); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog.GetTable(Identifier("foo", "bar$" + kind))); + for (bool direct : {false, true}) { + SCOPED_TRACE(direct); + const std::string path = table->GetRootPath() + "$" + kind; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(path).SetOptions(read_options).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + direct ? table->NewScan(std::move(scan_context)) + : TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder(path).SetOptions(read_options).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + direct ? table->NewRead(std::move(read_context)) + : TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadResultCollector::CollectResult(std::move(reader))); + auto array = SingleStructChunk(result); + ASSERT_EQ((std::vector{"rowkind", "_SEQUENCE_NUMBER", "pk", "v"}), + StructFieldNames(array)); + AssertStructArrayEqualsJson(array, kind == "audit_log" ? R"([["+I", 0, "a", 1]])" + : R"([["+I", 0, ["a"], [1]]])"); + } + } +} + TEST(SystemTableReadInteTest, TestReadBinlogSystemTable) { arrow::FieldVector fields = { arrow::field("pk", arrow::utf8()), @@ -4249,18 +4345,18 @@ Result> ReadGlobalSystemTable( ScanContextBuilder scan_context_builder(sys_path); scan_context_builder.SetOptions(options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr scan_context, + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - system_table->NewScan(scan_context)); + system_table->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); ReadContextBuilder read_context_builder(sys_path); read_context_builder.SetOptions(options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - system_table->NewRead(read_context)); + system_table->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result,