From c5ba731f91fc57c62028c2e1a846b8d96c0d360d Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:02:21 +0800 Subject: [PATCH 1/3] docs: correct license attribution and simplify Vortex comments (#518) (cherry picked from commit 520acdf79f2e3bef70cd2c9c8512b2119f202822) --- LICENSE | 15 +++- crates/vortex_callback_io/callback_io.rs | 92 +++++------------------- 2 files changed, 32 insertions(+), 75 deletions(-) diff --git a/LICENSE b/LICENSE index fc8d26538..a672c4a10 100644 --- a/LICENSE +++ b/LICENSE @@ -400,7 +400,8 @@ This product includes code from Apache ORC. * Upstream source patches: * cmake_modules/orc.diff * Adapted read-ahead cache: - * include/paimon/utils/read_ahead_cache.h (adapted from Cache.hh) + * include/paimon/utils/prefetch_cache_config.h (adapted from Cache.hh) + * src/paimon/common/utils/read_ahead_cache.h (adapted from Cache.hh) * src/paimon/common/utils/read_ahead_cache.cpp (adapted from Cache.cc) * Adapted byte range combiner: * src/paimon/common/utils/byte_range_combiner.h (adapted from Cache.hh) @@ -413,6 +414,18 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- +This product includes code from Vortex 0.75.0. + +* Callback I/O sink utilities in crates/vortex_callback_io/callback_io.rs + (adapted from vortex-ffi/src/sink.rs) +* Upstream source patch in cmake_modules/vortex.diff + +Copyright the Vortex contributors +Home page: https://github.com/vortex-data/vortex +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + This product includes code from xxHash. * MMH_rotl32 utility in src/paimon/common/utils/murmurhash_utils.h diff --git a/crates/vortex_callback_io/callback_io.rs b/crates/vortex_callback_io/callback_io.rs index f4d5fb857..d3d2faadc 100644 --- a/crates/vortex_callback_io/callback_io.rs +++ b/crates/vortex_callback_io/callback_io.rs @@ -14,37 +14,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Callback-based I/O for the Vortex C API. +//! Callback I/O for Vortex using Paimon's `InputStream` and `OutputStream`. //! -//! The stock C API can only read a whole file already materialized in memory -//! (`vx_data_source_new_buffer`) or a path that Vortex itself resolves -//! (`vx_data_source_new`), and it can only write to a local filesystem path -//! (`vx_array_sink_open_file`, which uses `async_fs::File::create`). paimon owns its own pluggable -//! filesystem, so none of these fit: reading forces the entire file into memory, and writing forces -//! a local temporary file that then has to be copied back. +//! Copied into `vortex-ffi/src/` at build time; see `cmake_modules/vortex.diff`. +//! Keep FFI types and signatures in sync with `src/paimon/format/vortex/vortex_ffi.h`. //! -//! This module adds entry points that take host callbacks instead, so Vortex issues positional -//! reads and sequential writes straight against paimon's `InputStream` / `OutputStream`. Vortex's -//! core I/O traits are the plug points; the shape of the bridge mirrors `vortex-jni`'s -//! `JavaReadable` and `JavaWrite`. -//! -//! The write side needs its own handle type rather than reusing `vx_array_sink`, whose fields are -//! private to `crate::sink`. -//! -//! The sink entry points below (`vx_callback_sink_push` / `_close` / `_abort`) are adapted from -//! vortex-ffi's `src/sink.rs` (SPDX-License-Identifier: Apache-2.0, Copyright the Vortex -//! contributors), reworked to drive the write pipeline through host callbacks instead of a -//! Vortex-resolved filesystem path. -//! -//! This file is maintained in the paimon-cpp tree (`crates/vortex_callback_io/`) and copied into -//! `vortex-ffi/src/` at build time; see `cmake_modules/vortex.diff`. -//! -//! The C declarations that mirror the types and entry points below live in paimon-cpp's -//! `src/paimon/format/vortex/vortex_ffi.h` (cbindgen does not see this module, so they are written -//! by hand). The two sides are only checked by human review, and a mismatch in field order or a -//! function signature is silent memory corruption, not a compile error. Any change to a -//! `#[repr(C)]` struct, a callback `type`, or a `#[no_mangle]` entry point here MUST be mirrored in -//! that header in the same change. +//! Sink entry points are adapted from Vortex's `vortex-ffi/src/sink.rs` +//! (SPDX-License-Identifier: Apache-2.0, Copyright the Vortex contributors). use std::ffi::c_void; use std::io; @@ -137,11 +113,7 @@ pub struct vx_output_callbacks { pub release_fn: Option, } -/// The host context, owned by the reader. -/// -/// Held behind an `Arc` so in-flight reads keep the context alive, and so `release_fn` runs exactly -/// once when the last reference goes away. It carries only the context, not the read callback, so -/// that ownership can be taken over before anything else is validated. +/// Shared host context, released when the last reader or writer reference is dropped. struct CallbackTarget { ctx: *mut c_void, release_fn: Option, @@ -164,11 +136,7 @@ impl Drop for CallbackTarget { } } -/// A [`VortexReadAt`] that forwards positional reads to host callbacks. -/// -/// Reads run on the runtime's blocking pool, since the host callback is synchronous and must not -/// occupy an async executor thread. The file size is supplied at construction time, so `size()` -/// never crosses the FFI boundary. +/// A [`VortexReadAt`] that runs host callbacks on the blocking pool. struct CallbackReadAt { target: Arc, read_at_fn: vx_read_at_fn, @@ -177,16 +145,12 @@ struct CallbackReadAt { concurrency: usize, } -/// Number of concurrent host read callbacks Vortex may have in flight for one file. -/// -/// The host filesystem may be remote, where concurrency hides latency; it is also the bound on -/// blocking threads occupied by one reader. +/// Maximum concurrent host reads per file, also bounding blocking-pool usage. const DEFAULT_CONCURRENCY: usize = 16; impl VortexReadAt for CallbackReadAt { fn coalesce_config(&self) -> Option { - // Each read costs an FFI hop plus a blocking-pool hand-off, and the host filesystem may be - // remote, so favor fewer and larger reads. + // Coalesce reads to reduce remote I/O and blocking-pool overhead. Some(CoalesceConfig::object_storage()) } @@ -255,8 +219,7 @@ unsafe fn data_source_new_callback( callbacks: vx_input_callbacks, size: u64, ) -> VortexResult<*const vx_data_source> { - // Take ownership of the host context before anything else, so an early error below still - // releases it exactly once when this `Arc` is dropped. + // Take ownership before validation so errors also release the context. let target = Arc::new(CallbackTarget { ctx: callbacks.ctx, release_fn: callbacks.release_fn, @@ -282,11 +245,8 @@ unsafe fn data_source_new_callback( /// Create a data source that reads through host callbacks. /// -/// `size` is the total length of the file in bytes; the host knows it up front, so it is passed -/// here instead of being fetched through another callback. -/// -/// The callbacks (and the context they carry) are owned by the returned data source and released -/// when it is freed, including when this call fails. +/// `size` is the total file length in bytes. Takes ownership of the host context, +/// which is released when the data source is freed or this call fails. /// /// On error, returns NULL and sets "err". /// @@ -306,11 +266,7 @@ pub unsafe extern "C-unwind" fn vx_data_source_new_callback( }) } -/// A [`VortexWrite`] that forwards sequential writes to host callbacks. -/// -/// Writes run inline on the runtime thread driving the write task, the same way `vortex-jni` -/// forwards to Java sinks: the host callback is synchronous and the layout writer hands over one -/// buffer at a time, so there is nothing to overlap. +/// A [`VortexWrite`] that calls host callbacks inline on the writer's runtime thread. struct CallbackWrite { target: Arc, write_fn: vx_write_fn, @@ -364,11 +320,7 @@ impl VortexWrite for CallbackWrite { } } -/// A sink that writes a Vortex file through host callbacks. -/// -/// Mirrors `vx_array_sink`: pushed arrays go through a channel feeding a write task on the session's -/// runtime, and errors surface when the sink is closed. This is a separate type because -/// `vx_array_sink`'s fields are private to its own module. +/// A Vortex file sink backed by host callbacks and a background write task. pub struct vx_callback_sink { sink: Sender>, writer: Task>, @@ -380,8 +332,7 @@ unsafe fn callback_sink_open( callbacks: vx_output_callbacks, dtype: *const vx_dtype, ) -> VortexResult<*mut vx_callback_sink> { - // Take ownership of the host context before anything else, so an early error below still - // releases it exactly once when this `Arc` is dropped. + // Take ownership before validation so errors also release the context. let target = Arc::new(CallbackTarget { ctx: callbacks.ctx, release_fn: callbacks.release_fn, @@ -401,7 +352,7 @@ unsafe fn callback_sink_open( flush_fn: callbacks.flush_fn, }; - // Small on purpose so the producing thread feels back-pressure, as in vortex-jni's writer. + // Limit queued arrays to apply backpressure to the producer. let (sink, rx) = mpsc::channel(4); let array_stream = ArrayStreamAdapter::new(dtype.clone(), rx.into_stream()); @@ -557,7 +508,6 @@ mod tests { use crate::sink::vx_array_sink_open_file; use crate::sink::vx_array_sink_push; - /// Host context standing in for paimon's `InputStream`: positional reads against a file. struct FileContext { file: File, } @@ -584,8 +534,7 @@ mod tests { drop(unsafe { Box::from_raw(ctx.cast::()) }); } - /// Writing a file with the stock sink and reading it back through the callback data source - /// must yield the same dtype and row count: the callback plug point is wired up correctly. + // Read a file written by the standard sink through host callbacks. #[test] #[cfg_attr(miri, ignore)] fn test_callback_data_source_round_trip() { @@ -638,8 +587,6 @@ mod tests { } } - /// A rejected request must still release the host context: the entry point takes ownership - /// before it validates anything, so the host never has to guess whether to clean up. #[test] #[cfg_attr(miri, ignore)] fn test_rejected_callbacks_release_context() { @@ -652,7 +599,6 @@ mod tests { unsafe { let session = vx_session_new(); let mut error = ptr::null_mut(); - // No read callback, so the call must fail. let callbacks = vx_input_callbacks { ctx: ptr::null_mut(), read_at_fn: None, @@ -672,12 +618,10 @@ mod tests { } } - /// Writing through the callback sink and reading the bytes back through the callback data source - /// must round-trip, with no filesystem involved in either direction. + // Round-trip through an in-memory buffer using only callbacks. #[test] #[cfg_attr(miri, ignore)] fn test_callback_sink_round_trip() { - /// Host context standing in for a paimon stream pair: an in-memory byte buffer. struct BufferContext { data: Mutex>, } From b52b0d4abf8c36362ff4fd416dc35014c9541db6 Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Fri, 6 Nov 2026 14:52:36 +0800 Subject: [PATCH 2/3] docs: align guides and CRoaring modification notices (#521) (cherry picked from commit c84520bb50dca0d5a7de6e083962a88f9e3bd7d9) --- docs/source/basic_concepts.rst | 10 +++-- docs/source/build_system.rst | 3 ++ docs/source/user_guide/append_only_table.rst | 14 ++++-- docs/source/user_guide/clean.rst | 27 +++++++++--- docs/source/user_guide/commit.rst | 35 +++++++++++---- docs/source/user_guide/manifest.rst | 14 ++++-- docs/source/user_guide/snapshot.rst | 46 +++++++++++++------- docs/source/user_guide/system_tables.rst | 9 +++- docs/source/user_guide/write.rst | 8 +++- third_party/roaring_bitmap/roaring.cpp | 2 + third_party/roaring_bitmap/roaring.h | 2 + third_party/roaring_bitmap/roaring.hh | 2 + 12 files changed, 126 insertions(+), 46 deletions(-) diff --git a/docs/source/basic_concepts.rst b/docs/source/basic_concepts.rst index 0af523056..42e62924f 100644 --- a/docs/source/basic_concepts.rst +++ b/docs/source/basic_concepts.rst @@ -23,8 +23,9 @@ Basic Concepts File Layouts ------------------------ -All files of a table are stored under one base directory. Paimon files are -organized in a layered style. The following image illustrates the file layout. +In a file-system deployment, table metadata is stored under the table's base +directory. Paimon files are organized in a layered style. The following image +illustrates the file layout. Starting from a snapshot file, Paimon readers can recursively access all records from the table. @@ -35,7 +36,10 @@ from the table. Snapshot ------------------- -All snapshot files are stored in the snapshot directory. +Direct file-system commits publish snapshot files in the snapshot directory. +A catalog that manages table versions may instead hold the latest snapshot +without publishing it there. See :doc:`user_guide/catalog` for the file-system +metadata still required by reads, recovery and maintenance with a catalog. A snapshot file is a JSON file containing information about this snapshot, including the schema file in use the manifest list containing all changes of diff --git a/docs/source/build_system.rst b/docs/source/build_system.rst index 44495f73c..3b168fdfc 100644 --- a/docs/source/build_system.rst +++ b/docs/source/build_system.rst @@ -48,6 +48,9 @@ for data format and file system. project(MyExample) + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + # Arrow's static package may export LZ4 under a different target name. find_package(lz4 CONFIG QUIET) if(TARGET LZ4::lz4_static AND NOT TARGET LZ4::lz4) diff --git a/docs/source/user_guide/append_only_table.rst b/docs/source/user_guide/append_only_table.rst index e1320ede8..4ed6061b5 100644 --- a/docs/source/user_guide/append_only_table.rst +++ b/docs/source/user_guide/append_only_table.rst @@ -20,7 +20,13 @@ Append Only Table ================= -If a table does not have a primary key defined, it is an append table. Compared -to the primary key table, it does not have the ability to directly receive changelogs. -It cannot be directly updated with data through upsert. It can only receive -incoming data from append data. +If a table does not have a primary key defined, it is an append table. +Ordinary append writes accept ``INSERT`` rows only and do not apply upserts +or deletes by key. + +The experimental real-time append deduplicate mode is an exception. With +``realtime.deduplicate-key-fields`` configured, it also accepts ``UPDATE_AFTER``, +``DELETE`` and ``UPDATE_BEFORE``, using deletion vectors to hide superseded +rows. The table still has no primary key; deduplication is scoped to each +partition-bucket. See :doc:`realtime` for the required schema, options and +shared context. diff --git a/docs/source/user_guide/clean.rst b/docs/source/user_guide/clean.rst index d0df8186f..5715c4b84 100644 --- a/docs/source/user_guide/clean.rst +++ b/docs/source/user_guide/clean.rst @@ -19,7 +19,9 @@ Data Cleanup =================================== .. note:: - The entire data cleanup feature supports only append tables, and cleanup of index manifests, changelog, statistics are not supported. Do not use this feature with primary key tables. + Cleanup supports append and primary-key tables, subject to the restrictions + below. Index manifests and index payloads are included. Cleanup of changelog + and statistics files is not supported. This document describes three cleanup capabilities: @@ -35,9 +37,15 @@ Orphan File Cleanup Description ~~~~~~~~~~~~~~~~~~~ -- Orphan file cleanup currently supports only append tables. +- Orphan file cleanup supports append and primary-key tables. - It runs as an independent task: construct a ``CleanContext`` and launch an ``OrphanFilesCleaner``. +- It covers table-local index manifests and payloads as well as data files, + preserving files referenced by retained snapshots. It does not enumerate a + potentially shared ``global-index.external-path``. +- Tables with tags or branches, or snapshots with a changelog manifest list, + are rejected with ``NotImplemented``. Tag retention during snapshot expiration + is supported separately, as described below. Detailed Steps ~~~~~~~~~~~~~~ @@ -62,7 +70,7 @@ Performance Considerations .. admonition:: TODO :class: tip - - Support cleanup of index manifests, changelog, statistics, etc. + - Support cleanup of changelog and statistics files. .. _expiring-partitions: @@ -109,10 +117,17 @@ Description - ``snapshot.time-retained``: The maximum age of completed snapshots to retain. Default: 1 hour. - ``snapshot.expire.limit``: The maximum number of snapshots allowed - to expire at a time. Default: 10. + to expire at a time. Default: 50. + - ``snapshot.clean-empty-directories``: Remove empty bucket and partition + directories after deleting their data files, without deleting the table + directory. Default: false. - The snapshot expiration interface deletes data files according to the expiration policy and returns the number of snapshots deleted. +- Data files, index manifests and index payloads referenced by retained + snapshots or current-branch tags are preserved. Retired index payloads are + deleted before their index manifests, including payloads stored under + ``global-index.external-path``. - Expiration on a branch other than main, or on a table with such a branch under ``branch/branch-``, is not supported: branches share the table's data files, so expiration returns ``NotImplemented`` before deleting anything. @@ -149,7 +164,7 @@ Detailed Steps * ``DELETE`` then ``ADD`` → the initial ``DELETE`` does not apply (file did not exist yet); the subsequent ``ADD`` ensures the file remains. 5. Clean meta files: - - Preserve manifests used by the last snapshot in the cleanup range (``end_exclusive_id``). + - Preserve manifests used by retained snapshots and current-branch tags. - Delete manifest files used by snapshots from ``begin_inclusive_id`` to ``end_exclusive_id`` (exclusive) and delete the snapshot files themselves. 6. Rewrite ``EarliestHint`` to ``end_exclusive_id``. 7. Return the number of snapshots deleted, i.e., ``end_exclusive_id - begin_inclusive_id``. @@ -164,6 +179,4 @@ Performance Considerations .. admonition:: TODO :class: tip - - Preserve tag (savepoint) data via ``tagManager``. - Delete changelog files. - - Remove empty directories. diff --git a/docs/source/user_guide/commit.rst b/docs/source/user_guide/commit.rst index 7e7ee1715..121462cb9 100644 --- a/docs/source/user_guide/commit.rst +++ b/docs/source/user_guide/commit.rst @@ -35,7 +35,7 @@ The Commit process consists of the following steps: 2. Compact (merge) Manifest files 3. Generate the Base Manifest List 4. Generate new Manifest files and the Delta Manifest List -5. Generate the Snapshot and HINT file +5. Prepare and publish the Snapshot Detailed Process ------------------- @@ -110,17 +110,34 @@ The initially collected file change information is written into new manifest fil Metadata for these newly created manifest files is then written into the delta manifest list. -Generate the Snapshot and HINT File -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Prepare and Publish the Snapshot +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ After the above steps are completed, the system generates a new Snapshot and performs the following operations: 1. Determine the new Snapshot ID based on the latest ``SnapshotId + 1``. 2. Record metadata such as ``schema id``, ``commit time``, and ``total record count``. -3. Atomicity guarantee: Generating a Snapshot is an atomic operation. If an exception - (e.g., an I/O error) occurs during the process, the manifest files and manifest list files - generated in Steps 2–4 are cleaned up and removed. -4. The Snapshot is written via a rename operation to ensure atomicity. -5. After the Snapshot is successfully written, the system writes the ``LATEST`` hint file - to reduce list operations when fetching the latest Snapshot. +3. If preparation fails before publication, the newly generated manifest files and + manifest list files are cleaned up. If publication returns an error and its + outcome is uncertain, the metadata files are retained because the snapshot may + already have been accepted. + +Snapshot publication depends on the commit mode: + +* **Direct file-system commit:** On non-object-store paths, the Snapshot is + published through an atomic rename. After publication, the committer writes + the ``LATEST`` hint file to reduce list operations when fetching the latest + Snapshot. +* **Catalog-mediated commit:** When ``CommitContextBuilder::WithCatalog()`` + supplies a catalog that manages table versions, the committer sends the + Snapshot and change statistics to that catalog for publication. This path + does not rename a local Snapshot file or write a local ``LATEST`` hint. +* **REST request preparation:** ``UseRESTCatalogCommit(true)`` builds a commit + request without publishing the Snapshot. The caller retrieves the JSON with + ``GetLastCommitTableRequest()`` and sends it to the REST catalog. A successful + local ``Commit()`` in this mode means the request was prepared, not that the + catalog accepted it. + +Object-store paths require one of the catalog modes. See :doc:`write` for +configuration and :doc:`catalog` for catalog publication and recovery semantics. diff --git a/docs/source/user_guide/manifest.rst b/docs/source/user_guide/manifest.rst index 4c1240da1..937c1c966 100644 --- a/docs/source/user_guide/manifest.rst +++ b/docs/source/user_guide/manifest.rst @@ -68,8 +68,8 @@ Data file metadata: 1. ``_FILE_NAME``: STRING — file name. 2. ``_FILE_SIZE``: BIGINT — file size. 3. ``_ROW_COUNT``: BIGINT — total number of rows (including add & delete) in this file. -4. ``_MIN_KEY``: STRING — minimum key of this file. -5. ``_MAX_KEY``: STRING — maximum key of this file. +4. ``_MIN_KEY``: BYTES — minimum key of this file, encoded as a BinaryRow. +5. ``_MAX_KEY``: BYTES — maximum key of this file, encoded as a BinaryRow. 6. ``_KEY_STATS``: SimpleStats — statistics of the key. 7. ``_VALUE_STATS``: SimpleStats — statistics of the value. 8. ``_MIN_SEQUENCE_NUMBER``: BIGINT — minimum sequence number. @@ -83,6 +83,11 @@ Data file metadata: 16. ``_FILE_SOURCE``: TINYINT — indicates whether this file is generated as an ``APPEND`` or ``COMPACT`` file. 17. ``_VALUE_STATS_COLS``: ARRAY — statistical columns in metadata. 18. ``_EXTERNAL_PATH``: STRING — external path of this file; ``null`` if it is in the warehouse. +19. ``_FIRST_ROW_ID``: BIGINT — first row id of this file for row tracking; nullable. +20. ``_WRITE_COLS``: ARRAY — columns written in this file for data evolution; nullable. +21. ``_WRITE_COLS_SEQUENCES``: ARRAY — maximum sequence number per physical table field, + in the order selected by ``_WRITE_COLS`` (excluding system fields), or the file schema order + when ``_WRITE_COLS`` is null. Nullable; when absent, only the file-level sequence range is available. Index Manifest -------------- @@ -99,7 +104,7 @@ Schema: 1. ``_KIND``: TINYINT — ``ADD`` or ``DELETE``. 2. ``_PARTITION``: BYTES — partition spec, a BinaryRow. 3. ``_BUCKET``: INT — bucket of this file. -4. ``_INDEX_TYPE``: STRING — ``HASH`` or ``DELETION_VECTORS``. +4. ``_INDEX_TYPE``: STRING — index type identifier, including global index types such as ``btree``. 5. ``_FILE_NAME``: STRING — file name. 6. ``_FILE_SIZE``: BIGINT — file size. 7. ``_ROW_COUNT``: BIGINT — total number of rows. @@ -108,6 +113,9 @@ Schema: - ``f1``: the starting offset of this deletion vector in the index file. - ``f2``: the length of this deletion vector in the index file. - ``_CARDINALITY``: the number of deleted rows. +9. ``_EXTERNAL_PATH``: STRING — external path of the index file; nullable. +10. ``_GLOBAL_INDEX``: global index metadata, including the row range, indexed field id, + optional extra field ids, index metadata and optional source metadata; nullable. Appendix -------- diff --git a/docs/source/user_guide/snapshot.rst b/docs/source/user_guide/snapshot.rst index 278e72af4..98c643038 100644 --- a/docs/source/user_guide/snapshot.rst +++ b/docs/source/user_guide/snapshot.rst @@ -21,10 +21,17 @@ Snapshot ======== -Each commit generates a snapshot file, and the version of the snapshot file starts from 1 and must be continuous. +Direct file-system commits publish snapshot files with consecutive snapshot ids +starting from 1. The layout and hint files below describe those published files. ``EARLIEST`` and ``LATEST`` are hint files at the beginning and end of the snapshot list, and they can be inaccurate. When hint files are inaccurate, the reader will scan all snapshot files to determine the beginning and end. +A catalog that manages table versions can hold the latest snapshot without a +snapshot file under the table path. Publication then depends on catalog +acceptance. ``UseRESTCatalogCommit(true)`` only prepares the request for the +caller to send; it does not publish a snapshot. See :doc:`commit` and +:doc:`catalog` for these modes and their historical metadata requirements. + Directory Layout ---------------- @@ -40,8 +47,8 @@ Directory Layout ├── snapshot-2 └── snapshot-3 -Writing commit will preempt the next snapshot id, and once the snapshot file is successfully written, this commit will -become visible. +A direct file-system commit attempts to publish the next snapshot id. Once the +snapshot file is successfully published, the commit becomes visible. Snapshot File ------------- @@ -53,16 +60,23 @@ Snapshot file is JSON and includes: 3. ``id``: Snapshot id, same as the file name. 4. ``schemaId``: The corresponding schema version for this commit. 5. ``baseManifestList``: A manifest list recording all changes from the previous snapshots. -6. ``deltaManifestList``: A manifest list recording all new changes occurred in this snapshot. -7. ``changelogManifestList``: A manifest list recording all changelog produced in this snapshot; ``null`` if no changelog is produced. -8. ``indexManifest``: A manifest recording all index files of this table; ``null`` if no table index file exists. -9. ``commitUser``: Usually generated by UUID; used for recovery of streaming writes—one stream write job with one user. -10. ``commitIdentifier``: Transaction id corresponding to streaming write; each transaction may result in multiple commits for different ``commitKind`` values. -11. ``commitKind``: Type of changes in this snapshot, including ``append``, ``compact``, ``overwrite`` and ``analyze``. -12. ``timeMillis``: Commit time in milliseconds. -13. ``logOffsets``: Commit log offsets. -14. ``totalRecordCount``: Record count of all changes occurred in this snapshot. -15. ``deltaRecordCount``: Record count of all new changes occurred in this snapshot. -16. ``changelogRecordCount``: Record count of all changelog produced in this snapshot. -17. ``watermark``: Watermark for input records, from Flink watermark mechanism; ``Long.MIN_VALUE`` if there is no watermark. -18. ``statistics``: Stats file name for statistics of this table. +6. ``baseManifestListSize``: Size in bytes of the base manifest list; optional. +7. ``deltaManifestList``: A manifest list recording all new changes occurred in this snapshot. +8. ``deltaManifestListSize``: Size in bytes of the delta manifest list; optional. +9. ``changelogManifestList``: A manifest list recording all changelog produced in this snapshot; omitted by C++ serialization if unset. +10. ``changelogManifestListSize``: Size in bytes of the changelog manifest list; optional. +11. ``indexManifest``: A manifest recording all index files of this table; omitted by C++ serialization if unset. +12. ``commitUser``: Usually generated by UUID; used for recovery of streaming writes—one stream write job with one user. +13. ``commitIdentifier``: Transaction id corresponding to streaming write; each transaction may result in multiple commits for different ``commitKind`` values. +14. ``commitKind``: Type of changes in this snapshot, including ``append``, ``compact``, ``overwrite`` and ``analyze``. +15. ``timeMillis``: Commit time in milliseconds. +16. ``totalRecordCount``: Record count of all changes occurred in this snapshot. +17. ``deltaRecordCount``: Record count of all new changes occurred in this snapshot. +18. ``changelogRecordCount``: Record count of all changelog produced in this snapshot; optional. +19. ``watermark``: Watermark for input records. When unset, C++ serialization omits this field. + A commit carries forward the previous snapshot's watermark if no new watermark is supplied. +20. ``statistics``: Stats file name for statistics of this table; optional. +21. ``properties``: Optional string-to-string map of snapshot properties, including real-time progress metadata when enabled. +22. ``nextRowId``: Next row id for row tracking; optional. + +The C++ snapshot serializer does not emit ``logOffsets``. diff --git a/docs/source/user_guide/system_tables.rst b/docs/source/user_guide/system_tables.rst index 843640eef..7df0311c9 100644 --- a/docs/source/user_guide/system_tables.rst +++ b/docs/source/user_guide/system_tables.rst @@ -39,12 +39,17 @@ case-insensitive, and an unrecognized suffix fails with * ``$tags``: tags and the snapshots they reference. * ``$branches``: table branches, including ``main``. * ``$consumers``: persisted streaming consumer offsets. -* ``$manifests``: data manifest metadata in the latest snapshot. -* ``$files``: data file metadata in the latest snapshot. +* ``$manifests``: data manifest metadata in the selected snapshot. +* ``$files``: data file metadata in the selected snapshot. * ``$audit_log``: row-level changelog records with rowkind. * ``$binlog``: changelog records with list-wrapped data columns. * ``$ro``: read-optimized data, described in the next section. +``$manifests`` and ``$files`` read the latest snapshot by default. Set +``scan.snapshot-id`` to read metadata from a specific snapshot of the selected +branch. If that snapshot does not exist, the read fails with +``snapshot does not exist`` instead of falling back to the latest snapshot. + Global System Tables -------------------- diff --git a/docs/source/user_guide/write.rst b/docs/source/user_guide/write.rst index 9d14c2414..97c4dd30b 100644 --- a/docs/source/user_guide/write.rst +++ b/docs/source/user_guide/write.rst @@ -89,8 +89,12 @@ whose values are ``LargeBinary``. Build the BLOB field with PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_element, paimon::Blob::ArrowField("element", /*nullable=*/true)); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr element, - arrow::ImportField(c_element.get())); + arrow::Result> element_result = + arrow::ImportField(c_element.get()); + if (!element_result.ok()) { + return paimon::Status::Invalid(element_result.status().ToString()); + } + std::shared_ptr element = element_result.ValueUnsafe(); std::shared_ptr schema = arrow::schema( {arrow::field("id", arrow::int32()), arrow::field("frames", arrow::list(element)), arrow::field("views", arrow::map(arrow::utf8(), element))}); diff --git a/third_party/roaring_bitmap/roaring.cpp b/third_party/roaring_bitmap/roaring.cpp index 2c704def0..03a32bef4 100644 --- a/third_party/roaring_bitmap/roaring.cpp +++ b/third_party/roaring_bitmap/roaring.cpp @@ -1,3 +1,5 @@ +// Modified by Apache Paimon to use the paimon::roaring namespace. + // !!! DO NOT EDIT - THIS IS AN AUTO-GENERATED FILE !!! // Created by amalgamation.sh on 2023-12-16T02:52:00Z diff --git a/third_party/roaring_bitmap/roaring.h b/third_party/roaring_bitmap/roaring.h index 2a9bf3b28..9bec9874c 100644 --- a/third_party/roaring_bitmap/roaring.h +++ b/third_party/roaring_bitmap/roaring.h @@ -1,3 +1,5 @@ +// Modified by Apache Paimon to use the paimon::roaring namespace. + // !!! DO NOT EDIT - THIS IS AN AUTO-GENERATED FILE !!! // Created by amalgamation.sh on 2023-12-16T02:52:00Z diff --git a/third_party/roaring_bitmap/roaring.hh b/third_party/roaring_bitmap/roaring.hh index 4e829cd32..4ddedec95 100644 --- a/third_party/roaring_bitmap/roaring.hh +++ b/third_party/roaring_bitmap/roaring.hh @@ -1,3 +1,5 @@ +// Modified by Apache Paimon to use the paimon::roaring namespace. + // !!! DO NOT EDIT - THIS IS AN AUTO-GENERATED FILE !!! // Created by amalgamation.sh on 2023-12-16T02:52:00Z From 86c0c16d2f5900dec5a40372107ae3e4291a16b9 Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Fri, 6 Nov 2026 14:53:44 +0800 Subject: [PATCH 3/3] fix(read): validate byte ranges and honor disabled prefetch cache (#522) (cherry picked from commit b1d588761512956f6d4f7e5028d853a2a657d0a0) --- ...e_materializing_file_batch_reader_test.cpp | 55 +++++++++++++++++++ .../prefetch_file_batch_reader_impl.cpp | 23 +++++--- src/paimon/common/utils/read_ahead_cache.cpp | 14 ++++- src/paimon/common/utils/read_ahead_cache.h | 8 ++- .../common/utils/read_ahead_cache_test.cpp | 52 ++++++++++++++++++ test/inte/read_inte_test.cpp | 6 +- .../write_and_read_inte_with_index_test.cpp | 9 ++- 7 files changed, 151 insertions(+), 16 deletions(-) diff --git a/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp b/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp index a95fd91a9..7b62862fa 100644 --- a/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp +++ b/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp @@ -32,6 +32,7 @@ #include "arrow/array/builder_dict.h" #include "arrow/array/builder_nested.h" #include "arrow/c/bridge.h" +#include "fmt/format.h" #include "gtest/gtest.h" #include "paimon/common/reader/late_materializing_reader_builder.h" #include "paimon/common/reader/prefetch_file_batch_reader_impl.h" @@ -942,6 +943,60 @@ TEST_F(LateMaterializingFileBatchReaderTest, PayloadPreBufferRangeErrorFailsRead EXPECT_EQ(calls, 0); } +TEST_F(LateMaterializingFileBatchReaderTest, DisabledPrefetchCacheSkipsRangeQueries) { + class UnavailableRangesBuilder : public MockFormatReaderBuilder { + public: + using MockFormatReaderBuilder::MockFormatReaderBuilder; + + Result> Build( + const std::shared_ptr& stream) const override { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + MockFormatReaderBuilder::Build(stream)); + checked_cast(reader.get()) + ->SetPreBufferRangeStatus(Status::IOError("pre-buffer range unavailable")); + return reader; + } + }; + + auto data = BuildData({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + auto mock_fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(2)); + for (WarmupLevel level : {WarmupLevel::NONE, WarmupLevel::RAW, WarmupLevel::DECODED}) { + LateMaterializingReaderBuilder builder( + std::make_unique(data, full_type_, /*batch_size=*/3), + GetArrowPool(pool_)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + PrefetchFileBatchReaderImpl::Create( + /*data_file_path=*/"", /*data_file_size=*/0, &builder, mock_fs, + /*prefetch_max_parallel_num=*/1, /*batch_size=*/3, /*prefetch_batch_count=*/2, + /*enable_adaptive_prefetch_strategy=*/false, executor, + /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/false, CacheConfig(), + /*enable_io_metrics=*/false, level, pool_, GetArrowPool(pool_))); + auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, + Literal(4l)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(full_fields_), &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, predicate, std::nullopt)); + reader->Warmup(); + + // Every range query fails, so successful probe/payload reads require skipping them all. + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + CollectStruct(reader.get())); + ASSERT_TRUE(result); + ASSERT_EQ(result->length(), 6); + auto keys = checked_pointer_cast(result->GetFieldByName("k")); + auto values = checked_pointer_cast(result->GetFieldByName("v")); + ASSERT_TRUE(keys); + ASSERT_TRUE(values); + for (int64_t i = 0; i < result->length(); ++i) { + ASSERT_EQ(keys->Value(i), i + 4); + ASSERT_EQ(values->GetString(i), fmt::format("v_{}", i + 4)); + } + reader->Close(); + } +} + // End to end below the prefetch layer: the ranges the inner LM reader reports mid-read must reach // the shared read-ahead cache and be fetched, on top of the probe ranges registered up front. TEST_F(LateMaterializingFileBatchReaderTest, PrefetchInnerRegistersPayloadPreBufferRanges) { diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp index 4e5e540e4..b3d89b2f7 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp @@ -322,15 +322,20 @@ Result> PrefetchFileBatchReaderImpl auto reader = std::unique_ptr(new PrefetchFileBatchReaderImpl( readers, batch_size, prefetch_queue_capacity, enable_adaptive_prefetch_strategy, executor, cache, io_metrics, warmup_level, arrow_pool)); - for (const auto& sub_reader : reader->readers_) { - // The sub-readers whose byte ranges only become known once they have read some data (the - // late-materialization payload pass) report them here instead of through PreBufferRange(), - // which is asked once per read-range generation before any read. Capturing the impl raw is - // safe: it owns readers_, and CleanUp() joins the background thread before it goes away. - sub_reader->SetPreBufferRangeCallback( - [impl = reader.get()](std::vector>&& ranges) { - impl->RegisterLatePreBufferRanges(std::move(ranges)); - }); + // LM uses callback presence to decide whether to query payload ranges. A disabled cache + // must not trigger those optional metadata queries. + if (cache) { + for (const auto& sub_reader : reader->readers_) { + // The sub-readers whose byte ranges only become known once they have read some data + // (the late-materialization payload pass) report them here instead of through + // PreBufferRange(), which is asked once per read-range generation before any read. + // Capturing the impl raw is safe: it owns readers_, and CleanUp() joins the background + // thread before it goes away. + sub_reader->SetPreBufferRangeCallback( + [impl = reader.get()](std::vector>&& ranges) { + impl->RegisterLatePreBufferRanges(std::move(ranges)); + }); + } } if (initialize_read_ranges) { // normally initialize read ranges should be false, as set read schema will refresh read diff --git a/src/paimon/common/utils/read_ahead_cache.cpp b/src/paimon/common/utils/read_ahead_cache.cpp index 68dc18034..a321c2e94 100644 --- a/src/paimon/common/utils/read_ahead_cache.cpp +++ b/src/paimon/common/utils/read_ahead_cache.cpp @@ -30,6 +30,7 @@ #include #include +#include "fmt/format.h" #include "paimon/common/memory/bytes_utils.h" #include "paimon/common/metrics/atomic_counter_pair.h" #include "paimon/common/utils/byte_range_combiner.h" @@ -69,6 +70,17 @@ void CopyRangeFromEntries(const std::vector& covering, const By } // namespace Result> ReadAheadCache::AddRanges(std::vector&& ranges) { + // Validate before coalescing can split an invalid range into many entries. Zero retains + // the existing unknown-size convention; the combiner still validates integer bounds. + if (file_size_ > 0) { + for (const auto& range : ranges) { + if (range.offset > file_size_ || range.length > file_size_ - range.offset) { + return Status::Invalid( + fmt::format("byte range (offset={}, length={}) exceeds file size {}", + range.offset, range.length, file_size_)); + } + } + } // Both an up-front registration and a mid-read one are cut at the plain size limit. PAIMON_ASSIGN_OR_RAISE( std::vector new_ranges, @@ -216,7 +228,7 @@ void ReadAheadCache::PreBuffer(uint64_t offset) { ReadAheadCache::ReadAheadCache(const std::shared_ptr& stream, const CacheConfig& config, uint64_t file_size, const std::shared_ptr& memory_pool) - : stream_(stream), config_(config), memory_pool_(memory_pool) { + : stream_(stream), config_(config), file_size_(file_size), memory_pool_(memory_pool) { // An unknown file size cannot be aligned to, and a zero limit or block size // means the block cache is turned off: leave it null in those cases. if (file_size > 0 && config_.GetBlockSize() > 0 && config_.GetBlockCacheLimit() > 0) { diff --git a/src/paimon/common/utils/read_ahead_cache.h b/src/paimon/common/utils/read_ahead_cache.h index 4eec73836..cf84ca09d 100644 --- a/src/paimon/common/utils/read_ahead_cache.h +++ b/src/paimon/common/utils/read_ahead_cache.h @@ -152,9 +152,9 @@ class PAIMON_EXPORT ReadAheadCache { /// Construct a read cache with given options /// @param stream The stream the cache fetches from. /// @param config The cache configuration. - /// @param file_size Size of the file behind `stream`, used to align the - /// block cache to the end of the file. Zero means unknown and disables the - /// block cache. + /// @param file_size Size of the file behind `stream`, used to validate registered ranges + /// and align the block cache to the end of the file. Zero means unknown, skipping file-size + /// validation and disabling the block cache. /// @param memory_pool The pool the cached buffers are allocated from. ReadAheadCache(const std::shared_ptr& stream, const CacheConfig& config, uint64_t file_size, const std::shared_ptr& memory_pool); @@ -174,6 +174,7 @@ class PAIMON_EXPORT ReadAheadCache { /// /// @param ranges The byte ranges to register. /// @return The offset of the first newly registered range, or nullopt when nothing was added. + /// Returns an error if a range exceeds the known file size or integer bounds. Result> AddRanges(std::vector&& ranges); /// Read a range previously registered through AddRanges(), copying the cached data @@ -261,6 +262,7 @@ class PAIMON_EXPORT ReadAheadCache { std::shared_ptr stream_; CacheConfig config_; + const uint64_t file_size_; // Every registered range, ordered by offset and disjoint (so a matching // region is found by binary search). An entry whose buffer is still null is // registered but not fetched yet; publishing fills buffer/future in place. diff --git a/src/paimon/common/utils/read_ahead_cache_test.cpp b/src/paimon/common/utils/read_ahead_cache_test.cpp index c9f7454a1..121db3469 100644 --- a/src/paimon/common/utils/read_ahead_cache_test.cpp +++ b/src/paimon/common/utils/read_ahead_cache_test.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -503,6 +504,57 @@ TEST(TestReadAheadCache, TestReinitAfterReset) { AssertReadMiss({20, 2}, &cache); } +TEST(TestReadAheadCache, TestAddRangesRejectsOutOfFileRanges) { + CacheConfig config = TestCacheConfig(/*range_size_limit=*/1024, + /*hole_size_limit=*/0, /*pre_buffer_limit=*/2048); + auto cache = CreateTestFileAndCache("data_file", std::string(1024, 'x'), config, {}, + /*file_size=*/1024); + constexpr uint64_t kMaxValue = std::numeric_limits::max(); + for (const ByteRange& range : std::vector{{512, 32 * 1024 * 1024}, + {1024, 1}, + {1025, 0}, + {1023, 2}, + {kMaxValue, 1}, + {1, kMaxValue}}) { + ASSERT_NOK_WITH_MSG(cache->AddRanges({range}), "exceeds file size 1024"); + } +} + +TEST(TestReadAheadCache, TestInvalidLateRangesPreserveRegisteredRanges) { + CacheConfig config = TestCacheConfig(/*range_size_limit=*/8, + /*hole_size_limit=*/0, /*pre_buffer_limit=*/32); + config.SetBlockCacheLimit(0); + auto cache = CreateTestFileAndCache("data_file", std::string(32, 'x'), config, {{0, 8}}, + /*file_size=*/32); + AssertReadEquals({0, 8}, std::string(8, 'x'), cache.get()); + + ASSERT_NOK_WITH_MSG(cache->AddRanges({{16, 8}, {31, 2}}), "exceeds file size 32"); + + AssertReadEquals({0, 8}, std::string(8, 'x'), cache.get()); + AssertReadMiss({16, 8}, cache.get()); +} + +TEST(TestReadAheadCache, TestAddRangesAcceptsFileEndBoundary) { + CacheConfig config = TestCacheConfig(/*range_size_limit=*/4, + /*hole_size_limit=*/0, /*pre_buffer_limit=*/16); + auto cache = CreateTestFileAndCache("data_file", "abcdefgh", config, {{4, 4}}, + /*file_size=*/8); + AssertReadEquals({4, 4}, "efgh", cache.get()); + ASSERT_OK_AND_ASSIGN(std::optional first_added, cache->AddRanges({{8, 0}})); + ASSERT_FALSE(first_added.has_value()); + ASSERT_OK(cache->AddRanges({{0, 8}}).status()); + AssertReadEquals({0, 8}, "abcdefgh", cache.get()); +} + +TEST(TestReadAheadCache, TestAddRangesPreservesUnknownFileSize) { + CacheConfig config = TestCacheConfig(/*range_size_limit=*/1024 * 1024, + /*hole_size_limit=*/0, /*pre_buffer_limit=*/2048); + auto cache = CreateTestFileAndCache("data_file", std::string(1024, 'x'), config, {}, + /*file_size=*/0); + // Zero retains its existing unknown-size meaning. Registration performs no I/O. + ASSERT_OK(cache->AddRanges({{512, 32 * 1024 * 1024}}).status()); +} + // Test that AddRanges() registers ranges that only become known after the cache was // initialized, so the reads that follow are served from the cache. TEST(TestReadAheadCache, TestAddRangesRegistersNewRanges) { diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 6f3aa0bea..ca889342c 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -172,8 +172,12 @@ class ReadInteTest : public testing::Test, public ::testing::WithParamInterface< EXPECT_TRUE(schema_ids.empty() || schema_ids.size() == data_files.size()); for (uint32_t j = 0; j < data_files.size(); j++) { const auto& data_file = data_files[j]; + EXPECT_OK_AND_ASSIGN( + FileStatus file_status, + LocalFileSystem().GetFileStatus(PathUtil::JoinPath(bucket_path, data_file))); data_file_metas.push_back(std::make_shared( - data_file, /*file_size=*/1, /*row_count=*/1, /*min_key=*/BinaryRow::EmptyRow(), + data_file, /*file_size=*/file_status.GetLen(), /*row_count=*/1, + /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(), /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0, /*max_sequence_number=*/0, /*schema_id=*/schema_ids.empty() ? 0 : schema_ids[j], diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp index efda3de03..7197bee75 100644 --- a/test/inte/write_and_read_inte_with_index_test.cpp +++ b/test/inte/write_and_read_inte_with_index_test.cpp @@ -56,6 +56,7 @@ #include "paimon/data/timestamp.h" #include "paimon/defs.h" #include "paimon/factories/factory_creator.h" +#include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" @@ -1749,11 +1750,13 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { auto create_data_file_meta = [&](const std::string& file_name, int32_t schema_id, const std::vector& embedded_bytes) { + EXPECT_OK_AND_ASSIGN(FileStatus file_status, + LocalFileSystem().GetFileStatus(path + "bucket-0/" + file_name)); auto embedded_index = std::make_shared(embedded_bytes.size(), pool_.get()); memcpy(embedded_index->data(), reinterpret_cast(embedded_bytes.data()), embedded_bytes.size()); return std::make_shared( - file_name, /*file_size=*/10, + file_name, /*file_size=*/file_status.GetLen(), /*row_count=*/10, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(), @@ -2431,8 +2434,10 @@ TEST_P(WriteAndReadInteWithIndexTest, TestBitmapPushDownWithMultiStripes) { } // each record is a stripe + ASSERT_OK_AND_ASSIGN(FileStatus file_status, + LocalFileSystem().GetFileStatus(path + "bucket-0/" + file_name)); auto data_file_meta = std::make_shared( - file_name, /*file_size=*/2637, + file_name, /*file_size=*/file_status.GetLen(), /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(), /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,