fix(release): backport license, docs and read-ahead fixes to 0.4

LICENSE

@@ -400,7 +400,8 @@ This product includes code from Apache ORC.
 * Upstream source patches:
   * cmake_modules/orc.diff
 * Adapted read-ahead cache:
-  * include/paimon/utils/read_ahead_cache.h (adapted from Cache.hh)
+  * include/paimon/utils/prefetch_cache_config.h (adapted from Cache.hh)
+  * src/paimon/common/utils/read_ahead_cache.h (adapted from Cache.hh)
   * src/paimon/common/utils/read_ahead_cache.cpp (adapted from Cache.cc)
 * Adapted byte range combiner:
   * src/paimon/common/utils/byte_range_combiner.h (adapted from Cache.hh)
@@ -413,6 +414,18 @@ License: https://www.apache.org/licenses/LICENSE-2.0
 
 --------------------------------------------------------------------------------
 
+This product includes code from Vortex 0.75.0.
+
+* Callback I/O sink utilities in crates/vortex_callback_io/callback_io.rs
+  (adapted from vortex-ffi/src/sink.rs)
+* Upstream source patch in cmake_modules/vortex.diff
+
+Copyright the Vortex contributors
+Home page: https://github.com/vortex-data/vortex
+License: https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
 This product includes code from xxHash.
 
 * MMH_rotl32 utility in src/paimon/common/utils/murmurhash_utils.h

crates/vortex_callback_io/callback_io.rs

@@ -14,37 +14,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Callback-based I/O for the Vortex C API.
+//! Callback I/O for Vortex using Paimon's `InputStream` and `OutputStream`.
 //!
-//! The stock C API can only read a whole file already materialized in memory
-//! (`vx_data_source_new_buffer`) or a path that Vortex itself resolves
-//! (`vx_data_source_new`), and it can only write to a local filesystem path
-//! (`vx_array_sink_open_file`, which uses `async_fs::File::create`). paimon owns its own pluggable
-//! filesystem, so none of these fit: reading forces the entire file into memory, and writing forces
-//! a local temporary file that then has to be copied back.
+//! Copied into `vortex-ffi/src/` at build time; see `cmake_modules/vortex.diff`.
+//! Keep FFI types and signatures in sync with `src/paimon/format/vortex/vortex_ffi.h`.
 //!
-//! This module adds entry points that take host callbacks instead, so Vortex issues positional
-//! reads and sequential writes straight against paimon's `InputStream` / `OutputStream`. Vortex's
-//! core I/O traits are the plug points; the shape of the bridge mirrors `vortex-jni`'s
-//! `JavaReadable` and `JavaWrite`.
-//!
-//! The write side needs its own handle type rather than reusing `vx_array_sink`, whose fields are
-//! private to `crate::sink`.
-//!
-//! The sink entry points below (`vx_callback_sink_push` / `_close` / `_abort`) are adapted from
-//! vortex-ffi's `src/sink.rs` (SPDX-License-Identifier: Apache-2.0, Copyright the Vortex
-//! contributors), reworked to drive the write pipeline through host callbacks instead of a
-//! Vortex-resolved filesystem path.
-//!
-//! This file is maintained in the paimon-cpp tree (`crates/vortex_callback_io/`) and copied into
-//! `vortex-ffi/src/` at build time; see `cmake_modules/vortex.diff`.
-//!
-//! The C declarations that mirror the types and entry points below live in paimon-cpp's
-//! `src/paimon/format/vortex/vortex_ffi.h` (cbindgen does not see this module, so they are written
-//! by hand). The two sides are only checked by human review, and a mismatch in field order or a
-//! function signature is silent memory corruption, not a compile error. Any change to a
-//! `#[repr(C)]` struct, a callback `type`, or a `#[no_mangle]` entry point here MUST be mirrored in
-//! that header in the same change.
+//! Sink entry points are adapted from Vortex's `vortex-ffi/src/sink.rs`
+//! (SPDX-License-Identifier: Apache-2.0, Copyright the Vortex contributors).
 
 use std::ffi::c_void;
 use std::io;
@@ -137,11 +113,7 @@ pub struct vx_output_callbacks {
     pub release_fn: Option<vx_release_fn>,
 }
 
-/// The host context, owned by the reader.
-///
-/// Held behind an `Arc` so in-flight reads keep the context alive, and so `release_fn` runs exactly
-/// once when the last reference goes away. It carries only the context, not the read callback, so
-/// that ownership can be taken over before anything else is validated.
+/// Shared host context, released when the last reader or writer reference is dropped.
 struct CallbackTarget {
     ctx: *mut c_void,
     release_fn: Option<vx_release_fn>,
@@ -164,11 +136,7 @@ impl Drop for CallbackTarget {
     }
 }
 
-/// A [`VortexReadAt`] that forwards positional reads to host callbacks.
-///
-/// Reads run on the runtime's blocking pool, since the host callback is synchronous and must not
-/// occupy an async executor thread. The file size is supplied at construction time, so `size()`
-/// never crosses the FFI boundary.
+/// A [`VortexReadAt`] that runs host callbacks on the blocking pool.
 struct CallbackReadAt {
     target: Arc<CallbackTarget>,
     read_at_fn: vx_read_at_fn,
@@ -177,16 +145,12 @@ struct CallbackReadAt {
     concurrency: usize,
 }
 
-/// Number of concurrent host read callbacks Vortex may have in flight for one file.
-///
-/// The host filesystem may be remote, where concurrency hides latency; it is also the bound on
-/// blocking threads occupied by one reader.
+/// Maximum concurrent host reads per file, also bounding blocking-pool usage.
 const DEFAULT_CONCURRENCY: usize = 16;
 
 impl VortexReadAt for CallbackReadAt {
     fn coalesce_config(&self) -> Option<CoalesceConfig> {
-        // Each read costs an FFI hop plus a blocking-pool hand-off, and the host filesystem may be
-        // remote, so favor fewer and larger reads.
+        // Coalesce reads to reduce remote I/O and blocking-pool overhead.
         Some(CoalesceConfig::object_storage())
     }
 
@@ -255,8 +219,7 @@ unsafe fn data_source_new_callback(
     callbacks: vx_input_callbacks,
     size: u64,
 ) -> VortexResult<*const vx_data_source> {
-    // Take ownership of the host context before anything else, so an early error below still
-    // releases it exactly once when this `Arc` is dropped.
+    // Take ownership before validation so errors also release the context.
     let target = Arc::new(CallbackTarget {
         ctx: callbacks.ctx,
         release_fn: callbacks.release_fn,
@@ -282,11 +245,8 @@ unsafe fn data_source_new_callback(
 
 /// Create a data source that reads through host callbacks.
 ///
-/// `size` is the total length of the file in bytes; the host knows it up front, so it is passed
-/// here instead of being fetched through another callback.
-///
-/// The callbacks (and the context they carry) are owned by the returned data source and released
-/// when it is freed, including when this call fails.
+/// `size` is the total file length in bytes. Takes ownership of the host context,
+/// which is released when the data source is freed or this call fails.
 ///
 /// On error, returns NULL and sets "err".
 ///
@@ -306,11 +266,7 @@ pub unsafe extern "C-unwind" fn vx_data_source_new_callback(
     })
 }
 
-/// A [`VortexWrite`] that forwards sequential writes to host callbacks.
-///
-/// Writes run inline on the runtime thread driving the write task, the same way `vortex-jni`
-/// forwards to Java sinks: the host callback is synchronous and the layout writer hands over one
-/// buffer at a time, so there is nothing to overlap.
+/// A [`VortexWrite`] that calls host callbacks inline on the writer's runtime thread.
 struct CallbackWrite {
     target: Arc<CallbackTarget>,
     write_fn: vx_write_fn,
@@ -364,11 +320,7 @@ impl VortexWrite for CallbackWrite {
     }
 }
 
-/// A sink that writes a Vortex file through host callbacks.
-///
-/// Mirrors `vx_array_sink`: pushed arrays go through a channel feeding a write task on the session's
-/// runtime, and errors surface when the sink is closed. This is a separate type because
-/// `vx_array_sink`'s fields are private to its own module.
+/// A Vortex file sink backed by host callbacks and a background write task.
 pub struct vx_callback_sink {
     sink: Sender<VortexResult<ArrayRef>>,
     writer: Task<VortexResult<WriteSummary>>,
@@ -380,8 +332,7 @@ unsafe fn callback_sink_open(
     callbacks: vx_output_callbacks,
     dtype: *const vx_dtype,
 ) -> VortexResult<*mut vx_callback_sink> {
-    // Take ownership of the host context before anything else, so an early error below still
-    // releases it exactly once when this `Arc` is dropped.
+    // Take ownership before validation so errors also release the context.
     let target = Arc::new(CallbackTarget {
         ctx: callbacks.ctx,
         release_fn: callbacks.release_fn,
@@ -401,7 +352,7 @@ unsafe fn callback_sink_open(
         flush_fn: callbacks.flush_fn,
     };
 
-    // Small on purpose so the producing thread feels back-pressure, as in vortex-jni's writer.
+    // Limit queued arrays to apply backpressure to the producer.
     let (sink, rx) = mpsc::channel(4);
     let array_stream = ArrayStreamAdapter::new(dtype.clone(), rx.into_stream());
 
@@ -557,7 +508,6 @@ mod tests {
     use crate::sink::vx_array_sink_open_file;
     use crate::sink::vx_array_sink_push;
 
-    /// Host context standing in for paimon's `InputStream`: positional reads against a file.
     struct FileContext {
         file: File,
     }
@@ -584,8 +534,7 @@ mod tests {
         drop(unsafe { Box::from_raw(ctx.cast::<FileContext>()) });
     }
 
-    /// Writing a file with the stock sink and reading it back through the callback data source
-    /// must yield the same dtype and row count: the callback plug point is wired up correctly.
+    // Read a file written by the standard sink through host callbacks.
     #[test]
     #[cfg_attr(miri, ignore)]
     fn test_callback_data_source_round_trip() {
@@ -638,8 +587,6 @@ mod tests {
         }
     }
 
-    /// A rejected request must still release the host context: the entry point takes ownership
-    /// before it validates anything, so the host never has to guess whether to clean up.
     #[test]
     #[cfg_attr(miri, ignore)]
     fn test_rejected_callbacks_release_context() {
@@ -652,7 +599,6 @@ mod tests {
         unsafe {
             let session = vx_session_new();
             let mut error = ptr::null_mut();
-            // No read callback, so the call must fail.
             let callbacks = vx_input_callbacks {
                 ctx: ptr::null_mut(),
                 read_at_fn: None,
@@ -672,12 +618,10 @@ mod tests {
         }
     }
 
-    /// Writing through the callback sink and reading the bytes back through the callback data source
-    /// must round-trip, with no filesystem involved in either direction.
+    // Round-trip through an in-memory buffer using only callbacks.
     #[test]
     #[cfg_attr(miri, ignore)]
     fn test_callback_sink_round_trip() {
-        /// Host context standing in for a paimon stream pair: an in-memory byte buffer.
         struct BufferContext {
             data: Mutex<Vec<u8>>,
         }

docs/source/basic_concepts.rst

@@ -23,8 +23,9 @@ Basic Concepts
 
 File Layouts
 ------------------------
-All files of a table are stored under one base directory. Paimon files are
-organized in a layered style. The following image illustrates the file layout.
+In a file-system deployment, table metadata is stored under the table's base
+directory. Paimon files are organized in a layered style. The following image
+illustrates the file layout.
 Starting from a snapshot file, Paimon readers can recursively access all records
 from the table.
 
@@ -35,7 +36,10 @@ from the table.
 
 Snapshot
 -------------------
-All snapshot files are stored in the snapshot directory.
+Direct file-system commits publish snapshot files in the snapshot directory.
+A catalog that manages table versions may instead hold the latest snapshot
+without publishing it there. See :doc:`user_guide/catalog` for the file-system
+metadata still required by reads, recovery and maintenance with a catalog.
 
 A snapshot file is a JSON file containing information about this snapshot,
 including the schema file in use the manifest list containing all changes of

docs/source/build_system.rst

@@ -48,6 +48,9 @@ for data format and file system.
 
    project(MyExample)
 
+   set(CMAKE_CXX_STANDARD 17)
+   set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
    # Arrow's static package may export LZ4 under a different target name.
    find_package(lz4 CONFIG QUIET)
    if(TARGET LZ4::lz4_static AND NOT TARGET LZ4::lz4)

docs/source/user_guide/append_only_table.rst

@@ -20,7 +20,13 @@
 
 Append Only Table
 =================
-If a table does not have a primary key defined, it is an append table. Compared
-to the primary key table, it does not have the ability to directly receive changelogs.
-It cannot be directly updated with data through upsert. It can only receive
-incoming data from append data.
+If a table does not have a primary key defined, it is an append table.
+Ordinary append writes accept ``INSERT`` rows only and do not apply upserts
+or deletes by key.
+
+The experimental real-time append deduplicate mode is an exception. With
+``realtime.deduplicate-key-fields`` configured, it also accepts ``UPDATE_AFTER``,
+``DELETE`` and ``UPDATE_BEFORE``, using deletion vectors to hide superseded
+rows. The table still has no primary key; deduplication is scoped to each
+partition-bucket. See :doc:`realtime` for the required schema, options and
+shared context.

docs/source/user_guide/clean.rst

@@ -19,7 +19,9 @@ Data Cleanup
 ===================================
 .. note::
 
-  The entire data cleanup feature supports only append tables, and cleanup of index manifests, changelog, statistics are not supported. Do not use this feature with primary key tables.
+  Cleanup supports append and primary-key tables, subject to the restrictions
+  below. Index manifests and index payloads are included. Cleanup of changelog
+  and statistics files is not supported.
 
 This document describes three cleanup capabilities:
 
@@ -35,9 +37,15 @@ Orphan File Cleanup
 Description
 ~~~~~~~~~~~~~~~~~~~
 
-- Orphan file cleanup currently supports only append tables.
+- Orphan file cleanup supports append and primary-key tables.
 - It runs as an independent task: construct a ``CleanContext`` and
   launch an ``OrphanFilesCleaner``.
+- It covers table-local index manifests and payloads as well as data files,
+  preserving files referenced by retained snapshots. It does not enumerate a
+  potentially shared ``global-index.external-path``.
+- Tables with tags or branches, or snapshots with a changelog manifest list,
+  are rejected with ``NotImplemented``. Tag retention during snapshot expiration
+  is supported separately, as described below.
 
 Detailed Steps
 ~~~~~~~~~~~~~~
@@ -62,7 +70,7 @@ Performance Considerations
 .. admonition:: TODO
   :class: tip
 
-  - Support cleanup of index manifests, changelog, statistics, etc.
+  - Support cleanup of changelog and statistics files.
 
 .. _expiring-partitions:
 
@@ -109,10 +117,17 @@ Description
   - ``snapshot.time-retained``: The maximum age of completed snapshots
     to retain. Default: 1 hour.
   - ``snapshot.expire.limit``: The maximum number of snapshots allowed
-    to expire at a time. Default: 10.
+    to expire at a time. Default: 50.
+  - ``snapshot.clean-empty-directories``: Remove empty bucket and partition
+    directories after deleting their data files, without deleting the table
+    directory. Default: false.
 
 - The snapshot expiration interface deletes data files according to the
   expiration policy and returns the number of snapshots deleted.
+- Data files, index manifests and index payloads referenced by retained
+  snapshots or current-branch tags are preserved. Retired index payloads are
+  deleted before their index manifests, including payloads stored under
+  ``global-index.external-path``.
 - Expiration on a branch other than main, or on a table with such a branch under
   ``branch/branch-<name>``, is not supported: branches share the table's data
   files, so expiration returns ``NotImplemented`` before deleting anything.
@@ -149,7 +164,7 @@ Detailed Steps
      * ``DELETE`` then ``ADD`` → the initial ``DELETE`` does not apply (file did not exist yet); the subsequent ``ADD`` ensures the file remains.
 
 5. Clean meta files:
-   - Preserve manifests used by the last snapshot in the cleanup range (``end_exclusive_id``).
+   - Preserve manifests used by retained snapshots and current-branch tags.
    - Delete manifest files used by snapshots from ``begin_inclusive_id`` to ``end_exclusive_id`` (exclusive) and delete the snapshot files themselves.
 6. Rewrite ``EarliestHint`` to ``end_exclusive_id``.
 7. Return the number of snapshots deleted, i.e., ``end_exclusive_id - begin_inclusive_id``.
@@ -164,6 +179,4 @@ Performance Considerations
 .. admonition:: TODO
   :class: tip
 
-  - Preserve tag (savepoint) data via ``tagManager``.
   - Delete changelog files.
-  - Remove empty directories.