From 5d8759d68f9900df3712e65ca9f712435f12204a Mon Sep 17 00:00:00 2001 From: Kshitij Prasad <236782705+kshitijprasaddev@users.noreply.github.com> Date: Thu, 22 Oct 2026 20:49:59 +0200 Subject: [PATCH] GH-47824: [Format] Clarify IPC footer offsets and metadata lengths --- docs/source/format/Columnar.rst | 14 ++++++++++++-- format/File.fbs | 7 +++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/source/format/Columnar.rst b/docs/source/format/Columnar.rst index f95d7e80769d..da2333f69f6d 100644 --- a/docs/source/format/Columnar.rst +++ b/docs/source/format/Columnar.rst @@ -1253,8 +1253,9 @@ The complete serialized message must be a multiple of 8 bytes so that messages can be relocated between streams. Otherwise the amount of padding between the metadata and the message body could be non-deterministic. -The ``metadata_size`` includes the size of the ``Message`` plus -padding. The ``metadata_flatbuffer`` contains a Flatbuffers-serialized +The ``metadata_size`` includes the size of the ``Message`` flatbuffer plus +padding, but excludes the continuation indicator and the metadata length prefix. +The ``metadata_flatbuffer`` contains a Flatbuffers-serialized ``Message`` value, which internally includes: * A version number @@ -1531,6 +1532,15 @@ sizes for each of the data blocks in the file. This enables random access to any record batch in the file. See `File.fbs`_ for the precise details of the file footer. +Each ``Block`` in the footer describes a dictionary or record batch IPC message. +Its ``offset`` points to the start of the encapsulated IPC message, including +the continuation indicator and metadata length prefix. Its ``metaDataLength`` +includes that prefix, the ``Message`` flatbuffer, and padding, so the message +body starts at ``offset + metaDataLength``. Thus, ``metaDataLength`` is +``8 + metadata_size`` for the encapsulated message format described above, +rather than ``metadata_size`` alone. For the legacy format without a continuation +indicator, the prefix is 4 bytes and ``metaDataLength`` is ``4 + metadata_size``. + Schematically we have: :: diff --git a/format/File.fbs b/format/File.fbs index 568b5482ebc2..82e9919de7ad 100644 --- a/format/File.fbs +++ b/format/File.fbs @@ -38,10 +38,13 @@ table Footer { struct Block { - /// Index to the start of the RecordBatch (note this is past the Message header) + /// Byte offset to the start of the encapsulated IPC message, including its + /// continuation indicator (if present) and metadata length prefix. offset: long; - /// Length of the metadata + /// Length of the encapsulated IPC message metadata, including the continuation + /// indicator (if present), metadata length prefix, Message flatbuffer, and + /// padding. This is the byte distance from offset to the start of the body. metaDataLength: int; /// Length of the data (this is aligned so there can be a gap between this and