[core] Optimize ordinary manifest merging with raw Avro blocks

docs/generated/core_configuration.html

@@ -1047,6 +1047,12 @@
             <td>Integer</td>
             <td>To avoid frequent manifest merges, this parameter specifies the minimum number of ManifestFileMeta to merge.<br />Note: when 'manifest-sort.enabled' is true, this minimum-count gate is only applied to the trailing sub-segment of a section that exceeds 'manifest-sort.max-rewrite-size'. Small under-budget sections are sorted and rewritten directly, so two small manifest files may be merged into one even when their count is below this threshold and full compaction is not triggered.</td>
         </tr>
+        <tr>
+            <td><h5>manifest.merge-optimize.enabled</h5></td>
+            <td style="word-wrap: break-word;">true</td>
+            <td>Boolean</td>
+            <td>Whether to enable block-aware ordinary manifest merging. When disabled, ordinary manifest compaction uses the legacy full-entry merger.</td>
+        </tr>
         <tr>
             <td><h5>manifest.target-file-size</h5></td>
             <td style="word-wrap: break-word;">8 mb</td>

paimon-api/src/main/java/org/apache/paimon/CoreOptions.java

@@ -589,6 +589,15 @@ public InlineElement getDescription() {
                                     + " skipped. Set to a larger value to allow more aggressive"
                                     + " sort rewriting. The cap only limits the sorted rewrite portion and full/minor cleanup may still happen beyond it.");
 
+    public static final ConfigOption<Boolean> MANIFEST_MERGE_OPTIMIZE_ENABLED =
+            key("manifest.merge-optimize.enabled")
+                    .booleanType()
+                    .defaultValue(true)
+                    .withDescription(
+                            "Whether to enable block-aware ordinary manifest merging. When"
+                                    + " disabled, ordinary manifest compaction uses the legacy"
+                                    + " full-entry merger.");
+
     public static final ConfigOption<String> PARTITION_DEFAULT_NAME =
             key("partition.default-name")
                     .stringType()
@@ -3066,6 +3075,10 @@ public long manifestSortMaxRewriteSize() {
         return options.get(MANIFEST_SORT_MAX_REWRITE_SIZE).getBytes();
     }
 
+    public boolean manifestMergeOptimizeEnabled() {
+        return options.get(MANIFEST_MERGE_OPTIMIZE_ENABLED);
+    }
+
     public String partitionDefaultName() {
         return options.get(PARTITION_DEFAULT_NAME);
     }

paimon-core/src/main/java/org/apache/paimon/io/ProjectedDataFileMeta.java

@@ -230,6 +230,16 @@ public boolean hasFirstRowId() {
         return !currentRow().isNullAt(requiredPosition(Fields.FIRST_ROW_ID));
     }
 
+    @Override
+    public long nonNullFirstRowId() {
+        // Read the primitive value directly on manifest scan hot paths. Calling firstRowId()
+        // here would box every value as Long before immediately unboxing it again.
+        int position = requiredPosition(Fields.FIRST_ROW_ID);
+        InternalRow row = currentRow();
+        checkState(!row.isNullAt(position), "First row id cannot be null.");
+        return row.getLong(position);
+    }
+
     @Nullable
     @Override
     public Long firstRowId() {

paimon-core/src/main/java/org/apache/paimon/manifest/CollectedDeletes.java

@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.manifest;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.io.ProjectedDataFileMeta;
+import org.apache.paimon.manifest.FileEntry.ReusableIdentifier;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.paimon.utils.Preconditions.checkState;
+
+/** DELETE identifiers and optional RowID and partition indexes collected for manifest merging. */
+public final class CollectedDeletes {
+
+    private final CompactFileIdentifierSet identifiers = new CompactFileIdentifierSet();
+    private final DeletedRowIdSet rowIds = new DeletedRowIdSet();
+    private Set<BinaryRow> partitions = new HashSet<>();
+    private final boolean useRowIdFilter;
+    private boolean immutable;
+
+    public CollectedDeletes(boolean useRowIdFilter) {
+        this.useRowIdFilter = useRowIdFilter;
+    }
+
+    public void add(
+            ProjectedManifestEntry entry, boolean collectRowIds, boolean collectPartitions) {
+        checkState(!immutable, "Cannot modify an immutable DELETE collection.");
+        identifiers.add(entry);
+        if (collectPartitions) {
+            partitions.add(entry.partition().copy());
+        }
+        if (collectRowIds) {
+            rowIds.add(entry.file().nonNullFirstRowId());
+        }
+    }
+
+    public void combine(CollectedDeletes other) {
+        checkState(!immutable, "Cannot modify an immutable DELETE collection.");
+        checkState(
+                useRowIdFilter == other.useRowIdFilter,
+                "Cannot combine DELETE collections with different RowID modes.");
+        identifiers.addAll(other.identifiers);
+        rowIds.addAll(other.rowIds);
+        partitions.addAll(other.partitions);
+    }
+
+    public CollectedDeletes toImmutable() {
+        checkState(!immutable, "Cannot modify an immutable DELETE collection.");
+        if (useRowIdFilter) {
+            rowIds.prepareRangeIndex();
+        }
+        partitions = Collections.unmodifiableSet(partitions);
+        immutable = true;
+        return this;
+    }
+
+    public boolean isEmpty() {
+        return identifiers.isEmpty();
+    }
+
+    public Set<BinaryRow> partitions() {
+        return partitions;
+    }
+
+    public boolean useRowIdFilter() {
+        return useRowIdFilter;
+    }
+
+    public boolean isDeleted(ProjectedManifestEntry entry, ReusableIdentifier reusableIdentifier) {
+        if (useRowIdFilter) {
+            ProjectedDataFileMeta file = entry.file();
+            checkState(file.hasFirstRowId(), "First row id should not be null.");
+            if (!rowIds.contains(file.nonNullFirstRowId())) {
+                return false;
+            }
+        }
+        return identifiers.contains(reusableIdentifier.replaceWithPartition(entry));
+    }
+
+    public boolean copyable(
+            ProjectedManifestEntry entry,
+            ReusableIdentifier reusableIdentifier,
+            boolean deferDeletedAddCheck) {
+        return entry.isAdd() && (deferDeletedAddCheck || !isDeleted(entry, reusableIdentifier));
+    }
+
+    public boolean intersectsRowIds(long minRowId, long maxRowId) {
+        return rowIds.intersects(minRowId, maxRowId);
+    }
+
+    public void release() {
+        identifiers.release();
+        rowIds.releaseRangeIndex();
+    }
+}

paimon-core/src/main/java/org/apache/paimon/manifest/CompactFileIdentifierSet.java

@@ -74,6 +74,13 @@ public void add(int partitionId, ReusableIdentifier identifier) {
         add(partitionId, identifier.bytes(), identifier.length());
     }
 
+    public void addAll(CompactFileIdentifierSet other) {
+        checkArgument(other != null, "Identifier set cannot be null.");
+        for (int entry = 0; entry < other.size; entry++) {
+            add(other.partitionIds[entry], other.arena, other.offsets[entry], other.lengths[entry]);
+        }
+    }
+
     public boolean contains(ProjectedManifestEntry entry) {
         return contains(reusableIdentifier().replaceWithPartition(entry));
     }
@@ -104,41 +111,47 @@ public void release() {
     }
 
     void add(int partitionId, byte[] identifier, int length) {
-        checkIdentifier(identifier, length);
-        long hash = hash(partitionId, identifier, length);
-        if (contains(partitionId, identifier, length, hash)) {
+        add(partitionId, identifier, 0, length);
+    }
+
+    private void add(int partitionId, byte[] identifier, int offset, int length) {
+        checkIdentifier(identifier, offset, length);
+        long hash = hash(partitionId, identifier, offset, length);
+        if (contains(partitionId, identifier, offset, length, hash)) {
             return;
         }
         if (size + 1 > (int) (buckets.length * LOAD_FACTOR)) {
             growBuckets();
         }
         ensureEntryCapacity(size + 1);
         ensureArenaCapacity(length);
-        int offset = arenaSize;
-        System.arraycopy(identifier, 0, arena, offset, length);
+        int arenaOffset = arenaSize;
+        System.arraycopy(identifier, offset, arena, arenaOffset, length);
         arenaSize = Math.addExact(arenaSize, length);
 
         int bucket = bucket(hash);
         hashes[size] = hash;
         partitionIds[size] = partitionId;
-        offsets[size] = offset;
+        offsets[size] = arenaOffset;
         lengths[size] = length;
         next[size] = buckets[bucket];
         buckets[bucket] = size;
         size++;
     }
 
     boolean contains(int partitionId, byte[] identifier, int length) {
-        checkIdentifier(identifier, length);
-        return contains(partitionId, identifier, length, hash(partitionId, identifier, length));
+        checkIdentifier(identifier, 0, length);
+        return contains(
+                partitionId, identifier, 0, length, hash(partitionId, identifier, 0, length));
     }
 
-    private boolean contains(int partitionId, byte[] identifier, int length, long hash) {
+    private boolean contains(
+            int partitionId, byte[] identifier, int offset, int length, long hash) {
         for (int entry = buckets[bucket(hash)]; entry >= 0; entry = next[entry]) {
             if (hashes[entry] == hash
                     && partitionIds[entry] == partitionId
                     && lengths[entry] == length
-                    && bytesEqual(arena, offsets[entry], identifier, length)) {
+                    && bytesEqual(arena, offsets[entry], identifier, offset, length)) {
                 return true;
             }
         }
@@ -194,20 +207,21 @@ private static int bucket(long hash, int bucketCount) {
         return ((int) (hash ^ (hash >>> 32))) & (bucketCount - 1);
     }
 
-    private static long hash(int partitionId, byte[] bytes, int length) {
+    private static long hash(int partitionId, byte[] bytes, int offset, int length) {
         long hash = 0xcbf29ce484222325L;
         hash ^= Integer.toUnsignedLong(partitionId);
         hash *= 0x100000001b3L;
         for (int i = 0; i < length; i++) {
-            hash ^= bytes[i] & 0xFFL;
+            hash ^= bytes[offset + i] & 0xFFL;
             hash *= 0x100000001b3L;
         }
         return hash;
     }
 
-    private static boolean bytesEqual(byte[] left, int leftOffset, byte[] right, int length) {
+    private static boolean bytesEqual(
+            byte[] left, int leftOffset, byte[] right, int rightOffset, int length) {
         for (int i = 0; i < length; i++) {
-            if (left[leftOffset + i] != right[i]) {
+            if (left[leftOffset + i] != right[rightOffset + i]) {
                 return false;
             }
         }
@@ -225,12 +239,13 @@ private ReusableIdentifier reusableIdentifier() {
         return reusableIdentifier;
     }
 
-    private static void checkIdentifier(byte[] identifier, int length) {
+    private static void checkIdentifier(byte[] identifier, int offset, int length) {
         checkArgument(identifier != null, "Identifier bytes cannot be null.");
         checkArgument(
-                length >= 0 && length <= identifier.length,
-                "Invalid identifier length %s.",
-                length);
+                offset >= 0 && length >= 0 && offset <= identifier.length - length,
+                "Invalid identifier range [%s, %s).",
+                offset,
+                offset + length);
     }
 
     private static int[] filledWithMinusOne(int length) {