Skip to content

Commit 7213570

Browse files
sundapengclaude
sundapeng
and
claude
committed
[core] Fix default partition handling and improve predicate pushdown correctness
- Handle __DEFAULT_PARTITION__ in buildPartitionPredicate() by generating isNull() instead of castFromString() which throws NumberFormatException on non-string partition types (e.g. INT) - Fix scan path to skip pushdown for unsupported predicates instead of returning empty results - Pass defaultPartitionName through all system table callers Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent e5185ab commit 7213570

File tree

6 files changed

+87
-38
lines changed

6 files changed

+87
-38
lines changed

‎paimon-core/src/main/java/org/apache/paimon/table/system/BucketsTable.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,8 @@ public RecordReader<InternalRow> createReader(Split split) {
234234
snapshotReader,
235235
bucketsSplit.partitionPredicate,
236236
partitionKeys,
237-
partitionType);
237+
partitionType,
238+
fileStoreTable.coreOptions().partitionDefaultName());
238239
if (!hasResults) {
239240
return new IteratorRecordReader<>(Collections.emptyIterator());
240241
}

‎paimon-core/src/main/java/org/apache/paimon/table/system/FileKeyRangesTable.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,11 @@ public Plan innerPlan() {
170170
RowType partitionType = fileStoreTable.schema().logicalPartitionType();
171171
boolean hasResults =
172172
PartitionPredicateHelper.applyPartitionFilter(
173-
snapshotReader, partitionPredicate, partitionKeys, partitionType);
173+
snapshotReader,
174+
partitionPredicate,
175+
partitionKeys,
176+
partitionType,
177+
fileStoreTable.coreOptions().partitionDefaultName());
174178
if (!hasResults) {
175179
return Collections::emptyList;
176180
}

‎paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,11 @@ public Plan innerPlan() {
196196
RowType partitionType = fileStoreTable.schema().logicalPartitionType();
197197
boolean hasResults =
198198
PartitionPredicateHelper.applyPartitionFilter(
199-
snapshotReader, partitionPredicate, partitionKeys, partitionType);
199+
snapshotReader,
200+
partitionPredicate,
201+
partitionKeys,
202+
partitionType,
203+
fileStoreTable.coreOptions().partitionDefaultName());
200204
if (!hasResults) {
201205
return Collections::emptyList;
202206
}

‎paimon-core/src/main/java/org/apache/paimon/table/system/PartitionsTable.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -455,13 +455,16 @@ private List<Partition> listPartitionEntries(@Nullable LeafPredicate partitionPr
455455
if (partitionPredicate != null) {
456456
List<String> partitionKeys = fileStoreTable.partitionKeys();
457457
RowType partitionType = fileStoreTable.schema().logicalPartitionType();
458-
Predicate partPred =
458+
String defaultPartitionName = fileStoreTable.coreOptions().partitionDefaultName();
459+
Predicate partitionFilter =
459460
PartitionPredicateHelper.buildPartitionPredicate(
460-
partitionPredicate, partitionKeys, partitionType);
461-
if (partPred == null) {
462-
return Collections.emptyList();
461+
partitionPredicate,
462+
partitionKeys,
463+
partitionType,
464+
defaultPartitionName);
465+
if (partitionFilter != null) {
466+
scan.withPartitionFilter(partitionFilter);
463467
}
464-
scan.withPartitionFilter(partPred);
465468
}
466469

467470
List<PartitionEntry> partitionEntries = scan.listPartitionEntries();

‎paimon-core/src/main/java/org/apache/paimon/utils/PartitionPredicateHelper.java‎

Lines changed: 58 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
package org.apache.paimon.utils;
2020

21+
import org.apache.paimon.predicate.AlwaysFalse;
2122
import org.apache.paimon.predicate.Equal;
2223
import org.apache.paimon.predicate.In;
2324
import org.apache.paimon.predicate.LeafBinaryFunction;
@@ -44,26 +45,33 @@ public class PartitionPredicateHelper {
4445
* Build a partition-typed predicate from a string-based leaf predicate on the "partition"
4546
* column.
4647
*
47-
* @return the predicate on partition fields, or {@code null} if the partition spec is invalid
48-
* (indicating no results should be returned)
48+
* @return {@code null} if the predicate type is unsupported for pushdown (caller should skip
49+
* pushdown), {@link PredicateBuilder#alwaysFalse()} if no partition can match (caller
50+
* should return empty), or a normal predicate to push down.
4951
*/
5052
@Nullable
5153
public static Predicate buildPartitionPredicate(
52-
LeafPredicate partitionPredicate, List<String> partitionKeys, RowType partitionType) {
54+
LeafPredicate partitionPredicate,
55+
List<String> partitionKeys,
56+
RowType partitionType,
57+
String defaultPartitionName) {
5358
if (partitionPredicate.function() instanceof Equal) {
5459
LinkedHashMap<String, String> partSpec =
5560
parsePartitionSpec(
5661
partitionPredicate.literals().get(0).toString(), partitionKeys);
5762
if (partSpec == null) {
58-
return null;
63+
return PredicateBuilder.alwaysFalse();
5964
}
6065
PredicateBuilder partBuilder = new PredicateBuilder(partitionType);
6166
List<Predicate> predicates = new ArrayList<>();
6267
for (int i = 0; i < partitionKeys.size(); i++) {
63-
Object value =
64-
TypeUtils.castFromString(
65-
partSpec.get(partitionKeys.get(i)), partitionType.getTypeAt(i));
66-
predicates.add(partBuilder.equal(i, value));
68+
String strValue = partSpec.get(partitionKeys.get(i));
69+
if (defaultPartitionName.equals(strValue)) {
70+
predicates.add(partBuilder.isNull(i));
71+
} else {
72+
Object value = TypeUtils.castFromString(strValue, partitionType.getTypeAt(i));
73+
predicates.add(partBuilder.equal(i, value));
74+
}
6775
}
6876
return PredicateBuilder.and(predicates);
6977
} else if (partitionPredicate.function() instanceof In) {
@@ -77,34 +85,44 @@ public static Predicate buildPartitionPredicate(
7785
}
7886
List<Predicate> andPredicates = new ArrayList<>();
7987
for (int i = 0; i < partitionKeys.size(); i++) {
80-
Object value =
81-
TypeUtils.castFromString(
82-
partSpec.get(partitionKeys.get(i)), partitionType.getTypeAt(i));
83-
andPredicates.add(partBuilder.equal(i, value));
88+
String strValue = partSpec.get(partitionKeys.get(i));
89+
if (defaultPartitionName.equals(strValue)) {
90+
andPredicates.add(partBuilder.isNull(i));
91+
} else {
92+
Object value =
93+
TypeUtils.castFromString(strValue, partitionType.getTypeAt(i));
94+
andPredicates.add(partBuilder.equal(i, value));
95+
}
8496
}
8597
orPredicates.add(PredicateBuilder.and(andPredicates));
8698
}
87-
return orPredicates.isEmpty() ? null : PredicateBuilder.or(orPredicates);
88-
} else if (partitionPredicate.function() instanceof LeafBinaryFunction) {
99+
return orPredicates.isEmpty()
100+
? PredicateBuilder.alwaysFalse()
101+
: PredicateBuilder.or(orPredicates);
102+
}
103+
if (partitionPredicate.function() instanceof LeafBinaryFunction) {
89104
LinkedHashMap<String, String> partSpec =
90105
parsePartitionSpec(
91106
partitionPredicate.literals().get(0).toString(), partitionKeys);
92107
if (partSpec == null) {
93-
return null;
108+
return PredicateBuilder.alwaysFalse();
94109
}
95110
PredicateBuilder partBuilder = new PredicateBuilder(partitionType);
96111
List<Predicate> predicates = new ArrayList<>();
97112
for (int i = 0; i < partitionKeys.size(); i++) {
98-
Object value =
99-
TypeUtils.castFromString(
100-
partSpec.get(partitionKeys.get(i)), partitionType.getTypeAt(i));
101-
predicates.add(
102-
new LeafPredicate(
103-
partitionPredicate.function(),
104-
partitionType.getTypeAt(i),
105-
i,
106-
partitionKeys.get(i),
107-
Collections.singletonList(value)));
113+
String strValue = partSpec.get(partitionKeys.get(i));
114+
if (defaultPartitionName.equals(strValue)) {
115+
predicates.add(partBuilder.isNull(i));
116+
} else {
117+
Object value = TypeUtils.castFromString(strValue, partitionType.getTypeAt(i));
118+
predicates.add(
119+
new LeafPredicate(
120+
partitionPredicate.function(),
121+
partitionType.getTypeAt(i),
122+
i,
123+
partitionKeys.get(i),
124+
Collections.singletonList(value)));
125+
}
108126
}
109127
return PredicateBuilder.and(predicates);
110128
}
@@ -115,20 +133,30 @@ public static boolean applyPartitionFilter(
115133
SnapshotReader snapshotReader,
116134
@Nullable LeafPredicate partitionPredicate,
117135
List<String> partitionKeys,
118-
RowType partitionType) {
136+
RowType partitionType,
137+
String defaultPartitionName) {
119138
if (partitionPredicate == null) {
120139
return true;
121140
}
122141

123-
Predicate predicate =
124-
buildPartitionPredicate(partitionPredicate, partitionKeys, partitionType);
125-
if (predicate == null) {
142+
Predicate result =
143+
buildPartitionPredicate(
144+
partitionPredicate, partitionKeys, partitionType, defaultPartitionName);
145+
if (result == null) {
146+
return true;
147+
}
148+
if (isAlwaysFalse(result)) {
126149
return false;
127150
}
128-
snapshotReader.withPartitionFilter(predicate);
151+
snapshotReader.withPartitionFilter(result);
129152
return true;
130153
}
131154

155+
private static boolean isAlwaysFalse(Predicate predicate) {
156+
return predicate instanceof LeafPredicate
157+
&& ((LeafPredicate) predicate).function().equals(AlwaysFalse.INSTANCE);
158+
}
159+
132160
@Nullable
133161
public static LinkedHashMap<String, String> parsePartitionSpec(
134162
String partitionStr, List<String> partitionKeys) {

‎paimon-core/src/test/java/org/apache/paimon/table/system/BucketsTableTest.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,15 @@ public void testBucketsTableWithCombinedFilter() throws Exception {
148148
assertThat(readWithFilter(bucketsTable, filter, new int[] {0, 1, 2, 4})).isEmpty();
149149
}
150150

151+
@Test
152+
public void testBucketsTableUnsupportedPredicateFallsBackToFullScan() throws Exception {
153+
PredicateBuilder builder = new PredicateBuilder(BucketsTable.TABLE_TYPE);
154+
155+
// isNotNull cannot be pushed down as partition filter — must fall back to full scan
156+
Predicate filter = builder.isNotNull(0);
157+
assertThat(readWithFilter(bucketsTable, filter, new int[] {0, 1, 2, 4})).hasSize(2);
158+
}
159+
151160
private List<InternalRow> readWithFilter(Table table, Predicate filter, int[] projection)
152161
throws Exception {
153162
ReadBuilder readBuilder = table.newReadBuilder().withFilter(filter);

0 commit comments

Comments
 (0)