Skip to content

Commit 3f41e81

Browse files
sundapengclaude
andcommitted
[core] Fix unsupported predicate returning empty results in applyPartitionFilter
buildPartitionPredicate returned null for both "unsupported predicate type" and "invalid partition spec", but applyPartitionFilter treated all nulls as "no matches". This caused BucketsTable/FilesTable/FileKeyRangesTable to return empty results for unsupported predicates like isNotNull. Introduce PushdownResult to distinguish three states: unsupported (skip pushdown), no-match (return empty), and success (apply predicate). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 62d378e commit 3f41e81

3 files changed

Lines changed: 62 additions & 20 deletions

File tree

paimon-core/src/main/java/org/apache/paimon/table/system/PartitionsTable.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -456,14 +456,14 @@ private List<Partition> listPartitionEntries(@Nullable LeafPredicate partitionPr
456456
List<String> partitionKeys = fileStoreTable.partitionKeys();
457457
RowType partitionType = fileStoreTable.schema().logicalPartitionType();
458458
String defaultPartitionName = fileStoreTable.coreOptions().partitionDefaultName();
459-
Predicate partPred =
459+
PartitionPredicateHelper.PushdownResult pushdownResult =
460460
PartitionPredicateHelper.buildPartitionPredicate(
461461
partitionPredicate,
462462
partitionKeys,
463463
partitionType,
464464
defaultPartitionName);
465-
if (partPred != null) {
466-
scan.withPartitionFilter(partPred);
465+
if (pushdownResult.predicate() != null) {
466+
scan.withPartitionFilter(pushdownResult.predicate());
467467
}
468468
}
469469

paimon-core/src/main/java/org/apache/paimon/utils/PartitionPredicateHelper.java

Lines changed: 50 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -40,15 +40,45 @@
4040
*/
4141
public class PartitionPredicateHelper {
4242

43+
/** Result of attempting to build a partition predicate for pushdown. */
44+
public static class PushdownResult {
45+
private static final PushdownResult UNSUPPORTED = new PushdownResult(null, false);
46+
47+
@Nullable private final Predicate predicate;
48+
private final boolean supported;
49+
50+
private PushdownResult(@Nullable Predicate predicate, boolean supported) {
51+
this.predicate = predicate;
52+
this.supported = supported;
53+
}
54+
55+
static PushdownResult unsupported() {
56+
return UNSUPPORTED;
57+
}
58+
59+
static PushdownResult noMatch() {
60+
return new PushdownResult(null, true);
61+
}
62+
63+
static PushdownResult of(Predicate predicate) {
64+
return new PushdownResult(predicate, true);
65+
}
66+
67+
public boolean isSupported() {
68+
return supported;
69+
}
70+
71+
@Nullable
72+
public Predicate predicate() {
73+
return predicate;
74+
}
75+
}
76+
4377
/**
4478
* Build a partition-typed predicate from a string-based leaf predicate on the "partition"
45-
* column. Only Equal and In predicates are supported for pushdown.
46-
*
47-
* @return the predicate on partition fields, or {@code null} if the predicate cannot be pushed
48-
* down (unsupported predicate type or invalid partition spec)
79+
* column.
4980
*/
50-
@Nullable
51-
public static Predicate buildPartitionPredicate(
81+
public static PushdownResult buildPartitionPredicate(
5282
LeafPredicate partitionPredicate,
5383
List<String> partitionKeys,
5484
RowType partitionType,
@@ -58,7 +88,7 @@ public static Predicate buildPartitionPredicate(
5888
parsePartitionSpec(
5989
partitionPredicate.literals().get(0).toString(), partitionKeys);
6090
if (partSpec == null) {
61-
return null;
91+
return PushdownResult.noMatch();
6292
}
6393
PredicateBuilder partBuilder = new PredicateBuilder(partitionType);
6494
List<Predicate> predicates = new ArrayList<>();
@@ -71,7 +101,7 @@ public static Predicate buildPartitionPredicate(
71101
predicates.add(partBuilder.equal(i, value));
72102
}
73103
}
74-
return PredicateBuilder.and(predicates);
104+
return PushdownResult.of(PredicateBuilder.and(predicates));
75105
} else if (partitionPredicate.function() instanceof In) {
76106
List<Predicate> orPredicates = new ArrayList<>();
77107
PredicateBuilder partBuilder = new PredicateBuilder(partitionType);
@@ -94,16 +124,16 @@ public static Predicate buildPartitionPredicate(
94124
}
95125
orPredicates.add(PredicateBuilder.and(andPredicates));
96126
}
97-
return orPredicates.isEmpty() ? null : PredicateBuilder.or(orPredicates);
127+
return orPredicates.isEmpty()
128+
? PushdownResult.noMatch()
129+
: PushdownResult.of(PredicateBuilder.or(orPredicates));
98130
}
99-
// Range predicates (>, >=, <, <=) can be pushed down for simple value formats
100-
// used by BucketsTable, FilesTable, FileKeyRangesTable.
101131
if (partitionPredicate.function() instanceof LeafBinaryFunction) {
102132
LinkedHashMap<String, String> partSpec =
103133
parsePartitionSpec(
104134
partitionPredicate.literals().get(0).toString(), partitionKeys);
105135
if (partSpec == null) {
106-
return null;
136+
return PushdownResult.noMatch();
107137
}
108138
PredicateBuilder partBuilder = new PredicateBuilder(partitionType);
109139
List<Predicate> predicates = new ArrayList<>();
@@ -122,9 +152,9 @@ public static Predicate buildPartitionPredicate(
122152
Collections.singletonList(value)));
123153
}
124154
}
125-
return PredicateBuilder.and(predicates);
155+
return PushdownResult.of(PredicateBuilder.and(predicates));
126156
}
127-
return null;
157+
return PushdownResult.unsupported();
128158
}
129159

130160
public static boolean applyPartitionFilter(
@@ -137,13 +167,16 @@ public static boolean applyPartitionFilter(
137167
return true;
138168
}
139169

140-
Predicate predicate =
170+
PushdownResult result =
141171
buildPartitionPredicate(
142172
partitionPredicate, partitionKeys, partitionType, defaultPartitionName);
143-
if (predicate == null) {
173+
if (!result.isSupported()) {
174+
return true;
175+
}
176+
if (result.predicate() == null) {
144177
return false;
145178
}
146-
snapshotReader.withPartitionFilter(predicate);
179+
snapshotReader.withPartitionFilter(result.predicate());
147180
return true;
148181
}
149182

paimon-core/src/test/java/org/apache/paimon/table/system/BucketsTableTest.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,15 @@ public void testBucketsTableWithCombinedFilter() throws Exception {
148148
assertThat(readWithFilter(bucketsTable, filter, new int[] {0, 1, 2, 4})).isEmpty();
149149
}
150150

151+
@Test
152+
public void testBucketsTableUnsupportedPredicateFallsBackToFullScan() throws Exception {
153+
PredicateBuilder builder = new PredicateBuilder(BucketsTable.TABLE_TYPE);
154+
155+
// isNotNull cannot be pushed down as partition filter — must fall back to full scan
156+
Predicate filter = builder.isNotNull(0);
157+
assertThat(readWithFilter(bucketsTable, filter, new int[] {0, 1, 2, 4})).hasSize(2);
158+
}
159+
151160
private List<InternalRow> readWithFilter(Table table, Predicate filter, int[] projection)
152161
throws Exception {
153162
ReadBuilder readBuilder = table.newReadBuilder().withFilter(filter);

0 commit comments

Comments
 (0)