From d098d859195d9cb8e45c8947b9875336a81244d8 Mon Sep 17 00:00:00 2001 From: seawinde Date: Tue, 28 Jul 2026 15:15:40 +0800 Subject: [PATCH 1/8] [fix](fe) Move SemiJoinCommute to rewrite phase ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: SemiJoinCommute structurally normalizes RIGHT OUTER, RIGHT SEMI, and RIGHT ANTI joins. Running it during analysis made analyzed plan shapes depend on join-reorder state, so consumers that identify analyzed plans could observe inconsistent signatures for the same query. Move the rule to both regular and CTE rewrite pipelines before the semi-join transpose rules, keeping analysis stable while preserving rewrite assumptions. ### Release note None ### Check List (For Author) - Test: Unit Test - ReorderJoinTest - SaltJoinTest - Behavior changed: Yes. Analyze preserves RIGHT JOIN shape and Rewrite performs the existing normalization. - Does this need documentation: No --- .../doris/nereids/jobs/executor/Analyzer.java | 2 -- .../doris/nereids/jobs/executor/Rewriter.java | 3 +++ .../nereids/rules/rewrite/ReorderJoinTest.java | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java index 1f83b242cf8489..b5c7078222a110 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java @@ -52,7 +52,6 @@ import org.apache.doris.nereids.rules.analysis.SubqueryToApply; import org.apache.doris.nereids.rules.rewrite.AdjustNullable; import org.apache.doris.nereids.rules.rewrite.MergeFilters; -import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute; import org.apache.doris.nereids.rules.rewrite.SimplifyAggGroupBy; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; import org.apache.doris.nereids.trees.plans.logical.LogicalView; @@ -188,7 +187,6 @@ private static List buildAnalyzerJobs() { topDown(new NormalizeAggregate()), topDown(new HavingToFilter()), topDown(new QualifyToFilter()), - bottomUp(new SemiJoinCommute()), bottomUp( new CollectSubQueryAlias(), new CollectJoinConstraint() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 833967447b16a5..f56f02f813d0b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -158,6 +158,7 @@ import org.apache.doris.nereids.rules.rewrite.RewriteSearchToSlots; import org.apache.doris.nereids.rules.rewrite.RewriteSimpleAggToConstantRule; import org.apache.doris.nereids.rules.rewrite.SaltJoin; +import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute; import org.apache.doris.nereids.rules.rewrite.SetPreAggStatus; import org.apache.doris.nereids.rules.rewrite.SimplifyEncodeDecode; import org.apache.doris.nereids.rules.rewrite.SimplifyWindowExpression; @@ -333,6 +334,7 @@ public class Rewriter extends AbstractBatchJobExecutor { ), // push down SEMI Join bottomUp( + new SemiJoinCommute(), new TransposeSemiJoinLogicalJoin(), new TransposeSemiJoinLogicalJoinProject(), new TransposeSemiJoinAgg(), @@ -572,6 +574,7 @@ public class Rewriter extends AbstractBatchJobExecutor { ), // push down SEMI Join bottomUp( + new SemiJoinCommute(), new TransposeSemiJoinLogicalJoin(), new TransposeSemiJoinLogicalJoinProject(), new TransposeSemiJoinAgg(), diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java index c8d8e8af7afe57..e2e1c0359ff9d8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java @@ -68,6 +68,23 @@ public void testRightOuterJoin() { testRightOuterJoinHelper(JoinType.RIGHT_OUTER_JOIN); } + @Test + public void testSemiJoinCommuteInRewrite() { + for (JoinType joinType : ImmutableList.of( + JoinType.RIGHT_OUTER_JOIN, JoinType.RIGHT_SEMI_JOIN, JoinType.RIGHT_ANTI_JOIN)) { + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + PlanChecker checker = PlanChecker.from(connectContext) + .analyze(new LogicalPlanBuilder(scan1) + .join(scan2, joinType, Pair.of(0, 0)) + .build()) + .matches(logicalJoin().when(join -> join.getJoinType() == joinType)); + + checker.rewrite() + .matches(logicalJoin().when(join -> join.getJoinType() == joinType.swap())); + } + } + private void testRightOuterJoinHelper(JoinType joinType) { ImmutableList plans = ImmutableList.of( new LogicalPlanBuilder(scan1) From a816222391f580c5bc4e0400ae5aa28ab5698f4e Mon Sep 17 00:00:00 2001 From: seawinde Date: Tue, 28 Jul 2026 17:28:10 +0800 Subject: [PATCH 2/8] [test](fe) Add SemiJoinCommute rewrite coverage ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: SemiJoinCommute now runs in rewrite pipelines instead of analysis. Add coverage for the disable_join_reorder boundary, the materialized-view pre-rewrite pipeline, and the ordering between SemiJoinCommute and semi-join transpose rules. ### Release note None ### Check List (For Author) - Test: Unit Test - ReorderJoinTest - SaltJoinTest - Behavior changed: No - Does this need documentation: No --- .../rules/rewrite/ReorderJoinTest.java | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java index e2e1c0359ff9d8..fc9011ff178644 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java @@ -18,6 +18,8 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.executor.Rewriter; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; @@ -85,6 +87,42 @@ public void testSemiJoinCommuteInRewrite() { } } + @Test + public void testDisableJoinReorderBeforeRewrite() { + for (JoinType joinType : ImmutableList.of( + JoinType.RIGHT_OUTER_JOIN, JoinType.RIGHT_SEMI_JOIN, JoinType.RIGHT_ANTI_JOIN)) { + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + PlanChecker checker = PlanChecker.from(connectContext) + .analyze(new LogicalPlanBuilder(scan1) + .join(scan2, joinType, Pair.of(0, 0)) + .build()); + + connectContext.getSessionVariable().setDisableJoinReorder(true); + checker.rewrite() + .matches(logicalJoin().when(join -> join.getJoinType() == joinType)); + } + } + + @Test + public void testSemiJoinCommuteInMvPreRewrite() { + for (JoinType joinType : ImmutableList.of( + JoinType.RIGHT_OUTER_JOIN, JoinType.RIGHT_SEMI_JOIN, JoinType.RIGHT_ANTI_JOIN)) { + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + PlanChecker checker = PlanChecker.from(connectContext) + .analyze(new LogicalPlanBuilder(scan1) + .join(scan2, joinType, Pair.of(0, 0)) + .build()); + CascadesContext cascadesContext = checker.getCascadesContext(); + + Rewriter.getCteChildrenRewriter( + cascadesContext, Rewriter.CTE_CHILDREN_REWRITE_JOBS_MV_REWRITE_USED, false).execute(); + MemoTestUtils.initMemoAndValidState(cascadesContext); + checker.matches(logicalJoin().when(join -> join.getJoinType() == joinType.swap())); + } + } + private void testRightOuterJoinHelper(JoinType joinType) { ImmutableList plans = ImmutableList.of( new LogicalPlanBuilder(scan1) @@ -170,7 +208,6 @@ public void testRightSemiJoin() { ConnectContext connectContext = MemoTestUtils.createConnectContext(); connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); PlanChecker.from(connectContext, plan2) - .applyBottomUp(new SemiJoinCommute()) .rewrite() .matchesFromRoot( logicalProject(innerLogicalJoin( From 24e03f5db9ffc1bbbe489b2c338b567b98113def Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 29 Jul 2026 10:06:30 +0800 Subject: [PATCH 3/8] [fix](fe) Fix leading constraints for right joins Issue Number: None Related PR: #66182 Problem Summary: Moving SemiJoinCommute out of analysis exposed that CollectJoinConstraint assumed right outer, semi, and anti joins had already been normalized to left joins. A preserved-side ON predicate could then be treated as a single-table filter and pushed below the join, changing query results. Canonicalize right-join constraint metadata to the existing left-join model and update unit-test expectations for the equivalent child order produced by the new rewrite stage. Fix incorrect LEADING query results for right outer, semi, and anti joins. - Test: Regression test / Unit Test - InferTest, ReorderJoinTest, SaltJoinTest - query_p0/hint/fix_leading - query_p0/hint/test_leading - Behavior changed: Yes, LEADING no longer pushes preserved-side right join ON predicates below the join. - Does this need documentation: No --- .../rules/analysis/CollectJoinConstraint.java | 31 ++++++++++++------- .../doris/nereids/sqltest/InferTest.java | 10 +++--- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java index b8b6d8adfabc37..7f78602e4c6efd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java @@ -61,6 +61,14 @@ public List buildRules() { Long leftHand = LongBitmap.computeTableBitmap(join.left().getInputRelations()); Long rightHand = LongBitmap.computeTableBitmap(join.right().getInputRelations()); join.setBitmap(LongBitmap.or(leftHand, rightHand)); + JoinType joinType = join.getJoinType(); + if (joinType.isRightSemiOrAntiJoin() || joinType.isRightOuterJoin()) { + // Leading constraints model the preserved/output side as the left child. + Long originalLeftHand = leftHand; + leftHand = rightHand; + rightHand = originalLeftHand; + joinType = joinType.swap(); + } List expressions = join.getHashJoinConjuncts(); Long totalFilterBitMap = 0L; Long nonNullableSlotBitMap = 0L; @@ -69,11 +77,11 @@ public List buildRules() { nonNullableSlotBitMap = LongBitmap.or(nonNullableSlotBitMap, nonNullable); Long filterBitMap = calSlotsTableBitMap(leading, expression.getInputSlots(), false); totalFilterBitMap = LongBitmap.or(totalFilterBitMap, filterBitMap); - if (join.getJoinType().isLeftJoin()) { + if (joinType.isLeftJoin()) { filterBitMap = LongBitmap.or(filterBitMap, rightHand); } leading.getFilters().add(Pair.of(filterBitMap, expression)); - leading.putConditionJoinType(expression, join.getJoinType()); + leading.putConditionJoinType(expression, joinType); } expressions = join.getOtherJoinConjuncts(); for (Expression expression : expressions) { @@ -81,13 +89,14 @@ public List buildRules() { nonNullableSlotBitMap = LongBitmap.or(nonNullableSlotBitMap, nonNullable); Long filterBitMap = calSlotsTableBitMap(leading, expression.getInputSlots(), false); totalFilterBitMap = LongBitmap.or(totalFilterBitMap, filterBitMap); - if (join.getJoinType().isLeftJoin()) { + if (joinType.isLeftJoin()) { filterBitMap = LongBitmap.or(filterBitMap, rightHand); } leading.getFilters().add(Pair.of(filterBitMap, expression)); - leading.putConditionJoinType(expression, join.getJoinType()); + leading.putConditionJoinType(expression, joinType); } - collectJoinConstraintList(leading, leftHand, rightHand, join, totalFilterBitMap, nonNullableSlotBitMap); + collectJoinConstraintList( + leading, leftHand, rightHand, joinType, totalFilterBitMap, nonNullableSlotBitMap); return ctx.root; }).toRule(RuleType.COLLECT_JOIN_CONSTRAINT), @@ -108,14 +117,14 @@ public List buildRules() { ); } - private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long rightHand, LogicalJoin join, + private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long rightHand, JoinType joinType, Long filterTableBitMap, Long nonNullableSlotBitMap) { Long totalTables = LongBitmap.or(leftHand, rightHand); - if (join.getJoinType().isInnerOrCrossJoin()) { + if (joinType.isInnerOrCrossJoin()) { leading.setInnerJoinBitmap(LongBitmap.or(leading.getInnerJoinBitmap(), totalTables)); return; } - if (join.getJoinType().isFullOuterJoin()) { + if (joinType.isFullOuterJoin()) { JoinConstraint newJoinConstraint = new JoinConstraint(leftHand, rightHand, leftHand, rightHand, JoinType.FULL_OUTER_JOIN, false); leading.getJoinConstraintList().add(newJoinConstraint); @@ -148,7 +157,7 @@ private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long if (LongBitmap.isOverlap(leftHand, other.getRightHand())) { if (LongBitmap.isOverlap(filterTableBitMap, other.getRightHand()) - && (join.getJoinType().isSemiOrAntiJoin() + && (joinType.isSemiOrAntiJoin() || !LongBitmap.isOverlap(nonNullableSlotBitMap, other.getMinRightHand()))) { minLeftHand = LongBitmap.or(minLeftHand, other.getLeftHand()); @@ -160,7 +169,7 @@ private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long if (LongBitmap.isOverlap(rightHand, other.getRightHand())) { if (LongBitmap.isOverlap(filterTableBitMap, other.getRightHand()) || !LongBitmap.isOverlap(filterTableBitMap, other.getMinLeftHand()) - || join.getJoinType().isSemiOrAntiJoin() + || joinType.isSemiOrAntiJoin() || other.getJoinType().isSemiOrAntiJoin() || !other.isLhsStrict()) { minRightHand = LongBitmap.or(minRightHand, other.getLeftHand()); @@ -176,7 +185,7 @@ private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long } JoinConstraint newJoinConstraint = new JoinConstraint(minLeftHand, minRightHand, leftHand, rightHand, - join.getJoinType(), isStrict); + joinType, isStrict); leading.getJoinConstraintList().add(newJoinConstraint); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java index cdc36164ae9f95..3d3fd49c428a62 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java @@ -52,13 +52,13 @@ void testInferNotNullFromFilterAndEliminateOuter2() { .printlnTree() .matches( innerLogicalJoin( - logicalFilter().when( - f -> f.getPredicate().toString().equals("(id#2 >= 4)")), logicalFilter().when( f -> ExpressionUtils.and(f.getConjuncts().stream() .sorted((a, b) -> a.toString().compareTo(b.toString())) .collect(Collectors.toList())) - .toString().equals("(id#0 >= 4)")) + .toString().equals("(id#0 >= 4)")), + logicalFilter().when( + f -> f.getPredicate().toString().equals("(id#2 >= 4)")) ) ); @@ -97,8 +97,8 @@ void testInferNotNullFromJoinAndEliminateOuter() { .rewrite() .matches( innerLogicalJoin( - logicalProject(), - logicalProject(leftSemiLogicalJoin()) + logicalProject(leftSemiLogicalJoin()), + logicalProject() ) ); } From eccd7e2b2adc15a4f4668859ff7c05ccc56bdcff Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 29 Jul 2026 10:56:39 +0800 Subject: [PATCH 4/8] [test](fe) Defer leading right join constraint coverage ### What problem does this PR solve? Issue Number: None Related PR: #66182 Problem Summary: Moving SemiJoinCommute out of analysis exposes a pre-existing LEADING dependency on Analyzer normalizing right joins to left joins. The attempted hint fix is independent of the rewrite-stage migration and needs dedicated coverage for right outer, semi, and anti joins. Remove that hint implementation from this PR and temporarily disable only the failing select4_2 regression case until the follow-up fix is ready. ### Release note None ### Check List (For Author) - Test: Regression test / Unit Test - ./run-fe-ut.sh --run org.apache.doris.nereids.sqltest.InferTest,org.apache.doris.nereids.rules.rewrite.ReorderJoinTest,org.apache.doris.nereids.rules.rewrite.SaltJoinTest - ./run-regression-test.sh --run -d query_p0 -s fix_leading - Behavior changed: Yes, defer LEADING handling for original right joins to a follow-up PR and temporarily skip its failing regression case. - Does this need documentation: No --- .../rules/analysis/CollectJoinConstraint.java | 31 +++++++------------ .../suites/nereids_p0/hint/fix_leading.groovy | 3 +- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java index 7f78602e4c6efd..b8b6d8adfabc37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java @@ -61,14 +61,6 @@ public List buildRules() { Long leftHand = LongBitmap.computeTableBitmap(join.left().getInputRelations()); Long rightHand = LongBitmap.computeTableBitmap(join.right().getInputRelations()); join.setBitmap(LongBitmap.or(leftHand, rightHand)); - JoinType joinType = join.getJoinType(); - if (joinType.isRightSemiOrAntiJoin() || joinType.isRightOuterJoin()) { - // Leading constraints model the preserved/output side as the left child. - Long originalLeftHand = leftHand; - leftHand = rightHand; - rightHand = originalLeftHand; - joinType = joinType.swap(); - } List expressions = join.getHashJoinConjuncts(); Long totalFilterBitMap = 0L; Long nonNullableSlotBitMap = 0L; @@ -77,11 +69,11 @@ public List buildRules() { nonNullableSlotBitMap = LongBitmap.or(nonNullableSlotBitMap, nonNullable); Long filterBitMap = calSlotsTableBitMap(leading, expression.getInputSlots(), false); totalFilterBitMap = LongBitmap.or(totalFilterBitMap, filterBitMap); - if (joinType.isLeftJoin()) { + if (join.getJoinType().isLeftJoin()) { filterBitMap = LongBitmap.or(filterBitMap, rightHand); } leading.getFilters().add(Pair.of(filterBitMap, expression)); - leading.putConditionJoinType(expression, joinType); + leading.putConditionJoinType(expression, join.getJoinType()); } expressions = join.getOtherJoinConjuncts(); for (Expression expression : expressions) { @@ -89,14 +81,13 @@ public List buildRules() { nonNullableSlotBitMap = LongBitmap.or(nonNullableSlotBitMap, nonNullable); Long filterBitMap = calSlotsTableBitMap(leading, expression.getInputSlots(), false); totalFilterBitMap = LongBitmap.or(totalFilterBitMap, filterBitMap); - if (joinType.isLeftJoin()) { + if (join.getJoinType().isLeftJoin()) { filterBitMap = LongBitmap.or(filterBitMap, rightHand); } leading.getFilters().add(Pair.of(filterBitMap, expression)); - leading.putConditionJoinType(expression, joinType); + leading.putConditionJoinType(expression, join.getJoinType()); } - collectJoinConstraintList( - leading, leftHand, rightHand, joinType, totalFilterBitMap, nonNullableSlotBitMap); + collectJoinConstraintList(leading, leftHand, rightHand, join, totalFilterBitMap, nonNullableSlotBitMap); return ctx.root; }).toRule(RuleType.COLLECT_JOIN_CONSTRAINT), @@ -117,14 +108,14 @@ public List buildRules() { ); } - private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long rightHand, JoinType joinType, + private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long rightHand, LogicalJoin join, Long filterTableBitMap, Long nonNullableSlotBitMap) { Long totalTables = LongBitmap.or(leftHand, rightHand); - if (joinType.isInnerOrCrossJoin()) { + if (join.getJoinType().isInnerOrCrossJoin()) { leading.setInnerJoinBitmap(LongBitmap.or(leading.getInnerJoinBitmap(), totalTables)); return; } - if (joinType.isFullOuterJoin()) { + if (join.getJoinType().isFullOuterJoin()) { JoinConstraint newJoinConstraint = new JoinConstraint(leftHand, rightHand, leftHand, rightHand, JoinType.FULL_OUTER_JOIN, false); leading.getJoinConstraintList().add(newJoinConstraint); @@ -157,7 +148,7 @@ private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long if (LongBitmap.isOverlap(leftHand, other.getRightHand())) { if (LongBitmap.isOverlap(filterTableBitMap, other.getRightHand()) - && (joinType.isSemiOrAntiJoin() + && (join.getJoinType().isSemiOrAntiJoin() || !LongBitmap.isOverlap(nonNullableSlotBitMap, other.getMinRightHand()))) { minLeftHand = LongBitmap.or(minLeftHand, other.getLeftHand()); @@ -169,7 +160,7 @@ private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long if (LongBitmap.isOverlap(rightHand, other.getRightHand())) { if (LongBitmap.isOverlap(filterTableBitMap, other.getRightHand()) || !LongBitmap.isOverlap(filterTableBitMap, other.getMinLeftHand()) - || joinType.isSemiOrAntiJoin() + || join.getJoinType().isSemiOrAntiJoin() || other.getJoinType().isSemiOrAntiJoin() || !other.isLhsStrict()) { minRightHand = LongBitmap.or(minRightHand, other.getLeftHand()); @@ -185,7 +176,7 @@ private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long } JoinConstraint newJoinConstraint = new JoinConstraint(minLeftHand, minRightHand, leftHand, rightHand, - joinType, isStrict); + join.getJoinType(), isStrict); leading.getJoinConstraintList().add(newJoinConstraint); } diff --git a/regression-test/suites/nereids_p0/hint/fix_leading.groovy b/regression-test/suites/nereids_p0/hint/fix_leading.groovy index d70d39c75b38fc..e6b1e0037a45a2 100644 --- a/regression-test/suites/nereids_p0/hint/fix_leading.groovy +++ b/regression-test/suites/nereids_p0/hint/fix_leading.groovy @@ -195,7 +195,8 @@ suite("fix_leading") { // check left right join result qt_select4_1 """select count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" - qt_select4_2 """select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" + // TODO: Re-enable after LEADING handles right join constraints without Analyzer normalization. + // qt_select4_2 """select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" qt_select4_3 """explain shape plan select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" // check whether we have all tables From cb13213d6ce6ebbd75d1df673de21bd7f5e71641 Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 29 Jul 2026 16:08:06 +0800 Subject: [PATCH 5/8] [test](regression) Defer leading right join shape coverage ### What problem does this PR solve? Issue Number: None Related PR: #66182 Problem Summary: Moving SemiJoinCommute out of analysis exposes a pre-existing LEADING dependency on Analyzer normalizing right joins. Temporarily disable the corresponding select4_3 shape assertion alongside select4_2 until the independent LEADING fix is ready. ### Release note None ### Check List (For Author) - Test: Regression test - Verified the P0 failure is isolated to select4_3; select2_2 and select2_5 complete before it - Behavior changed: No - Does this need documentation: No --- regression-test/suites/nereids_p0/hint/fix_leading.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/nereids_p0/hint/fix_leading.groovy b/regression-test/suites/nereids_p0/hint/fix_leading.groovy index e6b1e0037a45a2..9dc219042d6bcb 100644 --- a/regression-test/suites/nereids_p0/hint/fix_leading.groovy +++ b/regression-test/suites/nereids_p0/hint/fix_leading.groovy @@ -197,7 +197,7 @@ suite("fix_leading") { qt_select4_1 """select count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" // TODO: Re-enable after LEADING handles right join constraints without Analyzer normalization. // qt_select4_2 """select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" - qt_select4_3 """explain shape plan select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" + // qt_select4_3 """explain shape plan select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" // check whether we have all tables explain { From 9ff4e323c67c72d50cef6eea9c972a6117688768 Mon Sep 17 00:00:00 2001 From: seawinde Date: Mon, 3 Aug 2026 17:22:57 +0800 Subject: [PATCH 6/8] [fix](fe) Fix leading constraints for right joins ### What problem does this PR solve? Issue Number: None Related PR: #66182 Problem Summary: Moving SemiJoinCommute out of analysis exposes that LEADING constraint collection assumes right joins have already been normalized to left joins. RIGHT OUTER and RIGHT ANTI preserved-side predicates can then be pushed below their scans, while a RIGHT SEMI join can consume its condition before the complete non-output side is built and duplicate output rows. Canonicalize right-join constraint metadata to the equivalent left-join model without changing the analyzed plan, and restore or add focused regression coverage. ### Release note Fix incorrect LEADING query results for right outer, semi, and anti joins. ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.nereids.rules.rewrite.ReorderJoinTest,org.apache.doris.nereids.sqltest.SqlPlanSuiteTest (34 tests passed) - Regression test not run yet; waiting for FE to be built and restarted with this commit - Behavior changed: Yes. LEADING preserves right-join boundaries and ON predicate semantics. - Does this need documentation: No --- .../rules/analysis/CollectJoinConstraint.java | 31 ++++++++++++------- .../suites/nereids_p0/hint/fix_leading.groovy | 28 +++++++++++++++-- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java index b8b6d8adfabc37..eacdd3fdd41fe7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java @@ -61,6 +61,14 @@ public List buildRules() { Long leftHand = LongBitmap.computeTableBitmap(join.left().getInputRelations()); Long rightHand = LongBitmap.computeTableBitmap(join.right().getInputRelations()); join.setBitmap(LongBitmap.or(leftHand, rightHand)); + JoinType joinType = join.getJoinType(); + if (joinType.isRightJoin()) { + // LEADING constraints model the preserved/output side as the left child. + Long originalLeftHand = leftHand; + leftHand = rightHand; + rightHand = originalLeftHand; + joinType = joinType.swap(); + } List expressions = join.getHashJoinConjuncts(); Long totalFilterBitMap = 0L; Long nonNullableSlotBitMap = 0L; @@ -69,11 +77,11 @@ public List buildRules() { nonNullableSlotBitMap = LongBitmap.or(nonNullableSlotBitMap, nonNullable); Long filterBitMap = calSlotsTableBitMap(leading, expression.getInputSlots(), false); totalFilterBitMap = LongBitmap.or(totalFilterBitMap, filterBitMap); - if (join.getJoinType().isLeftJoin()) { + if (joinType.isLeftJoin()) { filterBitMap = LongBitmap.or(filterBitMap, rightHand); } leading.getFilters().add(Pair.of(filterBitMap, expression)); - leading.putConditionJoinType(expression, join.getJoinType()); + leading.putConditionJoinType(expression, joinType); } expressions = join.getOtherJoinConjuncts(); for (Expression expression : expressions) { @@ -81,13 +89,14 @@ public List buildRules() { nonNullableSlotBitMap = LongBitmap.or(nonNullableSlotBitMap, nonNullable); Long filterBitMap = calSlotsTableBitMap(leading, expression.getInputSlots(), false); totalFilterBitMap = LongBitmap.or(totalFilterBitMap, filterBitMap); - if (join.getJoinType().isLeftJoin()) { + if (joinType.isLeftJoin()) { filterBitMap = LongBitmap.or(filterBitMap, rightHand); } leading.getFilters().add(Pair.of(filterBitMap, expression)); - leading.putConditionJoinType(expression, join.getJoinType()); + leading.putConditionJoinType(expression, joinType); } - collectJoinConstraintList(leading, leftHand, rightHand, join, totalFilterBitMap, nonNullableSlotBitMap); + collectJoinConstraintList( + leading, leftHand, rightHand, joinType, totalFilterBitMap, nonNullableSlotBitMap); return ctx.root; }).toRule(RuleType.COLLECT_JOIN_CONSTRAINT), @@ -108,14 +117,14 @@ public List buildRules() { ); } - private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long rightHand, LogicalJoin join, + private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long rightHand, JoinType joinType, Long filterTableBitMap, Long nonNullableSlotBitMap) { Long totalTables = LongBitmap.or(leftHand, rightHand); - if (join.getJoinType().isInnerOrCrossJoin()) { + if (joinType.isInnerOrCrossJoin()) { leading.setInnerJoinBitmap(LongBitmap.or(leading.getInnerJoinBitmap(), totalTables)); return; } - if (join.getJoinType().isFullOuterJoin()) { + if (joinType.isFullOuterJoin()) { JoinConstraint newJoinConstraint = new JoinConstraint(leftHand, rightHand, leftHand, rightHand, JoinType.FULL_OUTER_JOIN, false); leading.getJoinConstraintList().add(newJoinConstraint); @@ -148,7 +157,7 @@ private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long if (LongBitmap.isOverlap(leftHand, other.getRightHand())) { if (LongBitmap.isOverlap(filterTableBitMap, other.getRightHand()) - && (join.getJoinType().isSemiOrAntiJoin() + && (joinType.isSemiOrAntiJoin() || !LongBitmap.isOverlap(nonNullableSlotBitMap, other.getMinRightHand()))) { minLeftHand = LongBitmap.or(minLeftHand, other.getLeftHand()); @@ -160,7 +169,7 @@ private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long if (LongBitmap.isOverlap(rightHand, other.getRightHand())) { if (LongBitmap.isOverlap(filterTableBitMap, other.getRightHand()) || !LongBitmap.isOverlap(filterTableBitMap, other.getMinLeftHand()) - || join.getJoinType().isSemiOrAntiJoin() + || joinType.isSemiOrAntiJoin() || other.getJoinType().isSemiOrAntiJoin() || !other.isLhsStrict()) { minRightHand = LongBitmap.or(minRightHand, other.getLeftHand()); @@ -176,7 +185,7 @@ private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long } JoinConstraint newJoinConstraint = new JoinConstraint(minLeftHand, minRightHand, leftHand, rightHand, - join.getJoinType(), isStrict); + joinType, isStrict); leading.getJoinConstraintList().add(newJoinConstraint); } diff --git a/regression-test/suites/nereids_p0/hint/fix_leading.groovy b/regression-test/suites/nereids_p0/hint/fix_leading.groovy index 9dc219042d6bcb..aa002bda91cd1a 100644 --- a/regression-test/suites/nereids_p0/hint/fix_leading.groovy +++ b/regression-test/suites/nereids_p0/hint/fix_leading.groovy @@ -40,6 +40,9 @@ suite("fix_leading") { sql """drop table if exists t2;""" sql """drop table if exists t3;""" sql """drop table if exists t4;""" + sql """drop table if exists right_join_a;""" + sql """drop table if exists right_join_b;""" + sql """drop table if exists right_join_c;""" sql """create table t1 (c1 int, c11 int) distributed by hash(c1) buckets 3 properties('replication_num' = '1');""" sql """create table t2 (c2 int, c22 int) distributed by hash(c2) buckets 3 properties('replication_num' = '1');""" @@ -47,6 +50,13 @@ suite("fix_leading") { sql """create table t4 (c4 int, c44 int) distributed by hash(c4) buckets 3 properties('replication_num' = '1');""" sql """create table t5 (c5 int, c55 int) distributed by hash(c5) buckets 3 properties('replication_num' = '1');""" sql """create table t6 (c6 int, c66 int) distributed by hash(c6) buckets 3 properties('replication_num' = '1');""" + sql """create table right_join_a (k int) distributed by hash(k) buckets 1 properties('replication_num' = '1');""" + sql """create table right_join_b (k int) distributed by hash(k) buckets 1 properties('replication_num' = '1');""" + sql """create table right_join_c (k int) distributed by hash(k) buckets 1 properties('replication_num' = '1');""" + + sql """insert into right_join_a values (1);""" + sql """insert into right_join_b values (0), (1);""" + sql """insert into right_join_c values (1), (2);""" streamLoad { table "t1" @@ -195,9 +205,21 @@ suite("fix_leading") { // check left right join result qt_select4_1 """select count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" - // TODO: Re-enable after LEADING handles right join constraints without Analyzer normalization. - // qt_select4_2 """select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" - // qt_select4_3 """explain shape plan select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" + qt_select4_2 """select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" + qt_select4_3 """explain shape plan select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" + + // check right semi join keeps its complete non-output side + qt_select4_4 """select /*+ leading(right_join_b right_join_a right_join_c) */ count(*) + from right_join_a cross join right_join_c + right semi join right_join_b on right_join_a.k = right_join_b.k;""" + qt_select4_5 """explain shape plan select /*+ leading(right_join_b right_join_a right_join_c) */ * + from right_join_a cross join right_join_c + right semi join right_join_b on right_join_a.k = right_join_b.k;""" + + // check right anti join does not push its preserved-side ON predicate below the join + qt_select4_6 """select /*+ leading(right_join_b right_join_a right_join_c) */ count(*) + from right_join_a cross join right_join_c + right anti join right_join_b on right_join_a.k = right_join_b.k and right_join_b.k > 0;""" // check whether we have all tables explain { From 88a7aaa5388088c3cce20b11fdc43a9e5b79ee35 Mon Sep 17 00:00:00 2001 From: seawinde Date: Tue, 4 Aug 2026 07:27:34 +0800 Subject: [PATCH 7/8] [test](regression) Refresh leading right join outputs ### What problem does this PR solve? Issue Number: None Related PR: #66182 Problem Summary: Refresh the fix_leading expected output after restoring right outer join coverage and adding focused right semi and right anti join cases. The generated results verify correct row counts and preserve the complete non-output side of the right semi join. ### Release note None ### Check List (For Author) - Test: Regression test - ./run-regression-test.sh --run -d query_p0 -s fix_leading -forceGenOut - ./run-regression-test.sh --run -d query_p0 -s fix_leading - Behavior changed: No - Does this need documentation: No --- .../data/nereids_p0/hint/fix_leading.out | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/regression-test/data/nereids_p0/hint/fix_leading.out b/regression-test/data/nereids_p0/hint/fix_leading.out index 73abf90eb058de..2e7f5c1e17379b 100644 --- a/regression-test/data/nereids_p0/hint/fix_leading.out +++ b/regression-test/data/nereids_p0/hint/fix_leading.out @@ -243,6 +243,25 @@ Used: leading(t1 t2 t3 ) UnUsed: SyntaxError: +-- !select4_4 -- +1 + +-- !select4_5 -- +PhysicalResultSink +--hashJoin[LEFT_SEMI_JOIN] hashCondition=((right_join_a.k = right_join_b.k)) otherCondition=() +----PhysicalOlapScan[right_join_b] +----NestedLoopJoin[CROSS_JOIN] +------PhysicalOlapScan[right_join_a] +------PhysicalStorageLayerAggregate[right_join_c] + +Hint log: +Used: +UnUsed: leading(right_join_b right_join_a right_join_c) +SyntaxError: + +-- !select4_6 -- +1 + -- !select6_1 -- PhysicalResultSink --hashAgg[GLOBAL] From 2056a14dcc3b876fd9e60ee5f3b538d3a33b41f0 Mon Sep 17 00:00:00 2001 From: seawinde Date: Tue, 4 Aug 2026 10:11:29 +0800 Subject: [PATCH 8/8] [test](regression) Remove unstable right semi join shape check ### What problem does this PR solve? Issue Number: None Related PR: #66182 Problem Summary: The right semi join regression test compared a complete physical plan shape even though equivalent CROSS JOIN children may be reordered by cost and statistics. Remove the unstable shape assertion while retaining result checks that reproduce the incorrect right semi and right anti join behavior. ### Release note None ### Check List (For Author) - Test: Regression test - ./run-regression-test.sh --run -d query_p0 -s fix_leading -forceGenOut - ./run-regression-test.sh --run -d query_p0 -s fix_leading - Behavior changed: No - Does this need documentation: No --- .../data/nereids_p0/hint/fix_leading.out | 13 ------------- .../suites/nereids_p0/hint/fix_leading.groovy | 3 --- 2 files changed, 16 deletions(-) diff --git a/regression-test/data/nereids_p0/hint/fix_leading.out b/regression-test/data/nereids_p0/hint/fix_leading.out index 2e7f5c1e17379b..b3e5989477c555 100644 --- a/regression-test/data/nereids_p0/hint/fix_leading.out +++ b/regression-test/data/nereids_p0/hint/fix_leading.out @@ -246,19 +246,6 @@ SyntaxError: -- !select4_4 -- 1 --- !select4_5 -- -PhysicalResultSink ---hashJoin[LEFT_SEMI_JOIN] hashCondition=((right_join_a.k = right_join_b.k)) otherCondition=() -----PhysicalOlapScan[right_join_b] -----NestedLoopJoin[CROSS_JOIN] -------PhysicalOlapScan[right_join_a] -------PhysicalStorageLayerAggregate[right_join_c] - -Hint log: -Used: -UnUsed: leading(right_join_b right_join_a right_join_c) -SyntaxError: - -- !select4_6 -- 1 diff --git a/regression-test/suites/nereids_p0/hint/fix_leading.groovy b/regression-test/suites/nereids_p0/hint/fix_leading.groovy index aa002bda91cd1a..dda3587a07356a 100644 --- a/regression-test/suites/nereids_p0/hint/fix_leading.groovy +++ b/regression-test/suites/nereids_p0/hint/fix_leading.groovy @@ -212,9 +212,6 @@ suite("fix_leading") { qt_select4_4 """select /*+ leading(right_join_b right_join_a right_join_c) */ count(*) from right_join_a cross join right_join_c right semi join right_join_b on right_join_a.k = right_join_b.k;""" - qt_select4_5 """explain shape plan select /*+ leading(right_join_b right_join_a right_join_c) */ * - from right_join_a cross join right_join_c - right semi join right_join_b on right_join_a.k = right_join_b.k;""" // check right anti join does not push its preserved-side ON predicate below the join qt_select4_6 """select /*+ leading(right_join_b right_join_a right_join_c) */ count(*)