branch:4.2: [fix](fe) Move SemiJoinCommute to rewrite phase #66182

fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java

@@ -52,7 +52,6 @@
 import org.apache.doris.nereids.rules.analysis.SubqueryToApply;
 import org.apache.doris.nereids.rules.rewrite.AdjustNullable;
 import org.apache.doris.nereids.rules.rewrite.MergeFilters;
-import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute;
 import org.apache.doris.nereids.rules.rewrite.SimplifyAggGroupBy;
 import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
 import org.apache.doris.nereids.trees.plans.logical.LogicalView;
@@ -188,7 +187,6 @@ private static List<RewriteJob> buildAnalyzerJobs() {
             topDown(new NormalizeAggregate()),
             topDown(new HavingToFilter()),
             topDown(new QualifyToFilter()),
-            bottomUp(new SemiJoinCommute()),
             bottomUp(
                     new CollectSubQueryAlias(),
                     new CollectJoinConstraint()

fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java

@@ -156,6 +156,7 @@
 import org.apache.doris.nereids.rules.rewrite.RewriteSearchToSlots;
 import org.apache.doris.nereids.rules.rewrite.RewriteSimpleAggToConstantRule;
 import org.apache.doris.nereids.rules.rewrite.SaltJoin;
+import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute;
 import org.apache.doris.nereids.rules.rewrite.SetPreAggStatus;
 import org.apache.doris.nereids.rules.rewrite.SimplifyEncodeDecode;
 import org.apache.doris.nereids.rules.rewrite.SimplifyWindowExpression;
@@ -331,6 +332,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
                                     ),
                                     // push down SEMI Join
                                     bottomUp(
+                                            new SemiJoinCommute(),
                                             new TransposeSemiJoinLogicalJoin(),
                                             new TransposeSemiJoinLogicalJoinProject(),
                                             new TransposeSemiJoinAgg(),
@@ -570,6 +572,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
                                 ),
                                 // push down SEMI Join
                                 bottomUp(
+                                        new SemiJoinCommute(),
                                         new TransposeSemiJoinLogicalJoin(),
                                         new TransposeSemiJoinLogicalJoinProject(),
                                         new TransposeSemiJoinAgg(),

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java

@@ -62,6 +62,14 @@ public List<Rule> buildRules() {
                 Long leftHand = LongBitmap.computeTableBitmap(join.left().getInputRelations());
                 Long rightHand = LongBitmap.computeTableBitmap(join.right().getInputRelations());
                 join.setBitmap(LongBitmap.or(leftHand, rightHand));
+                JoinType joinType = join.getJoinType();
+                if (joinType.isRightJoin()) {
+                    // LEADING constraints model the preserved/output side as the left child.
+                    Long originalLeftHand = leftHand;
+                    leftHand = rightHand;
+                    rightHand = originalLeftHand;
+                    joinType = joinType.swap();
+                }
                 List<Expression> expressions = join.getHashJoinConjuncts();
                 Long totalFilterBitMap = 0L;
                 Long nonNullableSlotBitMap = 0L;
@@ -70,25 +78,26 @@ public List<Rule> buildRules() {
                     nonNullableSlotBitMap = LongBitmap.or(nonNullableSlotBitMap, nonNullable);
                     Long filterBitMap = calSlotsTableBitMap(leading, expression.getInputSlots(), false);
                     totalFilterBitMap = LongBitmap.or(totalFilterBitMap, filterBitMap);
-                    if (join.getJoinType().isLeftJoin()) {
+                    if (joinType.isLeftJoin()) {
                         filterBitMap = LongBitmap.or(filterBitMap, rightHand);
                     }
                     leading.getFilters().add(Pair.of(filterBitMap, expression));
-                    leading.putConditionJoinType(expression, join.getJoinType());
+                    leading.putConditionJoinType(expression, joinType);
                 }
                 expressions = join.getOtherJoinConjuncts();
                 for (Expression expression : expressions) {
                     Long nonNullable = calSlotsTableBitMap(leading, expression.getInputSlots(), true);
                     nonNullableSlotBitMap = LongBitmap.or(nonNullableSlotBitMap, nonNullable);
                     Long filterBitMap = calSlotsTableBitMap(leading, expression.getInputSlots(), false);
                     totalFilterBitMap = LongBitmap.or(totalFilterBitMap, filterBitMap);
-                    if (join.getJoinType().isLeftJoin()) {
+                    if (joinType.isLeftJoin()) {
                         filterBitMap = LongBitmap.or(filterBitMap, rightHand);
                     }
                     leading.getFilters().add(Pair.of(filterBitMap, expression));
-                    leading.putConditionJoinType(expression, join.getJoinType());
+                    leading.putConditionJoinType(expression, joinType);
                 }
-                collectJoinConstraintList(leading, leftHand, rightHand, join, totalFilterBitMap, nonNullableSlotBitMap);
+                collectJoinConstraintList(
+                        leading, leftHand, rightHand, joinType, totalFilterBitMap, nonNullableSlotBitMap);
 
                 return ctx.root;
             }).toRule(RuleType.COLLECT_JOIN_CONSTRAINT)
@@ -115,14 +124,14 @@ private void collectLeafPlan(LeadingHint leading, LogicalPlan child) {
         leading.getRelationIdToScanMap().put(relationId, child);
     }
 
-    private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long rightHand, LogicalJoin join,
+    private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long rightHand, JoinType joinType,
                                             Long filterTableBitMap, Long nonNullableSlotBitMap) {
         Long totalTables = LongBitmap.or(leftHand, rightHand);
-        if (join.getJoinType().isInnerOrCrossJoin()) {
+        if (joinType.isInnerOrCrossJoin()) {
             leading.setInnerJoinBitmap(LongBitmap.or(leading.getInnerJoinBitmap(), totalTables));
             return;
         }
-        if (join.getJoinType().isFullOuterJoin()) {
+        if (joinType.isFullOuterJoin()) {
             JoinConstraint newJoinConstraint = new JoinConstraint(leftHand, rightHand, leftHand, rightHand,
                     JoinType.FULL_OUTER_JOIN, false);
             leading.getJoinConstraintList().add(newJoinConstraint);
@@ -155,7 +164,7 @@ private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long
 
             if (LongBitmap.isOverlap(leftHand, other.getRightHand())) {
                 if (LongBitmap.isOverlap(filterTableBitMap, other.getRightHand())
-                        && (join.getJoinType().isSemiOrAntiJoin()
+                        && (joinType.isSemiOrAntiJoin()
                         || !LongBitmap.isOverlap(nonNullableSlotBitMap, other.getMinRightHand()))) {
                     minLeftHand = LongBitmap.or(minLeftHand,
                         other.getLeftHand());
@@ -167,7 +176,7 @@ private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long
             if (LongBitmap.isOverlap(rightHand, other.getRightHand())) {
                 if (LongBitmap.isOverlap(filterTableBitMap, other.getRightHand())
                         || !LongBitmap.isOverlap(filterTableBitMap, other.getMinLeftHand())
-                        || join.getJoinType().isSemiOrAntiJoin()
+                        || joinType.isSemiOrAntiJoin()
                         || other.getJoinType().isSemiOrAntiJoin()
                         || !other.isLhsStrict()) {
                     minRightHand = LongBitmap.or(minRightHand, other.getLeftHand());
@@ -183,7 +192,7 @@ private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long
         }
 
         JoinConstraint newJoinConstraint = new JoinConstraint(minLeftHand, minRightHand, leftHand, rightHand,
-                join.getJoinType(), isStrict);
+                joinType, isStrict);
         leading.getJoinConstraintList().add(newJoinConstraint);
     }
 

fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java

@@ -18,6 +18,8 @@
 package org.apache.doris.nereids.rules.rewrite;
 
 import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.jobs.executor.Rewriter;
 import org.apache.doris.nereids.trees.expressions.EqualTo;
 import org.apache.doris.nereids.trees.plans.JoinType;
 import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
@@ -68,6 +70,59 @@ public void testRightOuterJoin() {
         testRightOuterJoinHelper(JoinType.RIGHT_OUTER_JOIN);
     }
 
+    @Test
+    public void testSemiJoinCommuteInRewrite() {
+        for (JoinType joinType : ImmutableList.of(
+                JoinType.RIGHT_OUTER_JOIN, JoinType.RIGHT_SEMI_JOIN, JoinType.RIGHT_ANTI_JOIN)) {
+            ConnectContext connectContext = MemoTestUtils.createConnectContext();
+            connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
+            PlanChecker checker = PlanChecker.from(connectContext)
+                    .analyze(new LogicalPlanBuilder(scan1)
+                            .join(scan2, joinType, Pair.of(0, 0))
+                            .build())
+                    .matches(logicalJoin().when(join -> join.getJoinType() == joinType));
+
+            checker.rewrite()
+                    .matches(logicalJoin().when(join -> join.getJoinType() == joinType.swap()));
+        }
+    }
+
+    @Test
+    public void testDisableJoinReorderBeforeRewrite() {
+        for (JoinType joinType : ImmutableList.of(
+                JoinType.RIGHT_OUTER_JOIN, JoinType.RIGHT_SEMI_JOIN, JoinType.RIGHT_ANTI_JOIN)) {
+            ConnectContext connectContext = MemoTestUtils.createConnectContext();
+            connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
+            PlanChecker checker = PlanChecker.from(connectContext)
+                    .analyze(new LogicalPlanBuilder(scan1)
+                            .join(scan2, joinType, Pair.of(0, 0))
+                            .build());
+
+            connectContext.getSessionVariable().setDisableJoinReorder(true);
+            checker.rewrite()
+                    .matches(logicalJoin().when(join -> join.getJoinType() == joinType));
+        }
+    }
+
+    @Test
+    public void testSemiJoinCommuteInMvPreRewrite() {
+        for (JoinType joinType : ImmutableList.of(
+                JoinType.RIGHT_OUTER_JOIN, JoinType.RIGHT_SEMI_JOIN, JoinType.RIGHT_ANTI_JOIN)) {
+            ConnectContext connectContext = MemoTestUtils.createConnectContext();
+            connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
+            PlanChecker checker = PlanChecker.from(connectContext)
+                    .analyze(new LogicalPlanBuilder(scan1)
+                            .join(scan2, joinType, Pair.of(0, 0))
+                            .build());
+            CascadesContext cascadesContext = checker.getCascadesContext();
+
+            Rewriter.getCteChildrenRewriter(
+                    cascadesContext, Rewriter.CTE_CHILDREN_REWRITE_JOBS_MV_REWRITE_USED, false).execute();
+            MemoTestUtils.initMemoAndValidState(cascadesContext);
+            checker.matches(logicalJoin().when(join -> join.getJoinType() == joinType.swap()));
+        }
+    }
+
     private void testRightOuterJoinHelper(JoinType joinType) {
         ImmutableList<LogicalPlan> plans = ImmutableList.of(
                 new LogicalPlanBuilder(scan1)
@@ -153,7 +208,6 @@ public void testRightSemiJoin() {
         ConnectContext connectContext = MemoTestUtils.createConnectContext();
         connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
         PlanChecker.from(connectContext, plan2)
-                .applyBottomUp(new SemiJoinCommute())
                 .rewrite()
                 .matchesFromRoot(
                         logicalProject(innerLogicalJoin(

fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java

@@ -52,13 +52,13 @@ void testInferNotNullFromFilterAndEliminateOuter2() {
                 .printlnTree()
                 .matches(
                     innerLogicalJoin(
-                        logicalFilter().when(
-                                f -> f.getPredicate().toString().equals("(id#2 >= 4)")),
                         logicalFilter().when(
                                 f -> ExpressionUtils.and(f.getConjuncts().stream()
                                         .sorted((a, b) -> a.toString().compareTo(b.toString()))
                                         .collect(Collectors.toList()))
-                                        .toString().equals("(id#0 >= 4)"))
+                                        .toString().equals("(id#0 >= 4)")),
+                        logicalFilter().when(
+                                f -> f.getPredicate().toString().equals("(id#2 >= 4)"))
                     )
 
                 );
@@ -97,8 +97,8 @@ void testInferNotNullFromJoinAndEliminateOuter() {
                 .rewrite()
                 .matches(
                         innerLogicalJoin(
-                                logicalProject(),
-                                logicalProject(leftSemiLogicalJoin())
+                                logicalProject(leftSemiLogicalJoin()),
+                                logicalProject()
                         )
                 );
     }

regression-test/data/nereids_p0/hint/fix_leading.out

@@ -243,6 +243,12 @@ Used: leading(t1 t2 t3 )
 UnUsed:
 SyntaxError:
 
+-- !select4_4 --
+1
+
+-- !select4_6 --
+1
+
 -- !select6_1 --
 PhysicalResultSink
 --hashAgg[GLOBAL]

regression-test/suites/nereids_p0/hint/fix_leading.groovy

@@ -40,13 +40,23 @@ suite("fix_leading") {
     sql """drop table if exists t2;"""
     sql """drop table if exists t3;"""
     sql """drop table if exists t4;"""
+    sql """drop table if exists right_join_a;"""
+    sql """drop table if exists right_join_b;"""
+    sql """drop table if exists right_join_c;"""
 
     sql """create table t1 (c1 int, c11 int) distributed by hash(c1) buckets 3 properties('replication_num' = '1');"""
     sql """create table t2 (c2 int, c22 int) distributed by hash(c2) buckets 3 properties('replication_num' = '1');"""
     sql """create table t3 (c3 int, c33 int) distributed by hash(c3) buckets 3 properties('replication_num' = '1');"""
     sql """create table t4 (c4 int, c44 int) distributed by hash(c4) buckets 3 properties('replication_num' = '1');"""
     sql """create table t5 (c5 int, c55 int) distributed by hash(c5) buckets 3 properties('replication_num' = '1');"""
     sql """create table t6 (c6 int, c66 int) distributed by hash(c6) buckets 3 properties('replication_num' = '1');"""
+    sql """create table right_join_a (k int) distributed by hash(k) buckets 1 properties('replication_num' = '1');"""
+    sql """create table right_join_b (k int) distributed by hash(k) buckets 1 properties('replication_num' = '1');"""
+    sql """create table right_join_c (k int) distributed by hash(k) buckets 1 properties('replication_num' = '1');"""
+
+    sql """insert into right_join_a values (1);"""
+    sql """insert into right_join_b values (0), (1);"""
+    sql """insert into right_join_c values (1), (2);"""
 
     streamLoad {
         table "t1"
@@ -198,6 +208,16 @@ suite("fix_leading") {
     qt_select4_2 """select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;"""
     qt_select4_3 """explain shape plan select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;"""
 
+    // check right semi join keeps its complete non-output side
+    qt_select4_4 """select /*+ leading(right_join_b right_join_a right_join_c) */ count(*)
+        from right_join_a cross join right_join_c
+        right semi join right_join_b on right_join_a.k = right_join_b.k;"""
+
+    // check right anti join does not push its preserved-side ON predicate below the join
+    qt_select4_6 """select /*+ leading(right_join_b right_join_a right_join_c) */ count(*)
+        from right_join_a cross join right_join_c
+        right anti join right_join_b on right_join_a.k = right_join_b.k and right_join_b.k > 0;"""
+
     // check whether we have all tables
     explain {
         sql """shape plan select /*+ leading(t1 t2)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;"""