diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java index 1f83b242cf8489..b5c7078222a110 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java @@ -52,7 +52,6 @@ import org.apache.doris.nereids.rules.analysis.SubqueryToApply; import org.apache.doris.nereids.rules.rewrite.AdjustNullable; import org.apache.doris.nereids.rules.rewrite.MergeFilters; -import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute; import org.apache.doris.nereids.rules.rewrite.SimplifyAggGroupBy; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; import org.apache.doris.nereids.trees.plans.logical.LogicalView; @@ -188,7 +187,6 @@ private static List buildAnalyzerJobs() { topDown(new NormalizeAggregate()), topDown(new HavingToFilter()), topDown(new QualifyToFilter()), - bottomUp(new SemiJoinCommute()), bottomUp( new CollectSubQueryAlias(), new CollectJoinConstraint() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 0c22fca5f83f70..738df92f2da1fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -156,6 +156,7 @@ import org.apache.doris.nereids.rules.rewrite.RewriteSearchToSlots; import org.apache.doris.nereids.rules.rewrite.RewriteSimpleAggToConstantRule; import org.apache.doris.nereids.rules.rewrite.SaltJoin; +import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute; import org.apache.doris.nereids.rules.rewrite.SetPreAggStatus; import org.apache.doris.nereids.rules.rewrite.SimplifyEncodeDecode; import org.apache.doris.nereids.rules.rewrite.SimplifyWindowExpression; @@ -331,6 +332,7 @@ public class Rewriter extends AbstractBatchJobExecutor { ), // push down SEMI Join bottomUp( + new SemiJoinCommute(), new TransposeSemiJoinLogicalJoin(), new TransposeSemiJoinLogicalJoinProject(), new TransposeSemiJoinAgg(), @@ -570,6 +572,7 @@ public class Rewriter extends AbstractBatchJobExecutor { ), // push down SEMI Join bottomUp( + new SemiJoinCommute(), new TransposeSemiJoinLogicalJoin(), new TransposeSemiJoinLogicalJoinProject(), new TransposeSemiJoinAgg(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java index 56e6b6f72350d4..60402be10241bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java @@ -62,6 +62,14 @@ public List buildRules() { Long leftHand = LongBitmap.computeTableBitmap(join.left().getInputRelations()); Long rightHand = LongBitmap.computeTableBitmap(join.right().getInputRelations()); join.setBitmap(LongBitmap.or(leftHand, rightHand)); + JoinType joinType = join.getJoinType(); + if (joinType.isRightJoin()) { + // LEADING constraints model the preserved/output side as the left child. + Long originalLeftHand = leftHand; + leftHand = rightHand; + rightHand = originalLeftHand; + joinType = joinType.swap(); + } List expressions = join.getHashJoinConjuncts(); Long totalFilterBitMap = 0L; Long nonNullableSlotBitMap = 0L; @@ -70,11 +78,11 @@ public List buildRules() { nonNullableSlotBitMap = LongBitmap.or(nonNullableSlotBitMap, nonNullable); Long filterBitMap = calSlotsTableBitMap(leading, expression.getInputSlots(), false); totalFilterBitMap = LongBitmap.or(totalFilterBitMap, filterBitMap); - if (join.getJoinType().isLeftJoin()) { + if (joinType.isLeftJoin()) { filterBitMap = LongBitmap.or(filterBitMap, rightHand); } leading.getFilters().add(Pair.of(filterBitMap, expression)); - leading.putConditionJoinType(expression, join.getJoinType()); + leading.putConditionJoinType(expression, joinType); } expressions = join.getOtherJoinConjuncts(); for (Expression expression : expressions) { @@ -82,13 +90,14 @@ public List buildRules() { nonNullableSlotBitMap = LongBitmap.or(nonNullableSlotBitMap, nonNullable); Long filterBitMap = calSlotsTableBitMap(leading, expression.getInputSlots(), false); totalFilterBitMap = LongBitmap.or(totalFilterBitMap, filterBitMap); - if (join.getJoinType().isLeftJoin()) { + if (joinType.isLeftJoin()) { filterBitMap = LongBitmap.or(filterBitMap, rightHand); } leading.getFilters().add(Pair.of(filterBitMap, expression)); - leading.putConditionJoinType(expression, join.getJoinType()); + leading.putConditionJoinType(expression, joinType); } - collectJoinConstraintList(leading, leftHand, rightHand, join, totalFilterBitMap, nonNullableSlotBitMap); + collectJoinConstraintList( + leading, leftHand, rightHand, joinType, totalFilterBitMap, nonNullableSlotBitMap); return ctx.root; }).toRule(RuleType.COLLECT_JOIN_CONSTRAINT) @@ -115,14 +124,14 @@ private void collectLeafPlan(LeadingHint leading, LogicalPlan child) { leading.getRelationIdToScanMap().put(relationId, child); } - private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long rightHand, LogicalJoin join, + private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long rightHand, JoinType joinType, Long filterTableBitMap, Long nonNullableSlotBitMap) { Long totalTables = LongBitmap.or(leftHand, rightHand); - if (join.getJoinType().isInnerOrCrossJoin()) { + if (joinType.isInnerOrCrossJoin()) { leading.setInnerJoinBitmap(LongBitmap.or(leading.getInnerJoinBitmap(), totalTables)); return; } - if (join.getJoinType().isFullOuterJoin()) { + if (joinType.isFullOuterJoin()) { JoinConstraint newJoinConstraint = new JoinConstraint(leftHand, rightHand, leftHand, rightHand, JoinType.FULL_OUTER_JOIN, false); leading.getJoinConstraintList().add(newJoinConstraint); @@ -155,7 +164,7 @@ private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long if (LongBitmap.isOverlap(leftHand, other.getRightHand())) { if (LongBitmap.isOverlap(filterTableBitMap, other.getRightHand()) - && (join.getJoinType().isSemiOrAntiJoin() + && (joinType.isSemiOrAntiJoin() || !LongBitmap.isOverlap(nonNullableSlotBitMap, other.getMinRightHand()))) { minLeftHand = LongBitmap.or(minLeftHand, other.getLeftHand()); @@ -167,7 +176,7 @@ private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long if (LongBitmap.isOverlap(rightHand, other.getRightHand())) { if (LongBitmap.isOverlap(filterTableBitMap, other.getRightHand()) || !LongBitmap.isOverlap(filterTableBitMap, other.getMinLeftHand()) - || join.getJoinType().isSemiOrAntiJoin() + || joinType.isSemiOrAntiJoin() || other.getJoinType().isSemiOrAntiJoin() || !other.isLhsStrict()) { minRightHand = LongBitmap.or(minRightHand, other.getLeftHand()); @@ -183,7 +192,7 @@ private void collectJoinConstraintList(LeadingHint leading, Long leftHand, Long } JoinConstraint newJoinConstraint = new JoinConstraint(minLeftHand, minRightHand, leftHand, rightHand, - join.getJoinType(), isStrict); + joinType, isStrict); leading.getJoinConstraintList().add(newJoinConstraint); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java index c8d8e8af7afe57..fc9011ff178644 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java @@ -18,6 +18,8 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.executor.Rewriter; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; @@ -68,6 +70,59 @@ public void testRightOuterJoin() { testRightOuterJoinHelper(JoinType.RIGHT_OUTER_JOIN); } + @Test + public void testSemiJoinCommuteInRewrite() { + for (JoinType joinType : ImmutableList.of( + JoinType.RIGHT_OUTER_JOIN, JoinType.RIGHT_SEMI_JOIN, JoinType.RIGHT_ANTI_JOIN)) { + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + PlanChecker checker = PlanChecker.from(connectContext) + .analyze(new LogicalPlanBuilder(scan1) + .join(scan2, joinType, Pair.of(0, 0)) + .build()) + .matches(logicalJoin().when(join -> join.getJoinType() == joinType)); + + checker.rewrite() + .matches(logicalJoin().when(join -> join.getJoinType() == joinType.swap())); + } + } + + @Test + public void testDisableJoinReorderBeforeRewrite() { + for (JoinType joinType : ImmutableList.of( + JoinType.RIGHT_OUTER_JOIN, JoinType.RIGHT_SEMI_JOIN, JoinType.RIGHT_ANTI_JOIN)) { + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + PlanChecker checker = PlanChecker.from(connectContext) + .analyze(new LogicalPlanBuilder(scan1) + .join(scan2, joinType, Pair.of(0, 0)) + .build()); + + connectContext.getSessionVariable().setDisableJoinReorder(true); + checker.rewrite() + .matches(logicalJoin().when(join -> join.getJoinType() == joinType)); + } + } + + @Test + public void testSemiJoinCommuteInMvPreRewrite() { + for (JoinType joinType : ImmutableList.of( + JoinType.RIGHT_OUTER_JOIN, JoinType.RIGHT_SEMI_JOIN, JoinType.RIGHT_ANTI_JOIN)) { + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + PlanChecker checker = PlanChecker.from(connectContext) + .analyze(new LogicalPlanBuilder(scan1) + .join(scan2, joinType, Pair.of(0, 0)) + .build()); + CascadesContext cascadesContext = checker.getCascadesContext(); + + Rewriter.getCteChildrenRewriter( + cascadesContext, Rewriter.CTE_CHILDREN_REWRITE_JOBS_MV_REWRITE_USED, false).execute(); + MemoTestUtils.initMemoAndValidState(cascadesContext); + checker.matches(logicalJoin().when(join -> join.getJoinType() == joinType.swap())); + } + } + private void testRightOuterJoinHelper(JoinType joinType) { ImmutableList plans = ImmutableList.of( new LogicalPlanBuilder(scan1) @@ -153,7 +208,6 @@ public void testRightSemiJoin() { ConnectContext connectContext = MemoTestUtils.createConnectContext(); connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); PlanChecker.from(connectContext, plan2) - .applyBottomUp(new SemiJoinCommute()) .rewrite() .matchesFromRoot( logicalProject(innerLogicalJoin( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java index cdc36164ae9f95..3d3fd49c428a62 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java @@ -52,13 +52,13 @@ void testInferNotNullFromFilterAndEliminateOuter2() { .printlnTree() .matches( innerLogicalJoin( - logicalFilter().when( - f -> f.getPredicate().toString().equals("(id#2 >= 4)")), logicalFilter().when( f -> ExpressionUtils.and(f.getConjuncts().stream() .sorted((a, b) -> a.toString().compareTo(b.toString())) .collect(Collectors.toList())) - .toString().equals("(id#0 >= 4)")) + .toString().equals("(id#0 >= 4)")), + logicalFilter().when( + f -> f.getPredicate().toString().equals("(id#2 >= 4)")) ) ); @@ -97,8 +97,8 @@ void testInferNotNullFromJoinAndEliminateOuter() { .rewrite() .matches( innerLogicalJoin( - logicalProject(), - logicalProject(leftSemiLogicalJoin()) + logicalProject(leftSemiLogicalJoin()), + logicalProject() ) ); } diff --git a/regression-test/data/nereids_p0/hint/fix_leading.out b/regression-test/data/nereids_p0/hint/fix_leading.out index 73abf90eb058de..b3e5989477c555 100644 --- a/regression-test/data/nereids_p0/hint/fix_leading.out +++ b/regression-test/data/nereids_p0/hint/fix_leading.out @@ -243,6 +243,12 @@ Used: leading(t1 t2 t3 ) UnUsed: SyntaxError: +-- !select4_4 -- +1 + +-- !select4_6 -- +1 + -- !select6_1 -- PhysicalResultSink --hashAgg[GLOBAL] diff --git a/regression-test/suites/nereids_p0/hint/fix_leading.groovy b/regression-test/suites/nereids_p0/hint/fix_leading.groovy index d70d39c75b38fc..dda3587a07356a 100644 --- a/regression-test/suites/nereids_p0/hint/fix_leading.groovy +++ b/regression-test/suites/nereids_p0/hint/fix_leading.groovy @@ -40,6 +40,9 @@ suite("fix_leading") { sql """drop table if exists t2;""" sql """drop table if exists t3;""" sql """drop table if exists t4;""" + sql """drop table if exists right_join_a;""" + sql """drop table if exists right_join_b;""" + sql """drop table if exists right_join_c;""" sql """create table t1 (c1 int, c11 int) distributed by hash(c1) buckets 3 properties('replication_num' = '1');""" sql """create table t2 (c2 int, c22 int) distributed by hash(c2) buckets 3 properties('replication_num' = '1');""" @@ -47,6 +50,13 @@ suite("fix_leading") { sql """create table t4 (c4 int, c44 int) distributed by hash(c4) buckets 3 properties('replication_num' = '1');""" sql """create table t5 (c5 int, c55 int) distributed by hash(c5) buckets 3 properties('replication_num' = '1');""" sql """create table t6 (c6 int, c66 int) distributed by hash(c6) buckets 3 properties('replication_num' = '1');""" + sql """create table right_join_a (k int) distributed by hash(k) buckets 1 properties('replication_num' = '1');""" + sql """create table right_join_b (k int) distributed by hash(k) buckets 1 properties('replication_num' = '1');""" + sql """create table right_join_c (k int) distributed by hash(k) buckets 1 properties('replication_num' = '1');""" + + sql """insert into right_join_a values (1);""" + sql """insert into right_join_b values (0), (1);""" + sql """insert into right_join_c values (1), (2);""" streamLoad { table "t1" @@ -198,6 +208,16 @@ suite("fix_leading") { qt_select4_2 """select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" qt_select4_3 """explain shape plan select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" + // check right semi join keeps its complete non-output side + qt_select4_4 """select /*+ leading(right_join_b right_join_a right_join_c) */ count(*) + from right_join_a cross join right_join_c + right semi join right_join_b on right_join_a.k = right_join_b.k;""" + + // check right anti join does not push its preserved-side ON predicate below the join + qt_select4_6 """select /*+ leading(right_join_b right_join_a right_join_c) */ count(*) + from right_join_a cross join right_join_c + right anti join right_join_b on right_join_a.k = right_join_b.k and right_join_b.k > 0;""" + // check whether we have all tables explain { sql """shape plan select /*+ leading(t1 t2)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;"""