diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index cedd031e1c6dc8..af630378f8c522 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -28,8 +28,6 @@ import org.apache.doris.nereids.rules.exploration.join.LogicalJoinSemiJoinTransposeProject; import org.apache.doris.nereids.rules.exploration.join.OuterJoinAssocProject; import org.apache.doris.nereids.rules.exploration.join.OuterJoinLAsscomProject; -import org.apache.doris.nereids.rules.exploration.join.PushDownProjectThroughInnerOuterJoin; -import org.apache.doris.nereids.rules.exploration.join.PushDownProjectThroughSemiJoin; import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTransposeProject; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewAggregateOnNoneAggregateRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewAggregateRule; @@ -156,8 +154,6 @@ public class RuleSet { .add(OuterJoinLAsscomProject.INSTANCE) .add(SemiJoinSemiJoinTransposeProject.INSTANCE) .add(LogicalJoinSemiJoinTransposeProject.INSTANCE) - .add(PushDownProjectThroughInnerOuterJoin.INSTANCE) - .add(PushDownProjectThroughSemiJoin.INSTANCE) .add(TransposeAggSemiJoinProject.INSTANCE) .addAll(new PushDownTopNThroughJoin().buildRules()) .addAll(new PushDownLimitDistinctThroughJoin().buildRules()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProject.java index 9ff7f01b829e0a..89e9552a72d513 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProject.java @@ -32,6 +32,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -62,16 +63,20 @@ public Rule build() { innerLogicalJoin(logicalProject(innerLogicalJoin()), group()) .when(topJoin -> checkReorder(topJoin, topJoin.left().child(), enableLeftZigZag)) - .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint()) - .when(join -> join.left().isAllSlots())) + .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint())) .then(topProject -> { LogicalJoin>, GroupPlan> topJoin = topProject.child(); + Optional>> + normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.left()); + if (!normalizedProject.isPresent()) { + return null; + } /* ********** init ********** */ - LogicalJoin bottomJoin = topJoin.left().child(); - GroupPlan a = bottomJoin.left(); - GroupPlan b = bottomJoin.right(); - GroupPlan c = topJoin.right(); + LogicalJoin bottomJoin = normalizedProject.get().child(); + Plan a = bottomJoin.left(); + Plan b = bottomJoin.right(); + Plan c = topJoin.right(); Set bExprIdSet = b.getOutputExprIdSet(); /* ********** split Conjuncts ********** */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateProject.java index e836eac3692f80..7deeb5507050d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateProject.java @@ -31,6 +31,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -50,15 +51,19 @@ public class InnerJoinLeftAssociateProject extends OneExplorationRuleFactory { public Rule build() { return logicalProject(innerLogicalJoin(group(), logicalProject(innerLogicalJoin())) .when(topJoin -> checkReorder(topJoin)) - .whenNot(join -> join.hasDistributeHint() || join.right().child().hasDistributeHint()) - .when(join -> join.right().isAllSlots())) + .whenNot(join -> join.hasDistributeHint() || join.right().child().hasDistributeHint())) .then(topProject -> { LogicalJoin>> topJoin = topProject.child(); - LogicalJoin bottomJoin = topJoin.right().child(); - GroupPlan a = topJoin.left(); - GroupPlan b = bottomJoin.left(); - GroupPlan c = bottomJoin.right(); + Optional>> + normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.right()); + if (!normalizedProject.isPresent()) { + return null; + } + LogicalJoin bottomJoin = normalizedProject.get().child(); + Plan a = topJoin.left(); + Plan b = bottomJoin.left(); + Plan c = bottomJoin.right(); Set cExprIdSet = c.getOutputExprIdSet(); // Split condition diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateProject.java index 841963e9a7b975..6c76465444a4d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateProject.java @@ -31,6 +31,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -48,15 +49,19 @@ public class InnerJoinRightAssociateProject extends OneExplorationRuleFactory { public Rule build() { return logicalProject(innerLogicalJoin(logicalProject(innerLogicalJoin()), group()) .when(topJoin -> checkReorder(topJoin)) - .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint()) - .when(join -> join.left().isAllSlots())) + .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint())) .then(topProject -> { LogicalJoin>, GroupPlan> topJoin = topProject.child(); - LogicalJoin bottomJoin = topJoin.left().child(); - GroupPlan a = bottomJoin.left(); - GroupPlan b = bottomJoin.right(); - GroupPlan c = topJoin.right(); + Optional>> + normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.left()); + if (!normalizedProject.isPresent()) { + return null; + } + LogicalJoin bottomJoin = normalizedProject.get().child(); + Plan a = bottomJoin.left(); + Plan b = bottomJoin.right(); + Plan c = topJoin.right(); Set aExprIdSet = a.getOutputExprIdSet(); // Split condition diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeBothProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeBothProject.java index bc79a715c0bcba..939bc2daac94fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeBothProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeBothProject.java @@ -37,6 +37,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; /** @@ -56,18 +57,24 @@ public class JoinExchangeBothProject extends OneExplorationRuleFactory { public Rule build() { return logicalProject(innerLogicalJoin(logicalProject(innerLogicalJoin()), logicalProject(innerLogicalJoin())) .when(JoinExchangeBothProject::checkReorder) - .when(join -> join.left().isAllSlots() && join.right().isAllSlots()) .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint() || join.right().child().hasDistributeHint())) .then(topProject -> { LogicalJoin>, LogicalProject>> topJoin = topProject.child(); - LogicalJoin leftJoin = topJoin.left().child(); - LogicalJoin rightJoin = topJoin.right().child(); - GroupPlan a = leftJoin.left(); - GroupPlan b = leftJoin.right(); - GroupPlan c = rightJoin.left(); - GroupPlan d = rightJoin.right(); + Optional>> + normalizedLeftProject = ProjectJoinReorderHelper.normalize(topJoin.left()); + Optional>> + normalizedRightProject = ProjectJoinReorderHelper.normalize(topJoin.right()); + if (!normalizedLeftProject.isPresent() || !normalizedRightProject.isPresent()) { + return null; + } + LogicalJoin leftJoin = normalizedLeftProject.get().child(); + LogicalJoin rightJoin = normalizedRightProject.get().child(); + Plan a = leftJoin.left(); + Plan b = leftJoin.right(); + Plan c = rightJoin.left(); + Plan d = rightJoin.right(); Set acOutputExprIdSet = JoinUtils.getJoinOutputExprIdSet(a, c); Set bdOutputExprIdSet = JoinUtils.getJoinOutputExprIdSet(b, d); @@ -92,10 +99,10 @@ public Rule build() { return null; } - LogicalJoin newLeftJoin = new LogicalJoin<>(JoinType.INNER_JOIN, + LogicalJoin newLeftJoin = new LogicalJoin<>(JoinType.INNER_JOIN, newLeftJoinHashJoinConjuncts, newLeftJoinOtherJoinConjuncts, new DistributeHint(DistributeType.NONE), a, c, null); - LogicalJoin newRightJoin = new LogicalJoin<>(JoinType.INNER_JOIN, + LogicalJoin newRightJoin = new LogicalJoin<>(JoinType.INNER_JOIN, newRightJoinHashJoinConjuncts, newRightJoinOtherJoinConjuncts, new DistributeHint(DistributeType.NONE), b, d, null); Set topUsedExprIds = new HashSet<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/LogicalJoinSemiJoinTransposeProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/LogicalJoinSemiJoinTransposeProject.java index 0531c6e54aca77..a3f6477d3cbb24 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/LogicalJoinSemiJoinTransposeProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/LogicalJoinSemiJoinTransposeProject.java @@ -32,6 +32,7 @@ import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; /** @@ -49,18 +50,22 @@ public List buildRules() { && (topJoin.getJoinType().isInnerJoin() || topJoin.getJoinType().isLeftOuterJoin()))) .whenNot(topJoin -> topJoin.hasDistributeHint() - || topJoin.left().child().hasDistributeHint()) - .when(join -> join.left().isAllSlots())) + || topJoin.left().child().hasDistributeHint())) .then(topProject -> { LogicalJoin>, GroupPlan> topJoin = topProject.child(); - LogicalJoin bottomJoin = topJoin.left().child(); + Optional>> + normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.left()); + if (!normalizedProject.isPresent()) { + return null; + } + LogicalJoin bottomJoin = normalizedProject.get().child(); if (!JoinUtils.checkReorderPrecondition(topJoin, bottomJoin)) { return null; } - GroupPlan a = bottomJoin.left(); - GroupPlan b = bottomJoin.right(); - GroupPlan c = topJoin.right(); + Plan a = bottomJoin.left(); + Plan b = bottomJoin.right(); + Plan c = topJoin.right(); Set topUsedExprIds = new HashSet<>(); topProject.getProjects().forEach(expr -> topUsedExprIds.addAll(expr.getInputSlotExprIds())); @@ -83,18 +88,22 @@ public List buildRules() { && (topJoin.getJoinType().isInnerJoin() || topJoin.getJoinType().isRightOuterJoin()))) .whenNot(topJoin -> topJoin.hasDistributeHint() - || topJoin.right().child().hasDistributeHint()) - .when(join -> join.right().isAllSlots())) + || topJoin.right().child().hasDistributeHint())) .then(topProject -> { LogicalJoin>> topJoin = topProject.child(); - LogicalJoin bottomJoin = topJoin.right().child(); + Optional>> + normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.right()); + if (!normalizedProject.isPresent()) { + return null; + } + LogicalJoin bottomJoin = normalizedProject.get().child(); if (!JoinUtils.checkReorderPrecondition(topJoin, bottomJoin)) { return null; } - GroupPlan a = topJoin.left(); - GroupPlan b = bottomJoin.left(); - GroupPlan c = bottomJoin.right(); + Plan a = topJoin.left(); + Plan b = bottomJoin.left(); + Plan c = bottomJoin.right(); Set topUsedExprIds = new HashSet<>(); topProject.getProjects().forEach(expr -> topUsedExprIds.addAll(expr.getInputSlotExprIds())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java index 0dcb3b8d344435..6ad9cf6a8612ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java @@ -38,6 +38,7 @@ import com.google.common.collect.ImmutableSet; import java.util.HashSet; +import java.util.Optional; import java.util.Set; import java.util.stream.Stream; @@ -66,19 +67,25 @@ public Rule build() { .when(join -> VALID_TYPE_PAIR_SET.contains( Pair.of(join.left().child().getJoinType(), join.getJoinType()))) .when(topJoin -> OuterJoinLAsscomProject.checkReorder(topJoin, topJoin.left().child())) - .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint()) - .when(join -> checkCondition(join, join.left().child().left().getOutputSet())) - .when(join -> join.left().isAllSlots())) + .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint())) .thenApply(ctx -> { LogicalProject>, GroupPlan>> topProject = ctx.root; LogicalJoin>, GroupPlan> topJoin = topProject.child(); + Optional>> + normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.left()); + if (!normalizedProject.isPresent()) { + return null; + } /* ********** init ********** */ - LogicalJoin bottomJoin = topJoin.left().child(); - GroupPlan a = bottomJoin.left(); - GroupPlan b = bottomJoin.right(); - GroupPlan c = topJoin.right(); + LogicalJoin bottomJoin = normalizedProject.get().child(); + Plan a = bottomJoin.left(); + Plan b = bottomJoin.right(); + Plan c = topJoin.right(); + if (!checkCondition(topJoin, a.getOutputSet())) { + return null; + } /* * Paper `On the Correct and Complete Enumeration of the Core Search Space`. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java index ac9787bed48dc1..af9fc9855af3f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java @@ -35,6 +35,7 @@ import com.google.common.collect.ImmutableSet; import java.util.HashSet; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -67,18 +68,23 @@ public Rule build() { .when(join -> OuterJoinLAsscomProject.VALID_TYPE_PAIR_SET.contains( Pair.of(join.left().child().getJoinType(), join.getJoinType()))) .when(topJoin -> OuterJoinLAsscomProject.checkReorder(topJoin, topJoin.left().child())) - .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint()) - .when(topJoin -> OuterJoinLAsscomProject.checkCondition(topJoin, - topJoin.left().child().right().getOutputExprIdSet())) - .when(join -> join.left().isAllSlots())) + .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint())) .then(topProject -> { LogicalJoin>, GroupPlan> topJoin = topProject.child(); + Optional>> + normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.left()); + if (!normalizedProject.isPresent()) { + return null; + } /* ********** init ********** */ - LogicalJoin bottomJoin = topJoin.left().child(); - GroupPlan a = bottomJoin.left(); - GroupPlan b = bottomJoin.right(); - GroupPlan c = topJoin.right(); + LogicalJoin bottomJoin = normalizedProject.get().child(); + Plan a = bottomJoin.left(); + Plan b = bottomJoin.right(); + Plan c = topJoin.right(); + if (!OuterJoinLAsscomProject.checkCondition(topJoin, b.getOutputExprIdSet())) { + return null; + } /* ********** new Plan ********** */ LogicalJoin newBottomJoin = topJoin.withChildrenNoContext(a, c, null); @@ -128,7 +134,7 @@ public static boolean checkCondition(LogicalJoin topJ * check join reorder masks. */ public static boolean checkReorder(LogicalJoin topJoin, - LogicalJoin bottomJoin) { + LogicalJoin bottomJoin) { // hasCommute will cause to lack of OuterJoinAssocRule:Left return !topJoin.getJoinReorderContext().hasLAsscom() && !topJoin.getJoinReorderContext().hasLeftAssociate() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/ProjectJoinReorderHelper.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/ProjectJoinReorderHelper.java new file mode 100644 index 00000000000000..223b271d0acd8f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/ProjectJoinReorderHelper.java @@ -0,0 +1,197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.join; + +import org.apache.doris.nereids.rules.exploration.CBOUtils; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** Normalize a Project over Join only when a Project-aware join reorder rule produces an alternative. */ +final class ProjectJoinReorderHelper { + private ProjectJoinReorderHelper() { + } + + /** + * Keep a slot-only Project unchanged, or push single-side complex expressions below its Join. + * + *

The returned Project is slot-only and has the same output as the input Project. Empty means that + * the Project cannot be moved without changing semantics.

+ */ + static Optional>> normalize(LogicalProject project) { + if (project.isAllSlots()) { + return Optional.of(castProject(project)); + } + + LogicalJoin join = childJoin(project); + JoinType joinType = join.getJoinType(); + if (joinType.isLeftSemiOrAntiJoin()) { + if (join.isMarkJoin() || projectBothJoinSides(project)) { + return Optional.empty(); + } + return Optional.of(pushDownLeftSemiProject(project, join)); + } + if (joinType.isInnerJoin() || joinType.isOuterJoin() + || joinType.isAsofInnerJoin() || joinType.isAsofOuterJoin()) { + return pushDownInnerOuterProject(project, join); + } + return Optional.empty(); + } + + private static boolean projectBothJoinSides(LogicalProject project) { + LogicalJoin join = childJoin(project); + Set projectOutput = project.getOutputSet(); + boolean containLeft = join.left().getOutput().stream().anyMatch(projectOutput::contains); + boolean containRight = join.right().getOutput().stream().anyMatch(projectOutput::contains); + return containLeft && containRight; + } + + private static LogicalProject> pushDownLeftSemiProject( + LogicalProject project, LogicalJoin join) { + Set conditionLeftSlots = CBOUtils.joinChildConditionSlots(join, true); + List newProjects = new ArrayList<>(project.getProjects()); + Set projectUsedSlots = project.getProjects().stream() + .map(NamedExpression::toSlot) + .collect(Collectors.toSet()); + conditionLeftSlots.stream() + .filter(slot -> !projectUsedSlots.contains(slot)) + .forEach(newProjects::add); + + Plan newLeft = new LogicalProject<>(newProjects, join.left()); + LogicalJoin newJoin = join.withChildren(ImmutableList.of(newLeft, join.right())); + return new LogicalProject<>(ImmutableList.copyOf(project.getOutput()), newJoin); + } + + private static Optional>> pushDownInnerOuterProject( + LogicalProject project, LogicalJoin join) { + Set leftOutputExprIds = join.left().getOutputExprIdSet(); + Set rightOutputExprIds = join.right().getOutputExprIdSet(); + + boolean containsHyperEdge = project.getProjects().stream().anyMatch(expression -> { + Set inputExprIds = expression.getInputSlotExprIds(); + return !leftOutputExprIds.containsAll(inputExprIds) + && !rightOutputExprIds.containsAll(inputExprIds); + }); + if (containsHyperEdge) { + return Optional.empty(); + } + + List projects = adjustProjectsNullable(project, join); + List leftProjects = new ArrayList<>(); + List rightProjects = new ArrayList<>(); + for (NamedExpression expression : projects) { + if (leftOutputExprIds.containsAll(expression.getInputSlotExprIds())) { + leftProjects.add(expression); + } else { + rightProjects.add(expression); + } + } + + boolean leftContainsComplexExpression = leftProjects.stream() + .anyMatch(expression -> !(expression instanceof Slot)); + boolean rightContainsComplexExpression = rightProjects.stream() + .anyMatch(expression -> !(expression instanceof Slot)); + // JoinCommute supplies the orientation in which a movable complex expression is on the left. + if (!leftContainsComplexExpression) { + return Optional.empty(); + } + JoinType joinType = join.getJoinType(); + boolean rightSideNullable = joinType.isLeftOuterJoin() + || joinType.isAsofLeftOuterJoin() || joinType.isFullOuterJoin(); + boolean leftSideNullable = joinType.isRightOuterJoin() + || joinType.isAsofRightOuterJoin() || joinType.isFullOuterJoin(); + if ((rightSideNullable && rightContainsComplexExpression) + || (leftSideNullable && leftContainsComplexExpression)) { + return Optional.empty(); + } + + Builder newLeftProjects = ImmutableList.builder() + .addAll(leftProjects); + Set leftConditionSlots = CBOUtils.joinChildConditionSlots(join, true); + Set leftProjectSlots = leftProjects.stream() + .map(NamedExpression::toSlot) + .collect(Collectors.toSet()); + leftConditionSlots.stream() + .filter(slot -> !leftProjectSlots.contains(slot)) + .forEach(newLeftProjects::add); + Plan newLeft = new LogicalProject<>(newLeftProjects.build(), join.left()); + + Plan newRight = join.right(); + if (rightContainsComplexExpression) { + Builder newRightProjects = ImmutableList.builder() + .addAll(rightProjects); + Set rightConditionSlots = CBOUtils.joinChildConditionSlots(join, false); + Set rightProjectSlots = rightProjects.stream() + .map(NamedExpression::toSlot) + .collect(Collectors.toSet()); + rightConditionSlots.stream() + .filter(slot -> !rightProjectSlots.contains(slot)) + .forEach(newRightProjects::add); + newRight = new LogicalProject<>(newRightProjects.build(), join.right()); + } + + LogicalJoin newJoin = join.withChildren(ImmutableList.of(newLeft, newRight)); + return Optional.of(new LogicalProject<>(ImmutableList.copyOf(project.getOutput()), newJoin)); + } + + private static List adjustProjectsNullable( + LogicalProject project, LogicalJoin join) { + if (join.getJoinType().isInnerJoin() || join.getJoinType().isAsofInnerJoin()) { + return project.getProjects(); + } + + Map childSlots = new HashMap<>(); + join.left().getOutputSet().forEach(slot -> childSlots.put(slot, slot)); + join.right().getOutputSet().forEach(slot -> childSlots.put(slot, slot)); + join.getOutputSet().forEach(slot -> { + if (childSlots.containsKey(slot)) { + childSlots.put(slot, childSlots.get(slot)); + } + }); + return project.getProjects().stream() + .map(expression -> expression.rewriteUp(child -> + child instanceof Slot ? childSlots.get((Slot) child) : child)) + .map(NamedExpression.class::cast) + .collect(Collectors.toList()); + } + + @SuppressWarnings("unchecked") + private static LogicalJoin childJoin(LogicalProject project) { + return (LogicalJoin) project.child(); + } + + @SuppressWarnings("unchecked") + private static LogicalProject> castProject(LogicalProject project) { + return (LogicalProject>) project; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoin.java index 905af662a2dd38..8a9c44015d5a97 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoin.java @@ -19,25 +19,15 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.exploration.CBOUtils; import org.apache.doris.nereids.rules.exploration.ExplorationRuleFactory; -import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableList.Builder; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; /** * Rule for pushdown project through inner/outer join @@ -66,7 +56,7 @@ public List buildRules() { .whenNot(j -> j.left().child().hasDistributeHint()) .then(topJoin -> { LogicalProject> project = topJoin.left(); - Plan newLeft = pushdownProject(project); + Plan newLeft = ProjectJoinReorderHelper.normalize(project).orElse(null); if (newLeft == null) { return null; } @@ -82,7 +72,7 @@ public List buildRules() { .whenNot(j -> j.right().child().hasDistributeHint()) .then(topJoin -> { LogicalProject> project = topJoin.right(); - Plan newRight = pushdownProject(project); + Plan newRight = ProjectJoinReorderHelper.normalize(project).orElse(null); if (newRight == null) { return null; } @@ -90,84 +80,4 @@ public List buildRules() { }).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_INNER_OUTER_JOIN_RIGHT) ); } - - private Plan pushdownProject(LogicalProject> project) { - LogicalJoin join = project.child(); - Set aOutputExprIdSet = join.left().getOutputExprIdSet(); - Set bOutputExprIdSet = join.right().getOutputExprIdSet(); - - // reject hyper edge in Project. - if (!project.getProjects().stream().allMatch(expr -> { - Set inputSlotExprIds = expr.getInputSlotExprIds(); - return aOutputExprIdSet.containsAll(inputSlotExprIds) - || bOutputExprIdSet.containsAll(inputSlotExprIds); - })) { - return null; - } - - List aProjects = new ArrayList<>(); - List bProjects = new ArrayList<>(); - List projects; - if (join.getJoinType().isInnerJoin() || join.getJoinType().isAsofInnerJoin()) { - projects = project.getProjects(); - } else { - Map childrenSlots = new HashMap<>(); - join.left().getOutputSet().forEach(slot -> childrenSlots.put(slot, slot)); - join.right().getOutputSet().forEach(slot -> childrenSlots.put(slot, slot)); - join.getOutputSet().forEach(slot -> { - if (childrenSlots.containsKey(slot)) { - childrenSlots.put(slot, childrenSlots.get(slot)); - } - }); - - projects = project.getProjects().stream().map(expr -> expr.rewriteUp(e -> - e instanceof Slot ? childrenSlots.get((Slot) e) : e - )).map(e -> (NamedExpression) e).collect(Collectors.toList()); - } - for (NamedExpression namedExpression : projects) { - Set usedExprIds = namedExpression.getInputSlotExprIds(); - if (aOutputExprIdSet.containsAll(usedExprIds)) { - aProjects.add(namedExpression); - } else { - bProjects.add(namedExpression); - } - } - - boolean leftContains = aProjects.stream().anyMatch(e -> !(e instanceof Slot)); - boolean rightContains = bProjects.stream().anyMatch(e -> !(e instanceof Slot)); - // due to JoinCommute, we don't need to consider just right contains. - if (!leftContains) { - return null; - } - // we could not push nullable side project - if (((join.getJoinType().isLeftOuterJoin() || join.getJoinType().isAsofLeftOuterJoin() - || join.getJoinType().isFullOuterJoin()) && rightContains) - || ((join.getJoinType().isRightOuterJoin() || join.getJoinType().isAsofRightOuterJoin() - || join.getJoinType().isFullOuterJoin()) && leftContains)) { - return null; - } - - Builder newAProject = ImmutableList.builder().addAll(aProjects); - Set aConditionSlots = CBOUtils.joinChildConditionSlots(join, true); - Set aProjectSlots = aProjects.stream().map(NamedExpression::toSlot) - .collect(Collectors.toSet()); - aConditionSlots.stream().filter(slot -> !aProjectSlots.contains(slot)).forEach(newAProject::add); - Plan newLeft = new LogicalProject<>(newAProject.build(), join.left()); - - if (!rightContains) { - Plan newJoin = join.withChildren(newLeft, join.right()); - return new LogicalProject<>(ImmutableList.copyOf(project.getOutput()), newJoin); - } - - Builder newBProject = ImmutableList.builder().addAll(bProjects); - Set bConditionSlots = CBOUtils.joinChildConditionSlots(join, false); - Set bProjectSlots = bProjects.stream().map(NamedExpression::toSlot) - .collect(Collectors.toSet()); - bConditionSlots.stream().filter(slot -> !bProjectSlots.contains(slot)).forEach(newBProject::add); - Plan newRight = new LogicalProject<>(newBProject.build(), join.right()); - - Plan newJoin = join.withChildren(newLeft, newRight); - return new LogicalProject<>(ImmutableList.copyOf(project.getOutput()), newJoin); - } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughSemiJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughSemiJoin.java index 11efc82fc21622..74d8befe3fae6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughSemiJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughSemiJoin.java @@ -19,10 +19,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.exploration.CBOUtils; import org.apache.doris.nereids.rules.exploration.ExplorationRuleFactory; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; @@ -30,10 +27,7 @@ import com.google.common.collect.ImmutableList; -import java.util.ArrayList; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; /** * Rule for pushdown project through left-semi/anti join @@ -62,10 +56,10 @@ public List buildRules() { .whenNot(j -> j.left().child().hasDistributeHint()) .then(topJoin -> { LogicalProject> project = topJoin.left(); - if (projectBothJoinSide(project)) { + Plan newLeft = ProjectJoinReorderHelper.normalize(project).orElse(null); + if (newLeft == null) { return null; } - Plan newLeft = pushdownProject(project); return topJoin.withChildren(newLeft, topJoin.right()); }).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_SEMI_JOIN_LEFT), @@ -76,39 +70,12 @@ public List buildRules() { .whenNot(j -> j.right().child().hasDistributeHint()) .then(topJoin -> { LogicalProject> project = topJoin.right(); - if (projectBothJoinSide(project)) { + Plan newRight = ProjectJoinReorderHelper.normalize(project).orElse(null); + if (newRight == null) { return null; } - Plan newRight = pushdownProject(project); return topJoin.withChildren(topJoin.left(), newRight); }).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_SEMI_JOIN_RIGHT) ); } - - private boolean projectBothJoinSide(LogicalProject> project) { - // if project contains both side of join, it can't be pushed. - // such as: - // Project(l, null as r) - // ------ L(l) left anti join R(r) - LogicalJoin join = project.child(); - Set projectOutput = project.getOutputSet(); - boolean containLeft = join.left().getOutput().stream().anyMatch(projectOutput::contains); - boolean containRight = join.right().getOutput().stream().anyMatch(projectOutput::contains); - return containRight && containLeft; - } - - private Plan pushdownProject(LogicalProject> project) { - LogicalJoin join = project.child(); - Set conditionLeftSlots = CBOUtils.joinChildConditionSlots(join, true); - - List newProject = new ArrayList<>(project.getProjects()); - Set projectUsedSlots = project.getProjects().stream().map(NamedExpression::toSlot) - .collect(Collectors.toSet()); - conditionLeftSlots.stream().filter(slot -> !projectUsedSlots.contains(slot)) - .forEach(newProject::add); - Plan newLeft = new LogicalProject<>(newProject, join.left()); - - Plan newJoin = join.withChildren(newLeft, join.right()); - return new LogicalProject<>(ImmutableList.copyOf(project.getOutput()), newJoin); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java index 8d1d67af5c0443..18c79ce6b203d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java @@ -33,6 +33,7 @@ import com.google.common.collect.ImmutableSet; import java.util.HashSet; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -68,7 +69,6 @@ public Rule build() { .when(this::typeChecker) .when(topSemi -> InnerJoinLAsscomProject.checkReorder(topSemi, topSemi.left().child(), false)) .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint()) - .when(join -> join.left().isAllSlots()) // the transpose swaps the bottom semi join to the top, so the mark slot // produced by the bottom mark join would be produced by the new top semi // join. if the top semi join references the mark slot in its conjuncts, @@ -80,11 +80,16 @@ public Rule build() { .then(topProject -> { LogicalJoin>, GroupPlan> topSemi = topProject.child(); - LogicalJoin bottomSemi = topSemi.left().child(); - LogicalProject> abProject = topSemi.left(); - GroupPlan a = bottomSemi.left(); - GroupPlan b = bottomSemi.right(); - GroupPlan c = topSemi.right(); + Optional>> + normalizedProject = ProjectJoinReorderHelper.normalize(topSemi.left()); + if (!normalizedProject.isPresent()) { + return null; + } + LogicalJoin bottomSemi = normalizedProject.get().child(); + LogicalProject> abProject = normalizedProject.get(); + Plan a = bottomSemi.left(); + Plan b = bottomSemi.right(); + Plan c = topSemi.right(); Set aOutputExprIdSet = a.getOutputExprIdSet(); // if bottom semi join is mark join, we need remove the mark join slot creating by bottom semi join // from the project list before swapping the bottom semi to top semi diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/ProjectAwareJoinReorderComplexProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/ProjectAwareJoinReorderComplexProjectTest.java new file mode 100644 index 00000000000000..e124f9e3b93453 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/ProjectAwareJoinReorderComplexProjectTest.java @@ -0,0 +1,354 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.join; + +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.rewrite.AddProjectForJoin; +import org.apache.doris.nereids.rules.rewrite.MergeProjectable; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoTestUtils; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.nereids.util.PlanConstructor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +class ProjectAwareJoinReorderComplexProjectTest { + + @Test + void completeClassicOptimizerReordersPlanAfterAddProjectForJoin() { + LogicalPlan barePlan = buildBarePlan(4); + List expectedOutput = outputSignature(barePlan); + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + SessionVariable sessionVariable = connectContext.getSessionVariable(); + sessionVariable.enableDPHypOptimizer = false; + sessionVariable.setMaxTableCountUseCascadesJoinReorder(64); + sessionVariable.joinReorderTimeLimit = 600_000; + + PlanChecker checker = PlanChecker.from(connectContext, barePlan) + .customRewrite(new AddProjectForJoin()) + .applyTopDown(new MergeProjectable()); + Plan rewrittenPlan = checker.getCascadesContext().getRewritePlan(); + + long joinCount = rewrittenPlan.collect(plan -> plan instanceof LogicalJoin).size(); + long projectOnJoinCount = rewrittenPlan.collect(plan -> plan instanceof LogicalProject + && plan.child(0) instanceof LogicalJoin).size(); + Assertions.assertEquals(3, joinCount); + Assertions.assertEquals(joinCount, projectOnJoinCount); + Assertions.assertEquals(expectedOutput, outputSignature(rewrittenPlan)); + + checker.optimize(); + Assertions.assertEquals(expectedOutput, outputSignature(checker.getBestPlanTree())); + Set joinTopologies = checker.getAllPlan().stream() + .map(ProjectAwareJoinReorderComplexProjectTest::joinTopology) + .filter(topology -> topology.startsWith("(")) + .collect(Collectors.toSet()); + Assertions.assertTrue(joinTopologies.size() > 1, joinTopologies::toString); + } + + @Test + void innerLAsscomMovesComplexExpressionWithItsInput() { + LogicalOlapScan a = scan(0, "a"); + LogicalOlapScan b = scan(1, "b"); + LogicalOlapScan c = scan(2, "c"); + Alias alias = alias(a, "a_alias"); + LogicalPlan ab = project(join(a, b, JoinType.INNER_JOIN, 0, 0), + alias, a.getOutput().get(1), b.getOutput().get(0), b.getOutput().get(1)); + LogicalPlan original = projectAll(join(ab, c, JoinType.INNER_JOIN, 0, 0)); + + Plan reordered = applyAndFind(InnerJoinLAsscomProject.INSTANCE.build(), original, + plan -> hasJoin(plan, JoinType.INNER_JOIN, names("a"), names("c")) + && hasJoin(plan, JoinType.INNER_JOIN, names("a", "c"), names("b"))); + + assertSemanticsPreserved(original, reordered); + assertAliasPlacedOnScan(reordered, alias, "a"); + } + + @Test + void innerAssociatesInBothDirectionsWithComplexExpression() { + LogicalOlapScan a = scan(0, "a"); + LogicalOlapScan b = scan(1, "b"); + LogicalOlapScan c = scan(2, "c"); + Alias rightAssociateAlias = alias(a, "a_alias"); + LogicalPlan ab = project(join(a, b, JoinType.INNER_JOIN, 0, 0), + rightAssociateAlias, a.getOutput().get(1), b.getOutput().get(0), b.getOutput().get(1)); + LogicalPlan rightAssociateOriginal = projectAll(join(ab, c, JoinType.INNER_JOIN, 2, 0)); + + Plan rightAssociated = applyAndFind(InnerJoinRightAssociateProject.INSTANCE.build(), + rightAssociateOriginal, + plan -> hasJoin(plan, JoinType.INNER_JOIN, names("b"), names("c")) + && hasJoin(plan, JoinType.INNER_JOIN, names("a"), names("b", "c"))); + assertSemanticsPreserved(rightAssociateOriginal, rightAssociated); + assertAliasPlacedOnScan(rightAssociated, rightAssociateAlias, "a"); + + Alias leftAssociateAlias = alias(b, "b_alias"); + LogicalPlan bc = project(join(b, c, JoinType.INNER_JOIN, 0, 0), + leftAssociateAlias, b.getOutput().get(0), b.getOutput().get(1), + c.getOutput().get(0), c.getOutput().get(1)); + LogicalPlan leftAssociateOriginal = projectAll(join(a, bc, JoinType.INNER_JOIN, 0, 1)); + + Plan leftAssociated = applyAndFind(InnerJoinLeftAssociateProject.INSTANCE.build(), + leftAssociateOriginal, + plan -> hasJoin(plan, JoinType.INNER_JOIN, names("a"), names("b")) + && hasJoin(plan, JoinType.INNER_JOIN, names("a", "b"), names("c"))); + assertSemanticsPreserved(leftAssociateOriginal, leftAssociated); + assertAliasPlacedOnScan(leftAssociated, leftAssociateAlias, "b"); + } + + @Test + void exchangePushesIndependentComplexExpressionsOnBothBranches() { + LogicalOlapScan a = scan(0, "a"); + LogicalOlapScan b = scan(1, "b"); + LogicalOlapScan c = scan(2, "c"); + LogicalOlapScan d = scan(3, "d"); + Alias aAlias = alias(a, "a_alias"); + Alias cAlias = alias(c, "c_alias"); + LogicalPlan ab = project(join(a, b, JoinType.INNER_JOIN, 0, 0), + aAlias, a.getOutput().get(0), a.getOutput().get(1), + b.getOutput().get(0), b.getOutput().get(1)); + LogicalPlan cd = project(join(c, d, JoinType.INNER_JOIN, 0, 0), + cAlias, c.getOutput().get(0), c.getOutput().get(1), + d.getOutput().get(0), d.getOutput().get(1)); + LogicalPlan original = projectAll(new LogicalPlanBuilder(ab) + .join(cd, JoinType.INNER_JOIN, ImmutableList.of(Pair.of(1, 1), Pair.of(3, 3))) + .build()); + + Plan reordered = applyAndFind(JoinExchangeBothProject.INSTANCE.build(), original, + plan -> hasJoin(plan, JoinType.INNER_JOIN, names("a"), names("c")) + && hasJoin(plan, JoinType.INNER_JOIN, names("b"), names("d"))); + + assertSemanticsPreserved(original, reordered); + assertAliasPlacedOnScan(reordered, aAlias, "a"); + assertAliasPlacedOnScan(reordered, cAlias, "c"); + } + + @Test + void transposeLogicalJoinAndSemiJoinWithComplexExpression() { + LogicalOlapScan a = scan(0, "a"); + LogicalOlapScan b = scan(1, "b"); + LogicalOlapScan c = scan(2, "c"); + Alias alias = alias(a, "a_alias"); + LogicalPlan semi = project(join(a, b, JoinType.LEFT_SEMI_JOIN, 0, 0), + alias, a.getOutput().get(0), a.getOutput().get(1)); + LogicalPlan original = projectAll(join(semi, c, JoinType.INNER_JOIN, 0, 0)); + + Plan reordered = applyAndFind( + LogicalJoinSemiJoinTransposeProject.INSTANCE.buildRules().get(0), original, + plan -> hasJoin(plan, JoinType.INNER_JOIN, names("a"), names("c")) + && hasJoin(plan, JoinType.LEFT_SEMI_JOIN, names("a", "c"), names("b"))); + + assertSemanticsPreserved(original, reordered); + assertAliasPlacedOnScan(reordered, alias, "a"); + } + + @Test + void transposeTwoSemiJoinsWithComplexExpression() { + LogicalOlapScan a = scan(0, "a"); + LogicalOlapScan b = scan(1, "b"); + LogicalOlapScan c = scan(2, "c"); + Alias alias = alias(a, "a_alias"); + LogicalPlan bottomSemi = project(join(a, b, JoinType.LEFT_SEMI_JOIN, 0, 0), + alias, a.getOutput().get(0), a.getOutput().get(1)); + LogicalPlan original = projectAll(join(bottomSemi, c, JoinType.LEFT_SEMI_JOIN, 0, 0)); + + Plan reordered = applyAndFind(SemiJoinSemiJoinTransposeProject.INSTANCE.build(), original, + plan -> hasJoin(plan, JoinType.LEFT_SEMI_JOIN, names("a"), names("c")) + && hasJoin(plan, JoinType.LEFT_SEMI_JOIN, names("a", "c"), names("b"))); + + assertSemanticsPreserved(original, reordered); + assertAliasPlacedOnScan(reordered, alias, "a"); + } + + @Test + void outerJoinReordersKeepComplexExpressionOnNonNullableInput() { + LogicalOlapScan a = scan(0, "a"); + LogicalOlapScan b = scan(1, "b"); + LogicalOlapScan c = scan(2, "c"); + Alias lAsscomAlias = alias(a, "a_lasscom_alias"); + LogicalPlan abForLAsscom = project(join(a, b, JoinType.INNER_JOIN, 0, 0), + lAsscomAlias, a.getOutput().get(0), a.getOutput().get(1), + b.getOutput().get(0), b.getOutput().get(1)); + LogicalPlan lAsscomOriginal = projectAll(join(abForLAsscom, c, JoinType.LEFT_OUTER_JOIN, 0, 0)); + + Plan lAsscom = applyAndFind(OuterJoinLAsscomProject.INSTANCE.build(), lAsscomOriginal, + plan -> hasJoin(plan, JoinType.LEFT_OUTER_JOIN, names("a"), names("c")) + && hasJoin(plan, JoinType.INNER_JOIN, names("a", "c"), names("b"))); + assertSemanticsPreserved(lAsscomOriginal, lAsscom); + assertAliasPlacedOnScan(lAsscom, lAsscomAlias, "a"); + + Alias assocAlias = alias(a, "a_assoc_alias"); + LogicalPlan abForAssoc = project(join(a, b, JoinType.INNER_JOIN, 0, 0), + assocAlias, a.getOutput().get(0), a.getOutput().get(1), + b.getOutput().get(0), b.getOutput().get(1)); + LogicalPlan assocOriginal = projectAll(join(abForAssoc, c, JoinType.LEFT_OUTER_JOIN, 3, 0)); + + Plan assoc = applyAndFind(OuterJoinAssocProject.INSTANCE.build(), assocOriginal, + plan -> hasJoin(plan, JoinType.LEFT_OUTER_JOIN, names("b"), names("c")) + && hasJoin(plan, JoinType.INNER_JOIN, names("a"), names("b", "c"))); + assertSemanticsPreserved(assocOriginal, assoc); + assertAliasPlacedOnScan(assoc, assocAlias, "a"); + } + + private static Plan applyAndFind(Rule rule, LogicalPlan original, Predicate predicate) { + List plans = PlanChecker.from(MemoTestUtils.createConnectContext(), original) + .applyExploration(rule) + .getAllPlan(); + return plans.stream() + .filter(predicate) + .findFirst() + .orElseThrow(() -> new AssertionError("Expected reordered alternative, found " + plans.size())); + } + + private static void assertSemanticsPreserved(Plan original, Plan reordered) { + Assertions.assertEquals(outputSignature(original), outputSignature(reordered)); + Assertions.assertEquals(conditionSignatures(original), conditionSignatures(reordered)); + } + + private static void assertAliasPlacedOnScan(Plan plan, Alias alias, String scanName) { + Assertions.assertTrue(anyPlan(plan, candidate -> candidate instanceof LogicalProject + && ((LogicalProject) candidate).getProjects().contains(alias) + && scanNames(candidate.child(0)).equals(names(scanName)))); + } + + private static boolean hasJoin(Plan plan, JoinType type, Set left, Set right) { + return anyPlan(plan, candidate -> candidate instanceof LogicalJoin + && ((LogicalJoin) candidate).getJoinType() == type + && scanNames(candidate.child(0)).equals(left) + && scanNames(candidate.child(1)).equals(right)); + } + + private static boolean anyPlan(Plan plan, Predicate predicate) { + if (predicate.test(plan)) { + return true; + } + for (Plan child : plan.children()) { + if (anyPlan(child, predicate)) { + return true; + } + } + return false; + } + + private static Set scanNames(Plan plan) { + ImmutableSet.Builder names = ImmutableSet.builder(); + collectScanNames(plan, names); + return names.build(); + } + + private static void collectScanNames(Plan plan, ImmutableSet.Builder names) { + if (plan instanceof LogicalOlapScan) { + names.add(((LogicalOlapScan) plan).getTable().getName()); + } + plan.children().forEach(child -> collectScanNames(child, names)); + } + + private static List conditionSignatures(Plan plan) { + List signatures = new ArrayList<>(); + collectConditionSignatures(plan, signatures); + Collections.sort(signatures); + return signatures; + } + + private static void collectConditionSignatures(Plan plan, List signatures) { + if (plan instanceof LogicalJoin) { + LogicalJoin join = (LogicalJoin) plan; + join.getHashJoinConjuncts().stream().map(Expression::toSql).forEach(signatures::add); + join.getOtherJoinConjuncts().stream().map(Expression::toSql).forEach(signatures::add); + join.getMarkJoinConjuncts().stream().map(Expression::toSql).forEach(signatures::add); + } + plan.children().forEach(child -> collectConditionSignatures(child, signatures)); + } + + private static List outputSignature(Plan plan) { + return plan.getOutput().stream() + .map(slot -> slot.getExprId() + ":" + slot.getName() + ":" + slot.nullable()) + .collect(Collectors.toList()); + } + + private static LogicalPlan buildBarePlan(int tableCount) { + LogicalPlan plan = scan(0, "t0"); + for (int i = 1; i < tableCount; i++) { + LogicalOlapScan right = scan(i, "t" + i); + plan = join(plan, right, JoinType.INNER_JOIN, 2 * (i - 1), 0); + } + return plan; + } + + private static String joinTopology(Plan plan) { + if (plan instanceof LogicalProject) { + return joinTopology(plan.child(0)); + } + if (plan instanceof LogicalOlapScan) { + return ((LogicalOlapScan) plan).getTable().getName(); + } + if (plan instanceof LogicalJoin) { + return "(" + joinTopology(plan.child(0)) + "," + joinTopology(plan.child(1)) + ")"; + } + return plan.getClass().getSimpleName(); + } + + private static LogicalOlapScan scan(int id, String name) { + return PlanConstructor.newLogicalOlapScan(id, name, 0); + } + + private static Alias alias(LogicalOlapScan scan, String name) { + return new Alias(new Add(scan.getOutput().get(0), new IntegerLiteral(1)), name); + } + + private static LogicalPlan join(LogicalPlan left, LogicalPlan right, JoinType type, + int leftIndex, int rightIndex) { + return new LogicalPlanBuilder(left) + .join(right, type, Pair.of(leftIndex, rightIndex)) + .build(); + } + + private static LogicalPlan project(LogicalPlan child, NamedExpression... projects) { + return new LogicalProject<>(ImmutableList.copyOf(projects), child); + } + + private static LogicalPlan projectAll(LogicalPlan child) { + return new LogicalProject<>(ImmutableList.copyOf(child.getOutput()), child); + } + + private static Set names(String... names) { + return ImmutableSet.copyOf(names); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/ProjectJoinReorderHelperTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/ProjectJoinReorderHelperTest.java new file mode 100644 index 00000000000000..1431074fd8d537 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/ProjectJoinReorderHelperTest.java @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.join; + +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleSet; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Random; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.PlanConstructor; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +class ProjectJoinReorderHelperTest { + private final LogicalOlapScan left = PlanConstructor.newLogicalOlapScan(0, "left", 0); + private final LogicalOlapScan right = PlanConstructor.newLogicalOlapScan(1, "right", 0); + + @Test + void keepSlotOnlyProjectUnchanged() { + LogicalJoin join = join(JoinType.INNER_JOIN); + LogicalProject> project + = new LogicalProject<>(ImmutableList.copyOf(join.getOutput()), join); + + Optional>> normalized + = ProjectJoinReorderHelper.normalize(project); + + Assertions.assertTrue(normalized.isPresent()); + Assertions.assertSame(project, normalized.get()); + } + + @Test + void removeStandalonePushDownRulesFromClassicReorder() { + List classicRuleTypes = RuleSet.OTHER_REORDER_RULES.stream() + .map(Rule::getRuleType) + .collect(Collectors.toList()); + + Assertions.assertFalse(classicRuleTypes.contains( + RuleType.PUSH_DOWN_PROJECT_THROUGH_INNER_OUTER_JOIN_LEFT)); + Assertions.assertFalse(classicRuleTypes.contains( + RuleType.PUSH_DOWN_PROJECT_THROUGH_INNER_OUTER_JOIN_RIGHT)); + Assertions.assertFalse(classicRuleTypes.contains( + RuleType.PUSH_DOWN_PROJECT_THROUGH_SEMI_JOIN_LEFT)); + Assertions.assertFalse(classicRuleTypes.contains( + RuleType.PUSH_DOWN_PROJECT_THROUGH_SEMI_JOIN_RIGHT)); + } + + @Test + void pushComplexExpressionsToBothInputsAndRestoreConditionSlots() { + LogicalJoin join = join(JoinType.INNER_JOIN); + Alias leftAlias = new Alias(new Add(left.getOutput().get(1), new IntegerLiteral(1)), "left_alias"); + Alias rightAlias = new Alias(new Add(right.getOutput().get(1), new IntegerLiteral(2)), "right_alias"); + LogicalProject> project + = new LogicalProject<>(ImmutableList.of(leftAlias, rightAlias), join); + + LogicalProject> normalized + = ProjectJoinReorderHelper.normalize(project).orElseThrow(AssertionError::new); + LogicalJoin normalizedJoin = normalized.child(); + + Assertions.assertTrue(normalized.getProjects().stream().allMatch(Slot.class::isInstance)); + Assertions.assertEquals(outputSignature(project), outputSignature(normalized)); + Assertions.assertEquals(join.getHashJoinConjuncts(), normalizedJoin.getHashJoinConjuncts()); + assertInputProject(normalizedJoin.left(), leftAlias, left.getOutput().get(0)); + assertInputProject(normalizedJoin.right(), rightAlias, right.getOutput().get(0)); + } + + @Test + void rejectHyperEdgeProject() { + LogicalJoin join = join(JoinType.INNER_JOIN); + Alias hyperEdge = new Alias(new Add(left.getOutput().get(0), right.getOutput().get(0)), "both"); + LogicalProject> project + = new LogicalProject<>(ImmutableList.of(hyperEdge), join); + + Assertions.assertFalse(ProjectJoinReorderHelper.normalize(project).isPresent()); + } + + @Test + void respectOuterJoinNullableSides() { + Alias leftAlias = new Alias(new Add(left.getOutput().get(1), new IntegerLiteral(1)), "left_alias"); + Alias rightAlias = new Alias(new Add(right.getOutput().get(1), new IntegerLiteral(1)), "right_alias"); + + LogicalJoin leftOuter = join(JoinType.LEFT_OUTER_JOIN); + LogicalProject> leftOnly + = new LogicalProject<>(ImmutableList.of(leftAlias, leftOuter.getOutput().get(2)), leftOuter); + LogicalProject> normalized + = ProjectJoinReorderHelper.normalize(leftOnly).orElseThrow(AssertionError::new); + Assertions.assertEquals(outputSignature(leftOnly), outputSignature(normalized)); + Assertions.assertFalse(normalized.child().right() instanceof LogicalProject); + + LogicalProject> leftAndNullableRight + = new LogicalProject<>(ImmutableList.of(leftAlias, rightAlias), leftOuter); + Assertions.assertFalse(ProjectJoinReorderHelper.normalize(leftAndNullableRight).isPresent()); + + LogicalJoin rightOuter = join(JoinType.RIGHT_OUTER_JOIN); + LogicalProject> nullableLeft + = new LogicalProject<>(ImmutableList.of(leftAlias), rightOuter); + Assertions.assertFalse(ProjectJoinReorderHelper.normalize(nullableLeft).isPresent()); + + LogicalJoin fullOuter = join(JoinType.FULL_OUTER_JOIN); + LogicalProject> fullOuterProject + = new LogicalProject<>(ImmutableList.of(leftAlias, rightAlias), fullOuter); + Assertions.assertFalse(ProjectJoinReorderHelper.normalize(fullOuterProject).isPresent()); + } + + @Test + void pushLeftSemiProjectAndRejectMarkJoin() { + LogicalJoin semiJoin = join(JoinType.LEFT_SEMI_JOIN); + Alias alias = new Alias(new Add(left.getOutput().get(1), new IntegerLiteral(1)), "semi_alias"); + LogicalProject> project + = new LogicalProject<>(ImmutableList.of(alias), semiJoin); + + LogicalProject> normalized + = ProjectJoinReorderHelper.normalize(project).orElseThrow(AssertionError::new); + assertInputProject(normalized.child().left(), alias, left.getOutput().get(0)); + Assertions.assertEquals(outputSignature(project), outputSignature(normalized)); + + LogicalJoin markJoin = (LogicalJoin) + new LogicalPlanBuilder(left) + .markJoin(right, JoinType.LEFT_SEMI_JOIN, Pair.of(0, 0)) + .build(); + LogicalProject> markProject + = new LogicalProject<>(ImmutableList.of(alias), markJoin); + Assertions.assertFalse(ProjectJoinReorderHelper.normalize(markProject).isPresent()); + } + + @Test + void preserveExistingLiteralAndVolatilePlacementBehavior() { + LogicalJoin join = join(JoinType.INNER_JOIN); + Alias literal = new Alias(new IntegerLiteral(1), "literal_alias"); + Alias random = new Alias(new Random(), "random_alias"); + LogicalProject> project + = new LogicalProject<>(ImmutableList.of(literal, random), join); + + LogicalProject> normalized + = ProjectJoinReorderHelper.normalize(project).orElseThrow(AssertionError::new); + + assertInputProject(normalized.child().left(), literal, random, left.getOutput().get(0)); + Assertions.assertEquals(outputSignature(project), outputSignature(normalized)); + } + + private LogicalJoin join(JoinType joinType) { + return (LogicalJoin) new LogicalPlanBuilder(left) + .join(right, joinType, Pair.of(0, 0)) + .build(); + } + + private static void assertInputProject(Plan input, NamedExpression... expectedProjects) { + Assertions.assertInstanceOf(LogicalProject.class, input); + LogicalProject project = (LogicalProject) input; + Assertions.assertEquals(ImmutableList.copyOf(expectedProjects), project.getProjects()); + } + + private static List outputSignature(Plan plan) { + return plan.getOutput().stream() + .map(slot -> slot.getExprId() + ":" + slot.getName() + ":" + slot.nullable()) + .collect(Collectors.toList()); + } +} diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query2.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query2.out index 19e82920b92474..3e4ed25b7cffc5 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query2.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query2.out @@ -23,17 +23,16 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ----------PhysicalProject ------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_week_seq = d_week_seq1)) otherCondition=() build RFs:RF3 d_week_seq->[d_week_seq] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 53))) otherCondition=() build RFs:RF2 expr_cast(d_week_seq1 as BIGINT)->[(cast(d_week_seq as BIGINT) - 53)] +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF2 d_week_seq->[d_week_seq] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffle] hashCondition=((date_dim.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF1 d_week_seq->[d_week_seq] +--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 53))) otherCondition=() build RFs:RF1 expr_cast(d_week_seq1 as BIGINT)->[(cast(d_week_seq as BIGINT) - 53)] ----------------------PhysicalProject ------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF1 RF2 ----------------------PhysicalProject -------------------------filter((date_dim.d_year = 1999)) ---------------------------PhysicalOlapScan[date_dim] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF3 ------------------PhysicalProject ---------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF3 +--------------------filter((date_dim.d_year = 1999)) +----------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------filter((date_dim.d_year = 1998)) ------------------PhysicalOlapScan[date_dim] - diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query59.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query59.out index 6091219b05a39a..3ab43e14ef4cfe 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query59.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query59.out @@ -19,24 +19,24 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((d.d_week_seq = d_week_seq1)) otherCondition=() build RFs:RF6 d_week_seq->[d_week_seq] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 52)) and (y.s_store_id1 = x.s_store_id2)) otherCondition=() build RFs:RF4 s_store_id2->[s_store_id];RF5 expr_(cast(d_week_seq2 as BIGINT) - 52)->[cast(d_week_seq as BIGINT)] +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((wss.ss_store_sk = store.s_store_sk) and (y.s_store_id1 = x.s_store_id2)) otherCondition=() --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((wss.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->[ss_store_sk] +----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF3 d_week_seq->[d_week_seq] ------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF5 RF6 -------------------------PhysicalProject ---------------------------PhysicalOlapScan[store] apply RFs: RF4 ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF2 d_week_seq->[d_week_seq] -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((wss.ss_store_sk = store.s_store_sk)) otherCondition=() +--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((wss.ss_store_sk = store.s_store_sk)) otherCondition=() ----------------------------PhysicalProject -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF2 +------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 52))) otherCondition=() build RFs:RF1 expr_cast(d_week_seq1 as BIGINT)->[(cast(d_week_seq as BIGINT) - 52)] +--------------------------------PhysicalProject +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF1 RF3 +--------------------------------PhysicalProject +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF6 ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject --------------------------filter((d.d_month_seq <= 1219) and (d.d_month_seq >= 1208)) ----------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalProject +----------------------PhysicalOlapScan[store] ----------------PhysicalProject ------------------filter((d.d_month_seq <= 1207) and (d.d_month_seq >= 1196)) --------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query2.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query2.out index 5ed4762bd44d7c..380de4da6753c8 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query2.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query2.out @@ -23,17 +23,16 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ----------PhysicalProject ------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_week_seq = d_week_seq1)) otherCondition=() build RFs:RF3 d_week_seq->[d_week_seq] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 53))) otherCondition=() build RFs:RF2 expr_cast(d_week_seq1 as BIGINT)->[(cast(d_week_seq as BIGINT) - 53)] +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF2 d_week_seq->[d_week_seq] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffle] hashCondition=((date_dim.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF1 d_week_seq->[d_week_seq] +--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 53))) otherCondition=() build RFs:RF1 expr_cast(d_week_seq1 as BIGINT)->[(cast(d_week_seq as BIGINT) - 53)] ----------------------PhysicalProject ------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF1 RF2 ----------------------PhysicalProject -------------------------filter((date_dim.d_year = 1999)) ---------------------------PhysicalOlapScan[date_dim] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF3 ------------------PhysicalProject ---------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF3 +--------------------filter((date_dim.d_year = 1999)) +----------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------filter((date_dim.d_year = 1998)) ------------------PhysicalOlapScan[date_dim] - diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query59.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query59.out index eef04971d4123e..b668c85f5f678d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query59.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query59.out @@ -19,24 +19,24 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((d.d_week_seq = d_week_seq1)) otherCondition=() build RFs:RF6 d_week_seq->[d_week_seq] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 52)) and (y.s_store_id1 = x.s_store_id2)) otherCondition=() build RFs:RF4 s_store_id2->[s_store_id];RF5 expr_(cast(d_week_seq2 as BIGINT) - 52)->[cast(d_week_seq as BIGINT)] +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((wss.ss_store_sk = store.s_store_sk) and (y.s_store_id1 = x.s_store_id2)) otherCondition=() build RFs:RF4 s_store_id1->[s_store_id];RF5 s_store_sk->[ss_store_sk] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((wss.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->[ss_store_sk] +----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF3 d_week_seq->[d_week_seq] ------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF5 RF6 -------------------------PhysicalProject ---------------------------PhysicalOlapScan[store] apply RFs: RF4 ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF2 d_week_seq->[d_week_seq] -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((wss.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] +--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((wss.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk] ----------------------------PhysicalProject -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF1 RF2 +------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 52))) otherCondition=() build RFs:RF1 expr_cast(d_week_seq1 as BIGINT)->[(cast(d_week_seq as BIGINT) - 52)] +--------------------------------PhysicalProject +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF1 RF2 RF3 +--------------------------------PhysicalProject +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5 RF6 ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[store] +------------------------------PhysicalOlapScan[store] apply RFs: RF4 ------------------------PhysicalProject --------------------------filter((d.d_month_seq <= 1219) and (d.d_month_seq >= 1208)) ----------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalProject +----------------------PhysicalOlapScan[store] ----------------PhysicalProject ------------------filter((d.d_month_seq <= 1207) and (d.d_month_seq >= 1196)) --------------------PhysicalOlapScan[date_dim]