Skip to content

branch:4.2: [improvement](cascades) Fuse project pushdown into join reorder #67541 #68289

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 3 commits into from
Sep 22, 2026
Merged

branch:4.2: [improvement](cascades) Fuse project pushdown into join reorder #67541 #68289

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7611819
[improvement](cascades) Fuse project pushdown into join reorder
morrySnow Sep 4, 2026
251b016
Merge upstream/branch-4.2 into pick-67541-to-branch-4.2
morrySnow Sep 21, 2026
8e439bd
[test](regression) Refresh branch-4.2 TPC-DS query59 no-stats shapes
morrySnow Sep 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@
import org.apache.doris.nereids.rules.exploration.join.LogicalJoinSemiJoinTransposeProject;
import org.apache.doris.nereids.rules.exploration.join.OuterJoinAssocProject;
import org.apache.doris.nereids.rules.exploration.join.OuterJoinLAsscomProject;
import org.apache.doris.nereids.rules.exploration.join.PushDownProjectThroughInnerOuterJoin;
import org.apache.doris.nereids.rules.exploration.join.PushDownProjectThroughSemiJoin;
import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTransposeProject;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewAggregateOnNoneAggregateRule;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewAggregateRule;
Expand Down Expand Up @@ -156,8 +154,6 @@ public class RuleSet {
.add(OuterJoinLAsscomProject.INSTANCE)
.add(SemiJoinSemiJoinTransposeProject.INSTANCE)
.add(LogicalJoinSemiJoinTransposeProject.INSTANCE)
.add(PushDownProjectThroughInnerOuterJoin.INSTANCE)
.add(PushDownProjectThroughSemiJoin.INSTANCE)
.add(TransposeAggSemiJoinProject.INSTANCE)
.addAll(new PushDownTopNThroughJoin().buildRules())
.addAll(new PushDownLimitDistinctThroughJoin().buildRules())
Expand Down
17 changes: 11 additions & 6 deletions ...rc/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProject.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -62,16 +63,20 @@ public Rule build() {
innerLogicalJoin(logicalProject(innerLogicalJoin()), group())
.when(topJoin -> checkReorder(topJoin, topJoin.left().child(),
enableLeftZigZag))
.whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint())
.when(join -> join.left().isAllSlots()))
.whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint()))
.then(topProject -> {
LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>, GroupPlan> topJoin
= topProject.child();
Optional<LogicalProject<LogicalJoin<Plan, Plan>>>
normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.left());
if (!normalizedProject.isPresent()) {
return null;
}
/* ********** init ********** */
LogicalJoin<GroupPlan, GroupPlan> bottomJoin = topJoin.left().child();
GroupPlan a = bottomJoin.left();
GroupPlan b = bottomJoin.right();
GroupPlan c = topJoin.right();
LogicalJoin<Plan, Plan> bottomJoin = normalizedProject.get().child();
Plan a = bottomJoin.left();
Plan b = bottomJoin.right();
Plan c = topJoin.right();
Set<ExprId> bExprIdSet = b.getOutputExprIdSet();

/* ********** split Conjuncts ********** */
Expand Down
17 changes: 11 additions & 6 deletions ...n/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateProject.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

/**
Expand All @@ -50,15 +51,19 @@ public class InnerJoinLeftAssociateProject extends OneExplorationRuleFactory {
public Rule build() {
return logicalProject(innerLogicalJoin(group(), logicalProject(innerLogicalJoin()))
.when(topJoin -> checkReorder(topJoin))
.whenNot(join -> join.hasDistributeHint() || join.right().child().hasDistributeHint())
.when(join -> join.right().isAllSlots()))
.whenNot(join -> join.hasDistributeHint() || join.right().child().hasDistributeHint()))
.then(topProject -> {
LogicalJoin<GroupPlan, LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>> topJoin
= topProject.child();
LogicalJoin<GroupPlan, GroupPlan> bottomJoin = topJoin.right().child();
GroupPlan a = topJoin.left();
GroupPlan b = bottomJoin.left();
GroupPlan c = bottomJoin.right();
Optional<LogicalProject<LogicalJoin<Plan, Plan>>>
normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.right());
if (!normalizedProject.isPresent()) {
return null;
}
LogicalJoin<Plan, Plan> bottomJoin = normalizedProject.get().child();
Plan a = topJoin.left();
Plan b = bottomJoin.left();
Plan c = bottomJoin.right();
Set<ExprId> cExprIdSet = c.getOutputExprIdSet();

// Split condition
Expand Down
17 changes: 11 additions & 6 deletions .../java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateProject.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

/**
Expand All @@ -48,15 +49,19 @@ public class InnerJoinRightAssociateProject extends OneExplorationRuleFactory {
public Rule build() {
return logicalProject(innerLogicalJoin(logicalProject(innerLogicalJoin()), group())
.when(topJoin -> checkReorder(topJoin))
.whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint())
.when(join -> join.left().isAllSlots()))
.whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint()))
.then(topProject -> {
LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>, GroupPlan> topJoin
= topProject.child();
LogicalJoin<GroupPlan, GroupPlan> bottomJoin = topJoin.left().child();
GroupPlan a = bottomJoin.left();
GroupPlan b = bottomJoin.right();
GroupPlan c = topJoin.right();
Optional<LogicalProject<LogicalJoin<Plan, Plan>>>
normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.left());
if (!normalizedProject.isPresent()) {
return null;
}
LogicalJoin<Plan, Plan> bottomJoin = normalizedProject.get().child();
Plan a = bottomJoin.left();
Plan b = bottomJoin.right();
Plan c = topJoin.right();
Set<ExprId> aExprIdSet = a.getOutputExprIdSet();

// Split condition
Expand Down
25 changes: 16 additions & 9 deletions ...rc/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeBothProject.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;

/**
Expand All @@ -56,18 +57,24 @@ public class JoinExchangeBothProject extends OneExplorationRuleFactory {
public Rule build() {
return logicalProject(innerLogicalJoin(logicalProject(innerLogicalJoin()), logicalProject(innerLogicalJoin()))
.when(JoinExchangeBothProject::checkReorder)
.when(join -> join.left().isAllSlots() && join.right().isAllSlots())
.whenNot(join -> join.hasDistributeHint()
|| join.left().child().hasDistributeHint() || join.right().child().hasDistributeHint()))
.then(topProject -> {
LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>,
LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>> topJoin = topProject.child();
LogicalJoin<GroupPlan, GroupPlan> leftJoin = topJoin.left().child();
LogicalJoin<GroupPlan, GroupPlan> rightJoin = topJoin.right().child();
GroupPlan a = leftJoin.left();
GroupPlan b = leftJoin.right();
GroupPlan c = rightJoin.left();
GroupPlan d = rightJoin.right();
Optional<LogicalProject<LogicalJoin<Plan, Plan>>>
normalizedLeftProject = ProjectJoinReorderHelper.normalize(topJoin.left());
Optional<LogicalProject<LogicalJoin<Plan, Plan>>>
normalizedRightProject = ProjectJoinReorderHelper.normalize(topJoin.right());
if (!normalizedLeftProject.isPresent() || !normalizedRightProject.isPresent()) {
return null;
}
LogicalJoin<Plan, Plan> leftJoin = normalizedLeftProject.get().child();
LogicalJoin<Plan, Plan> rightJoin = normalizedRightProject.get().child();
Plan a = leftJoin.left();
Plan b = leftJoin.right();
Plan c = rightJoin.left();
Plan d = rightJoin.right();

Set<ExprId> acOutputExprIdSet = JoinUtils.getJoinOutputExprIdSet(a, c);
Set<ExprId> bdOutputExprIdSet = JoinUtils.getJoinOutputExprIdSet(b, d);
Expand All @@ -92,10 +99,10 @@ public Rule build() {
return null;
}

LogicalJoin<GroupPlan, GroupPlan> newLeftJoin = new LogicalJoin<>(JoinType.INNER_JOIN,
LogicalJoin<Plan, Plan> newLeftJoin = new LogicalJoin<>(JoinType.INNER_JOIN,
newLeftJoinHashJoinConjuncts, newLeftJoinOtherJoinConjuncts,
new DistributeHint(DistributeType.NONE), a, c, null);
LogicalJoin<GroupPlan, GroupPlan> newRightJoin = new LogicalJoin<>(JoinType.INNER_JOIN,
LogicalJoin<Plan, Plan> newRightJoin = new LogicalJoin<>(JoinType.INNER_JOIN,
newRightJoinHashJoinConjuncts, newRightJoinOtherJoinConjuncts,
new DistributeHint(DistributeType.NONE), b, d, null);
Set<ExprId> topUsedExprIds = new HashSet<>();
Expand Down
33 changes: 21 additions & 12 deletions .../org/apache/doris/nereids/rules/exploration/join/LogicalJoinSemiJoinTransposeProject.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;

/**
Expand All @@ -49,18 +50,22 @@ public List<Rule> buildRules() {
&& (topJoin.getJoinType().isInnerJoin()
|| topJoin.getJoinType().isLeftOuterJoin())))
.whenNot(topJoin -> topJoin.hasDistributeHint()
|| topJoin.left().child().hasDistributeHint())
.when(join -> join.left().isAllSlots()))
|| topJoin.left().child().hasDistributeHint()))
.then(topProject -> {
LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>, GroupPlan> topJoin
= topProject.child();
LogicalJoin<GroupPlan, GroupPlan> bottomJoin = topJoin.left().child();
Optional<LogicalProject<LogicalJoin<Plan, Plan>>>
normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.left());
if (!normalizedProject.isPresent()) {
return null;
}
LogicalJoin<Plan, Plan> bottomJoin = normalizedProject.get().child();
if (!JoinUtils.checkReorderPrecondition(topJoin, bottomJoin)) {
return null;
}
GroupPlan a = bottomJoin.left();
GroupPlan b = bottomJoin.right();
GroupPlan c = topJoin.right();
Plan a = bottomJoin.left();
Plan b = bottomJoin.right();
Plan c = topJoin.right();

Set<ExprId> topUsedExprIds = new HashSet<>();
topProject.getProjects().forEach(expr -> topUsedExprIds.addAll(expr.getInputSlotExprIds()));
Expand All @@ -83,18 +88,22 @@ public List<Rule> buildRules() {
&& (topJoin.getJoinType().isInnerJoin()
|| topJoin.getJoinType().isRightOuterJoin())))
.whenNot(topJoin -> topJoin.hasDistributeHint()
|| topJoin.right().child().hasDistributeHint())
.when(join -> join.right().isAllSlots()))
|| topJoin.right().child().hasDistributeHint()))
.then(topProject -> {
LogicalJoin<GroupPlan, LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>> topJoin
= topProject.child();
LogicalJoin<GroupPlan, GroupPlan> bottomJoin = topJoin.right().child();
Optional<LogicalProject<LogicalJoin<Plan, Plan>>>
normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.right());
if (!normalizedProject.isPresent()) {
return null;
}
LogicalJoin<Plan, Plan> bottomJoin = normalizedProject.get().child();
if (!JoinUtils.checkReorderPrecondition(topJoin, bottomJoin)) {
return null;
}
GroupPlan a = topJoin.left();
GroupPlan b = bottomJoin.left();
GroupPlan c = bottomJoin.right();
Plan a = topJoin.left();
Plan b = bottomJoin.left();
Plan c = bottomJoin.right();

Set<ExprId> topUsedExprIds = new HashSet<>();
topProject.getProjects().forEach(expr -> topUsedExprIds.addAll(expr.getInputSlotExprIds()));
Expand Down
21 changes: 14 additions & 7 deletions .../src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import com.google.common.collect.ImmutableSet;

import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Stream;

Expand Down Expand Up @@ -66,19 +67,25 @@ public Rule build() {
.when(join -> VALID_TYPE_PAIR_SET.contains(
Pair.of(join.left().child().getJoinType(), join.getJoinType())))
.when(topJoin -> OuterJoinLAsscomProject.checkReorder(topJoin, topJoin.left().child()))
.whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint())
.when(join -> checkCondition(join, join.left().child().left().getOutputSet()))
.when(join -> join.left().isAllSlots()))
.whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint()))
.thenApply(ctx -> {
LogicalProject<LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>, GroupPlan>> topProject
= ctx.root;
LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>, GroupPlan> topJoin
= topProject.child();
Optional<LogicalProject<LogicalJoin<Plan, Plan>>>
normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.left());
if (!normalizedProject.isPresent()) {
return null;
}
/* ********** init ********** */
LogicalJoin<GroupPlan, GroupPlan> bottomJoin = topJoin.left().child();
GroupPlan a = bottomJoin.left();
GroupPlan b = bottomJoin.right();
GroupPlan c = topJoin.right();
LogicalJoin<Plan, Plan> bottomJoin = normalizedProject.get().child();
Plan a = bottomJoin.left();
Plan b = bottomJoin.right();
Plan c = topJoin.right();
if (!checkCondition(topJoin, a.getOutputSet())) {
return null;
}

/*
* Paper `On the Correct and Complete Enumeration of the Core Search Space`.
Expand Down
24 changes: 15 additions & 9 deletions ...rc/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import com.google.common.collect.ImmutableSet;

import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
Expand Down Expand Up @@ -67,18 +68,23 @@ public Rule build() {
.when(join -> OuterJoinLAsscomProject.VALID_TYPE_PAIR_SET.contains(
Pair.of(join.left().child().getJoinType(), join.getJoinType())))
.when(topJoin -> OuterJoinLAsscomProject.checkReorder(topJoin, topJoin.left().child()))
.whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint())
.when(topJoin -> OuterJoinLAsscomProject.checkCondition(topJoin,
topJoin.left().child().right().getOutputExprIdSet()))
.when(join -> join.left().isAllSlots()))
.whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint()))
.then(topProject -> {
LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>, GroupPlan> topJoin
= topProject.child();
Optional<LogicalProject<LogicalJoin<Plan, Plan>>>
normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.left());
if (!normalizedProject.isPresent()) {
return null;
}
/* ********** init ********** */
LogicalJoin<GroupPlan, GroupPlan> bottomJoin = topJoin.left().child();
GroupPlan a = bottomJoin.left();
GroupPlan b = bottomJoin.right();
GroupPlan c = topJoin.right();
LogicalJoin<Plan, Plan> bottomJoin = normalizedProject.get().child();
Plan a = bottomJoin.left();
Plan b = bottomJoin.right();
Plan c = topJoin.right();
if (!OuterJoinLAsscomProject.checkCondition(topJoin, b.getOutputExprIdSet())) {
return null;
}

/* ********** new Plan ********** */
LogicalJoin newBottomJoin = topJoin.withChildrenNoContext(a, c, null);
Expand Down Expand Up @@ -128,7 +134,7 @@ public static boolean checkCondition(LogicalJoin<? extends Plan, GroupPlan> topJ
* check join reorder masks.
*/
public static boolean checkReorder(LogicalJoin<? extends Plan, GroupPlan> topJoin,
LogicalJoin<GroupPlan, GroupPlan> bottomJoin) {
LogicalJoin<? extends Plan, ? extends Plan> bottomJoin) {
// hasCommute will cause to lack of OuterJoinAssocRule:Left
return !topJoin.getJoinReorderContext().hasLAsscom()
&& !topJoin.getJoinReorderContext().hasLeftAssociate()
Expand Down
Loading
Loading