From 990efa5913f35fa72fbb3b4c10281dbfa8fa6aec Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Sat, 9 May 2026 19:57:31 +0800 Subject: [PATCH 1/8] fix --- .../doris/nereids/StatementContext.java | 13 + .../processor/post/PlanPostProcessors.java | 1 + .../post/PrunePartitionPredicate.java | 146 ++++++++++ .../rules/PartitionPrunablePredicate.java | 97 +++++++ .../expression/rules/PartitionPruner.java | 23 -- .../rules/rewrite/PruneOlapScanPartition.java | 34 ++- .../trees/plans/logical/LogicalOlapScan.java | 6 + .../rules/rewrite/PartitionPrunerTest.java | 256 ------------------ .../prune_predicates_mv_test.out | 9 + .../prune_predicates_mv_test.groovy | 138 ++++++++++ 10 files changed, 439 insertions(+), 284 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPrunablePredicate.java create mode 100644 regression-test/data/nereids_rules_p0/partition_prune/prune_predicates_mv_test.out create mode 100644 regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 520ef17fb0bb18..47c67d0ae62584 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -27,6 +27,7 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.View; +import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.common.Id; import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; @@ -42,6 +43,7 @@ import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.analysis.ColumnAliasGenerator; +import org.apache.doris.nereids.rules.expression.rules.PartitionPrunablePredicate; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; @@ -166,6 +168,13 @@ public enum TableFrom { private final Map> cteIdToProducer = new HashMap<>(); private final Map> consumerIdToFilters = new HashMap<>(); + // Partition-prunable conjuncts recorded by PruneOlapScanPartition. The actual + // predicate removal happens later in PrunePartitionPredicate post-processor + // so that materialized-view rewrite still sees the original predicates. + // Multiple entries may share the same TableIdentifier when the same physical + // table is referenced more than once (self-join, CTE expansion, etc.). + private final Map> partitionPrunablePredicates + = new HashMap<>(); // Used to update consumer's stats private final Map, Group>>> cteIdToConsumerGroup = new HashMap<>(); private final Map rewrittenCteProducer = new HashMap<>(); @@ -633,6 +642,10 @@ public Map> getConsumerIdToFilters() { return consumerIdToFilters; } + public Map> getPartitionPrunablePredicates() { + return partitionPrunablePredicates; + } + public PlaceholderId getNextPlaceholderId() { return placeHolderIdGenerator.getNextId(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java index 4f305554685260..31f8d716c75820 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java @@ -63,6 +63,7 @@ public List getProcessors() { // add processor if we need Builder builder = ImmutableList.builder(); builder.add(new PushDownFilterThroughProject()); + builder.add(new PrunePartitionPredicate()); builder.add(new RemoveUselessProjectPostProcessor()); builder.add(new ShuffleKeyPruner()); builder.add(new RecomputeLogicalPropertiesProcessor()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java new file mode 100644 index 00000000000000..58af220fc29adb --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.processor.post; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.constraint.TableIdentifier; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.expression.rules.PartitionPrunablePredicate; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.util.ExpressionUtils; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Removes partition-prunable conjuncts that were registered by {@link + * org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition} but kept in + * the logical plan during cascades. Doing the removal here, after + * materialized-view rewrite has finished, ensures MV matching observes the + * original predicates; otherwise the MV view-predicate may incorrectly cover + * the dropped partition predicate and produce extra rows. + * + *

Matching is keyed by {@link TableIdentifier} (catalog/db/table) plus the + * surviving partition id set. Because intermediate rewrites can rebuild scans + * with fresh slot ids, the recorded snapshot slots are remapped onto the + * actual scan output by column name before each conjunct is rewritten and + * removed from the filter. + */ +public class PrunePartitionPredicate extends PlanPostProcessor { + + @Override + public Plan processRoot(Plan plan, CascadesContext ctx) { + boolean skipPrunePredicate = ctx.getConnectContext().getSessionVariable().skipPrunePredicate + || ctx.getStatementContext().isDelete(); + Map> registry = + ctx.getStatementContext().getPartitionPrunablePredicates(); + if (skipPrunePredicate || registry.isEmpty()) { + return plan; + } + return plan.accept(this, ctx); + } + + @Override + public Plan visitPhysicalFilter(PhysicalFilter filter, CascadesContext context) { + filter = (PhysicalFilter) super.visit(filter, context); + Plan child = filter.child(); + if (!(child instanceof PhysicalOlapScan)) { + return filter; + } + Map> registry = + context.getStatementContext().getPartitionPrunablePredicates(); + if (registry.isEmpty()) { + return filter; + } + PhysicalOlapScan scan = (PhysicalOlapScan) child; + TableIdentifier scanIdentifier = new TableIdentifier(scan.getTable()); + Set entries = registry.get(scanIdentifier); + if (entries == null || entries.isEmpty()) { + return filter; + } + Set scanPartitions = new HashSet<>(scan.getSelectedPartitionIds()); + Map nameToOutputSlot = buildNameToSlotMap(scan.getOutput()); + + Set remaining = new LinkedHashSet<>(filter.getConjuncts()); + boolean changed = false; + for (PartitionPrunablePredicate entry : entries) { + if (!entry.getSelectedPartitionIds().containsAll(scanPartitions)) { + continue; + } + Map slotReplaceMap = + buildSlotReplaceMap(entry.getSnapshotPartitionSlots(), nameToOutputSlot); + if (slotReplaceMap == null) { + continue; + } + for (Expression conjunct : entry.getPrunableConjuncts()) { + Expression rewritten = slotReplaceMap.isEmpty() + ? conjunct : ExpressionUtils.replace(conjunct, slotReplaceMap); + if (remaining.remove(rewritten)) { + changed = true; + } + } + } + if (!changed) { + return filter; + } + if (remaining.isEmpty()) { + return scan; + } + return filter.withConjunctsAndChild(remaining, scan) + .copyStatsAndGroupIdFrom((AbstractPhysicalPlan) filter); + } + + private static Map buildNameToSlotMap(List slots) { + Map map = new HashMap<>(slots.size()); + for (Slot slot : slots) { + map.put(slot.getName().toLowerCase(), slot); + } + return map; + } + + /** + * Map each recorded snapshot slot to the scan's current output slot of the + * same column name. Returns null when any snapshot slot cannot be located, + * so the caller can skip the entry. + */ + private static Map buildSlotReplaceMap( + List snapshotSlots, Map nameToOutputSlot) { + Map replaceMap = new HashMap<>(snapshotSlots.size()); + for (Slot snapshot : snapshotSlots) { + Slot current = nameToOutputSlot.get(snapshot.getName().toLowerCase()); + if (current == null) { + return null; + } + if (!snapshot.equals(current)) { + replaceMap.put(snapshot, current); + } + } + return replaceMap; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPrunablePredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPrunablePredicate.java new file mode 100644 index 00000000000000..f420e082b07b6b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPrunablePredicate.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.expression.rules; + +import org.apache.doris.catalog.constraint.TableIdentifier; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.List; +import java.util.Set; + +/** + * Records that, on a specific physical {@link TableIdentifier} restricted to + * {@link #selectedPartitionIds}, the {@link #prunableConjuncts} are guaranteed + * to evaluate to TRUE for every surviving row. + * + *

The predicate is registered by {@link + * org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition} but kept in + * the logical filter during cascades. The actual removal happens later in + * {@link org.apache.doris.nereids.processor.post.PrunePartitionPredicate} so + * that materialized-view rewrite still sees the original predicates. Keeping + * the predicate in the plan avoids the wrong-result problem in which the MV + * view-predicate happens to cover the remaining conjuncts after the partition + * predicate has been silently dropped. + * + *

Because rewrites between recording and removal may rebuild the scan with + * fresh slot ids, {@link #snapshotPartitionSlots} captures the slots that + * appear in the recorded conjuncts. The post-processor maps them onto the + * actual scan's output slots by column name before performing the conjunct + * removal. + */ +public class PartitionPrunablePredicate { + private final TableIdentifier tableIdentifier; + private final Set selectedPartitionIds; + private final List snapshotPartitionSlots; + private final Set prunableConjuncts; + + public PartitionPrunablePredicate(TableIdentifier tableIdentifier, + Set selectedPartitionIds, + List snapshotPartitionSlots, + Set prunableConjuncts) { + this.tableIdentifier = tableIdentifier; + this.selectedPartitionIds = ImmutableSet.copyOf(selectedPartitionIds); + this.snapshotPartitionSlots = ImmutableList.copyOf(snapshotPartitionSlots); + this.prunableConjuncts = ImmutableSet.copyOf(prunableConjuncts); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PartitionPrunablePredicate that = (PartitionPrunablePredicate) o; + return tableIdentifier.equals(that.tableIdentifier) + && selectedPartitionIds.equals(that.selectedPartitionIds) + && snapshotPartitionSlots.equals(that.snapshotPartitionSlots) + && prunableConjuncts.equals(that.prunableConjuncts); + + } + + public TableIdentifier getTableIdentifier() { + return tableIdentifier; + } + + public Set getSelectedPartitionIds() { + return selectedPartitionIds; + } + + public List getSnapshotPartitionSlots() { + return snapshotPartitionSlots; + } + + public Set getPrunableConjuncts() { + return prunableConjuncts; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java index 83e7ad6e9cd0c8..b49c8b7436210e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java @@ -36,11 +36,7 @@ import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; import org.apache.doris.nereids.types.DateTimeType; -import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; @@ -51,7 +47,6 @@ import com.google.common.collect.RangeSet; import com.google.common.collect.Sets; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -390,22 +385,4 @@ private static Pair canBePrunedOut(Expression partitionPre return Pair.of(true, false); } } - - /** remove predicates that are always true*/ - public static Plan prunePredicate(boolean skipPrunePredicate, Optional prunedPredicates, - LogicalFilter filter, LogicalRelation scan) { - if (!skipPrunePredicate && prunedPredicates.isPresent()) { - Set conjuncts = new LinkedHashSet<>(filter.getConjuncts()); - Expression deletedPredicate = prunedPredicates.get(); - Set deletedPredicateSet = ExpressionUtils.extractConjunctionToSet(deletedPredicate); - conjuncts.removeAll(deletedPredicateSet); - if (conjuncts.isEmpty()) { - return scan; - } else { - return filter.withConjunctsAndChild(conjuncts, scan); - } - } else { - return filter.withChildren(ImmutableList.of(scan)); - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java index 23e0c3971919c2..02696d204e7abb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java @@ -25,11 +25,13 @@ import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.Tablet; +import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.common.Pair; import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager; import org.apache.doris.nereids.pattern.MatchingContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.expression.rules.PartitionPrunablePredicate; import org.apache.doris.nereids.rules.expression.rules.PartitionPruner; import org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionPruneResult; import org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionTableType; @@ -40,6 +42,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; +import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; @@ -47,6 +50,7 @@ import com.google.common.collect.ImmutableSet; import java.util.ArrayList; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -92,12 +96,32 @@ public List buildRules() { } if (rewrittenLogicalRelation instanceof LogicalEmptyRelation) { return rewrittenLogicalRelation; - } else { - return PartitionPruner.prunePredicate( - ctx.connectContext.getSessionVariable().skipPrunePredicate - || ctx.statementContext.isDelete(), - prunedRes.second, filter, rewrittenLogicalRelation); } + boolean skipPrunePredicate = ctx.connectContext.getSessionVariable().skipPrunePredicate + || ctx.statementContext.isDelete(); + if (!skipPrunePredicate && prunedRes.second.isPresent()) { + // Defer the predicate removal to PlanPostProcessor so that materialized-view + // rewrite still sees the original predicates. Otherwise, partition predicates + // that are equivalent to the surviving partition list would be silently + // dropped, leading to wrong results when an MV definition predicate matches + // the remaining conjuncts. + LogicalOlapScan prunedScan = (LogicalOlapScan) rewrittenLogicalRelation; + Set prunableConjuncts = ExpressionUtils.extractConjunctionToSet( + prunedRes.second.get()); + List partitionSlots = getPartitionSlots(prunedScan, prunedScan.getTable()); + if (partitionSlots != null) { + TableIdentifier tableIdentifier = new TableIdentifier(prunedScan.getTable()); + PartitionPrunablePredicate entry = new PartitionPrunablePredicate( + tableIdentifier, + new HashSet<>(prunedScan.getSelectedPartitionIds()), + partitionSlots, + prunableConjuncts); + ctx.statementContext.getPartitionPrunablePredicates() + .computeIfAbsent(tableIdentifier, k -> new HashSet<>()) + .add(entry); + } + } + return filter.withChildren(ImmutableList.of(rewrittenLogicalRelation)); }).toRule(RuleType.OLAP_SCAN_PARTITION_PRUNE) ); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 415b2a9817ef37..347908bfed1608 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -334,7 +334,13 @@ public OlapTable getTable() { @Override public String toString() { + String partitions = ""; + int partitionCount = this.table.getPartitionNames().size(); + if (selectedPartitionIds.size() != partitionCount) { + partitions = " partitions(" + selectedPartitionIds.size() + "/" + partitionCount + ")"; + } return Utils.toSqlStringSkipNull("LogicalOlapScan[" + id.asInt() + "]", + "partitions", partitions, "qualified", qualifiedName(), "alias", tableAlias, "indexName", getSelectedMaterializedIndexName().orElse(""), diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java index 137791ba5a5653..f4f78da1e30ac6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java @@ -20,7 +20,6 @@ import org.apache.doris.analysis.PartitionValue; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.ListPartitionItem; -import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PrimitiveType; @@ -38,16 +37,11 @@ import org.apache.doris.nereids.trees.expressions.GreaterThan; import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.IsNull; -import org.apache.doris.nereids.trees.expressions.LessThan; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.RelationId; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.utframe.TestWithFeService; @@ -59,11 +53,9 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; public class PartitionPrunerTest extends TestWithFeService { private Method canBePrunedOutMethod; @@ -324,254 +316,6 @@ public void testComplexNestedPredicate() Assertions.assertFalse(result.second); } - // test prune predicate - // Test basis: some predicates are pruned - @Test - public void testPrunePartialPredicates() { - Set predicates = new LinkedHashSet<>(); - GreaterThan gt = new GreaterThan(slotA, Literal.of(10)); - LessThan lt = new LessThan(slotB, Literal.of(20)); - predicates.add(gt); - predicates.add(lt); - - LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new OlapTable()); - LogicalFilter filter = new LogicalFilter<>(predicates, scan); - - Plan prunedPlan = PartitionPruner.prunePredicate(false, Optional.of(gt), filter, scan); - - Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan); - LogicalFilter prunedFilter = (LogicalFilter) prunedPlan; - Assertions.assertEquals(1, prunedFilter.getConjuncts().size()); - Assertions.assertTrue(prunedFilter.getConjuncts().contains(lt)); - Assertions.assertFalse(prunedFilter.getConjuncts().contains(gt)); - } - - // all predicates are pruned - @Test - public void testPruneAllPredicates() { - Set predicates = new LinkedHashSet<>(); - GreaterThan gt = new GreaterThan(slotA, Literal.of(10)); - predicates.add(gt); - - LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new OlapTable()); - LogicalFilter filter = new LogicalFilter<>(predicates, scan); - - Plan prunedPlan = PartitionPruner.prunePredicate(false, Optional.of(gt), filter, scan); - - Assertions.assertInstanceOf(LogicalOlapScan.class, prunedPlan); - } - - // no predicates are pruned - @Test - public void testPruneNoPredicates() { - Set predicates = new LinkedHashSet<>(); - GreaterThan gt = new GreaterThan(slotA, Literal.of(10)); - LessThan lt = new LessThan(slotB, Literal.of(20)); - predicates.add(gt); - predicates.add(lt); - - LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new OlapTable()); - LogicalFilter filter = new LogicalFilter<>(predicates, scan); - - EqualTo nonExistentPredicate = new EqualTo(slotC, Literal.of(30)); - Plan prunedPlan = PartitionPruner.prunePredicate(false, Optional.of(nonExistentPredicate), filter, scan); - - Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan); - LogicalFilter prunedFilter = (LogicalFilter) prunedPlan; - Assertions.assertEquals(2, prunedFilter.getConjuncts().size()); - Assertions.assertTrue(prunedFilter.getConjuncts().contains(gt)); - Assertions.assertTrue(prunedFilter.getConjuncts().contains(lt)); - } - - @Test - public void testPruneCompoundPredicate() { - Set predicates = new LinkedHashSet<>(); - GreaterThan gt = new GreaterThan(slotA, Literal.of(10)); - LessThan lt = new LessThan(slotB, Literal.of(20)); - EqualTo eq = new EqualTo(slotC, Literal.of(30)); - predicates.add(gt); - predicates.add(lt); - predicates.add(eq); - - LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new OlapTable()); - LogicalFilter filter = new LogicalFilter<>(predicates, scan); - - // (a > 10 AND b < 20) - And compoundPredicate = new And(gt, lt); - Plan prunedPlan = PartitionPruner.prunePredicate(false, Optional.of(compoundPredicate), filter, scan); - - Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan); - LogicalFilter prunedFilter = (LogicalFilter) prunedPlan; - Assertions.assertEquals(1, prunedFilter.getConjuncts().size()); - Assertions.assertTrue(prunedFilter.getConjuncts().contains(eq)); - Assertions.assertFalse(prunedFilter.getConjuncts().contains(gt)); - Assertions.assertFalse(prunedFilter.getConjuncts().contains(lt)); - } - - @Test - public void testSkipPrunePredicate() { - Set predicates = new LinkedHashSet<>(); - GreaterThan gt = new GreaterThan(slotA, Literal.of(10)); - predicates.add(gt); - - LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new OlapTable()); - LogicalFilter filter = new LogicalFilter<>(predicates, scan); - - Plan prunedPlan = PartitionPruner.prunePredicate(true, Optional.of(gt), filter, scan); - - Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan); - LogicalFilter prunedFilter = (LogicalFilter) prunedPlan; - Assertions.assertEquals(1, prunedFilter.getConjuncts().size()); - Assertions.assertTrue(prunedFilter.getConjuncts().contains(gt)); - } - - @Test - public void testEmptyPrunedPredicates() { - Set predicates = new LinkedHashSet<>(); - GreaterThan gt = new GreaterThan(slotA, Literal.of(10)); - predicates.add(gt); - - LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new OlapTable()); - LogicalFilter filter = new LogicalFilter<>(predicates, scan); - - // prunedPredicates is empty - Plan prunedPlan = PartitionPruner.prunePredicate(false, Optional.empty(), filter, scan); - - Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan); - LogicalFilter prunedFilter = (LogicalFilter) prunedPlan; - Assertions.assertEquals(1, prunedFilter.getConjuncts().size()); - Assertions.assertTrue(prunedFilter.getConjuncts().contains(gt)); - } - - @Test - public void testPruneDuplicatePredicates() { - Set predicates = new LinkedHashSet<>(); - GreaterThan gt1 = new GreaterThan(slotA, Literal.of(10)); - GreaterThan gt2 = new GreaterThan(slotA, Literal.of(10)); // duplicated predicate - predicates.add(gt1); - predicates.add(gt2); - - LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new OlapTable()); - LogicalFilter filter = new LogicalFilter<>(predicates, scan); - - Plan prunedPlan = PartitionPruner.prunePredicate(false, Optional.of(gt1), filter, scan); - - Assertions.assertInstanceOf(LogicalOlapScan.class, prunedPlan); - } - - @Test - public void testPruneWithNullLiteral() { - Set predicates = new LinkedHashSet<>(); - GreaterThan gt = new GreaterThan(slotA, Literal.of(10)); - EqualTo nullEq = new EqualTo(slotB, new NullLiteral()); - predicates.add(gt); - predicates.add(nullEq); - - LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new OlapTable()); - LogicalFilter filter = new LogicalFilter<>(predicates, scan); - - Plan prunedPlan = PartitionPruner.prunePredicate(false, Optional.of(gt), filter, scan); - - Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan); - LogicalFilter prunedFilter = (LogicalFilter) prunedPlan; - Assertions.assertEquals(1, prunedFilter.getConjuncts().size()); - Assertions.assertTrue(prunedFilter.getConjuncts().contains(nullEq)); - } - - @Test - public void testPruneMultiplePredicatesPartially() { - Set predicates = new LinkedHashSet<>(); - GreaterThan gt = new GreaterThan(slotA, Literal.of(10)); - LessThan lt = new LessThan(slotB, Literal.of(20)); - EqualTo eq1 = new EqualTo(slotC, Literal.of(30)); - EqualTo eq2 = new EqualTo(slotC, Literal.of(40)); - predicates.add(gt); - predicates.add(lt); - predicates.add(eq1); - predicates.add(eq2); - - LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new OlapTable()); - LogicalFilter filter = new LogicalFilter<>(predicates, scan); - - // (a > 10 AND b < 20) - And compoundPredicate = new And(gt, lt); - Plan prunedPlan = PartitionPruner.prunePredicate(false, Optional.of(compoundPredicate), filter, scan); - - Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan); - LogicalFilter prunedFilter = (LogicalFilter) prunedPlan; - Assertions.assertEquals(2, prunedFilter.getConjuncts().size()); - Assertions.assertTrue(prunedFilter.getConjuncts().contains(eq1)); - Assertions.assertTrue(prunedFilter.getConjuncts().contains(eq2)); - Assertions.assertFalse(prunedFilter.getConjuncts().contains(gt)); - Assertions.assertFalse(prunedFilter.getConjuncts().contains(lt)); - } - - @Test - public void testPruneNestedCompoundPredicate() { - Set predicates = new LinkedHashSet<>(); - GreaterThan gt = new GreaterThan(slotA, Literal.of(10)); - LessThan lt = new LessThan(slotB, Literal.of(20)); - EqualTo eq = new EqualTo(slotC, Literal.of(30)); - predicates.add(gt); - predicates.add(lt); - predicates.add(eq); - - LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new OlapTable()); - LogicalFilter filter = new LogicalFilter<>(predicates, scan); - - // (a > 10 AND (b < 20 AND c = 30)) - And innerAnd = new And(lt, eq); - And outerAnd = new And(gt, innerAnd); - - Plan prunedPlan = PartitionPruner.prunePredicate(false, Optional.of(outerAnd), filter, scan); - - Assertions.assertInstanceOf(LogicalOlapScan.class, prunedPlan); - } - - @Test - public void testPruneWhenFilterContainsOr() { - Set predicates = new LinkedHashSet<>(); - Or orPredicate = new Or( - new GreaterThan(slotA, Literal.of(10)), - new LessThan(slotB, Literal.of(20)) - ); - predicates.add(orPredicate); - LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new OlapTable()); - LogicalFilter filter = new LogicalFilter<>(predicates, scan); - - GreaterThan gt = new GreaterThan(slotA, Literal.of(10)); - Plan prunedPlan = PartitionPruner.prunePredicate(false, Optional.of(gt), filter, scan); - - Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan); - LogicalFilter prunedFilter = (LogicalFilter) prunedPlan; - Assertions.assertEquals(1, prunedFilter.getConjuncts().size()); - Assertions.assertTrue(prunedFilter.getConjuncts().contains(orPredicate)); - } - - @Test - public void testPruneWhenFilterContainsAndOrMix() { - Set predicates = new LinkedHashSet<>(); - // filter :a > 10 AND (b < 20 OR c = 30) - Or orPredicate = new Or( - new LessThan(slotB, Literal.of(20)), - new EqualTo(slotC, Literal.of(30)) - ); - GreaterThan gt = new GreaterThan(slotA, Literal.of(10)); - - predicates.add(gt); - predicates.add(orPredicate); - - LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new OlapTable()); - LogicalFilter filter = new LogicalFilter<>(predicates, scan); - // a > 10 - Plan prunedPlan = PartitionPruner.prunePredicate(false, Optional.of(gt), filter, scan); - Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan); - LogicalFilter prunedFilter = (LogicalFilter) prunedPlan; - - Assertions.assertEquals(1, prunedFilter.getConjuncts().size()); - Assertions.assertTrue(prunedFilter.getConjuncts().contains(orPredicate)); - } - @Test public void testPruneWithResultIgnoresNonPruningPartitionPredicate() throws AnalysisException { Map idToPartitions = ImmutableMap.of( diff --git a/regression-test/data/nereids_rules_p0/partition_prune/prune_predicates_mv_test.out b/regression-test/data/nereids_rules_p0/partition_prune/prune_predicates_mv_test.out new file mode 100644 index 00000000000000..5aab6559698dc4 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/partition_prune/prune_predicates_mv_test.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !mv_1 -- +1 +2 + +-- !mv_2 -- +1 +2 + diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy new file mode 100644 index 00000000000000..7ec06911fc9500 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("prune_predicates_mv_test") { + String currentDb = context.config.getDbNameByFile(context.file) + sql """ + drop table if exists base_t; + CREATE TABLE base_t ( + top_asset varchar(64) NOT NULL, + tag_key int NOT NULL, + tag_value int NOT NULL, + frame_count int NOT NULL + ) ENGINE=OLAP + UNIQUE KEY(top_asset, tag_key, tag_value) + AUTO PARTITION BY LIST (tag_key) () + DISTRIBUTED BY HASH(top_asset) BUCKETS 4 + PROPERTIES ( + "replication_num" = "1" + ); + + INSERT INTO base_t VALUES + ('a', 1, 100, 5), ('a', 1, 101, 0), + ('a', 2, 200, 7), ('a', 2, 201, 0), + ('a', 3, 300, 0), + ('a', 4, 400, 9), + ('a', 5, 500, 1), + ('a', 6, 600, 2); + """ + + // case 1: + def mv_1 = """ + SELECT top_asset, tag_key, SUM(frame_count) AS frame_count + FROM base_t + WHERE frame_count != 0 + GROUP BY top_asset, tag_key; + """ + + def query_1 = """ + SELECT /*+ USE_MV(mv_1) */ tag_key FROM base_t + WHERE tag_key IN (1, 2, 3) AND frame_count != 0 + GROUP BY tag_key + ORDER BY tag_key; + """ + + //执行（需要强制改写）： + //1.检查结果正确 + //2.检查shape plan里面有filter + //3.检查改写成功了 async_mv_rewrite_success + + // table有分区，分区裁剪之后谓词被删除掉了，mv没有分区，检查mv没有把谓词给删除掉 + async_mv_rewrite_success(currentDb, mv_1, query_1, "mv_1") + order_qt_mv_1 query_1 + explain { + sql "shape plan ${query_1}" + contains "filter" + } + + // case2: 物化视图也是带有分区的，检查对物化视图进行分区裁剪，检查在分区裁剪后将恒true谓词删除 + + def async_partition_mv_rewrite_success = { db, mv_sql, query_sql, mv_name, partition, expected_pre_rewrite_strategys = [] -> + if (!mvShouldContinueCheck(expected_pre_rewrite_strategys)) { + return; + } + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_name} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + ${partition} + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS ${mv_sql} + """ + def job_name = getJobName(db, mv_name); + waitingMTMVTaskFinished(job_name) + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" + mv_rewrite_success(query_sql, mv_name, true, expected_pre_rewrite_strategys) + } + + sql """ + drop table if exists base_t2; + CREATE TABLE base_t2 ( + top_asset varchar(64) NOT NULL, + tag_key int NOT NULL, + tag_value int NOT NULL, + frame_count int NOT NULL + ) ENGINE=OLAP + UNIQUE KEY(top_asset, tag_key, tag_value) + AUTO PARTITION BY LIST (tag_key) () + DISTRIBUTED BY HASH(top_asset) BUCKETS 4 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "replication_num" = "1" + ); + + INSERT INTO base_t2 VALUES + ('a', 1, 100, 5), ('a', 1, 101, 0), + ('a', 2, 200, 7), ('a', 2, 201, 0), + ('a', 3, 300, 0), + ('a', 4, 400, 9), + ('a', 5, 500, 1), + ('a', 6, 600, 2); + """ + def mv_2 = """ + SELECT top_asset, tag_key, SUM(frame_count) AS frame_count + FROM base_t2 + WHERE frame_count != 0 + GROUP BY top_asset, tag_key; + """ + def query_2 = """ + SELECT /*+use_mv(mv_2)*/ tag_key FROM base_t2 + WHERE tag_key IN (1, 2, 3) AND frame_count != 0 + GROUP BY tag_key + ORDER BY tag_key; + """ + + async_partition_mv_rewrite_success(currentDb, mv_2, query_2, "mv_2", "PARTITION BY (tag_key)") + order_qt_mv_2 query_2 + explain { + sql "physical plan ${query_2}" + contains "partitions(2/6)" + notContains "PhysicalFilter" + } +} \ No newline at end of file From b2a3401a37fdf46b57a24a4eebd1ba702f665473 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Sat, 9 May 2026 20:44:18 +0800 Subject: [PATCH 2/8] fix --- .../doris/nereids/processor/post/PrunePartitionPredicate.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java index 58af220fc29adb..eb7f7b1e622b77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java @@ -17,13 +17,11 @@ package org.apache.doris.nereids.processor.post; -import org.apache.doris.catalog.Column; import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.expression.rules.PartitionPrunablePredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; @@ -35,7 +33,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; /** From 7836b856c1b48e3431234a73507a0d7ed23a80d8 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 11 May 2026 15:09:16 +0800 Subject: [PATCH 3/8] fix --- .../post/PrunePartitionPredicate.java | 34 +++++++++++++-- .../prune_predicates_mv_test.out | 5 +++ .../prune_predicates_mv_test.groovy | 42 +++++++++++++++++++ 3 files changed, 77 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java index eb7f7b1e622b77..89fa637b4af2d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java @@ -17,11 +17,16 @@ package org.apache.doris.nereids.processor.post; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.expression.rules.PartitionPrunablePredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; @@ -33,6 +38,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -82,7 +88,7 @@ public Plan visitPhysicalFilter(PhysicalFilter filter, CascadesC return filter; } Set scanPartitions = new HashSet<>(scan.getSelectedPartitionIds()); - Map nameToOutputSlot = buildNameToSlotMap(scan.getOutput()); + Map nameToOutputSlot = buildNameToSlotMap(scan); Set remaining = new LinkedHashSet<>(filter.getConjuncts()); boolean changed = false; @@ -113,10 +119,30 @@ public Plan visitPhysicalFilter(PhysicalFilter filter, CascadesC .copyStatsAndGroupIdFrom((AbstractPhysicalPlan) filter); } - private static Map buildNameToSlotMap(List slots) { + private static Map buildNameToSlotMap(PhysicalOlapScan scan) { + OlapTable table = scan.getTable(); + List slots = scan.getOutput(); Map map = new HashMap<>(slots.size()); - for (Slot slot : slots) { - map.put(slot.getName().toLowerCase(), slot); + if (scan.getSelectedIndexId() == table.getBaseIndexId()) { + for (Slot slot : slots) { + map.put(slot.getName().toLowerCase(), slot); + } + } else { + for (Slot slot : slots) { + if (!(slot instanceof SlotReference)) { + continue; + } + SlotReference slotReference = (SlotReference) slot; + Optional columnOptional = slotReference.getOriginalColumn(); + if (!columnOptional.isPresent()) { + continue; + } + Expr expr = columnOptional.get().getDefineExpr(); + if (!(expr instanceof SlotRef)) { + continue; + } + map.put(((SlotRef) expr).getColumnName().toLowerCase(), slot); + } } return map; } diff --git a/regression-test/data/nereids_rules_p0/partition_prune/prune_predicates_mv_test.out b/regression-test/data/nereids_rules_p0/partition_prune/prune_predicates_mv_test.out index 5aab6559698dc4..cc704a5c7fe213 100644 --- a/regression-test/data/nereids_rules_p0/partition_prune/prune_predicates_mv_test.out +++ b/regression-test/data/nereids_rules_p0/partition_prune/prune_predicates_mv_test.out @@ -7,3 +7,8 @@ 1 2 +-- !query_3 -- +a 1 5 +a 2 7 +a 3 0 + diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy index 7ec06911fc9500..6d45ebde8f42c2 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy @@ -135,4 +135,46 @@ suite("prune_predicates_mv_test") { contains "partitions(2/6)" notContains "PhysicalFilter" } + + sql """ + drop table if exists base_t3; + CREATE TABLE base_t3 ( + top_asset varchar(64) NOT NULL, + tag_key int NOT NULL, + tag_value int NOT NULL, + frame_count int NOT NULL + ) ENGINE=OLAP + duplicate KEY(top_asset, tag_key, tag_value) + AUTO PARTITION BY LIST (tag_key) () + DISTRIBUTED BY HASH(top_asset) BUCKETS 4 + PROPERTIES ( + "replication_num" = "1" + ); + + INSERT INTO base_t3 VALUES + ('a', 1, 100, 5), ('a', 1, 101, 0), + ('a', 2, 200, 7), ('a', 2, 201, 0), + ('a', 3, 300, 0), + ('a', 4, 400, 9), + ('a', 5, 500, 1), + ('a', 6, 600, 2); + """ + create_sync_mv(currentDb, "base_t3", "mv_3", """ + SELECT top_asset as mv_ta, tag_key as mv_tk, SUM(frame_count) AS mv_sum + FROM base_t3 + GROUP BY top_asset, tag_key; + """) + + def query_3 = """ + SELECT top_asset as mv_ta, tag_key as mv_tk, SUM(frame_count) AS mv_sum + FROM base_t3 + where tag_key in (1,2,3) + GROUP BY top_asset, tag_key; + """ + mv_rewrite_success(query_3, "mv_3") + order_qt_query_3 query_3 + explain { + sql "physical plan ${query_3}" + notContains "PhysicalFilter" + } } \ No newline at end of file From c4285dd1e1e0b0a698bc81ff430ac62bff58cb5b Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 11 May 2026 21:23:21 +0800 Subject: [PATCH 4/8] fix --- .../doris/nereids/trees/plans/logical/LogicalOlapScan.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 347908bfed1608..415b2a9817ef37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -334,13 +334,7 @@ public OlapTable getTable() { @Override public String toString() { - String partitions = ""; - int partitionCount = this.table.getPartitionNames().size(); - if (selectedPartitionIds.size() != partitionCount) { - partitions = " partitions(" + selectedPartitionIds.size() + "/" + partitionCount + ")"; - } return Utils.toSqlStringSkipNull("LogicalOlapScan[" + id.asInt() + "]", - "partitions", partitions, "qualified", qualifiedName(), "alias", tableAlias, "indexName", getSelectedMaterializedIndexName().orElse(""), From b2a01c9402816a0b0b966eee0c04cff0b57d4eb4 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Tue, 12 May 2026 14:56:46 +0800 Subject: [PATCH 5/8] [refactor](nereids) Carry partition-prunable predicates on the scan instead of StatementContext ### What problem does this PR solve? Problem Summary: Move PartitionPrunablePredicate info from StatementContext (keyed by TableIdentifier) onto LogicalOlapScan / PhysicalOlapScan so it follows the scan through with*/deepCopy and is included in equals. Also propagate the info onto the MV scan rewritten by SyncMaterializationContext#getScanPlan so the post-processor can still strip the deferred conjuncts after sync MV rewrite. ### Release note None ### Check List (For Author) - Test: Regression test (prune_predicates_mv_test.groovy) - Behavior changed: No - Does this need documentation: No Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../doris/nereids/StatementContext.java | 13 --- .../post/PrunePartitionPredicate.java | 40 +++---- .../mv/SyncMaterializationContext.java | 11 +- .../rules/PartitionPrunablePredicate.java | 34 +++--- .../LogicalOlapScanToPhysicalOlapScan.java | 3 +- .../rules/rewrite/PruneOlapScanPartition.java | 12 +-- .../trees/plans/logical/LogicalOlapScan.java | 100 +++++++++++++++--- .../plans/physical/PhysicalOlapScan.java | 65 +++++++++++- 8 files changed, 191 insertions(+), 87 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 47c67d0ae62584..520ef17fb0bb18 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -27,7 +27,6 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.View; -import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.common.Id; import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; @@ -43,7 +42,6 @@ import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.analysis.ColumnAliasGenerator; -import org.apache.doris.nereids.rules.expression.rules.PartitionPrunablePredicate; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; @@ -168,13 +166,6 @@ public enum TableFrom { private final Map> cteIdToProducer = new HashMap<>(); private final Map> consumerIdToFilters = new HashMap<>(); - // Partition-prunable conjuncts recorded by PruneOlapScanPartition. The actual - // predicate removal happens later in PrunePartitionPredicate post-processor - // so that materialized-view rewrite still sees the original predicates. - // Multiple entries may share the same TableIdentifier when the same physical - // table is referenced more than once (self-join, CTE expansion, etc.). - private final Map> partitionPrunablePredicates - = new HashMap<>(); // Used to update consumer's stats private final Map, Group>>> cteIdToConsumerGroup = new HashMap<>(); private final Map rewrittenCteProducer = new HashMap<>(); @@ -642,10 +633,6 @@ public Map> getConsumerIdToFilters() { return consumerIdToFilters; } - public Map> getPartitionPrunablePredicates() { - return partitionPrunablePredicates; - } - public PlaceholderId getNextPlaceholderId() { return placeHolderIdGenerator.getNextId(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java index 89fa637b4af2d8..9a530d27543679 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java @@ -21,7 +21,6 @@ import org.apache.doris.analysis.SlotRef; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.expression.rules.PartitionPrunablePredicate; import org.apache.doris.nereids.trees.expressions.Expression; @@ -49,26 +48,18 @@ * original predicates; otherwise the MV view-predicate may incorrectly cover * the dropped partition predicate and produce extra rows. * - *

Matching is keyed by {@link TableIdentifier} (catalog/db/table) plus the - * surviving partition id set. Because intermediate rewrites can rebuild scans - * with fresh slot ids, the recorded snapshot slots are remapped onto the - * actual scan output by column name before each conjunct is rewritten and - * removed from the filter. + *

The {@link PartitionPrunablePredicate} entries live directly on the + * {@link PhysicalOlapScan} so this processor no longer needs a side-channel + * registry keyed by table identifier. Because intermediate rewrites can + * rebuild scans with fresh slot ids, the recorded snapshot slots are remapped + * onto the actual scan output by column name before each conjunct is rewritten + * and removed from the filter. Each recorded entry is only applied when its + * recorded partition id set is a superset of the scan's surviving partitions + * - otherwise a later partition pruning step would have invalidated the + * implication. */ public class PrunePartitionPredicate extends PlanPostProcessor { - @Override - public Plan processRoot(Plan plan, CascadesContext ctx) { - boolean skipPrunePredicate = ctx.getConnectContext().getSessionVariable().skipPrunePredicate - || ctx.getStatementContext().isDelete(); - Map> registry = - ctx.getStatementContext().getPartitionPrunablePredicates(); - if (skipPrunePredicate || registry.isEmpty()) { - return plan; - } - return plan.accept(this, ctx); - } - @Override public Plan visitPhysicalFilter(PhysicalFilter filter, CascadesContext context) { filter = (PhysicalFilter) super.visit(filter, context); @@ -76,15 +67,14 @@ public Plan visitPhysicalFilter(PhysicalFilter filter, CascadesC if (!(child instanceof PhysicalOlapScan)) { return filter; } - Map> registry = - context.getStatementContext().getPartitionPrunablePredicates(); - if (registry.isEmpty()) { + PhysicalOlapScan scan = (PhysicalOlapScan) child; + Set entries = scan.getPartitionPrunablePredicates(); + if (entries.isEmpty()) { return filter; } - PhysicalOlapScan scan = (PhysicalOlapScan) child; - TableIdentifier scanIdentifier = new TableIdentifier(scan.getTable()); - Set entries = registry.get(scanIdentifier); - if (entries == null || entries.isEmpty()) { + boolean skipPrunePredicate = context.getConnectContext().getSessionVariable().skipPrunePredicate + || context.getStatementContext().isDelete(); + if (skipPrunePredicate) { return filter; } Set scanPartitions = new HashSet<>(scan.getSelectedPartitionIds()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SyncMaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SyncMaterializationContext.java index 95c8d3726654d3..0aa5108efdf738 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SyncMaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SyncMaterializationContext.java @@ -122,13 +122,18 @@ public Plan getScanPlan(StructInfo queryStructInfo, CascadesContext cascadesCont return scanPlan.accept(new DefaultPlanRewriter() { @Override public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, Void context) { - if (!queryStructInfoRelations.get(0).getTable().getFullQualifiers().equals( + LogicalOlapScan queryScan = (LogicalOlapScan) queryStructInfoRelations.get(0); + if (!queryScan.getTable().getFullQualifiers().equals( olapScan.getTable().getFullQualifiers())) { // Only the same table, we can do partition prue return olapScan; } - return olapScan.withSelectedPartitionIds( - ((LogicalOlapScan) queryStructInfoRelations.get(0)).getSelectedPartitionIds()); + // Carry partition-prunable predicates from the original query scan onto + // the rewritten MV scan so the post-processor can still drop the + // predicates that have already been enforced by partition pruning. + return olapScan + .withSelectedPartitionIds(queryScan.getSelectedPartitionIds()) + .withPartitionPrunablePredicates(queryScan.getPartitionPrunablePredicates()); } }, null); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPrunablePredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPrunablePredicate.java index f420e082b07b6b..1285526f26f46c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPrunablePredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPrunablePredicate.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.rules.expression.rules; -import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -25,12 +24,13 @@ import com.google.common.collect.ImmutableSet; import java.util.List; +import java.util.Objects; import java.util.Set; /** - * Records that, on a specific physical {@link TableIdentifier} restricted to - * {@link #selectedPartitionIds}, the {@link #prunableConjuncts} are guaranteed - * to evaluate to TRUE for every surviving row. + * Records that, on the scan whose partition list equals {@link + * #selectedPartitionIds}, the {@link #prunableConjuncts} are guaranteed to + * evaluate to TRUE for every surviving row. * *

The predicate is registered by {@link * org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition} but kept in @@ -41,23 +41,22 @@ * view-predicate happens to cover the remaining conjuncts after the partition * predicate has been silently dropped. * - *

Because rewrites between recording and removal may rebuild the scan with - * fresh slot ids, {@link #snapshotPartitionSlots} captures the slots that - * appear in the recorded conjuncts. The post-processor maps them onto the - * actual scan's output slots by column name before performing the conjunct - * removal. + *

The predicate lives on the scan itself (see {@code LogicalOlapScan} and + * {@code PhysicalOlapScan}) so we no longer need to match it back to its scan + * via a table identifier. Because rewrites between recording and removal may + * rebuild the scan with fresh slot ids, {@link #snapshotPartitionSlots} + * captures the slots that appear in the recorded conjuncts. The post-processor + * maps them onto the actual scan's output slots by column name before + * performing the conjunct removal. */ public class PartitionPrunablePredicate { - private final TableIdentifier tableIdentifier; private final Set selectedPartitionIds; private final List snapshotPartitionSlots; private final Set prunableConjuncts; - public PartitionPrunablePredicate(TableIdentifier tableIdentifier, - Set selectedPartitionIds, + public PartitionPrunablePredicate(Set selectedPartitionIds, List snapshotPartitionSlots, Set prunableConjuncts) { - this.tableIdentifier = tableIdentifier; this.selectedPartitionIds = ImmutableSet.copyOf(selectedPartitionIds); this.snapshotPartitionSlots = ImmutableList.copyOf(snapshotPartitionSlots); this.prunableConjuncts = ImmutableSet.copyOf(prunableConjuncts); @@ -72,15 +71,14 @@ public boolean equals(Object o) { return false; } PartitionPrunablePredicate that = (PartitionPrunablePredicate) o; - return tableIdentifier.equals(that.tableIdentifier) - && selectedPartitionIds.equals(that.selectedPartitionIds) + return selectedPartitionIds.equals(that.selectedPartitionIds) && snapshotPartitionSlots.equals(that.snapshotPartitionSlots) && prunableConjuncts.equals(that.prunableConjuncts); - } - public TableIdentifier getTableIdentifier() { - return tableIdentifier; + @Override + public int hashCode() { + return Objects.hash(selectedPartitionIds, snapshotPartitionSlots, prunableConjuncts); } public Set getSelectedPartitionIds() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java index 0b1e483fc21432..6a8b24c2c1184d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java @@ -72,7 +72,8 @@ public Rule build() { olapScan.getScoreRangeInfo(), olapScan.getAnnOrderKeys(), olapScan.getAnnLimit(), - olapScan.getTableAlias()) + olapScan.getTableAlias(), + olapScan.getPartitionPrunablePredicates()) ).toRule(RuleType.LOGICAL_OLAP_SCAN_TO_PHYSICAL_OLAP_SCAN_RULE); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java index 02696d204e7abb..08f3919ed4e5b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java @@ -25,7 +25,6 @@ import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.Tablet; -import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.common.Pair; import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager; import org.apache.doris.nereids.pattern.MatchingContext; @@ -110,15 +109,16 @@ public List buildRules() { prunedRes.second.get()); List partitionSlots = getPartitionSlots(prunedScan, prunedScan.getTable()); if (partitionSlots != null) { - TableIdentifier tableIdentifier = new TableIdentifier(prunedScan.getTable()); PartitionPrunablePredicate entry = new PartitionPrunablePredicate( - tableIdentifier, new HashSet<>(prunedScan.getSelectedPartitionIds()), partitionSlots, prunableConjuncts); - ctx.statementContext.getPartitionPrunablePredicates() - .computeIfAbsent(tableIdentifier, k -> new HashSet<>()) - .add(entry); + Set merged + = ImmutableSet.builder() + .addAll(prunedScan.getPartitionPrunablePredicates()) + .add(entry) + .build(); + rewrittenLogicalRelation = prunedScan.withPartitionPrunablePredicates(merged); } } return filter.withChildren(ImmutableList.of(rewrittenLogicalRelation)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 415b2a9817ef37..081fe358a652a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.properties.DataTrait; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.rules.expression.rules.PartitionPrunablePredicate; import org.apache.doris.nereids.trees.TableSample; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -161,6 +162,16 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan, private final List annOrderKeys; private final Optional annLimit; + /** + * Conjuncts that are guaranteed to be TRUE on the current scan because they + * were already enforced by partition pruning. The deferred removal happens + * in the {@link org.apache.doris.nereids.processor.post.PrunePartitionPredicate} + * post-processor so that materialized-view rewrite still observes the + * original predicates. The set is preserved through {@code with*} rewrites + * and copied onto MV rewrite outputs. + */ + private final Set partitionPrunablePredicates; + public LogicalOlapScan(RelationId id, OlapTable table) { this(id, table, ImmutableList.of()); } @@ -256,6 +267,29 @@ public LogicalOlapScan(RelationId id, Table table, List qualifier, Collection operativeSlots, List virtualColumns, List scoreOrderKeys, Optional scoreLimit, Optional scoreRangeInfo, List annOrderKeys, Optional annLimit, String tableAlias) { + this(id, table, qualifier, groupExpression, logicalProperties, selectedPartitionIds, partitionPruned, + hasPartitionPredicate, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, + specifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, + colToSubPathsMap, specifiedTabletIds, operativeSlots, virtualColumns, + scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, + ImmutableSet.of()); + } + + /** + * Constructor for LogicalOlapScan. + */ + public LogicalOlapScan(RelationId id, Table table, List qualifier, + Optional groupExpression, Optional logicalProperties, + List selectedPartitionIds, boolean partitionPruned, boolean hasPartitionPredicate, + List selectedTabletIds, long selectedIndexId, boolean indexSelected, + PreAggStatus preAggStatus, List specifiedPartitions, + List hints, Map, Slot> cacheSlotWithSlotName, + Optional> cachedOutput, Optional tableSample, boolean directMvScan, + Map>> colToSubPathsMap, List specifiedTabletIds, + Collection operativeSlots, List virtualColumns, + List scoreOrderKeys, Optional scoreLimit, Optional scoreRangeInfo, + List annOrderKeys, Optional annLimit, String tableAlias, + Set partitionPrunablePredicates) { super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier, operativeSlots, virtualColumns, groupExpression, logicalProperties, tableAlias); Preconditions.checkArgument(selectedPartitionIds != null, @@ -294,6 +328,9 @@ public LogicalOlapScan(RelationId id, Table table, List qualifier, this.scoreRangeInfo = scoreRangeInfo; this.annOrderKeys = Utils.fastToImmutableList(annOrderKeys); this.annLimit = annLimit; + this.partitionPrunablePredicates = partitionPrunablePredicates == null + ? ImmutableSet.of() + : ImmutableSet.copyOf(partitionPrunablePredicates); } public List getSelectedPartitionIds() { @@ -304,6 +341,29 @@ public boolean hasPartitionPredicate() { return hasPartitionPredicate; } + public Set getPartitionPrunablePredicates() { + return partitionPrunablePredicates; + } + + /** + * Returns a new {@code LogicalOlapScan} carrying the supplied + * {@link PartitionPrunablePredicate} set. The set is preserved across all + * other {@code with*} builders so partition-derived conjuncts can be + * removed safely after MV rewrite has had a chance to match the plan. + */ + public LogicalOlapScan withPartitionPrunablePredicates( + Set partitionPrunablePredicates) { + return AbstractPlan.copyWithSameId(this, () -> + new LogicalOlapScan(relationId, (Table) table, qualifier, + groupExpression, Optional.of(getLogicalProperties()), + selectedPartitionIds, partitionPruned, hasPartitionPredicate, selectedTabletIds, + selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, + hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, + colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns, + scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, + partitionPrunablePredicates)); + } + @Override public String getFingerprint() { String partitions = ""; @@ -371,7 +431,8 @@ public boolean equals(Object o) { && Objects.equals(scoreLimit, that.scoreLimit) && Objects.equals(scoreRangeInfo, that.scoreRangeInfo) && Objects.equals(annOrderKeys, that.annOrderKeys) - && Objects.equals(annLimit, that.annLimit); + && Objects.equals(annLimit, that.annLimit) + && Objects.equals(partitionPrunablePredicates, that.partitionPrunablePredicates); } @Override @@ -388,7 +449,8 @@ public LogicalOlapScan withGroupExpression(Optional groupExpres selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns, - scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias)); + scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, + partitionPrunablePredicates)); } @Override @@ -400,7 +462,8 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns, - scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias)); + scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, + partitionPrunablePredicates)); } /** @@ -422,7 +485,8 @@ public LogicalOlapScan withSelectedPartitionIds(List selectedPartitionIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns, - scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias)); + scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, + partitionPrunablePredicates)); } /** @@ -438,7 +502,7 @@ public LogicalOlapScan withMaterializedIndexSelected(long indexId) { indexId, true, PreAggStatus.unset(), manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, scoreRangeInfo, - annOrderKeys, annLimit, tableAlias)); + annOrderKeys, annLimit, tableAlias, partitionPrunablePredicates)); } /** @@ -452,7 +516,7 @@ public LogicalOlapScan withSelectedTabletIds(List selectedTabletIds) { selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys, - scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias)); + scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, partitionPrunablePredicates)); } /** @@ -466,7 +530,8 @@ public LogicalOlapScan withPreAggStatus(PreAggStatus preAggStatus) { selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns, - scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias)); + scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, + partitionPrunablePredicates)); } /** @@ -480,7 +545,8 @@ public LogicalOlapScan withColToSubPathsMap(Map>> colTo selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns, - scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias)); + scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, + partitionPrunablePredicates)); } /** @@ -494,7 +560,8 @@ public LogicalOlapScan withManuallySpecifiedTabletIds(List manuallySpecifi selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns, - scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias)); + scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, + partitionPrunablePredicates)); } @Override @@ -507,7 +574,7 @@ public LogicalOlapScan withRelationId(RelationId relationId) { selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, Maps.newHashMap(), Optional.empty(), tableSample, directMvScan, colToSubPathsMap, selectedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys, - scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias)); + scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, partitionPrunablePredicates)); } @Override @@ -519,7 +586,8 @@ public LogicalOlapScan withTableAlias(String tableAlias) { selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns, - scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias)); + scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, + partitionPrunablePredicates)); } /** @@ -540,7 +608,7 @@ public LogicalOlapScan withVirtualColumns(List virtualColumns) selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, - scoreRangeInfo, annOrderKeys, annLimit, tableAlias)); + scoreRangeInfo, annOrderKeys, annLimit, tableAlias, partitionPrunablePredicates)); } /** @@ -563,7 +631,7 @@ public LogicalOlapScan appendVirtualColumns(List additionalVirt selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, mergedVirtualColumns, scoreOrderKeys, scoreLimit, - scoreRangeInfo, annOrderKeys, annLimit, tableAlias); + scoreRangeInfo, annOrderKeys, annLimit, tableAlias, partitionPrunablePredicates); } /** @@ -592,7 +660,7 @@ public LogicalOlapScan appendVirtualColumnsAndTopN( selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, mergedVirtualColumns, scoreOrderKeys, scoreLimit, - scoreRangeInfo, annOrderKeys, annLimit, tableAlias); + scoreRangeInfo, annOrderKeys, annLimit, tableAlias, partitionPrunablePredicates); } @Override @@ -939,7 +1007,7 @@ public CatalogRelation withOperativeSlots(Collection operativeSlots) { selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, - scoreRangeInfo, annOrderKeys, annLimit, tableAlias)); + scoreRangeInfo, annOrderKeys, annLimit, tableAlias, partitionPrunablePredicates)); } @VisibleForTesting @@ -1021,7 +1089,7 @@ public LogicalOlapScan withCachedOutput(List outputSlots) { selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, Optional.of(outputSlots), tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, - scoreRangeInfo, annOrderKeys, annLimit, tableAlias)); + scoreRangeInfo, annOrderKeys, annLimit, tableAlias, partitionPrunablePredicates)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java index d2d4bcd8fd365c..47a14b64c7327d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.expression.rules.PartitionPrunablePredicate; import org.apache.doris.nereids.trees.TableSample; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; @@ -42,6 +43,7 @@ import org.apache.doris.statistics.Statistics; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONObject; @@ -51,6 +53,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; /** @@ -78,6 +81,15 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca private final List annOrderKeys; private final Optional annLimit; + /** + * Predicates known to be TRUE on this scan thanks to partition pruning. + * Carried alongside the scan so the + * {@link org.apache.doris.nereids.processor.post.PrunePartitionPredicate} + * post-processor can strip them from the surrounding filter after MV + * rewrite has finished its matching work. + */ + private final Set partitionPrunablePredicates; + /** * Constructor for PhysicalOlapScan. */ @@ -144,6 +156,26 @@ public PhysicalOlapScan(RelationId id, OlapTable olapTable, List qualifi Collection operativeSlots, List virtualColumns, List scoreOrderKeys, Optional scoreLimit, Optional scoreRangeInfo, List annOrderKeys, Optional annLimit, String tableAlias) { + this(id, olapTable, qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, + hasPartitionPredicate, distributionSpec, preAggStatus, baseOutputs, groupExpression, + logicalProperties, physicalProperties, statistics, tableSample, operativeSlots, virtualColumns, + scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, + ImmutableSet.of()); + } + + /** + * Ultimate constructor for PhysicalOlapScan. + */ + public PhysicalOlapScan(RelationId id, OlapTable olapTable, List qualifier, long selectedIndexId, + List selectedTabletIds, List selectedPartitionIds, boolean hasPartitionPredicate, + DistributionSpec distributionSpec, PreAggStatus preAggStatus, List baseOutputs, + Optional groupExpression, LogicalProperties logicalProperties, + PhysicalProperties physicalProperties, Statistics statistics, + Optional tableSample, + Collection operativeSlots, List virtualColumns, + List scoreOrderKeys, Optional scoreLimit, Optional scoreRangeInfo, + List annOrderKeys, Optional annLimit, String tableAlias, + Set partitionPrunablePredicates) { super(id, PlanType.PHYSICAL_OLAP_SCAN, olapTable, qualifier, groupExpression, logicalProperties, physicalProperties, statistics, operativeSlots, tableAlias); this.selectedIndexId = selectedIndexId; @@ -161,6 +193,9 @@ public PhysicalOlapScan(RelationId id, OlapTable olapTable, List qualifi this.scoreRangeInfo = scoreRangeInfo; this.annOrderKeys = ImmutableList.copyOf(annOrderKeys); this.annLimit = annLimit; + this.partitionPrunablePredicates = partitionPrunablePredicates == null + ? ImmutableSet.of() + : ImmutableSet.copyOf(partitionPrunablePredicates); } @Override @@ -181,6 +216,25 @@ public boolean hasPartitionPredicate() { return hasPartitionPredicate; } + public Set getPartitionPrunablePredicates() { + return partitionPrunablePredicates; + } + + /** + * Returns a new {@code PhysicalOlapScan} carrying the supplied + * {@link PartitionPrunablePredicate} set. The set is preserved through all + * other {@code with*} builders so the post-processor can remove the + * derived conjuncts after MV rewrite has had a chance to match them. + */ + public PhysicalOlapScan withPartitionPrunablePredicates( + Set partitionPrunablePredicates) { + return AbstractPlan.copyWithSameId(this, () -> new PhysicalOlapScan(relationId, getTable(), qualifier, + selectedIndexId, selectedTabletIds, selectedPartitionIds, hasPartitionPredicate, + distributionSpec, preAggStatus, baseOutputs, groupExpression, getLogicalProperties(), + getPhysicalProperties(), statistics, tableSample, operativeSlots, virtualColumns, scoreOrderKeys, + scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, partitionPrunablePredicates)); + } + @Override public OlapTable getTable() { return (OlapTable) table; @@ -306,7 +360,8 @@ public boolean equals(Object o) { && Objects.equals(scoreLimit, olapScan.scoreLimit) && Objects.equals(scoreRangeInfo, olapScan.scoreRangeInfo) && Objects.equals(annOrderKeys, olapScan.annOrderKeys) - && Objects.equals(annLimit, olapScan.annLimit); + && Objects.equals(annLimit, olapScan.annLimit) + && Objects.equals(partitionPrunablePredicates, olapScan.partitionPrunablePredicates); } @Override @@ -325,7 +380,7 @@ public PhysicalOlapScan withGroupExpression(Optional groupExpre selectedIndexId, selectedTabletIds, selectedPartitionIds, hasPartitionPredicate, distributionSpec, preAggStatus, baseOutputs, groupExpression, getLogicalProperties(), null, null, tableSample, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, scoreRangeInfo, - annOrderKeys, annLimit, tableAlias)); + annOrderKeys, annLimit, tableAlias, partitionPrunablePredicates)); } @Override @@ -335,7 +390,7 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr selectedIndexId, selectedTabletIds, selectedPartitionIds, hasPartitionPredicate, distributionSpec, preAggStatus, baseOutputs, groupExpression, logicalProperties.get(), null, null, tableSample, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, scoreRangeInfo, - annOrderKeys, annLimit, tableAlias)); + annOrderKeys, annLimit, tableAlias, partitionPrunablePredicates)); } @Override @@ -345,7 +400,7 @@ public PhysicalOlapScan withPhysicalPropertiesAndStats( selectedIndexId, selectedTabletIds, selectedPartitionIds, hasPartitionPredicate, distributionSpec, preAggStatus, baseOutputs, groupExpression, getLogicalProperties(), physicalProperties, statistics, tableSample, operativeSlots, virtualColumns, scoreOrderKeys, - scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias)); + scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, partitionPrunablePredicates)); } @Override @@ -373,7 +428,7 @@ public CatalogRelation withOperativeSlots(Collection operativeSlots) { distributionSpec, preAggStatus, baseOutputs, groupExpression, getLogicalProperties(), getPhysicalProperties(), statistics, tableSample, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, - scoreRangeInfo, annOrderKeys, annLimit, tableAlias)); + scoreRangeInfo, annOrderKeys, annLimit, tableAlias, partitionPrunablePredicates)); } @Override From 4c371b5d965b0bd3c957c8196c6be17956b96535 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Tue, 12 May 2026 15:26:24 +0800 Subject: [PATCH 6/8] [refactor](nereids) Use Optional on scan One olap scan can carry at most one PartitionPrunablePredicate, so use Optional instead of Set. Simplifies merge in PruneOlapScanPartition and iteration in PrunePartitionPredicate. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../post/PrunePartitionPredicate.java | 35 ++++++------------- .../rules/rewrite/PruneOlapScanPartition.java | 8 ++--- .../trees/plans/logical/LogicalOlapScan.java | 20 +++++------ .../plans/physical/PhysicalOlapScan.java | 16 ++++----- .../prune_predicates_mv_test.groovy | 14 ++++---- 5 files changed, 38 insertions(+), 55 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java index 9a530d27543679..8820f3a0f683fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java @@ -47,16 +47,6 @@ * materialized-view rewrite has finished, ensures MV matching observes the * original predicates; otherwise the MV view-predicate may incorrectly cover * the dropped partition predicate and produce extra rows. - * - *

The {@link PartitionPrunablePredicate} entries live directly on the - * {@link PhysicalOlapScan} so this processor no longer needs a side-channel - * registry keyed by table identifier. Because intermediate rewrites can - * rebuild scans with fresh slot ids, the recorded snapshot slots are remapped - * onto the actual scan output by column name before each conjunct is rewritten - * and removed from the filter. Each recorded entry is only applied when its - * recorded partition id set is a superset of the scan's surviving partitions - * - otherwise a later partition pruning step would have invalidated the - * implication. */ public class PrunePartitionPredicate extends PlanPostProcessor { @@ -68,8 +58,8 @@ public Plan visitPhysicalFilter(PhysicalFilter filter, CascadesC return filter; } PhysicalOlapScan scan = (PhysicalOlapScan) child; - Set entries = scan.getPartitionPrunablePredicates(); - if (entries.isEmpty()) { + Optional entryOpt = scan.getPartitionPrunablePredicates(); + if (!entryOpt.isPresent()) { return filter; } boolean skipPrunePredicate = context.getConnectContext().getSessionVariable().skipPrunePredicate @@ -82,20 +72,17 @@ public Plan visitPhysicalFilter(PhysicalFilter filter, CascadesC Set remaining = new LinkedHashSet<>(filter.getConjuncts()); boolean changed = false; - for (PartitionPrunablePredicate entry : entries) { - if (!entry.getSelectedPartitionIds().containsAll(scanPartitions)) { - continue; - } + PartitionPrunablePredicate entry = entryOpt.get(); + if (entry.getSelectedPartitionIds().containsAll(scanPartitions)) { Map slotReplaceMap = buildSlotReplaceMap(entry.getSnapshotPartitionSlots(), nameToOutputSlot); - if (slotReplaceMap == null) { - continue; - } - for (Expression conjunct : entry.getPrunableConjuncts()) { - Expression rewritten = slotReplaceMap.isEmpty() - ? conjunct : ExpressionUtils.replace(conjunct, slotReplaceMap); - if (remaining.remove(rewritten)) { - changed = true; + if (slotReplaceMap != null) { + for (Expression conjunct : entry.getPrunableConjuncts()) { + Expression rewritten = slotReplaceMap.isEmpty() + ? conjunct : ExpressionUtils.replace(conjunct, slotReplaceMap); + if (remaining.remove(rewritten)) { + changed = true; + } } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java index 08f3919ed4e5b3..0fef5a166b8433 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java @@ -113,12 +113,8 @@ public List buildRules() { new HashSet<>(prunedScan.getSelectedPartitionIds()), partitionSlots, prunableConjuncts); - Set merged - = ImmutableSet.builder() - .addAll(prunedScan.getPartitionPrunablePredicates()) - .add(entry) - .build(); - rewrittenLogicalRelation = prunedScan.withPartitionPrunablePredicates(merged); + rewrittenLogicalRelation = prunedScan.withPartitionPrunablePredicates( + Optional.of(entry)); } } return filter.withChildren(ImmutableList.of(rewrittenLogicalRelation)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 081fe358a652a1..9246cb3da3009e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -170,7 +170,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan, * original predicates. The set is preserved through {@code with*} rewrites * and copied onto MV rewrite outputs. */ - private final Set partitionPrunablePredicates; + private final Optional partitionPrunablePredicates; public LogicalOlapScan(RelationId id, OlapTable table) { this(id, table, ImmutableList.of()); @@ -272,7 +272,7 @@ public LogicalOlapScan(RelationId id, Table table, List qualifier, specifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap, specifiedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, - ImmutableSet.of()); + Optional.empty()); } /** @@ -289,7 +289,7 @@ public LogicalOlapScan(RelationId id, Table table, List qualifier, Collection operativeSlots, List virtualColumns, List scoreOrderKeys, Optional scoreLimit, Optional scoreRangeInfo, List annOrderKeys, Optional annLimit, String tableAlias, - Set partitionPrunablePredicates) { + Optional partitionPrunablePredicates) { super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier, operativeSlots, virtualColumns, groupExpression, logicalProperties, tableAlias); Preconditions.checkArgument(selectedPartitionIds != null, @@ -329,8 +329,8 @@ public LogicalOlapScan(RelationId id, Table table, List qualifier, this.annOrderKeys = Utils.fastToImmutableList(annOrderKeys); this.annLimit = annLimit; this.partitionPrunablePredicates = partitionPrunablePredicates == null - ? ImmutableSet.of() - : ImmutableSet.copyOf(partitionPrunablePredicates); + ? Optional.empty() + : partitionPrunablePredicates; } public List getSelectedPartitionIds() { @@ -341,18 +341,18 @@ public boolean hasPartitionPredicate() { return hasPartitionPredicate; } - public Set getPartitionPrunablePredicates() { + public Optional getPartitionPrunablePredicates() { return partitionPrunablePredicates; } /** * Returns a new {@code LogicalOlapScan} carrying the supplied - * {@link PartitionPrunablePredicate} set. The set is preserved across all - * other {@code with*} builders so partition-derived conjuncts can be - * removed safely after MV rewrite has had a chance to match the plan. + * {@link PartitionPrunablePredicate}. It is preserved across all other + * {@code with*} builders so partition-derived conjuncts can be removed + * safely after MV rewrite has had a chance to match the plan. */ public LogicalOlapScan withPartitionPrunablePredicates( - Set partitionPrunablePredicates) { + Optional partitionPrunablePredicates) { return AbstractPlan.copyWithSameId(this, () -> new LogicalOlapScan(relationId, (Table) table, qualifier, groupExpression, Optional.of(getLogicalProperties()), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java index 47a14b64c7327d..11dd2fe4825a24 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java @@ -43,7 +43,6 @@ import org.apache.doris.statistics.Statistics; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONObject; @@ -53,7 +52,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; /** @@ -88,7 +86,7 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca * post-processor can strip them from the surrounding filter after MV * rewrite has finished its matching work. */ - private final Set partitionPrunablePredicates; + private final Optional partitionPrunablePredicates; /** * Constructor for PhysicalOlapScan. @@ -160,7 +158,7 @@ public PhysicalOlapScan(RelationId id, OlapTable olapTable, List qualifi hasPartitionPredicate, distributionSpec, preAggStatus, baseOutputs, groupExpression, logicalProperties, physicalProperties, statistics, tableSample, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys, annLimit, tableAlias, - ImmutableSet.of()); + Optional.empty()); } /** @@ -175,7 +173,7 @@ public PhysicalOlapScan(RelationId id, OlapTable olapTable, List qualifi Collection operativeSlots, List virtualColumns, List scoreOrderKeys, Optional scoreLimit, Optional scoreRangeInfo, List annOrderKeys, Optional annLimit, String tableAlias, - Set partitionPrunablePredicates) { + Optional partitionPrunablePredicates) { super(id, PlanType.PHYSICAL_OLAP_SCAN, olapTable, qualifier, groupExpression, logicalProperties, physicalProperties, statistics, operativeSlots, tableAlias); this.selectedIndexId = selectedIndexId; @@ -194,8 +192,8 @@ public PhysicalOlapScan(RelationId id, OlapTable olapTable, List qualifi this.annOrderKeys = ImmutableList.copyOf(annOrderKeys); this.annLimit = annLimit; this.partitionPrunablePredicates = partitionPrunablePredicates == null - ? ImmutableSet.of() - : ImmutableSet.copyOf(partitionPrunablePredicates); + ? Optional.empty() + : partitionPrunablePredicates; } @Override @@ -216,7 +214,7 @@ public boolean hasPartitionPredicate() { return hasPartitionPredicate; } - public Set getPartitionPrunablePredicates() { + public Optional getPartitionPrunablePredicates() { return partitionPrunablePredicates; } @@ -227,7 +225,7 @@ public Set getPartitionPrunablePredicates() { * derived conjuncts after MV rewrite has had a chance to match them. */ public PhysicalOlapScan withPartitionPrunablePredicates( - Set partitionPrunablePredicates) { + Optional partitionPrunablePredicates) { return AbstractPlan.copyWithSameId(this, () -> new PhysicalOlapScan(relationId, getTable(), qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, hasPartitionPredicate, distributionSpec, preAggStatus, baseOutputs, groupExpression, getLogicalProperties(), diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy index 6d45ebde8f42c2..cf33f0a5f1d71f 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy @@ -56,12 +56,13 @@ suite("prune_predicates_mv_test") { ORDER BY tag_key; """ - //执行（需要强制改写）： - //1.检查结果正确 - //2.检查shape plan里面有filter - //3.检查改写成功了 async_mv_rewrite_success + //Execute (force rewrite): + //1. verify result is correct + //2. verify shape plan contains filter + //3. verify rewrite succeeded: async_mv_rewrite_success - // table有分区，分区裁剪之后谓词被删除掉了，mv没有分区，检查mv没有把谓词给删除掉 + // The base table is partitioned and the predicate was removed after partition prune; + // the MV is not partitioned, so verify the MV did not drop the predicate. async_mv_rewrite_success(currentDb, mv_1, query_1, "mv_1") order_qt_mv_1 query_1 explain { @@ -69,7 +70,8 @@ suite("prune_predicates_mv_test") { contains "filter" } - // case2: 物化视图也是带有分区的，检查对物化视图进行分区裁剪，检查在分区裁剪后将恒true谓词删除 + // case2: the MV is also partitioned; verify that partition pruning on the MV + // is performed and the always-true predicate is removed after the prune. def async_partition_mv_rewrite_success = { db, mv_sql, query_sql, mv_name, partition, expected_pre_rewrite_strategys = [] -> if (!mvShouldContinueCheck(expected_pre_rewrite_strategys)) { From 36a3ecedd7edd31c0d6ab416879e14a9800f384a Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 14 May 2026 12:41:00 +0800 Subject: [PATCH 7/8] fix --- .../post/PrunePartitionPredicate.java | 2 +- .../rules/PartitionPrunablePredicate.java | 95 ------------------- .../rules/rewrite/PruneOlapScanPartition.java | 2 +- .../trees/plans/logical/LogicalOlapScan.java | 2 +- .../plans/physical/PhysicalOlapScan.java | 2 +- 5 files changed, 4 insertions(+), 99 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPrunablePredicate.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java index 8820f3a0f683fa..01ecf88e595ac4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java @@ -22,10 +22,10 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.OlapTable; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.expression.rules.PartitionPrunablePredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.PartitionPrunablePredicate; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPrunablePredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPrunablePredicate.java deleted file mode 100644 index 1285526f26f46c..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPrunablePredicate.java +++ /dev/null @@ -1,95 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.expression.rules; - -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.Slot; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; - -import java.util.List; -import java.util.Objects; -import java.util.Set; - -/** - * Records that, on the scan whose partition list equals {@link - * #selectedPartitionIds}, the {@link #prunableConjuncts} are guaranteed to - * evaluate to TRUE for every surviving row. - * - *

The predicate is registered by {@link - * org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition} but kept in - * the logical filter during cascades. The actual removal happens later in - * {@link org.apache.doris.nereids.processor.post.PrunePartitionPredicate} so - * that materialized-view rewrite still sees the original predicates. Keeping - * the predicate in the plan avoids the wrong-result problem in which the MV - * view-predicate happens to cover the remaining conjuncts after the partition - * predicate has been silently dropped. - * - *

The predicate lives on the scan itself (see {@code LogicalOlapScan} and - * {@code PhysicalOlapScan}) so we no longer need to match it back to its scan - * via a table identifier. Because rewrites between recording and removal may - * rebuild the scan with fresh slot ids, {@link #snapshotPartitionSlots} - * captures the slots that appear in the recorded conjuncts. The post-processor - * maps them onto the actual scan's output slots by column name before - * performing the conjunct removal. - */ -public class PartitionPrunablePredicate { - private final Set selectedPartitionIds; - private final List snapshotPartitionSlots; - private final Set prunableConjuncts; - - public PartitionPrunablePredicate(Set selectedPartitionIds, - List snapshotPartitionSlots, - Set prunableConjuncts) { - this.selectedPartitionIds = ImmutableSet.copyOf(selectedPartitionIds); - this.snapshotPartitionSlots = ImmutableList.copyOf(snapshotPartitionSlots); - this.prunableConjuncts = ImmutableSet.copyOf(prunableConjuncts); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - PartitionPrunablePredicate that = (PartitionPrunablePredicate) o; - return selectedPartitionIds.equals(that.selectedPartitionIds) - && snapshotPartitionSlots.equals(that.snapshotPartitionSlots) - && prunableConjuncts.equals(that.prunableConjuncts); - } - - @Override - public int hashCode() { - return Objects.hash(selectedPartitionIds, snapshotPartitionSlots, prunableConjuncts); - } - - public Set getSelectedPartitionIds() { - return selectedPartitionIds; - } - - public List getSnapshotPartitionSlots() { - return snapshotPartitionSlots; - } - - public Set getPrunableConjuncts() { - return prunableConjuncts; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java index 0fef5a166b8433..b0cfde50962979 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java @@ -30,13 +30,13 @@ import org.apache.doris.nereids.pattern.MatchingContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.expression.rules.PartitionPrunablePredicate; import org.apache.doris.nereids.rules.expression.rules.PartitionPruner; import org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionPruneResult; import org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionTableType; import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.PartitionPrunablePredicate; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 9246cb3da3009e..080a22b9d92ed4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -28,7 +28,6 @@ import org.apache.doris.nereids.properties.DataTrait; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.OrderKey; -import org.apache.doris.nereids.rules.expression.rules.PartitionPrunablePredicate; import org.apache.doris.nereids.trees.TableSample; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -36,6 +35,7 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.AbstractPlan; +import org.apache.doris.nereids.trees.plans.PartitionPrunablePredicate; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.PreAggStatus; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java index 11dd2fe4825a24..9205f008cf4b1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java @@ -23,13 +23,13 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.rules.expression.rules.PartitionPrunablePredicate; import org.apache.doris.nereids.trees.TableSample; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.AbstractPlan; +import org.apache.doris.nereids.trees.plans.PartitionPrunablePredicate; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.PreAggStatus; From 2d5071c247addf7c1280f4c8401675858dedc9f7 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 14 May 2026 14:32:33 +0800 Subject: [PATCH 8/8] fix --- .../plans/PartitionPrunablePredicate.java | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PartitionPrunablePredicate.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PartitionPrunablePredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PartitionPrunablePredicate.java new file mode 100644 index 00000000000000..54ba19493b86bc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PartitionPrunablePredicate.java @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans; + +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * Records that, on the scan whose partition list equals {@link + * #selectedPartitionIds}, the {@link #prunableConjuncts} are guaranteed to + * evaluate to TRUE for every surviving row. + * + *

The predicate is registered by {@link + * org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition} but kept in + * the logical filter during cascades. The actual removal happens later in + * {@link org.apache.doris.nereids.processor.post.PrunePartitionPredicate} so + * that materialized-view rewrite still sees the original predicates. Keeping + * the predicate in the plan avoids the wrong-result problem in which the MV + * view-predicate happens to cover the remaining conjuncts after the partition + * predicate has been silently dropped. + * + *

The predicate lives on the scan itself (see {@code LogicalOlapScan} and + * {@code PhysicalOlapScan}) so we no longer need to match it back to its scan + * via a table identifier. Because rewrites between recording and removal may + * rebuild the scan with fresh slot ids, {@link #snapshotPartitionSlots} + * captures the slots that appear in the recorded conjuncts. The post-processor + * maps them onto the actual scan's output slots by column name before + * performing the conjunct removal. + */ +public class PartitionPrunablePredicate { + private final Set selectedPartitionIds; + private final List snapshotPartitionSlots; + private final Set prunableConjuncts; + + public PartitionPrunablePredicate(Set selectedPartitionIds, + List snapshotPartitionSlots, + Set prunableConjuncts) { + this.selectedPartitionIds = ImmutableSet.copyOf(selectedPartitionIds); + this.snapshotPartitionSlots = ImmutableList.copyOf(snapshotPartitionSlots); + this.prunableConjuncts = ImmutableSet.copyOf(prunableConjuncts); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PartitionPrunablePredicate that = (PartitionPrunablePredicate) o; + return selectedPartitionIds.equals(that.selectedPartitionIds) + && snapshotPartitionSlots.equals(that.snapshotPartitionSlots) + && prunableConjuncts.equals(that.prunableConjuncts); + } + + @Override + public int hashCode() { + return Objects.hash(selectedPartitionIds, snapshotPartitionSlots, prunableConjuncts); + } + + public Set getSelectedPartitionIds() { + return selectedPartitionIds; + } + + public List getSnapshotPartitionSlots() { + return snapshotPartitionSlots; + } + + public Set getPrunableConjuncts() { + return prunableConjuncts; + } +}