From 29f20aeaf1eee791dc7f3ca54447f8097acb296c Mon Sep 17 00:00:00 2001 From: AilinKid <314806019@qq.com> Date: Thu, 15 Aug 2024 14:24:19 +0800 Subject: [PATCH 1/4] . Signed-off-by: AilinKid <314806019@qq.com> --- pkg/planner/core/exhaust_physical_plans.go | 5 +++-- pkg/planner/core/expression_rewriter.go | 4 ++-- pkg/planner/core/logical_plan_builder.go | 14 +++++++------- pkg/planner/core/logical_plans.go | 2 +- .../{ => operator/logicalop}/logical_expand.go | 17 +++++++++-------- pkg/planner/core/planbuilder.go | 4 ++-- .../core/rule_resolve_grouping_expand.go | 3 ++- pkg/planner/util/utilfuncp/func_pointer_misc.go | 4 ++++ 8 files changed, 30 insertions(+), 23 deletions(-) rename pkg/planner/core/{ => operator/logicalop}/logical_expand.go (97%) diff --git a/pkg/planner/core/exhaust_physical_plans.go b/pkg/planner/core/exhaust_physical_plans.go index 9673c4b9f9b5c..48951754e540e 100644 --- a/pkg/planner/core/exhaust_physical_plans.go +++ b/pkg/planner/core/exhaust_physical_plans.go @@ -2044,7 +2044,8 @@ func exhaustPhysicalPlans4LogicalJoin(lp base.LogicalPlan, prop *property.Physic return joins, true, nil } -func exhaustPhysicalPlans4LogicalExpand(p *LogicalExpand, prop *property.PhysicalProperty) ([]base.PhysicalPlan, bool, error) { +func exhaustPhysicalPlans4LogicalExpand(lp base.LogicalPlan, prop *property.PhysicalProperty) ([]base.PhysicalPlan, bool, error) { + p := lp.(*logicalop.LogicalExpand) // under the mpp task type, if the sort item is not empty, refuse it, cause expanded data doesn't support any sort items. if !prop.IsSortItemEmpty() { // false, meaning we can add a sort enforcer. @@ -2461,7 +2462,7 @@ func canPushToCopImpl(lp base.LogicalPlan, storeTp kv.StoreType, considerDual bo return false } ret = ret && canPushToCopImpl(&c.BaseLogicalPlan, storeTp, considerDual) - case *LogicalExpand: + case *logicalop.LogicalExpand: // Expand itself only contains simple col ref and literal projection. (always ok, check its child) if storeTp != kv.TiFlash { return false diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index c1e126b19817c..228d96b4542f3 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -334,7 +334,7 @@ type exprRewriterPlanCtx struct { // of the "INSERT" statement. insertPlan *Insert - rollExpand *LogicalExpand + rollExpand *logicalop.LogicalExpand } type expressionRewriter struct { @@ -2342,7 +2342,7 @@ func (er *expressionRewriter) funcCallToExpressionWithPlanCtx(planCtx *exprRewri return } // resolve grouping args in group by items or not. - resolvedCols, err := planCtx.rollExpand.resolveGroupingFuncArgsInGroupBy(args) + resolvedCols, err := planCtx.rollExpand.ResolveGroupingFuncArgsInGroupBy(args) if err != nil { er.err = err er.ctxStackAppend(nil, types.EmptyName) diff --git a/pkg/planner/core/logical_plan_builder.go b/pkg/planner/core/logical_plan_builder.go index d6628099b10db..856fdaa179376 100644 --- a/pkg/planner/core/logical_plan_builder.go +++ b/pkg/planner/core/logical_plan_builder.go @@ -186,7 +186,7 @@ func (b *PlanBuilder) buildExpand(p base.LogicalPlan, gbyItems []expression.Expr // for grouping set {}, project it as: [null, null, null, d, gid] expandSchema := proj.Schema().Clone() expression.AdjustNullabilityFromGroupingSets(rollupGroupingSets, expandSchema) - expand := LogicalExpand{ + expand := logicalop.LogicalExpand{ RollupGroupingSets: rollupGroupingSets, DistinctGroupByCol: distinctGbyCols, DistinctGbyColNames: distinctGbyColNames, @@ -261,8 +261,8 @@ func (b *PlanBuilder) buildAggregation(ctx context.Context, p base.LogicalPlan, if b.buildingCTE { b.outerCTEs[len(b.outerCTEs)-1].containAggOrWindow = true } - var rollupExpand *LogicalExpand - if expand, ok := p.(*LogicalExpand); ok { + var rollupExpand *logicalop.LogicalExpand + if expand, ok := p.(*logicalop.LogicalExpand); ok { rollupExpand = expand } @@ -1210,7 +1210,7 @@ func findColFromNaturalUsingJoin(p base.LogicalPlan, col *expression.Column) (na } type resolveGroupingTraverseAction struct { - CurrentBlockExpand *LogicalExpand + CurrentBlockExpand *logicalop.LogicalExpand } func (r resolveGroupingTraverseAction) Transform(expr expression.Expression) (res expression.Expression) { @@ -1219,18 +1219,18 @@ func (r resolveGroupingTraverseAction) Transform(expr expression.Expression) (re // when meeting a column, judge whether it's a relate grouping set col. // eg: select a, b from t group by a, c with rollup, here a is, while b is not. // in underlying Expand schema (a,b,c,a',c'), a select list should be resolved to a'. - res, _ = r.CurrentBlockExpand.trySubstituteExprWithGroupingSetCol(x) + res, _ = r.CurrentBlockExpand.TrySubstituteExprWithGroupingSetCol(x) case *expression.CorrelatedColumn: // select 1 in (select t2.a from t group by t2.a, b with rollup) from t2; // in this case: group by item has correlated column t2.a, and it's select list contains t2.a as well. - res, _ = r.CurrentBlockExpand.trySubstituteExprWithGroupingSetCol(x) + res, _ = r.CurrentBlockExpand.TrySubstituteExprWithGroupingSetCol(x) case *expression.Constant: // constant just keep it real: select 1 from t group by a, b with rollup. res = x case *expression.ScalarFunction: // scalar function just try to resolve itself first, then if not changed, trying resolve its children. var substituted bool - res, substituted = r.CurrentBlockExpand.trySubstituteExprWithGroupingSetCol(x) + res, substituted = r.CurrentBlockExpand.TrySubstituteExprWithGroupingSetCol(x) if !substituted { // if not changed, try to resolve it children. // select a+1, grouping(b) from t group by a+1 (projected as c), b with rollup: in this case, a+1 is resolved as c as a whole. diff --git a/pkg/planner/core/logical_plans.go b/pkg/planner/core/logical_plans.go index d7b08397a4bbc..1ce76663b5398 100644 --- a/pkg/planner/core/logical_plans.go +++ b/pkg/planner/core/logical_plans.go @@ -37,7 +37,7 @@ var ( _ base.LogicalPlan = &logicalop.LogicalLock{} _ base.LogicalPlan = &logicalop.LogicalLimit{} _ base.LogicalPlan = &logicalop.LogicalWindow{} - _ base.LogicalPlan = &LogicalExpand{} + _ base.LogicalPlan = &logicalop.LogicalExpand{} _ base.LogicalPlan = &logicalop.LogicalUnionScan{} _ base.LogicalPlan = &logicalop.LogicalMemTable{} _ base.LogicalPlan = &logicalop.LogicalShow{} diff --git a/pkg/planner/core/logical_expand.go b/pkg/planner/core/operator/logicalop/logical_expand.go similarity index 97% rename from pkg/planner/core/logical_expand.go rename to pkg/planner/core/operator/logicalop/logical_expand.go index 83260219596c2..bd5cc3ba2a054 100644 --- a/pkg/planner/core/logical_expand.go +++ b/pkg/planner/core/operator/logicalop/logical_expand.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package core +package logicalop import ( "bytes" @@ -20,11 +20,11 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop" fd "github.com/pingcap/tidb/pkg/planner/funcdep" "github.com/pingcap/tidb/pkg/planner/property" "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/planner/util/optimizetrace/logicaltrace" + "github.com/pingcap/tidb/pkg/planner/util/utilfuncp" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/dbterror/plannererrors" "github.com/pingcap/tidb/pkg/util/plancodec" @@ -33,7 +33,7 @@ import ( // LogicalExpand represents a logical Expand OP serves for data replication requirement. type LogicalExpand struct { - logicalop.LogicalSchemaProducer + LogicalSchemaProducer // distinct group by columns. (maybe projected below if it's a non-col) DistinctGroupByCol []*expression.Column @@ -66,7 +66,7 @@ type LogicalExpand struct { // Init initializes LogicalProjection. func (p LogicalExpand) Init(ctx base.PlanContext, offset int) *LogicalExpand { - p.BaseLogicalPlan = logicalop.NewBaseLogicalPlan(ctx, plancodec.TypeExpand, &p, offset) + p.BaseLogicalPlan = NewBaseLogicalPlan(ctx, plancodec.TypeExpand, &p, offset) return &p } @@ -139,7 +139,7 @@ func (p *LogicalExpand) PruneColumns(parentUsedCols []*expression.Column, opt *o // ExhaustPhysicalPlans implements base.LogicalPlan.<14th> interface. func (p *LogicalExpand) ExhaustPhysicalPlans(prop *property.PhysicalProperty) ([]base.PhysicalPlan, bool, error) { - return exhaustPhysicalPlans4LogicalExpand(p, prop) + return utilfuncp.ExhaustPhysicalPlans4LogicalExpand(p, prop) } // ExtractCorrelatedCols implements base.LogicalPlan.<15th> interface. @@ -294,7 +294,8 @@ func (p *LogicalExpand) GenerateGroupingMarks(sourceCols []*expression.Column) [ return resSliceMap } -func (p *LogicalExpand) trySubstituteExprWithGroupingSetCol(expr expression.Expression) (expression.Expression, bool) { +// TrySubstituteExprWithGroupingSetCol is used to substitute the original gby expression with new gby col. +func (p *LogicalExpand) TrySubstituteExprWithGroupingSetCol(expr expression.Expression) (expression.Expression, bool) { // since all the original group items has been projected even single col, // let's check the origin gby expression here, and map it to new gby col. for i, oneExpr := range p.DistinctGbyExprs { @@ -307,8 +308,8 @@ func (p *LogicalExpand) trySubstituteExprWithGroupingSetCol(expr expression.Expr return expr, false } -// CheckGroupingFuncArgsInGroupBy checks whether grouping function args is in grouping items. -func (p *LogicalExpand) resolveGroupingFuncArgsInGroupBy(groupingFuncArgs []expression.Expression) ([]*expression.Column, error) { +// ResolveGroupingFuncArgsInGroupBy checks whether grouping function args is in grouping items. +func (p *LogicalExpand) ResolveGroupingFuncArgsInGroupBy(groupingFuncArgs []expression.Expression) ([]*expression.Column, error) { // build GBYColMap distinctGBYColMap := make(map[int64]struct{}, len(p.DistinctGroupByCol)) for _, oneDistinctGBYCol := range p.DistinctGroupByCol { diff --git a/pkg/planner/core/planbuilder.go b/pkg/planner/core/planbuilder.go index 16c30a1f02d4c..c966aa02a2cbd 100644 --- a/pkg/planner/core/planbuilder.go +++ b/pkg/planner/core/planbuilder.go @@ -199,8 +199,8 @@ type PlanBuilder struct { outerNames [][]*types.FieldName outerCTEs []*cteInfo // outerBlockExpand register current Expand OP for rollup syntax in every select query block. - outerBlockExpand []*LogicalExpand - currentBlockExpand *LogicalExpand + outerBlockExpand []*logicalop.LogicalExpand + currentBlockExpand *logicalop.LogicalExpand // colMapper stores the column that must be pre-resolved. colMapper map[*ast.ColumnNameExpr]int // visitInfo is used for privilege check. diff --git a/pkg/planner/core/rule_resolve_grouping_expand.go b/pkg/planner/core/rule_resolve_grouping_expand.go index 363eca21fa10b..3817397dc4a34 100644 --- a/pkg/planner/core/rule_resolve_grouping_expand.go +++ b/pkg/planner/core/rule_resolve_grouping_expand.go @@ -16,6 +16,7 @@ package core import ( "context" + "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop" "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" @@ -97,7 +98,7 @@ func genExpand(p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.L } p.Children()[i] = np } - if expand, ok := p.(*LogicalExpand); ok { + if expand, ok := p.(*logicalop.LogicalExpand); ok { expand.GenLevelProjections() } return p, nil diff --git a/pkg/planner/util/utilfuncp/func_pointer_misc.go b/pkg/planner/util/utilfuncp/func_pointer_misc.go index c8ac26fa24f51..c653e13eb7ea8 100644 --- a/pkg/planner/util/utilfuncp/func_pointer_misc.go +++ b/pkg/planner/util/utilfuncp/func_pointer_misc.go @@ -161,6 +161,10 @@ var ExhaustPhysicalPlans4LogicalPartitionUnionAll func(lp base.LogicalPlan, prop var ExhaustPhysicalPlans4LogicalUnionAll func(lp base.LogicalPlan, prop *property.PhysicalProperty) ( []base.PhysicalPlan, bool, error) +// ExhaustPhysicalPlans4LogicalExpand will be called by LogicalExpand in logicalOp pkg. +var ExhaustPhysicalPlans4LogicalExpand func(lp base.LogicalPlan, prop *property.PhysicalProperty) ( + []base.PhysicalPlan, bool, error) + // *************************************** physical op related ******************************************* // GetEstimatedProbeCntFromProbeParents will be called by BasePhysicalPlan in physicalOp pkg. From 02168152eb62db6de2cf67a58002e106cb5ee165 Mon Sep 17 00:00:00 2001 From: AilinKid <314806019@qq.com> Date: Thu, 15 Aug 2024 14:27:12 +0800 Subject: [PATCH 2/4] . Signed-off-by: AilinKid <314806019@qq.com> --- pkg/planner/core/rule_resolve_grouping_expand.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/planner/core/rule_resolve_grouping_expand.go b/pkg/planner/core/rule_resolve_grouping_expand.go index 3817397dc4a34..0a42a459e5da7 100644 --- a/pkg/planner/core/rule_resolve_grouping_expand.go +++ b/pkg/planner/core/rule_resolve_grouping_expand.go @@ -16,9 +16,9 @@ package core import ( "context" - "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop" "github.com/pingcap/tidb/pkg/planner/core/base" + "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop" "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" ) From 88e6bd1a7d12012603675f0e1aeb30f93bce0b26 Mon Sep 17 00:00:00 2001 From: AilinKid <314806019@qq.com> Date: Fri, 16 Aug 2024 15:37:35 +0800 Subject: [PATCH 3/4] . Signed-off-by: AilinKid <314806019@qq.com> --- pkg/planner/core/BUILD.bazel | 1 - pkg/planner/core/operator/logicalop/BUILD.bazel | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/planner/core/BUILD.bazel b/pkg/planner/core/BUILD.bazel index 0be151392f4df..a5d58fd511c21 100644 --- a/pkg/planner/core/BUILD.bazel +++ b/pkg/planner/core/BUILD.bazel @@ -24,7 +24,6 @@ go_library( "initialize.go", "logical_cte.go", "logical_datasource.go", - "logical_expand.go", "logical_index_scan.go", "logical_initialize.go", "logical_plan_builder.go", diff --git a/pkg/planner/core/operator/logicalop/BUILD.bazel b/pkg/planner/core/operator/logicalop/BUILD.bazel index f53c23229914e..c58bffccc93d7 100644 --- a/pkg/planner/core/operator/logicalop/BUILD.bazel +++ b/pkg/planner/core/operator/logicalop/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "logical_aggregation.go", "logical_apply.go", "logical_cte_table.go", + "logical_expand.go", "logical_join.go", "logical_limit.go", "logical_lock.go", From ab6f8916cf853ae66f341e449324737ce6c12004 Mon Sep 17 00:00:00 2001 From: AilinKid <314806019@qq.com> Date: Mon, 19 Aug 2024 11:35:47 +0800 Subject: [PATCH 4/4] . Signed-off-by: AilinKid <314806019@qq.com> --- pkg/planner/core/core_init.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/planner/core/core_init.go b/pkg/planner/core/core_init.go index 718a6b9437305..aa64607250bef 100644 --- a/pkg/planner/core/core_init.go +++ b/pkg/planner/core/core_init.go @@ -46,6 +46,7 @@ func init() { utilfuncp.ExhaustPhysicalPlans4LogicalApply = exhaustPhysicalPlans4LogicalApply utilfuncp.ExhaustPhysicalPlans4LogicalLimit = exhaustPhysicalPlans4LogicalLimit utilfuncp.ExhaustPhysicalPlans4LogicalWindow = exhaustPhysicalPlans4LogicalWindow + utilfuncp.ExhaustPhysicalPlans4LogicalExpand = exhaustPhysicalPlans4LogicalExpand utilfuncp.ExhaustPhysicalPlans4LogicalUnionAll = exhaustPhysicalPlans4LogicalUnionAll utilfuncp.ExhaustPhysicalPlans4LogicalSequence = exhaustPhysicalPlans4LogicalSequence utilfuncp.ExhaustPhysicalPlans4LogicalSelection = exhaustPhysicalPlans4LogicalSelection