From 61b66aa83769f9d8812dd1095f45bfdac276a1b0 Mon Sep 17 00:00:00 2001 From: Arenatlx <314806019@qq.com> Date: Tue, 5 Mar 2024 20:41:04 +0800 Subject: [PATCH] planner: make index merge union case aware of order property (#49632) close pingcap/tidb#48359 --- pkg/planner/core/casetest/dag/dag_test.go | 1 + .../casetest/dag/testdata/plan_suite_out.json | 2 +- pkg/planner/core/find_best_task.go | 248 ++++++++++++++++++ .../core/indexmerge_intersection_test.go | 29 ++ pkg/planner/core/indexmerge_path.go | 148 ++++++----- pkg/planner/core/indexmerge_test.go | 12 +- pkg/planner/core/mock.go | 7 + .../core/testdata/index_merge_suite_in.json | 25 ++ .../core/testdata/index_merge_suite_out.json | 165 +++++++++++- pkg/planner/util/path.go | 74 ++++-- .../r/executor/index_merge_reader.result | 4 +- tests/integrationtest/r/index_merge.result | 4 +- .../core/issuetest/planner_issue.result | 11 +- 13 files changed, 624 insertions(+), 106 deletions(-) diff --git a/pkg/planner/core/casetest/dag/dag_test.go b/pkg/planner/core/casetest/dag/dag_test.go index 3b6ed17cd2e39..9fc497b8c3ef9 100644 --- a/pkg/planner/core/casetest/dag/dag_test.go +++ b/pkg/planner/core/casetest/dag/dag_test.go @@ -67,6 +67,7 @@ func TestDAGPlanBuilderSimpleCase(t *testing.T) { stmt, err := p.ParseOneStmt(tt, "", "") require.NoError(t, err, comment) require.NoError(t, sessiontxn.NewTxn(context.Background(), tk.Session())) + tk.Session().GetSessionVars().StmtCtx.OriginalSQL = tt p, _, err := planner.Optimize(context.TODO(), tk.Session(), stmt, is) require.NoError(t, err) testdata.OnRecord(func() { diff --git a/pkg/planner/core/casetest/dag/testdata/plan_suite_out.json b/pkg/planner/core/casetest/dag/testdata/plan_suite_out.json index d3c45f8fea7cc..02c188b7b090a 100644 --- a/pkg/planner/core/casetest/dag/testdata/plan_suite_out.json +++ b/pkg/planner/core/casetest/dag/testdata/plan_suite_out.json @@ -16,7 +16,7 @@ }, { "SQL": "select * from t where (t.c > 0 and t.c < 2) or (t.c > 4 and t.c < 6) or (t.c > 8 and t.c < 10) or (t.c > 12 and t.c < 14) or (t.c > 16 and t.c < 18)", - "Best": "TableReader(Table(t)->Sel([or(or(and(gt(test.t.c, 0), lt(test.t.c, 2)), and(gt(test.t.c, 4), lt(test.t.c, 6))), or(and(gt(test.t.c, 8), lt(test.t.c, 10)), or(and(gt(test.t.c, 12), lt(test.t.c, 14)), and(gt(test.t.c, 16), lt(test.t.c, 18)))))]))" + "Best": "IndexLookUp(Index(t.c_d_e)[[1,1] [5,5] [9,9] [13,13] [17,17]], Table(t))" }, { "SQL": "select * from t where (t.c > 0 and t.c < 1) or (t.c > 2 and t.c < 3) or (t.c > 4 and t.c < 5) or (t.c > 6 and t.c < 7) or (t.c > 9 and t.c < 10)", diff --git a/pkg/planner/core/find_best_task.go b/pkg/planner/core/find_best_task.go index ed1d2610038a0..050169022c0d8 100644 --- a/pkg/planner/core/find_best_task.go +++ b/pkg/planner/core/find_best_task.go @@ -15,6 +15,7 @@ package core import ( + "cmp" "fmt" "math" "slices" @@ -839,6 +840,235 @@ func (ds *DataSource) isMatchProp(path *util.AccessPath, prop *property.Physical return isMatchProp } +// matchPropForIndexMergeAlternatives will match the prop with inside PartialAlternativeIndexPaths, and choose +// 1 matched alternative to be a determined index merge partial path for each dimension in PartialAlternativeIndexPaths. +// finally, after we collected the all decided index merge partial paths, we will output a concrete index merge path +// with field PartialIndexPaths is fulfilled here. +// +// as we mentioned before, after deriveStats is done, the normal index OR path will be generated like below: +// +// `create table t (a int, b int, c int, key a(a), key b(b), key ac(a, c), key bc(b, c))` +// `explain format='verbose' select * from t where a=1 or b=1 order by c` +// +// like the case here: +// normal index merge OR path should be: +// for a=1, it has two partial alternative paths: [a, ac] +// for b=1, it has two partial alternative paths: [b, bc] +// and the index merge path: +// +// indexMergePath: { +// PartialIndexPaths: empty // 1D array here, currently is not decided yet. +// PartialAlternativeIndexPaths: [[a, ac], [b, bc]] // 2D array here, each for one DNF item choices. +// } +// +// let's say we have a prop requirement like sort by [c] here, we will choose the better one [ac] (because it can keep +// order) for the first batch [a, ac] from PartialAlternativeIndexPaths; and choose the better one [bc] (because it can +// keep order too) for the second batch [b, bc] from PartialAlternativeIndexPaths. Finally we output a concrete index +// merge path as +// +// indexMergePath: { +// PartialIndexPaths: [ac, bc] // just collected since they match the prop. +// ... +// } +// +// how about the prop is empty? that means the choice to be decided from [a, ac] and [b, bc] is quite random just according +// to their countAfterAccess. That's why we use a slices.SortFunc(matchIdxes, func(a, b int){}) inside there. After sort, +// the ASC order of matchIdxes of matched paths are ordered by their countAfterAccess, choosing the first one is straight forward. +// +// there is another case shown below, just the pick the first one after matchIdxes is ordered is not always right, as shown: +// special logic for alternative paths: +// +// index merge: +// matched paths-1: {pk, index1} +// matched paths-2: {pk} +// +// if we choose first one as we talked above, says pk here in the first matched paths, then path2 has no choice(avoiding all same +// index logic inside) but pk, this will result in all single index failure. so we need to sort the matchIdxes again according to +// their matched paths length, here mean: +// +// index merge: +// matched paths-1: {pk, index1} +// matched paths-2: {pk} +// +// and let matched paths-2 to be the first to make their determination --- choosing pk here, then next turn is matched paths-1 to +// make their choice, since pk is occupied, avoiding-all-same-index-logic inside will try to pick index1 here, so work can be done. +// +// at last, according to determinedIndexPartialPaths to rewrite their real countAfterAccess, this part is move from deriveStats to +// here. +func (ds *DataSource) matchPropForIndexMergeAlternatives(path *util.AccessPath, prop *property.PhysicalProperty) (*util.AccessPath, bool) { + // target: + // 1: index merge case, try to match the every alternative partial path to the order property as long as + // possible, and generate that property-matched index merge path out if any. + // 2: If the prop is empty (means no sort requirement), we will generate a random index partial combination + // path from all alternatives in case that no index merge path comes out. + + // Execution part doesn't support the merge operation for intersection case yet. + if path.IndexMergeIsIntersection { + return nil, false + } + + noSortItem := prop.IsSortItemEmpty() + allSame, _ := prop.AllSameOrder() + if !allSame { + return nil, false + } + // step1: match the property from all the index partial alternative paths. + determinedIndexPartialPaths := make([]*util.AccessPath, 0, len(path.PartialAlternativeIndexPaths)) + usedIndexMap := make(map[int64]struct{}, 1) + type idxWrapper struct { + // matchIdx is those match alternative paths from one alternative paths set. + // like we said above, for a=1, it has two partial alternative paths: [a, ac] + // if we met an empty property here, matchIdx from [a, ac] for a=1 will be both. = [0,1] + // if we met an sort[c] property here, matchIdx from [a, ac] for a=1 will be both. = [1] + matchIdx []int + // pathIdx actually is original position offset indicates where current matchIdx is + // computed from. eg: [[a, ac], [b, bc]] for sort[c] property: + // idxWrapper{[ac], 0}, 0 is the offset in first dimension of PartialAlternativeIndexPaths + // idxWrapper{[bc], 1}, 1 is the offset in first dimension of PartialAlternativeIndexPaths + pathIdx int + } + allMatchIdxes := make([]idxWrapper, 0, len(path.PartialAlternativeIndexPaths)) + // special logic for alternative paths: + // index merge: + // path1: {pk, index1} + // path2: {pk} + // if we choose pk in the first path, then path2 has no choice but pk, this will result in all single index failure. + // so we should collect all match prop paths down, stored as matchIdxes here. + for pathIdx, oneItemAlternatives := range path.PartialAlternativeIndexPaths { + matchIdxes := make([]int, 0, 1) + for i, oneIndexAlternativePath := range oneItemAlternatives { + // if there is some sort items and this path doesn't match this prop, continue. + if !noSortItem && !ds.isMatchProp(oneIndexAlternativePath, prop) { + continue + } + // two possibility here: + // 1. no sort items requirement. + // 2. matched with sorted items. + matchIdxes = append(matchIdxes, i) + } + if len(matchIdxes) == 0 { + // if all index alternative of one of the cnf item's couldn't match the sort property, + // the entire index merge union path can be ignored for this sort property, return false. + return nil, false + } + if len(matchIdxes) > 1 { + // if matchIdxes greater than 1, we should sort this match alternative path by its CountAfterAccess. + tmpOneItemAlternatives := oneItemAlternatives + slices.SortStableFunc(matchIdxes, func(a, b int) int { + lhsCountAfter := tmpOneItemAlternatives[a].CountAfterAccess + if len(tmpOneItemAlternatives[a].IndexFilters) > 0 { + lhsCountAfter = tmpOneItemAlternatives[a].CountAfterIndex + } + rhsCountAfter := tmpOneItemAlternatives[b].CountAfterAccess + if len(tmpOneItemAlternatives[b].IndexFilters) > 0 { + rhsCountAfter = tmpOneItemAlternatives[b].CountAfterIndex + } + return cmp.Compare(lhsCountAfter, rhsCountAfter) + }) + } + allMatchIdxes = append(allMatchIdxes, idxWrapper{matchIdxes, pathIdx}) + } + // sort allMatchIdxes by its element length. + // index merge: index merge: + // path1: {pk, index1} ==> path2: {pk} + // path2: {pk} path1: {pk, index1} + // here for the fixed choice pk of path2, let it be the first one to choose, left choice of index1 to path1. + slices.SortStableFunc(allMatchIdxes, func(a, b idxWrapper) int { + lhsLen := len(a.matchIdx) + rhsLen := len(b.matchIdx) + return cmp.Compare(lhsLen, rhsLen) + }) + for _, matchIdxes := range allMatchIdxes { + // since matchIdxes are ordered by matchIdxes's length, + // we should use matchIdxes.pathIdx to locate where it comes from. + alternatives := path.PartialAlternativeIndexPaths[matchIdxes.pathIdx] + found := false + // pick a most suitable index partial alternative from all matched alternative paths according to asc CountAfterAccess, + // By this way, a distinguished one is better. + for _, oneIdx := range matchIdxes.matchIdx { + var indexID int64 + if alternatives[oneIdx].IsTablePath() { + indexID = -1 + } else { + indexID = alternatives[oneIdx].Index.ID + } + if _, ok := usedIndexMap[indexID]; !ok { + // try to avoid all index partial paths are all about a single index. + determinedIndexPartialPaths = append(determinedIndexPartialPaths, alternatives[oneIdx].Clone()) + usedIndexMap[indexID] = struct{}{} + found = true + break + } + } + if !found { + // just pick the same name index (just using the first one is ok), in case that there may be some other + // picked distinctive index path for other partial paths latter. + determinedIndexPartialPaths = append(determinedIndexPartialPaths, alternatives[matchIdxes.matchIdx[0]].Clone()) + // uedIndexMap[oneItemAlternatives[oneIdx].Index.ID] = struct{}{} must already be colored. + } + } + if len(usedIndexMap) == 1 { + // if all partial path are using a same index, meaningless and fail over. + return nil, false + } + // step2: gen a new **concrete** index merge path. + indexMergePath := &util.AccessPath{ + PartialIndexPaths: determinedIndexPartialPaths, + IndexMergeIsIntersection: false, + // inherit those determined can't pushed-down table filters. + TableFilters: path.TableFilters, + } + // path.ShouldBeKeptCurrentFilter record that whether there are some part of the cnf item couldn't be pushed down to tikv already. + shouldKeepCurrentFilter := path.KeepIndexMergeORSourceFilter + for _, path := range determinedIndexPartialPaths { + // If any partial path contains table filters, we need to keep the whole DNF filter in the Selection. + if len(path.TableFilters) > 0 { + if !expression.CanExprsPushDown(ds.SCtx().GetExprCtx(), path.TableFilters, ds.SCtx().GetClient(), kv.TiKV) { + // if this table filters can't be pushed down, all of them should be kept in the table side, cleaning the lookup side here. + path.TableFilters = nil + } + shouldKeepCurrentFilter = true + } + // If any partial path's index filter cannot be pushed to TiKV, we should keep the whole DNF filter. + if len(path.IndexFilters) != 0 && !expression.CanExprsPushDown(ds.SCtx().GetExprCtx(), path.IndexFilters, ds.SCtx().GetClient(), kv.TiKV) { + shouldKeepCurrentFilter = true + // Clear IndexFilter, the whole filter will be put in indexMergePath.TableFilters. + path.IndexFilters = nil + } + } + // Keep this filter as a part of table filters for safety if it has any parameter. + if expression.MaybeOverOptimized4PlanCache(ds.SCtx().GetExprCtx(), []expression.Expression{path.IndexMergeORSourceFilter}) { + shouldKeepCurrentFilter = true + } + if shouldKeepCurrentFilter { + // add the cnf expression back as table filer. + indexMergePath.TableFilters = append(indexMergePath.TableFilters, path.IndexMergeORSourceFilter) + } + + // step3: after the index merge path is determined, compute the countAfterAccess as usual. + accessConds := make([]expression.Expression, 0, len(determinedIndexPartialPaths)) + for _, p := range determinedIndexPartialPaths { + indexCondsForP := p.AccessConds[:] + indexCondsForP = append(indexCondsForP, p.IndexFilters...) + if len(indexCondsForP) > 0 { + accessConds = append(accessConds, expression.ComposeCNFCondition(ds.SCtx().GetExprCtx(), indexCondsForP...)) + } + } + accessDNF := expression.ComposeDNFCondition(ds.SCtx().GetExprCtx(), accessConds...) + sel, _, err := cardinality.Selectivity(ds.SCtx(), ds.tableStats.HistColl, []expression.Expression{accessDNF}, nil) + if err != nil { + logutil.BgLogger().Debug("something wrong happened, use the default selectivity", zap.Error(err)) + sel = SelectionFactor + } + indexMergePath.CountAfterAccess = sel * ds.tableStats.RowCount + if noSortItem { + // since there is no sort property, index merge case is generated by random combination, each alternative with the lower/lowest + // countAfterAccess, here the returned matchProperty should be false. + return indexMergePath, false + } + return indexMergePath, true +} + func (ds *DataSource) isMatchPropForIndexMerge(path *util.AccessPath, prop *property.PhysicalProperty) bool { // Execution part doesn't support the merge operation for intersection case yet. if path.IndexMergeIsIntersection { @@ -871,6 +1101,16 @@ func (ds *DataSource) getIndexCandidate(path *util.AccessPath, prop *property.Ph return candidate } +func (ds *DataSource) convergeIndexMergeCandidate(path *util.AccessPath, prop *property.PhysicalProperty) *candidatePath { + // since the all index path alternative paths is collected and undetermined, and we should determine a possible and concrete path for this prop. + possiblePath, match := ds.matchPropForIndexMergeAlternatives(path, prop) + if possiblePath == nil { + return nil + } + candidate := &candidatePath{path: possiblePath, isMatchProp: match} + return candidate +} + func (ds *DataSource) getIndexMergeCandidate(path *util.AccessPath, prop *property.PhysicalProperty) *candidatePath { candidate := &candidatePath{path: path} candidate.isMatchProp = ds.isMatchPropForIndexMerge(path, prop) @@ -886,6 +1126,14 @@ func (ds *DataSource) skylinePruning(prop *property.PhysicalProperty) []*candida if path.StoreType != kv.TiFlash && prop.IsFlashProp() { continue } + if len(path.PartialAlternativeIndexPaths) > 0 { + // OR normal index merge path, try to determine every index partial path for this property. + candidate := ds.convergeIndexMergeCandidate(path, prop) + if candidate != nil { + candidates = append(candidates, candidate) + } + continue + } if path.PartialIndexPaths != nil { candidates = append(candidates, ds.getIndexMergeCandidate(path, prop)) continue diff --git a/pkg/planner/core/indexmerge_intersection_test.go b/pkg/planner/core/indexmerge_intersection_test.go index 209cb4af00401..df056566e0e87 100644 --- a/pkg/planner/core/indexmerge_intersection_test.go +++ b/pkg/planner/core/indexmerge_intersection_test.go @@ -43,6 +43,35 @@ func TestPlanCacheForIntersectionIndexMerge(t *testing.T) { require.True(t, tk.HasPlanForLastExecution("IndexMerge")) } +func TestIndexMergeWithOrderProperty(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int, b int, c int, d int, e int, key a(a), key b(b), key c(c), key ac(a, c), key bc(b, c), key ae(a, e), key be(b, e)," + + " key abd(a, b, d), key cd(c, d))") + tk.MustExec("create table t2 (a int, b int, c int, key a(a), key b(b), key ac(a, c))") + + var ( + input []string + output []struct { + SQL string + Plan []string + } + ) + planSuiteData := core.GetIndexMergeSuiteData() + planSuiteData.LoadTestCases(t, &input, &output) + for i, ts := range input { + testdata.OnRecord(func() { + output[i].SQL = ts + output[i].Plan = testdata.ConvertRowsToStrings(tk.MustQuery("explain format = 'brief' " + ts).Rows()) + }) + tk.MustQuery("explain format = 'brief' " + ts).Check(testkit.Rows(output[i].Plan...)) + // Expect no warnings. + tk.MustQuery("show warnings").Check(testkit.Rows()) + } +} + func TestHintForIntersectionIndexMerge(t *testing.T) { store, domain := testkit.CreateMockStoreAndDomain(t) handle := domain.StatsHandle() diff --git a/pkg/planner/core/indexmerge_path.go b/pkg/planner/core/indexmerge_path.go index d0dbc594fc764..9e2083fcfef6e 100644 --- a/pkg/planner/core/indexmerge_path.go +++ b/pkg/planner/core/indexmerge_path.go @@ -197,9 +197,27 @@ func (ds *DataSource) generateNormalIndexPartialPaths4DNF(dnfItems []expression. } // getIndexMergeOrPath generates all possible IndexMergeOrPaths. +// For index merge union case, the order property from its partial +// path can be kept and multi-way merged and output. So we don't +// generate a concrete index merge path out, but an un-determined +// alternatives set index merge path instead. +// +// `create table t (a int, b int, c int, key a(a), key b(b), key ac(a, c), key bc(b, c))` +// `explain format='verbose' select * from t where a=1 or b=1 order by c` +// +// like the case here: +// normal index merge OR path should be: +// for a=1, it has two partial alternative paths: [a, ac] +// for b=1, it has two partial alternative paths: [b, bc] +// and the index merge path: +// +// indexMergePath: { +// PartialIndexPaths: empty // 1D array here, currently is not decided yet. +// PartialAlternativeIndexPaths: [[a, ac], [b, bc]] // 2D array here, each for one DNF item choices. +// } func (ds *DataSource) generateIndexMergeOrPaths(filters []expression.Expression) error { usedIndexCount := len(ds.possibleAccessPaths) - for i, cond := range filters { + for k, cond := range filters { sf, ok := cond.(*expression.ScalarFunction) if !ok || sf.FuncName.L != ast.LogicOr { continue @@ -207,7 +225,7 @@ func (ds *DataSource) generateIndexMergeOrPaths(filters []expression.Expression) // shouldKeepCurrentFilter means the partial paths can't cover the current filter completely, so we must add // the current filter into a Selection after partial paths. shouldKeepCurrentFilter := false - var partialPaths = make([]*util.AccessPath, 0, usedIndexCount) + var partialAlternativePaths = make([][]*util.AccessPath, 0, usedIndexCount) dnfItems := expression.FlattenDNFConditions(sf) for _, item := range dnfItems { cnfItems := expression.SplitCNFItems(item) @@ -227,50 +245,58 @@ func (ds *DataSource) generateIndexMergeOrPaths(filters []expression.Expression) itemPaths := ds.accessPathsForConds(pushedDownCNFItems, usedIndexCount) if len(itemPaths) == 0 { - partialPaths = nil - break - } - partialPath := ds.buildIndexMergePartialPath(itemPaths) - if partialPath == nil { - partialPaths = nil + partialAlternativePaths = nil break } - partialPaths = append(partialPaths, partialPath) + // we don't prune other possible index merge path here. + // keep all the possible index merge partial paths here to let the property choose. + partialAlternativePaths = append(partialAlternativePaths, itemPaths) } - // If all of the partialPaths use the same index, we will not use the indexMerge. - singlePath := true - for i := len(partialPaths) - 1; i >= 1; i-- { - if partialPaths[i].Index != partialPaths[i-1].Index { - singlePath = false - break - } - } - if singlePath { + if len(partialAlternativePaths) <= 1 { continue } - if len(partialPaths) > 1 { - possiblePath := ds.buildIndexMergeOrPath(filters, partialPaths, i, shouldKeepCurrentFilter) - if possiblePath == nil { - return nil - } - - accessConds := make([]expression.Expression, 0, len(partialPaths)) - for _, p := range partialPaths { - indexCondsForP := p.AccessConds[:] - indexCondsForP = append(indexCondsForP, p.IndexFilters...) - if len(indexCondsForP) > 0 { - accessConds = append(accessConds, expression.ComposeCNFCondition(ds.SCtx().GetExprCtx(), indexCondsForP...)) + // in this loop we do two things. + // 1: If all the partialPaths use the same index, we will not use the indexMerge. + // 2: Compute a theoretical best countAfterAccess(pick its accessConds) for every alternative path(s). + indexMap := make(map[int64]struct{}, 1) + accessConds := make([]expression.Expression, 0, len(partialAlternativePaths)) + for _, oneAlternativeSet := range partialAlternativePaths { + // 1: mark used map. + for _, oneAlternativePath := range oneAlternativeSet { + if oneAlternativePath.IsTablePath() { + // table path + indexMap[-1] = struct{}{} + } else { + // index path + indexMap[oneAlternativePath.Index.ID] = struct{}{} } } - accessDNF := expression.ComposeDNFCondition(ds.SCtx().GetExprCtx(), accessConds...) - sel, _, err := cardinality.Selectivity(ds.SCtx(), ds.tableStats.HistColl, []expression.Expression{accessDNF}, nil) - if err != nil { - logutil.BgLogger().Debug("something wrong happened, use the default selectivity", zap.Error(err)) - sel = SelectionFactor + // 2.1: trade off on countAfterAccess. + minCountAfterAccessPath := ds.buildIndexMergePartialPath(oneAlternativeSet) + indexCondsForP := minCountAfterAccessPath.AccessConds[:] + indexCondsForP = append(indexCondsForP, minCountAfterAccessPath.IndexFilters...) + if len(indexCondsForP) > 0 { + accessConds = append(accessConds, expression.ComposeCNFCondition(ds.SCtx().GetExprCtx(), indexCondsForP...)) } - possiblePath.CountAfterAccess = sel * ds.tableStats.RowCount - ds.possibleAccessPaths = append(ds.possibleAccessPaths, possiblePath) } + if len(indexMap) == 1 { + continue + } + // 2.2 get the theoretical whole count after access for index merge. + accessDNF := expression.ComposeDNFCondition(ds.SCtx().GetExprCtx(), accessConds...) + sel, _, err := cardinality.Selectivity(ds.SCtx(), ds.tableStats.HistColl, []expression.Expression{accessDNF}, nil) + if err != nil { + logutil.BgLogger().Debug("something wrong happened, use the default selectivity", zap.Error(err)) + sel = SelectionFactor + } + + possiblePath := ds.buildIndexMergeOrPath(filters, partialAlternativePaths, k, shouldKeepCurrentFilter) + if possiblePath == nil { + return nil + } + possiblePath.CountAfterAccess = sel * ds.tableStats.RowCount + // only after all partial path is determined, can the countAfterAccess be done, delay it to converging. + ds.possibleAccessPaths = append(ds.possibleAccessPaths, possiblePath) } return nil } @@ -420,46 +446,36 @@ func (*DataSource) buildIndexMergePartialPath(indexAccessPaths []*util.AccessPat // buildIndexMergeOrPath generates one possible IndexMergePath. func (ds *DataSource) buildIndexMergeOrPath( filters []expression.Expression, - partialPaths []*util.AccessPath, + partialAlternativePaths [][]*util.AccessPath, current int, shouldKeepCurrentFilter bool, ) *util.AccessPath { - indexMergePath := &util.AccessPath{PartialIndexPaths: partialPaths} + indexMergePath := &util.AccessPath{PartialAlternativeIndexPaths: partialAlternativePaths} indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[:current]...) indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[current+1:]...) // If global index exists, index merge is not allowed. // Global index is not compatible with IndexMergeReaderExecutor. - for i := range partialPaths { - if partialPaths[i].Index != nil && partialPaths[i].Index.Global { + for i := range partialAlternativePaths { + // if one path's all alternatives are global index, warning it. + allGlobal := true + for _, oneAlternative := range partialAlternativePaths[i] { + // once we have a table alternative path + if oneAlternative.IsTablePath() { + allGlobal = false + } + if oneAlternative.Index != nil && !oneAlternative.Index.Global { + allGlobal = false + } + } + if allGlobal { ds.SCtx().GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("global index is not compatible with index merge, so ignore it")) return nil } } - - for _, path := range partialPaths { - // If any partial path contains table filters, we need to keep the whole DNF filter in the Selection. - if len(path.TableFilters) > 0 { - shouldKeepCurrentFilter = true - } - // If any partial path's index filter cannot be pushed to TiKV, we should keep the whole DNF filter. - if len(path.IndexFilters) != 0 && !expression.CanExprsPushDown(ds.SCtx().GetExprCtx(), path.IndexFilters, ds.SCtx().GetClient(), kv.TiKV) { - shouldKeepCurrentFilter = true - // Clear IndexFilter, the whole filter will be put in indexMergePath.TableFilters. - path.IndexFilters = nil - } - if len(path.TableFilters) != 0 && !expression.CanExprsPushDown(ds.SCtx().GetExprCtx(), path.TableFilters, ds.SCtx().GetClient(), kv.TiKV) { - shouldKeepCurrentFilter = true - path.TableFilters = nil - } - } - - // Keep this filter as a part of table filters for safety if it has any parameter. - if expression.MaybeOverOptimized4PlanCache(ds.SCtx().GetExprCtx(), filters[current:current+1]) { - shouldKeepCurrentFilter = true - } - if shouldKeepCurrentFilter { - indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[current]) - } + // since shouldKeepCurrentFilter may be changed in alternative paths converging, kept the filer expression anyway here. + indexMergePath.KeepIndexMergeORSourceFilter = shouldKeepCurrentFilter + // this filter will be merged into indexPath's table filters when converging. + indexMergePath.IndexMergeORSourceFilter = filters[current] return indexMergePath } diff --git a/pkg/planner/core/indexmerge_test.go b/pkg/planner/core/indexmerge_test.go index bb89671cbb58e..f75ff1a8ccb7b 100644 --- a/pkg/planner/core/indexmerge_test.go +++ b/pkg/planner/core/indexmerge_test.go @@ -39,11 +39,19 @@ func getIndexMergePathDigest(paths []*util.AccessPath, startIndex int) string { } path := paths[i] idxMergeDisgest += "{Idxs:[" - for j := 0; j < len(path.PartialIndexPaths); j++ { + for j := 0; j < len(path.PartialAlternativeIndexPaths); j++ { if j > 0 { idxMergeDisgest += "," } - idxMergeDisgest += path.PartialIndexPaths[j].Index.Name.L + idxMergeDisgest += "{" + // for every ONE index partial alternatives, output a set. + for k, one := range path.PartialAlternativeIndexPaths[j] { + if k != 0 { + idxMergeDisgest += "," + } + idxMergeDisgest += one.Index.Name.L + } + idxMergeDisgest += "}" } idxMergeDisgest += "],TbFilters:[" for j := 0; j < len(path.TableFilters); j++ { diff --git a/pkg/planner/core/mock.go b/pkg/planner/core/mock.go index 99b1fe4de11b6..0b63002c6c5e8 100644 --- a/pkg/planner/core/mock.go +++ b/pkg/planner/core/mock.go @@ -50,6 +50,7 @@ func MockSignedTable() *model.TableInfo { // indices: c_d_e, e, f, g, f_g, c_d_e_str, e_d_c_str_prefix indices := []*model.IndexInfo{ { + ID: 1, Name: model.NewCIStr("c_d_e"), Columns: []*model.IndexColumn{ { @@ -72,6 +73,7 @@ func MockSignedTable() *model.TableInfo { Unique: true, }, { + ID: 2, Name: model.NewCIStr("x"), Columns: []*model.IndexColumn{ { @@ -84,6 +86,7 @@ func MockSignedTable() *model.TableInfo { Unique: true, }, { + ID: 3, Name: model.NewCIStr("f"), Columns: []*model.IndexColumn{ { @@ -96,6 +99,7 @@ func MockSignedTable() *model.TableInfo { Unique: true, }, { + ID: 4, Name: model.NewCIStr("g"), Columns: []*model.IndexColumn{ { @@ -107,6 +111,7 @@ func MockSignedTable() *model.TableInfo { State: model.StatePublic, }, { + ID: 5, Name: model.NewCIStr("f_g"), Columns: []*model.IndexColumn{ { @@ -124,6 +129,7 @@ func MockSignedTable() *model.TableInfo { Unique: true, }, { + ID: 6, Name: model.NewCIStr("c_d_e_str"), Columns: []*model.IndexColumn{ { @@ -145,6 +151,7 @@ func MockSignedTable() *model.TableInfo { State: model.StatePublic, }, { + ID: 7, Name: model.NewCIStr("e_d_c_str_prefix"), Columns: []*model.IndexColumn{ { diff --git a/pkg/planner/core/testdata/index_merge_suite_in.json b/pkg/planner/core/testdata/index_merge_suite_in.json index db7ebacdb29c7..30ecf13285165 100644 --- a/pkg/planner/core/testdata/index_merge_suite_in.json +++ b/pkg/planner/core/testdata/index_merge_suite_in.json @@ -39,5 +39,30 @@ "select /*+ use_index_merge(t3, ia, ibc, id, ie) */ * from t3 where a > 10 and b = 20 and c < 35 and d < 45 and e = 100", "select /*+ use_index_merge(t4, ia, ibc, id, ie) */ * from t4 where a > 10 and b = 20 and c < 35 and d in (1,3,8,9) and e = 100" ] + }, + { + "name": "TestIndexMergeWithOrderProperty", + "cases": [ + "select * from t where a=1 or b=1 order by c; -- 1. auto pick the most suitable index ac and bc", + "select /*+ use_index_merge(t, ac, bc) */ * from t where a=1 or b=1 order by c; -- 2. hint instruction to pick the suitable index ac and bc", + "select * from t2 where a=1 or b=1 order by c; -- 3. if one of the index order path can't satisfied, back to normal choice", + "select /*+ use_index_merge(t, ac, b) */ * from t where a=1 or b=1 order by c; -- 4. hint to use ac strongly, but sort OP is still needed", + + "select * from t where a=1 or a=2 order by c; -- 5. duplicated column a's conditions and lack of column b's conditions", + "select * from t use index(a) where a=1 or a=2 order by c; -- 6. duplicated column a's conditions and lack of column b's conditions", + + "select * from t where a=1 or a=2 or b=3 order by c; -- 7. duplicate column a's conditions", + "select /*+ use_index_merge(t, ac, bc) */ * from t where a=1 or a=2 or b=3 order by c; -- 8. duplicate column a's conditions", + + "select * from t where a=1 or b=1 or c=1 order by d; -- 9. more than 2 index related columns, here c condition is interference", + "select /*+ use_index_merge(t, a, b, c) */ * from t where a=1 or b=1 or c=1 order by d; -- 10. more than 2 index related columns, here c condition is interference", + + "select * from t where a=1 or b=1 or c=1 or d=1 order by e; -- 11. more index definition about idx(a,e), idx(b,e)", + "select /*+ use_index_merge(t, ae, be, c) */ * from t where a=1 or b=1 or c=1 order by e; -- 12. more index definition about idx(a,e), idx(b,e)", + + "select * from t where (a=1 and b=1) or c=1 order by d; -- 13. composite predicates and more index defined, idx(a, b, d), idx(c, d)", + "select * from t where (a=1 and b=1) or (c=1 and d=2) order by d; -- 14. composite predicates and more index defined, idx(a, b, d), idx(c, d)", + "select * from t where (a=1 and b=1) or (c=1 and d=1) order by e" + ] } ] diff --git a/pkg/planner/core/testdata/index_merge_suite_out.json b/pkg/planner/core/testdata/index_merge_suite_out.json index cfa13020f7c1a..0d98061beedd1 100644 --- a/pkg/planner/core/testdata/index_merge_suite_out.json +++ b/pkg/planner/core/testdata/index_merge_suite_out.json @@ -4,10 +4,10 @@ "Cases": [ "[]", "[]", - "[{Idxs:[c_d_e,f],TbFilters:[]}]", - "[{Idxs:[c_d_e,f],TbFilters:[or(gt(test.t.c, 5), lt(test.t.f, 7))]},{Idxs:[c_d_e,f],TbFilters:[or(lt(test.t.c, 1), gt(test.t.f, 2))]}]", - "[{Idxs:[c_d_e,f],TbFilters:[or(gt(test.t.c, 5), lt(test.t.f, 7)),or(lt(test.t.c, 1), gt(test.t.g, 2))]},{Idxs:[c_d_e,f],TbFilters:[or(lt(test.t.c, 1), gt(test.t.f, 2)),or(lt(test.t.c, 1), gt(test.t.g, 2))]},{Idxs:[c_d_e,g],TbFilters:[or(lt(test.t.c, 1), gt(test.t.f, 2)),or(gt(test.t.c, 5), lt(test.t.f, 7))]}]", - "[{Idxs:[c_d_e,f],TbFilters:[or(gt(test.t.c, 5), lt(test.t.f, 7)),or(lt(test.t.e, 1), gt(test.t.f, 2))]},{Idxs:[c_d_e,f],TbFilters:[or(lt(test.t.c, 1), gt(test.t.f, 2)),or(lt(test.t.e, 1), gt(test.t.f, 2))]}]" + "[{Idxs:[{c_d_e},{f,f_g}],TbFilters:[]}]", + "[{Idxs:[{c_d_e},{f,f_g}],TbFilters:[or(gt(test.t.c, 5), lt(test.t.f, 7))]},{Idxs:[{c_d_e},{f,f_g}],TbFilters:[or(lt(test.t.c, 1), gt(test.t.f, 2))]}]", + "[{Idxs:[{c_d_e},{f,f_g}],TbFilters:[or(gt(test.t.c, 5), lt(test.t.f, 7)),or(lt(test.t.c, 1), gt(test.t.g, 2))]},{Idxs:[{c_d_e},{f,f_g}],TbFilters:[or(lt(test.t.c, 1), gt(test.t.f, 2)),or(lt(test.t.c, 1), gt(test.t.g, 2))]},{Idxs:[{c_d_e},{g}],TbFilters:[or(lt(test.t.c, 1), gt(test.t.f, 2)),or(gt(test.t.c, 5), lt(test.t.f, 7))]}]", + "[{Idxs:[{c_d_e},{f,f_g}],TbFilters:[or(gt(test.t.c, 5), lt(test.t.f, 7)),or(lt(test.t.e, 1), gt(test.t.f, 2))]},{Idxs:[{c_d_e},{f,f_g}],TbFilters:[or(lt(test.t.c, 1), gt(test.t.f, 2)),or(lt(test.t.e, 1), gt(test.t.f, 2))]}]" ] }, { @@ -432,5 +432,162 @@ ] } ] + }, + { + "Name": "TestIndexMergeWithOrderProperty", + "Cases": [ + { + "SQL": "select * from t where a=1 or b=1 order by c; -- 1. auto pick the most suitable index ac and bc", + "Plan": [ + "Projection 19.99 root test.t.a, test.t.b, test.t.c, test.t.d, test.t.e", + "└─IndexMerge 19.99 root type: union", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ac(a, c) range:[1,1], keep order:true, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:bc(b, c) range:[1,1], keep order:true, stats:pseudo", + " └─TableRowIDScan(Probe) 19.99 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "select /*+ use_index_merge(t, ac, bc) */ * from t where a=1 or b=1 order by c; -- 2. hint instruction to pick the suitable index ac and bc", + "Plan": [ + "Projection 19.99 root test.t.a, test.t.b, test.t.c, test.t.d, test.t.e", + "└─IndexMerge 19.99 root type: union", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ac(a, c) range:[1,1], keep order:true, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:bc(b, c) range:[1,1], keep order:true, stats:pseudo", + " └─TableRowIDScan(Probe) 19.99 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "select * from t2 where a=1 or b=1 order by c; -- 3. if one of the index order path can't satisfied, back to normal choice", + "Plan": [ + "Sort 19.99 root test.t2.c", + "└─IndexMerge 19.99 root type: union", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t2, index:b(b) range:[1,1], keep order:false, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t2, index:a(a) range:[1,1], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 19.99 cop[tikv] table:t2 keep order:false, stats:pseudo" + ] + }, + { + "SQL": "select /*+ use_index_merge(t, ac, b) */ * from t where a=1 or b=1 order by c; -- 4. hint to use ac strongly, but sort OP is still needed", + "Plan": [ + "Sort 19.99 root test.t.c", + "└─IndexMerge 19.99 root type: union", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ac(a, c) range:[1,1], keep order:false, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:b(b) range:[1,1], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 19.99 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "select * from t where a=1 or a=2 order by c; -- 5. duplicated column a's conditions and lack of column b's conditions", + "Plan": [ + "Sort 250.00 root test.t.c", + "└─TableReader 250.00 root data:Selection", + " └─Selection 250.00 cop[tikv] or(eq(test.t.a, 1), eq(test.t.a, 2))", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "select * from t use index(a) where a=1 or a=2 order by c; -- 6. duplicated column a's conditions and lack of column b's conditions", + "Plan": [ + "Sort 250.00 root test.t.c", + "└─IndexLookUp 250.00 root ", + " ├─IndexRangeScan(Build) 250.00 cop[tikv] table:t, index:a(a) range:[1,2], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 250.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "select * from t where a=1 or a=2 or b=3 order by c; -- 7. duplicate column a's conditions", + "Plan": [ + "Projection 259.75 root test.t.a, test.t.b, test.t.c, test.t.d, test.t.e", + "└─IndexMerge 259.75 root type: union", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ac(a, c) range:[1,1], keep order:true, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ac(a, c) range:[2,2], keep order:true, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:bc(b, c) range:[3,3], keep order:true, stats:pseudo", + " └─TableRowIDScan(Probe) 259.75 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "select /*+ use_index_merge(t, ac, bc) */ * from t where a=1 or a=2 or b=3 order by c; -- 8. duplicate column a's conditions", + "Plan": [ + "Projection 259.75 root test.t.a, test.t.b, test.t.c, test.t.d, test.t.e", + "└─IndexMerge 259.75 root type: union", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ac(a, c) range:[1,1], keep order:true, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ac(a, c) range:[2,2], keep order:true, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:bc(b, c) range:[3,3], keep order:true, stats:pseudo", + " └─TableRowIDScan(Probe) 259.75 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "select * from t where a=1 or b=1 or c=1 order by d; -- 9. more than 2 index related columns, here c condition is interference", + "Plan": [ + "Sort 29.97 root test.t.d", + "└─IndexMerge 29.97 root type: union", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:c(c) range:[1,1], keep order:false, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:b(b) range:[1,1], keep order:false, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:a(a) range:[1,1], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 29.97 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "select /*+ use_index_merge(t, a, b, c) */ * from t where a=1 or b=1 or c=1 order by d; -- 10. more than 2 index related columns, here c condition is interference", + "Plan": [ + "Sort 29.97 root test.t.d", + "└─IndexMerge 29.97 root type: union", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:a(a) range:[1,1], keep order:false, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:b(b) range:[1,1], keep order:false, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:c(c) range:[1,1], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 29.97 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "select * from t where a=1 or b=1 or c=1 or d=1 order by e; -- 11. more index definition about idx(a,e), idx(b,e)", + "Plan": [ + "Sort 39.94 root test.t.e", + "└─TableReader 39.94 root data:Selection", + " └─Selection 39.94 cop[tikv] or(or(eq(test.t.a, 1), eq(test.t.b, 1)), or(eq(test.t.c, 1), eq(test.t.d, 1)))", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "select /*+ use_index_merge(t, ae, be, c) */ * from t where a=1 or b=1 or c=1 order by e; -- 12. more index definition about idx(a,e), idx(b,e)", + "Plan": [ + "Sort 29.97 root test.t.e", + "└─IndexMerge 29.97 root type: union", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ae(a, e) range:[1,1], keep order:false, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:be(b, e) range:[1,1], keep order:false, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:c(c) range:[1,1], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 29.97 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "select * from t where (a=1 and b=1) or c=1 order by d; -- 13. composite predicates and more index defined, idx(a, b, d), idx(c, d)", + "Plan": [ + "Projection 10.10 root test.t.a, test.t.b, test.t.c, test.t.d, test.t.e", + "└─IndexMerge 10.10 root type: union", + " ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:abd(a, b, d) range:[1 1,1 1], keep order:true, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:cd(c, d) range:[1,1], keep order:true, stats:pseudo", + " └─TableRowIDScan(Probe) 10.10 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "select * from t where (a=1 and b=1) or (c=1 and d=2) order by d; -- 14. composite predicates and more index defined, idx(a, b, d), idx(c, d)", + "Plan": [ + "Projection 0.20 root test.t.a, test.t.b, test.t.c, test.t.d, test.t.e", + "└─IndexMerge 0.20 root type: union", + " ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:abd(a, b, d) range:[1 1,1 1], keep order:true, stats:pseudo", + " ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:cd(c, d) range:[1 2,1 2], keep order:true, stats:pseudo", + " └─TableRowIDScan(Probe) 0.20 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "select * from t where (a=1 and b=1) or (c=1 and d=1) order by e", + "Plan": [ + "Sort 0.20 root test.t.e", + "└─IndexMerge 0.20 root type: union", + " ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:cd(c, d) range:[1 1,1 1], keep order:false, stats:pseudo", + " ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:abd(a, b, d) range:[1 1,1 1], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 0.20 cop[tikv] table:t keep order:false, stats:pseudo" + ] + } + ] } ] diff --git a/pkg/planner/util/path.go b/pkg/planner/util/path.go index 48b90029e12d6..4818d70ff3ad4 100644 --- a/pkg/planner/util/path.go +++ b/pkg/planner/util/path.go @@ -51,6 +51,19 @@ type AccessPath struct { // PartialIndexPaths store all index access paths. // If there are extra filters, store them in TableFilters. PartialIndexPaths []*AccessPath + + // ************************************************** special field below ********************************************************* + // For every dnf/cnf item, there maybe several matched partial index paths to be determined later in property detecting and cost model. + // when PartialAlternativeIndexPaths is not empty, it means a special state for index merge path, and it can't have PartialIndexPaths + // at same time. Normal single index or table path also doesn't use this field. + PartialAlternativeIndexPaths [][]*AccessPath + // KeepIndexMergeORSourceFilter and IndexMergeORSourceFilter are only used with PartialAlternativeIndexPaths, which means for + // the new state/type of access path. (undetermined index merge path) + KeepIndexMergeORSourceFilter bool + // IndexMergeORSourceFilter indicates that there are some expression inside this dnf that couldn't be pushed down, and we should keep the entire dnf above. + IndexMergeORSourceFilter expression.Expression + // ******************************************************************************************************************************** + // IndexMergeIsIntersection means whether it's intersection type or union type IndexMerge path. // It's only valid for a IndexMerge path. // Intersection type is for expressions connected by `AND` and union type is for `OR`. @@ -81,35 +94,46 @@ type AccessPath struct { // some fields like FieldType are not deep-copied. func (path *AccessPath) Clone() *AccessPath { ret := &AccessPath{ - Index: path.Index.Clone(), - FullIdxCols: CloneCols(path.FullIdxCols), - FullIdxColLens: slices.Clone(path.FullIdxColLens), - IdxCols: CloneCols(path.IdxCols), - IdxColLens: slices.Clone(path.IdxColLens), - ConstCols: slices.Clone(path.ConstCols), - Ranges: CloneRanges(path.Ranges), - CountAfterAccess: path.CountAfterAccess, - CountAfterIndex: path.CountAfterIndex, - AccessConds: CloneExprs(path.AccessConds), - EqCondCount: path.EqCondCount, - EqOrInCondCount: path.EqOrInCondCount, - IndexFilters: CloneExprs(path.IndexFilters), - TableFilters: CloneExprs(path.TableFilters), - IndexMergeIsIntersection: path.IndexMergeIsIntersection, - PartialIndexPaths: nil, - StoreType: path.StoreType, - IsDNFCond: path.IsDNFCond, - IsIntHandlePath: path.IsIntHandlePath, - IsCommonHandlePath: path.IsCommonHandlePath, - Forced: path.Forced, - ForceKeepOrder: path.ForceKeepOrder, - ForceNoKeepOrder: path.ForceNoKeepOrder, - IsSingleScan: path.IsSingleScan, - IsUkShardIndexPath: path.IsUkShardIndexPath, + Index: path.Index.Clone(), + FullIdxCols: CloneCols(path.FullIdxCols), + FullIdxColLens: slices.Clone(path.FullIdxColLens), + IdxCols: CloneCols(path.IdxCols), + IdxColLens: slices.Clone(path.IdxColLens), + ConstCols: slices.Clone(path.ConstCols), + Ranges: CloneRanges(path.Ranges), + CountAfterAccess: path.CountAfterAccess, + CountAfterIndex: path.CountAfterIndex, + AccessConds: CloneExprs(path.AccessConds), + EqCondCount: path.EqCondCount, + EqOrInCondCount: path.EqOrInCondCount, + IndexFilters: CloneExprs(path.IndexFilters), + TableFilters: CloneExprs(path.TableFilters), + IndexMergeIsIntersection: path.IndexMergeIsIntersection, + PartialIndexPaths: nil, + StoreType: path.StoreType, + IsDNFCond: path.IsDNFCond, + IsIntHandlePath: path.IsIntHandlePath, + IsCommonHandlePath: path.IsCommonHandlePath, + Forced: path.Forced, + ForceKeepOrder: path.ForceKeepOrder, + ForceNoKeepOrder: path.ForceNoKeepOrder, + IsSingleScan: path.IsSingleScan, + IsUkShardIndexPath: path.IsUkShardIndexPath, + KeepIndexMergeORSourceFilter: path.KeepIndexMergeORSourceFilter, + } + if path.IndexMergeORSourceFilter != nil { + ret.IndexMergeORSourceFilter = path.IndexMergeORSourceFilter.Clone() } for _, partialPath := range path.PartialIndexPaths { ret.PartialIndexPaths = append(ret.PartialIndexPaths, partialPath.Clone()) } + for _, onePartialAlternative := range path.PartialAlternativeIndexPaths { + tmp := make([]*AccessPath, 0, len(onePartialAlternative)) + for _, oneAlternative := range onePartialAlternative { + tmp = append(tmp, oneAlternative.Clone()) + } + ret.PartialAlternativeIndexPaths = append(ret.PartialAlternativeIndexPaths, tmp) + } return ret } diff --git a/tests/integrationtest/r/executor/index_merge_reader.result b/tests/integrationtest/r/executor/index_merge_reader.result index 8b08c1710dd2f..2fdca48d7c16c 100644 --- a/tests/integrationtest/r/executor/index_merge_reader.result +++ b/tests/integrationtest/r/executor/index_merge_reader.result @@ -539,8 +539,8 @@ Projection 2666.67 root istrue(or(not(ge(executor__index_merge_reader.t.i, j筧 └─StreamAgg 2666.67 root group by:executor__index_merge_reader.t.i, funcs:max(executor__index_merge_reader.t.e)->Column#10, funcs:firstrow(executor__index_merge_reader.t.i)->executor__index_merge_reader.t.i └─Sort 3333.33 root executor__index_merge_reader.t.i └─IndexMerge 3333.33 root partition:all type: union - ├─TableRangeScan(Build) 3333.33 cop[tikv] table:t range:(240817,+inf], keep order:false, stats:pseudo ├─IndexFullScan(Build) 0.00 cop[tikv] table:t, index:idx_25(h, i, e) keep order:false, stats:pseudo + ├─TableRangeScan(Build) 3333.33 cop[tikv] table:t range:(240817,+inf], keep order:false, stats:pseudo └─TableRowIDScan(Probe) 3333.33 cop[tikv] table:t keep order:false, stats:pseudo select count(*) from (SELECT /*+ AGG_TO_COP() STREAM_AGG()*/ (NOT (`t`.`i`>=_UTF8MB4'j筧8') OR NOT (`t`.`i`=_UTF8MB4'暈lH忧ll6')) IS TRUE,MAX(`t`.`e`) AS `r0`,QUOTE(`t`.`i`) AS `r1` FROM `t` WHERE `t`.`h`>240817 OR `t`.`i` BETWEEN _UTF8MB4'WVz' AND _UTF8MB4'G#駧褉ZC領*lov' GROUP BY `t`.`i`) derived; count(*) @@ -550,8 +550,8 @@ id estRows task access object operator info Projection 2666.67 root istrue(or(not(ge(executor__index_merge_reader.t.i, j筧8)), not(eq(executor__index_merge_reader.t.i, 暈lH忧ll6))))->Column#11, Column#10, quote(executor__index_merge_reader.t.i)->Column#12 └─HashAgg 2666.67 root group by:executor__index_merge_reader.t.i, funcs:max(executor__index_merge_reader.t.e)->Column#10, funcs:firstrow(executor__index_merge_reader.t.i)->executor__index_merge_reader.t.i └─IndexMerge 3333.33 root partition:all type: union - ├─TableRangeScan(Build) 3333.33 cop[tikv] table:t range:(240817,+inf], keep order:false, stats:pseudo ├─IndexFullScan(Build) 0.00 cop[tikv] table:t, index:idx_25(h, i, e) keep order:false, stats:pseudo + ├─TableRangeScan(Build) 3333.33 cop[tikv] table:t range:(240817,+inf], keep order:false, stats:pseudo └─TableRowIDScan(Probe) 3333.33 cop[tikv] table:t keep order:false, stats:pseudo select count(*) from (SELECT /*+ AGG_TO_COP() */ (NOT (`t`.`i`>=_UTF8MB4'j筧8') OR NOT (`t`.`i`=_UTF8MB4'暈lH忧ll6')) IS TRUE,MAX(`t`.`e`) AS `r0`,QUOTE(`t`.`i`) AS `r1` FROM `t` WHERE `t`.`h`>240817 OR `t`.`i` BETWEEN _UTF8MB4'WVz' AND _UTF8MB4'G#駧褉ZC領*lov' GROUP BY `t`.`i`) derived; count(*) diff --git a/tests/integrationtest/r/index_merge.result b/tests/integrationtest/r/index_merge.result index 261363f92873c..986112abaee60 100644 --- a/tests/integrationtest/r/index_merge.result +++ b/tests/integrationtest/r/index_merge.result @@ -598,8 +598,8 @@ explain select /*+ use_index_merge(t1) */ * from t1 where ((c1 < 10 and c4 < 10) id estRows task access object operator info Sort_5 2250.55 root index_merge.t1.c1 └─IndexMerge_12 1247.30 root type: union - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo └─Selection_11(Probe) 1247.30 cop[tikv] or(and(lt(index_merge.t1.c1, 10), lt(index_merge.t1.c4, 10)), lt(index_merge.t1.c2, 10)), or(lt(index_merge.t1.c3, 10), lt(index_merge.t1.c5, 10)) └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select /*+ use_index_merge(t1) */ * from t1 where ((c1 < 10 and c4 < 10) or c2 < 10) and (c3 < 10 or c4 < 10) order by 1; diff --git a/tests/integrationtest/r/planner/core/issuetest/planner_issue.result b/tests/integrationtest/r/planner/core/issuetest/planner_issue.result index 72bfff299b387..fa909e1c03283 100644 --- a/tests/integrationtest/r/planner/core/issuetest/planner_issue.result +++ b/tests/integrationtest/r/planner/core/issuetest/planner_issue.result @@ -113,10 +113,13 @@ explain select /*+ use_index_merge( tbl_39) */ col_239 from tbl_39 where not( id estRows task access object operator info Projection_8 382.00 root planner__core__issuetest__planner_issue.tbl_39.col_239 └─Limit_15 382.00 root offset:0, count:382 - └─UnionScan_26 382.00 root or(and(not(not(eq(planner__core__issuetest__planner_issue.tbl_39.col_239, 1994))), not(in(planner__core__issuetest__planner_issue.tbl_39.col_239, 2004, 2010, 2010))), and(not(le(planner__core__issuetest__planner_issue.tbl_39.col_239, 1996)), not(and(ge(cast(planner__core__issuetest__planner_issue.tbl_39.col_239, double UNSIGNED BINARY), 2026), le(cast(planner__core__issuetest__planner_issue.tbl_39.col_239, double UNSIGNED BINARY), 2011))))) - └─IndexReader_29 382.00 root index:Selection_28 - └─Selection_28 382.00 cop[tikv] or(and(eq(planner__core__issuetest__planner_issue.tbl_39.col_239, 1994), not(in(planner__core__issuetest__planner_issue.tbl_39.col_239, 2004, 2010, 2010))), and(gt(planner__core__issuetest__planner_issue.tbl_39.col_239, 1996), or(lt(cast(planner__core__issuetest__planner_issue.tbl_39.col_239, double UNSIGNED BINARY), 2026), gt(cast(planner__core__issuetest__planner_issue.tbl_39.col_239, double UNSIGNED BINARY), 2011)))) - └─IndexRangeScan_27 477.50 cop[tikv] table:tbl_39, index:PRIMARY(col_239) range:[1994,1994], (1996,+inf], keep order:true, stats:pseudo + └─UnionScan_23 382.00 root or(and(not(not(eq(planner__core__issuetest__planner_issue.tbl_39.col_239, 1994))), not(in(planner__core__issuetest__planner_issue.tbl_39.col_239, 2004, 2010, 2010))), and(not(le(planner__core__issuetest__planner_issue.tbl_39.col_239, 1996)), not(and(ge(cast(planner__core__issuetest__planner_issue.tbl_39.col_239, double UNSIGNED BINARY), 2026), le(cast(planner__core__issuetest__planner_issue.tbl_39.col_239, double UNSIGNED BINARY), 2011))))) + └─IndexMerge_29 382.00 root type: union + ├─Selection_25(Build) 0.05 cop[tikv] not(in(planner__core__issuetest__planner_issue.tbl_39.col_239, 2004, 2010, 2010)) + │ └─IndexRangeScan_24 0.14 cop[tikv] table:tbl_39, index:PRIMARY(col_239) range:[1994,1994], keep order:true, stats:pseudo + ├─Selection_27(Build) 458.26 cop[tikv] or(lt(cast(planner__core__issuetest__planner_issue.tbl_39.col_239, double UNSIGNED BINARY), 2026), gt(cast(planner__core__issuetest__planner_issue.tbl_39.col_239, double UNSIGNED BINARY), 2011)) + │ └─IndexRangeScan_26 477.36 cop[tikv] table:tbl_39, index:idx_223(col_239) range:(1996,+inf], keep order:true, stats:pseudo + └─TableRowIDScan_28(Probe) 382.00 cop[tikv] table:tbl_39 keep order:false, stats:pseudo select /*+ use_index_merge( tbl_39) */ col_239 from tbl_39 where not( tbl_39.col_239 not in ( '1994' ) ) and tbl_39.col_239 not in ( '2004' , '2010' , '2010' ) or not( tbl_39.col_239 <= '1996' ) and not( tbl_39.col_239 between '2026' and '2011' ) order by tbl_39.col_239 limit 382; col_239 1994