planner: fix uncorrect behavior of index join (#11724) (#11759)

executor/join_test.go

@@ -1390,4 +1390,13 @@ func (s *testSuite2) TestIssue11544(c *C) {
 	tk.MustExec("insert into 11544t values(1)")
 	tk.MustExec("insert into 11544tt values(1, 'aaaaaaa'), (1, 'aaaabbb'), (1, 'aaaacccc')")
 	tk.MustQuery("select /*+ TIDB_INLJ(tt) */ * from 11544t t, 11544tt tt where t.a=tt.a and (tt.b = 'aaaaaaa' or tt.b = 'aaaabbb')").Check(testkit.Rows("1 1 aaaaaaa", "1 1 aaaabbb"))
+	tk.MustQuery("select /*+ TIDB_INLJ(tt) */ * from 11544t t, 11544tt tt where t.a=tt.a and tt.b in ('aaaaaaa', 'aaaabbb', 'aaaacccc')").Check(testkit.Rows("1 1 aaaaaaa", "1 1 aaaabbb", "1 1 aaaacccc"))
+}
+
+func (s *testSuite2) TestIssue11390(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("create table 11390t (k1 int unsigned, k2 int unsigned, key(k1, k2))")
+	tk.MustExec("insert into 11390t values(1, 1)")
+	tk.MustQuery("select /*+ TIDB_INLJ(t1, t2) */ * from 11390t t1, 11390t t2 where t1.k2 > 0 and t1.k2 = t2.k2 and t2.k1=1;").Check(testkit.Rows("1 1 1 1"))
 }

planner/cascades/optimize_test.go

@@ -43,7 +43,7 @@ type testCascadesSuite struct {
 
 func (s *testCascadesSuite) SetUpSuite(c *C) {
 	testleak.BeforeTest()
-	s.is = infoschema.MockInfoSchema([]*model.TableInfo{plannercore.MockTable()})
+	s.is = infoschema.MockInfoSchema([]*model.TableInfo{plannercore.MockSignedTable()})
 	s.sctx = plannercore.MockContext()
 	s.Parser = parser.New()
 }

planner/core/exhaust_physical_plans.go

@@ -292,34 +292,6 @@ func (p *LogicalJoin) getHashJoin(prop *property.PhysicalProperty, innerIdx int)
 	return hashJoin
 }
 
-// joinKeysMatchIndex checks whether the join key is in the index.
-// It returns a slice a[] what a[i] means keys[i] is related with indexCols[a[i]], -1 for no matching column.
-// It will return nil if there's no column that matches index.
-func joinKeysMatchIndex(keys, indexCols []*expression.Column, colLengths []int) []int {
-	keyOff2IdxOff := make([]int, len(keys))
-	for i := range keyOff2IdxOff {
-		keyOff2IdxOff[i] = -1
-	}
-	// There should be at least one column in join keys which can match the index's column.
-	matched := false
-	tmpSchema := expression.NewSchema(keys...)
-	for i, idxCol := range indexCols {
-		if colLengths[i] != types.UnspecifiedLength {
-			continue
-		}
-		keyOff := tmpSchema.ColumnIndex(idxCol)
-		if keyOff == -1 {
-			continue
-		}
-		matched = true
-		keyOff2IdxOff[keyOff] = i
-	}
-	if !matched {
-		return nil
-	}
-	return keyOff2IdxOff
-}
-
 // When inner plan is TableReader, the parameter `ranges` will be nil. Because pk only have one column. So all of its range
 // is generated during execution time.
 func (p *LogicalJoin) constructIndexJoin(
@@ -448,7 +420,10 @@ func (p *LogicalJoin) getIndexJoinByOuterIdx(prop *property.PhysicalProperty, ou
 			continue
 		}
 		indexInfo := path.index
-		err := helper.analyzeLookUpFilters(indexInfo, ds, innerJoinKeys)
+		emptyRange, err := helper.analyzeLookUpFilters(indexInfo, ds, innerJoinKeys)
+		if emptyRange {
+			return nil
+		}
 		if err != nil {
 			logutil.Logger(context.Background()).Warn("build index join failed", zap.Error(err))
 		}
@@ -809,10 +784,10 @@ func (ijHelper *indexJoinBuildHelper) removeUselessEqAndInFunc(
 	return notKeyEqAndIn, nil
 }
 
-func (ijHelper *indexJoinBuildHelper) analyzeLookUpFilters(indexInfo *model.IndexInfo, innerPlan *DataSource, innerJoinKeys []*expression.Column) error {
+func (ijHelper *indexJoinBuildHelper) analyzeLookUpFilters(indexInfo *model.IndexInfo, innerPlan *DataSource, innerJoinKeys []*expression.Column) (emptyRange bool, err error) {
 	idxCols, colLengths := expression.IndexInfo2Cols(innerPlan.schema.Columns, indexInfo)
 	if len(idxCols) == 0 {
-		return nil
+		return false, nil
 	}
 	accesses := make([]expression.Expression, 0, len(idxCols))
 	ijHelper.resetContextForIndex(innerJoinKeys, idxCols, colLengths)
@@ -822,31 +797,34 @@ func (ijHelper *indexJoinBuildHelper) analyzeLookUpFilters(indexInfo *model.Inde
 	matchedKeyCnt := len(ijHelper.curPossibleUsedKeys)
 	// If no join key is matched while join keys actually are not empty. We don't choose index join for now.
 	if matchedKeyCnt <= 0 && len(innerJoinKeys) > 0 {
-		return nil
+		return false, nil
 	}
 	accesses = append(accesses, notKeyEqAndIn...)
 	remained = append(remained, remainedEqAndIn...)
 	lastColPos := matchedKeyCnt + len(notKeyEqAndIn)
 	// There should be some equal conditions. But we don't need that there must be some join key in accesses here.
 	// A more strict check is applied later.
 	if lastColPos <= 0 {
-		return nil
+		return false, nil
 	}
 	// If all the index columns are covered by eq/in conditions, we don't need to consider other conditions anymore.
 	if lastColPos == len(idxCols) {
 		// If there's join key matched index column. Then choose hash join is always a better idea.
 		// e.g. select * from t1, t2 where t2.a=1 and t2.b=1. And t2 has index(a, b).
 		//      If we don't have the following check, TiDB will build index join for this case.
 		if matchedKeyCnt <= 0 {
-			return nil
+			return false, nil
 		}
 		remained = append(remained, rangeFilterCandidates...)
-		ranges, err := ijHelper.buildTemplateRange(matchedKeyCnt, notKeyEqAndIn, nil, false)
+		ranges, emptyRange, err := ijHelper.buildTemplateRange(matchedKeyCnt, notKeyEqAndIn, nil, false)
 		if err != nil {
-			return err
+			return false, err
+		}
+		if emptyRange {
+			return true, nil
 		}
 		ijHelper.updateBestChoice(ranges, indexInfo, accesses, remained, nil)
-		return nil
+		return false, nil
 	}
 	lastPossibleCol := idxCols[lastColPos]
 	lastColManager := &ColWithCmpFuncManager{
@@ -861,37 +839,43 @@ func (ijHelper *indexJoinBuildHelper) analyzeLookUpFilters(indexInfo *model.Inde
 		// e.g. select * from t1, t2 where t2.a=1 and t2.b=1 and t2.c > 10 and t2.c < 20. And t2 has index(a, b, c).
 		//      If we don't have the following check, TiDB will build index join for this case.
 		if matchedKeyCnt <= 0 {
-			return nil
+			return false, nil
 		}
 		colAccesses, colRemained := ranger.DetachCondsForColumn(ijHelper.join.ctx, rangeFilterCandidates, lastPossibleCol)
 		var ranges, nextColRange []*ranger.Range
 		var err error
 		if len(colAccesses) > 0 {
 			nextColRange, err = ranger.BuildColumnRange(colAccesses, ijHelper.join.ctx.GetSessionVars().StmtCtx, lastPossibleCol.RetType, colLengths[lastColPos])
 			if err != nil {
-				return err
+				return false, err
 			}
 		}
-		ranges, err = ijHelper.buildTemplateRange(matchedKeyCnt, notKeyEqAndIn, nextColRange, false)
+		ranges, emptyRange, err = ijHelper.buildTemplateRange(matchedKeyCnt, notKeyEqAndIn, nextColRange, false)
 		if err != nil {
-			return err
+			return false, err
+		}
+		if emptyRange {
+			return true, nil
 		}
 		remained = append(remained, colRemained...)
 		if colLengths[lastColPos] != types.UnspecifiedLength {
 			remained = append(remained, colAccesses...)
 		}
 		accesses = append(accesses, colAccesses...)
 		ijHelper.updateBestChoice(ranges, indexInfo, accesses, remained, nil)
-		return nil
+		return false, nil
 	}
 	accesses = append(accesses, lastColAccess...)
 	remained = append(remained, rangeFilterCandidates...)
-	ranges, err := ijHelper.buildTemplateRange(matchedKeyCnt, notKeyEqAndIn, nil, true)
+	ranges, emptyRange, err := ijHelper.buildTemplateRange(matchedKeyCnt, notKeyEqAndIn, nil, true)
 	if err != nil {
-		return err
+		return false, err
+	}
+	if emptyRange {
+		return true, nil
 	}
 	ijHelper.updateBestChoice(ranges, indexInfo, accesses, remained, lastColManager)
-	return nil
+	return false, nil
 }
 
 func (ijHelper *indexJoinBuildHelper) updateBestChoice(ranges []*ranger.Range, idxInfo *model.IndexInfo, accesses,
@@ -910,7 +894,7 @@ func (ijHelper *indexJoinBuildHelper) updateBestChoice(ranges []*ranger.Range, i
 	}
 }
 
-func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAndInFuncs []expression.Expression, nextColRange []*ranger.Range, haveExtraCol bool) (ranges []*ranger.Range, err error) {
+func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAndInFuncs []expression.Expression, nextColRange []*ranger.Range, haveExtraCol bool) (ranges []*ranger.Range, emptyRange bool, err error) {
 	pointLength := matchedKeyCnt + len(eqAndInFuncs)
 	if nextColRange != nil {
 		for _, colRan := range nextColRange {
@@ -937,49 +921,37 @@ func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAn
 			HighVal: make([]types.Datum, pointLength, pointLength),
 		})
 	}
-	emptyRow := chunk.Row{}
+	sc := ijHelper.join.ctx.GetSessionVars().StmtCtx
 	for i, j := 0, 0; j < len(eqAndInFuncs); i++ {
 		// This position is occupied by join key.
 		if ijHelper.curIdxOff2KeyOff[i] != -1 {
 			continue
 		}
-		sf := eqAndInFuncs[j].(*expression.ScalarFunction)
-		// Deal with the first two args.
-		if _, ok := sf.GetArgs()[0].(*expression.Column); ok {
-			for _, ran := range ranges {
-				ran.LowVal[i], err = sf.GetArgs()[1].Eval(emptyRow)
-				if err != nil {
-					return nil, err
-				}
-				ran.HighVal[i] = ran.LowVal[i]
-			}
-		} else {
-			for _, ran := range ranges {
-				ran.LowVal[i], err = sf.GetArgs()[0].Eval(emptyRow)
-				if err != nil {
-					return nil, err
-				}
-				ran.HighVal[i] = ran.LowVal[i]
-			}
+		oneColumnRan, err := ranger.BuildColumnRange([]expression.Expression{eqAndInFuncs[j]}, sc, ijHelper.curNotUsedIndexCols[j].RetType, ijHelper.curNotUsedColLens[j])
+		if err != nil {
+			return nil, false, err
+		}
+		if len(oneColumnRan) == 0 {
+			return nil, true, nil
+		}
+		for _, ran := range ranges {
+			ran.LowVal[i] = oneColumnRan[0].LowVal[0]
+			ran.HighVal[i] = oneColumnRan[0].HighVal[0]
 		}
-		// If the length of in function's constant list is more than one, we will expand ranges.
 		curRangeLen := len(ranges)
-		for argIdx := 2; argIdx < len(sf.GetArgs()); argIdx++ {
+		for ranIdx := 1; ranIdx < len(oneColumnRan); ranIdx++ {
 			newRanges := make([]*ranger.Range, 0, curRangeLen)
 			for oldRangeIdx := 0; oldRangeIdx < curRangeLen; oldRangeIdx++ {
 				newRange := ranges[oldRangeIdx].Clone()
-				newRange.LowVal[i], err = sf.GetArgs()[argIdx].Eval(emptyRow)
-				if err != nil {
-					return nil, err
-				}
-				newRange.HighVal[i] = newRange.LowVal[i]
+				newRange.LowVal[i] = oneColumnRan[ranIdx].LowVal[0]
+				newRange.HighVal[i] = oneColumnRan[ranIdx].HighVal[0]
 				newRanges = append(newRanges, newRange)
 			}
 			ranges = append(ranges, newRanges...)
 		}
 		j++
 	}
-	return ranges, nil
+	return ranges, false, nil
 }
 
 // tryToGetIndexJoin will get index join by hints. If we can generate a valid index join by hint, the second return value

planner/core/exhaust_physical_plans_test.go

@@ -227,7 +227,7 @@ func (s *testUnitTestSuit) TestIndexJoinAnalyzeLookUpFilters(c *C) {
 		c.Assert(err, IsNil)
 		joinNode.OtherConditions = others
 		helper := &indexJoinBuildHelper{join: joinNode, lastColManager: nil}
-		err = helper.analyzeLookUpFilters(idxInfo, dataSourceNode, tt.innerKeys)
+		_, err = helper.analyzeLookUpFilters(idxInfo, dataSourceNode, tt.innerKeys)
 		c.Assert(err, IsNil)
 		c.Assert(fmt.Sprintf("%v", helper.chosenRanges), Equals, tt.ranges, Commentf("test case: #%v", i))
 		c.Assert(fmt.Sprintf("%v", helper.idxOff2KeyOff), Equals, tt.idxOff2KeyOff)

planner/core/logical_plan_test.go

@@ -48,7 +48,7 @@ type testPlanSuite struct {
 }
 
 func (s *testPlanSuite) SetUpSuite(c *C) {
-	s.is = infoschema.MockInfoSchema([]*model.TableInfo{MockTable(), MockView()})
+	s.is = infoschema.MockInfoSchema([]*model.TableInfo{MockSignedTable(), MockView()})
 	s.ctx = MockContext()
 	s.Parser = parser.New()
 }
@@ -577,7 +577,7 @@ func (s *testPlanSuite) TestDeriveNotNullConds(c *C) {
 
 func buildLogicPlan4GroupBy(s *testPlanSuite, c *C, sql string) (Plan, error) {
 	sqlMode := s.ctx.GetSessionVars().SQLMode
-	mockedTableInfo := MockTable()
+	mockedTableInfo := MockSignedTable()
 	// mock the table info here for later use
 	// enable only full group by
 	s.ctx.GetSessionVars().SQLMode = sqlMode | mysql.ModeOnlyFullGroupBy

planner/core/mock.go

@@ -39,8 +39,8 @@ func newDateType() types.FieldType {
 	return *ft
 }
 
-// MockTable is only used for plan related tests.
-func MockTable() *model.TableInfo {
+// MockSignedTable is only used for plan related tests.
+func MockSignedTable() *model.TableInfo {
 	// column: a, b, c, d, e, c_str, d_str, e_str, f, g
 	// PK: a
 	// indeices: c_d_e, e, f, g, f_g, c_d_e_str, c_d_e_str_prefix
@@ -263,6 +263,75 @@ func MockTable() *model.TableInfo {
 	return table
 }
 
+// MockUnsignedTable is only used for plan related tests.
+func MockUnsignedTable() *model.TableInfo {
+	// column: a, b
+	// PK: a
+	// indeices: b
+	indices := []*model.IndexInfo{
+		{
+			Name: model.NewCIStr("b"),
+			Columns: []*model.IndexColumn{
+				{
+					Name:   model.NewCIStr("b"),
+					Length: types.UnspecifiedLength,
+					Offset: 1,
+				},
+			},
+			State:  model.StatePublic,
+			Unique: true,
+		},
+		{
+			Name: model.NewCIStr("b_c"),
+			Columns: []*model.IndexColumn{
+				{
+					Name:   model.NewCIStr("b"),
+					Length: types.UnspecifiedLength,
+					Offset: 1,
+				},
+				{
+					Name:   model.NewCIStr("c"),
+					Length: types.UnspecifiedLength,
+					Offset: 2,
+				},
+			},
+			State: model.StatePublic,
+		},
+	}
+	pkColumn := &model.ColumnInfo{
+		State:     model.StatePublic,
+		Offset:    0,
+		Name:      model.NewCIStr("a"),
+		FieldType: newLongType(),
+		ID:        1,
+	}
+	col0 := &model.ColumnInfo{
+		State:     model.StatePublic,
+		Offset:    1,
+		Name:      model.NewCIStr("b"),
+		FieldType: newLongType(),
+		ID:        2,
+	}
+	col1 := &model.ColumnInfo{
+		State:     model.StatePublic,
+		Offset:    2,
+		Name:      model.NewCIStr("c"),
+		FieldType: newLongType(),
+		ID:        3,
+	}
+	pkColumn.Flag = mysql.PriKeyFlag | mysql.NotNullFlag | mysql.UnsignedFlag
+	// Column 'b', 'c', 'd', 'f', 'g' is not null.
+	col0.Flag = mysql.NotNullFlag
+	col1.Flag = mysql.UnsignedFlag
+	table := &model.TableInfo{
+		Columns:    []*model.ColumnInfo{pkColumn, col0, col1},
+		Indices:    indices,
+		Name:       model.NewCIStr("t2"),
+		PKIsHandle: true,
+	}
+	return table
+}
+
 // MockView is only used for plan related tests.
 func MockView() *model.TableInfo {
 	selectStmt := "select b,c,d from t"
@@ -308,7 +377,7 @@ func MockContext() sessionctx.Context {
 
 // MockPartitionInfoSchema mocks an info schema for partition table.
 func MockPartitionInfoSchema(definitions []model.PartitionDefinition) infoschema.InfoSchema {
-	tableInfo := MockTable()
+	tableInfo := MockSignedTable()
 	cols := make([]*model.ColumnInfo, 0, len(tableInfo.Columns))
 	cols = append(cols, tableInfo.Columns...)
 	last := tableInfo.Columns[len(tableInfo.Columns)-1]