Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ddl: support add global index operation on partition table #18402

Merged
merged 23 commits into from
Aug 21, 2020
Merged

ddl: support add global index operation on partition table #18402

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
47f82e1
ddl: support add global index
ldeng-ustc Jul 5, 2020
55b09e2
Merge branch 'master' of github.com:pingcap/tidb
ldeng-ustc Jul 7, 2020
58f90b7
ddl: add config value 'EnableGlobalIndex'
ldeng-ustc Jul 9, 2020
5d47ec6
Merge branch 'master' of github.com:pingcap/tidb into global_index
ldeng-ustc Jul 9, 2020
b15b1f2
fix
ldeng-ustc Jul 9, 2020
2e04b6c
tests: update testcase for creating global index
ldeng-ustc Jul 9, 2020
e5284c6
go.mod: updade parser to support IndexInfo.Global
ldeng-ustc Jul 9, 2020
2faf246
go.mod: updade parser to support IndexInfo.Global
ldeng-ustc Jul 9, 2020
505a620
fix
ldeng-ustc Jul 10, 2020
19b0a97
Merge branch 'master' of github.com:pingcap/tidb into global_index
ldeng-ustc Jul 27, 2020
2febc3a
tablecodec: refactor GenIndexValue for extensibility
ldeng-ustc Jul 27, 2020
9ef4eab
tablecodec: add decodeIndexKvGeneral for extensible index decoding
ldeng-ustc Aug 1, 2020
c933af0
tablecodec: use decodeIndexKvGeneral in DecodeIndexKV
ldeng-ustc Aug 2, 2020
5616d86
Merge branch 'master' of github.com:pingcap/tidb into global_index
ldeng-ustc Aug 2, 2020
be23f72
table: refine comments of index value layout
ldeng-ustc Aug 2, 2020
2ad6198
fix comment
ldeng-ustc Aug 2, 2020
4d99cc3
fix fmt
ldeng-ustc Aug 3, 2020
3dab738
Merge remote-tracking branch 'pingcap/master' into global_index
ldeng-ustc Aug 17, 2020
cf92788
fix comment
ldeng-ustc Aug 18, 2020
3d75458
fix make check fail
ldeng-ustc Aug 18, 2020
4286a90
Merge remote-tracking branch 'pingcap/master' into global_index
ldeng-ustc Aug 20, 2020
516bbf0
Merge branch 'master' of github.com:pingcap/tidb into global_index
ldeng-ustc Aug 20, 2020
2862e06
Merge branch 'master' into global_index
ti-srebot Aug 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions ddl/index.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -468,6 +468,9 @@ func (w *worker) onCreateIndex(d *ddlCtx, t *meta.Meta, job *model.Job, isPK boo
} else {
indexInfo.Tp = indexOption.Tp
}
if indexOption.Locality == ast.IndexLocalityGlobal {
indexInfo.Global = true
tiancaiamao marked this conversation as resolved.
Show resolved Hide resolved
}
} else {
// Use btree as default index type.
indexInfo.Tp = model.IndexTypeBtree
Expand Down
184 changes: 111 additions & 73 deletions table/tables/index.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,11 +99,19 @@ func (c *index) checkContainNonBinaryString() bool {

// NewIndex builds a new Index object.
func NewIndex(physicalID int64, tblInfo *model.TableInfo, indexInfo *model.IndexInfo) table.Index {
// The prefix can't encode from tblInfo.ID, because table partition may change the id to partition id.
var prefix kv.Key
if indexInfo.Global {
// In glabal index of partition table, prefix start with tblInfo.ID.
prefix = tablecodec.EncodeTableIndexPrefix(tblInfo.ID, indexInfo.ID)
} else {
// Otherwise, start with physicalID.
prefix = tablecodec.EncodeTableIndexPrefix(physicalID, indexInfo.ID)
}
index := &index{
idxInfo: indexInfo,
tblInfo: tblInfo,
// The prefix can't encode from tblInfo.ID, because table partition may change the id to partition id.
prefix: tablecodec.EncodeTableIndexPrefix(physicalID, indexInfo.ID),
idxInfo: indexInfo,
tblInfo: tblInfo,
prefix: prefix,
phyTblID: physicalID,
}
index.containNonBinaryString = index.checkContainNonBinaryString()
Expand All @@ -118,99 +126,129 @@ func (c *index) Meta() *model.IndexInfo {
// GenIndexKey generates storage key for index values. Returned distinct indicates whether the
// indexed values should be distinct in storage (i.e. whether handle is encoded in the key).
func (c *index) GenIndexKey(sc *stmtctx.StatementContext, indexedValues []types.Datum, h kv.Handle, buf []byte) (key []byte, distinct bool, err error) {
return tablecodec.GenIndexKey(sc, c.tblInfo, c.idxInfo, c.phyTblID, indexedValues, h, buf)
idxTblID := c.phyTblID
if c.idxInfo.Global {
idxTblID = c.tblInfo.ID
}
return tablecodec.GenIndexKey(sc, c.tblInfo, c.idxInfo, idxTblID, indexedValues, h, buf)
}

// Create creates a new entry in the kvIndex data.
// If the index is unique and there is an existing entry with the same key,
// Create will return the existing entry's handle as the first return value, ErrKeyExists as the second return value.
// Value layout:
// +--With Restore Data(for indices on string columns)
// +--Local Index
// | |
// | +--Non Unique (TailLen = len(PaddingData) + len(Flag), TailLen < 8 always)
// | | |
// | | +--Without Untouched Flag:
// | +--With Restore Data(for indices on string columns)
// | | |
// | | | Layout: TailLen | RestoreData | PaddingData
// | | | Length: 1 | size(RestoreData) | size(paddingData)
// | | +--Non Unique (TailLen = len(PaddingData) + len(Flag), TailLen < 8 always)
// | | | |
// | | | +--Without Untouched Flag:
// | | | |
// | | | | Layout: TailLen | RestoreData | PaddingData
// | | | | Length: 1 | size(RestoreData) | size(paddingData)
// | | | |
// | | | | The length >= 10 always because of padding.
// | | | |
// | | | +--With Untouched Flag:
// | | |
// | | | The length >= 10 always because of padding.
// | | | Layout: TailLen | RestoreData | PaddingData | Flag
// | | | Length: 1 | size(RestoreData) | size(paddingData) | 1
// | | |
// | | +--With Untouched Flag:
// | |
// | | Layout: TailLen | RestoreData | PaddingData | Flag
// | | Length: 1 | size(RestoreData) | size(paddingData) | 1
// | |
// | | The length >= 11 always because of padding.
// | |
// | +--Unique Common Handle
// | | | The length >= 11 always because of padding.
// | | |
// | | +--Without Untouched Flag:
// | | +--Unique Common Handle
// | | | |
// | | | +--Without Untouched Flag:
// | | | |
// | | | | Layout: 0x00 | CHandle Flag | CHandle Len | CHandle | RestoreData
// | | | | Length: 1 | 1 | 2 | size(CHandle) | size(RestoreData)
// | | | |
// | | | | The length > 10 always because of CHandle size.
// | | | |
// | | | +--With Untouched Flag:
// | | |
// | | | Layout: 0x00 | CHandle Flag | CHandle Len | CHandle | RestoreData
// | | | Length: 1 | 1 | 2 | size(CHandle) | size(RestoreData)
// | | | Layout: 0x01 | CHandle Flag | CHandle Len | CHandle | RestoreData | Flag
// | | | Length: 1 | 1 | 2 | size(CHandle) | size(RestoreData) | 1
// | | |
// | | | The length > 10 always because of CHandle size.
// | | | The length > 10 always because of CHandle size.
// | | |
// | | +--With Untouched Flag:
// | | +--Unique Integer Handle (TailLen = len(Handle) + len(Flag), TailLen == 8 || TailLen == 9)
// | | |
// | | +--Without Untouched Flag:
// | | |
// | | | Layout: 0x08 | RestoreData | Handle
// | | | Length: 1 | size(RestoreData) | 8
// | | |
// | | | The length >= 10 always since size(RestoreData) > 0.
// | | |
// | | +--With Untouched Flag:
// | |
// | | Layout: 0x01 | CHandle Flag | CHandle Len | CHandle | RestoreData | Flag
// | | Length: 1 | 1 | 2 | size(CHandle) | size(RestoreData) | 1
// | | Layout: 0x09 | RestoreData | Handle | Flag
// | | Length: 1 | size(RestoreData) | 8 | 1
// | |
// | | The length > 10 always because of CHandle size.
// | | The length >= 11 always since size(RestoreData) > 0.
// | |
// | +--Unique Integer Handle (TailLen = len(Handle) + len(Flag), TailLen == 8 || TailLen == 9)
// | +--Without Restore Data
// | |
// | +--Without Untouched Flag:
// | +--Non Unique
// | | |
// | | +--Without Untouched Flag:
// | | |
// | | | Layout: '0'
// | | | Length: 1
// | | |
// | | +--With Untouched Flag:
// | |
// | | Layout: 0x08 | RestoreData | Handle
// | | Length: 1 | size(RestoreData) | 8
// | | Layout: Flag
// | | Length: 1
// | |
// | | The length >= 10 always since size(RestoreData) > 0.
// | +--Unique Common Handle
// | | |
// | | +--Without Untouched Flag:
// | | |
// | | | Layout: 0x00 | CHandle Flag | CHandle Len | CHandle
// | | | Length: 1 | 1 | 2 | size(CHandle)
// | | |
// | | +--With Untouched Flag:
// | |
// | +--With Untouched Flag:
// |
// | Layout: 0x09 | RestoreData | Handle | Flag
// | Length: 1 | size(RestoreData) | 8 | 1
// |
// | The length >= 11 always since size(RestoreData) > 0.
// | | Layout: 0x01 | CHandle Flag | CHandle Len | CHandle | Flag
// | | Length: 1 | 1 | 2 | size(CHandle) | 1
// | |
// | +--Unique Integer Handle
// | |
// | +--Without Untouched Flag:
// | |
// | | Layout: Handle
// | | Length: 8
// | |
// | +--With Untouched Flag:
// |
// +--Without Restore Data
// | Layout: Handle | Flag
// | Length: 8 | 1
// |
// +--Non Unique
// +--Global Index (Similar to Local Index, except partitionID is append to the end.)
tiancaiamao marked this conversation as resolved.
Show resolved Hide resolved
// | |
// | +--Without Untouched Flag:
// | |
// | | Layout: '0'
// | | Length: 1
// | |
// | +--With Untouched Flag:
// |
// | Layout: Flag
// | Length: 1
// +--Unique Common Handle
// | |
// | +--Without Untouched Flag:
// | |
// | | Layout: 0x00 | CHandle Flag | CHandle Len | CHandle
// | | Length: 1 | 1 | 2 | size(CHandle)
// | |
// | +--With Untouched Flag:
// |
// | Layout: 0x01 | CHandle Flag | CHandle Len | CHandle | Flag
// | Length: 1 | 1 | 2 | size(CHandle) | 1
// |
// +--Unique Integer Handle
// |
// +--Without Untouched Flag:
// |
// | Layout: Handle
// | Length: 8
// |
// +--With Untouched Flag:
// | +--With Restore Data(for indices on string columns)
// | | |
// | | +--Non Unique (TailLen = len(PaddingData) + len(Flag), TailLen < 8 always)
// | | | |
// | | | +--Without Untouched Flag:
// | | | |
// | | | | Layout: TailLen | RestoreData | PaddingData | PartitionID |
// | | | | Length: 1 | size(RestoreData) | size(paddingData) | 8 |
// | | | |
// | | | | The length >= 18 always because of padding.
// | | | |
// | | | +--With Untouched Flag:
// | | |
// | | | Layout: TailLen | RestoreData | PaddingData | Flag | PartitionID |
// | | | Length: 1 | size(RestoreData) | size(paddingData) | 1 | 8 |
// | | |
// | | | The length >= 19 always because of padding.
// | | |
// ... (Truncated because of length.)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why truncated?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

New layout of local index is just add 8 bytes partitionID at the end, rest part is same as local index . I think 2 examples is enough. If write whole layout here, it seems too long.

//
// Layout: Handle | Flag
// Length: 8 | 1
func (c *index) Create(sctx sessionctx.Context, rm kv.RetrieverMutator, indexedValues []types.Datum, h kv.Handle, opts ...table.CreateIdxOptFunc) (kv.Handle, error) {
var opt table.CreateIdxOpt
for _, fn := range opts {
Expand Down Expand Up @@ -242,7 +280,7 @@ func (c *index) Create(sctx sessionctx.Context, rm kv.RetrieverMutator, indexedV
// save the key buffer to reuse.
writeBufs.IndexKeyBuf = key
idxVal, err := tablecodec.GenIndexValue(sctx.GetSessionVars().StmtCtx, c.tblInfo, c.idxInfo,
c.containNonBinaryString, distinct, opt.Untouched, indexedValues, h)
c.containNonBinaryString, distinct, opt.Untouched, indexedValues, h, c.phyTblID)
if err != nil {
return nil, err
}
Expand Down
7 changes: 5 additions & 2 deletions tablecodec/tablecodec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -971,8 +971,8 @@ func GenIndexKey(sc *stmtctx.StatementContext, tblInfo *model.TableInfo, idxInfo
}

// GenIndexValue creates encoded index value and returns the result
func GenIndexValue(sc *stmtctx.StatementContext, tblInfo *model.TableInfo, idxInfo *model.IndexInfo,
containNonBinaryString bool, distinct bool, untouched bool, indexedValues []types.Datum, h kv.Handle) ([]byte, error) {
func GenIndexValue(sc *stmtctx.StatementContext, tblInfo *model.TableInfo, idxInfo *model.IndexInfo, containNonBinaryString bool,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If there are multiple reference of this function in other packages, it's better to introduce a new function such as GenGlobalIndexValue instead of changing the API

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This API does not change, I just split a long single line into two.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You add a partitionID int64, right?

distinct bool, untouched bool, indexedValues []types.Datum, h kv.Handle, partitionID int64) ([]byte, error) {
var idxVal []byte
if collate.NewCollationEnabled() && containNonBinaryString {
colIds := make([]int64, len(idxInfo.Columns))
Expand Down Expand Up @@ -1025,6 +1025,9 @@ func GenIndexValue(sc *stmtctx.StatementContext, tblInfo *model.TableInfo, idxIn
idxVal = []byte{'0'}
}
}
if idxInfo.Global {
idxVal = append(idxVal, codec.EncodeInt(nil, partitionID)...)
}
return idxVal, nil
}

Expand Down