Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

backup: implement store change notifier #20

Open
wants to merge 229 commits into
base: store_backup
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
229 commits
Select commit Hold shift + click to select a range
65817ac
placement: fix alter placement policy cannot update the relative rang…
jiyfhust May 7, 2024
b1b0995
ddl: check local file existence before resume checkpoint (#53072)
lance6716 May 7, 2024
81a7cf5
variable: disable tidb_enable_historical_stats by default (#53046)
hawkingrei May 7, 2024
264cd4e
planner: move QueryTimeRange structure to util pkg (#53071)
AilinKid May 7, 2024
5798dc6
Revert "planner: merge fast plan cache and normal plan cache" (#53084)
qw4990 May 7, 2024
96f9797
tablecodec: fix the issue that decoding an index value might panic (#…
zyguan May 7, 2024
013a4e6
expression: update md5 hash to use hex (#53003)
qichengzx May 7, 2024
554a5ff
CI: Update actions (#53078)
dveeden May 7, 2024
2b754c7
planner: simplify plan cache for fast point get (#53090)
qw4990 May 8, 2024
a177111
infoschema: keep the timestamp of infoschema v2 updating (#52983)
tiancaiamao May 8, 2024
fe7c324
extension: make `RelatedTables` work when the statement fails (#50989)
CbcWestwolf May 8, 2024
788d505
planner: move handle cols to planner util (#53093)
AilinKid May 8, 2024
9c5981c
store/copr: fix tracing for `copr.buildCopTasks` (#53086)
time-and-fate May 8, 2024
57b382b
executor,types: Fix truncate function behavior when second param is l…
yibin87 May 8, 2024
50b8dfa
planner: simplify plan cache for fast point get (#53094)
qw4990 May 8, 2024
a5c4031
planner: move table sample logic to planner util (#53100)
AilinKid May 8, 2024
184c76b
br: fix checkpoint cannot advance after pause->stop->start (#53091)
RidRisR May 8, 2024
1a1e7b5
testkit: enable infoschema v2 for realtikv test (#52896)
tiancaiamao May 8, 2024
f304f04
br: Fail backup if any snapshots failed (#53038)
michaelmdeng May 8, 2024
c33b6cd
expression: `PropagateType` should never let orignal data overflow (#…
windtalker May 9, 2024
12b37d8
ddl: fix the primary key in index is not in restored format (#53118)
djshow832 May 9, 2024
11ccc2e
infoschema: fix the nil pointer (#53096)
rleungx May 9, 2024
f0a7447
planner: move cost factors to core/cost pkg (#53120)
AilinKid May 9, 2024
9bf3500
executor: fix mppIterator memory leak when got error (#53106)
guo-shaoge May 9, 2024
58469bb
planner: simplify plan cache for fast point get (#53135)
qw4990 May 9, 2024
97d4705
planner: resolve panic when the update lists contain complex subquery…
winoros May 9, 2024
203e5f2
executor: Fix push downed topN won't replace correlated column proble…
yibin87 May 10, 2024
341503b
planner: merge fast plan cache and normal plan cache (#53143)
qw4990 May 10, 2024
dded4ea
build(deps): bump golang.org/x/oauth2 from 0.18.0 to 0.20.0 (#53160)
dependabot[bot] May 10, 2024
87247c1
planner: move code of rebuilding cached plans into a separate file (#…
qw4990 May 10, 2024
43fd1b2
*: upgrade go1.21.10 (#53123)
hawkingrei May 10, 2024
49f09fe
statistics: support adaptive sync load concurrency (#53142)
hawkingrei May 10, 2024
f46db20
*: caller can use a fixed TS for fast reorg (#52993)
lance6716 May 10, 2024
9831208
lightning/config: align the behaviour of tidb.tls to doc (#53140)
lance6716 May 10, 2024
5d780eb
executor: disable probe short path in hash join (#53079)
windtalker May 11, 2024
1bac22a
infoschema: fix flashback integration test to make it more stable (#5…
tiancaiamao May 11, 2024
6d92e7a
point-get: calculate the row level checksum on the real time to avoid…
3AceShowHand May 11, 2024
af825ac
ddl,mdl: fix the issue that the table_ids overlap regex is not correc…
YangKeao May 11, 2024
536cf00
expression: wrong result of timestampadd(month,1,date '2024-01-31') (…
xzhangxian1008 May 11, 2024
09c8f96
planner: fix the issue that UnionAll didn't handle the range bump cas…
YangKeao May 11, 2024
ce74191
planner: move hash encoding logic and HasMaxOneRow to util (#53161)
AilinKid May 11, 2024
f311d77
expression: unify casting real to string in tidb (tikv#16975) (#53129)
solotzg May 11, 2024
dcd1fa9
expression: fix the collation of functions with json arguments (#53126)
YangKeao May 11, 2024
dd177a8
lightning: update util.go (#53204)
eltociear May 13, 2024
3239d12
executor: remove unnecessary memory allocation in (*HashAggExec).init…
xzhangxian1008 May 13, 2024
601e21c
ddl: add context cancel check before commit (#53134)
tangenta May 13, 2024
e5651c4
planner: optimize performance of `IndexUsage` utilities (#53195)
YangKeao May 13, 2024
fa94f49
planner: move loical plan functionality AppendCandidate4PhysicalOptim…
AilinKid May 13, 2024
4674b12
expression: fix charset conversion warning and error behavior (#51191)
YangKeao May 13, 2024
8ef6667
ttl: use static expression context to calculate the TTL expire time (…
lcwangchao May 13, 2024
41ce0a5
executor: add a panic guard for stmtsummary (#53216)
xhebox May 13, 2024
b1818cd
*: global index support `admin check table | index` (#53156)
Defined2014 May 13, 2024
4fe70da
planner: move getTaskPlanCost to planner core util (#53218)
AilinKid May 13, 2024
37b29bd
expression: clarify the `cast(... as char array)` error message (#53220)
YangKeao May 13, 2024
3750495
planner: fix wrong empty projection (#53235)
winoros May 13, 2024
78549f3
executor: ddl jobs microsecond (#52930)
dveeden May 13, 2024
4b91fee
expression: Report error when empty pattern in regexp related functio…
yibin87 May 14, 2024
041f700
*: fix issue of doesn't fast fail request when no available connectio…
crazycs520 May 14, 2024
98e5cfb
planner: fix wrong behavior for = all() (#52801)
hawkingrei May 14, 2024
a2037fe
planner: collect all columns meta by sync load (#53137)
hawkingrei May 14, 2024
46d842f
lightning: fix unstable test for conflict detection (#53225)
lyzx2001 May 14, 2024
a011759
planner: encapsulate baseLogicalPlan push down topN logic and move ad…
AilinKid May 14, 2024
be80474
*: some tiny optimizations to reduce infoschema v2 memory (#53242)
tiancaiamao May 14, 2024
39383e3
test: stabilize TestTruncateTableAndSchemaDependence (#53245)
tangenta May 14, 2024
045a5b5
expression: reverts #53129 (#53259)
solotzg May 14, 2024
acdb6f5
planner: UPDATE's select plan's output col IDs should be stable (#53268)
winoros May 14, 2024
6aef624
planner: move invalidTask to task base pkg. (#53267)
AilinKid May 15, 2024
2d552c0
planner: make `var_samp` can be used as a window function (#53130)
Rustin170506 May 15, 2024
f0af776
stmtctx, context: remove the redundant implementation of warnings/ext…
YangKeao May 15, 2024
76133dc
infoschema: speedup SchemaByID (#53283)
D3Hunter May 15, 2024
201f223
planner: remove useless code (#53280)
hawkingrei May 15, 2024
63cf3e5
ddl: Better error when check constraint is off (#52814)
dveeden May 15, 2024
4eca4eb
executor: Support TopN Spill (#51809)
xzhangxian1008 May 15, 2024
9eece67
add unit test for main loop
3pointer May 15, 2024
59c7da1
ddl: fix a bug about setting MaxIndexLength (#53284)
zimulala May 15, 2024
4538a21
ddl: add owner_id field to tidb_mdl_info to avoid unexpected writes (…
tangenta May 15, 2024
b9b330a
br/restore: use a more relax check for changefeeds when restoring (#5…
YuJuncen May 15, 2024
687f39c
statistics: fix wrong behavior for primary key' non-lite init stats (…
hawkingrei May 15, 2024
5d27b73
planner, statistics: async load should load all column meta info for …
winoros May 15, 2024
9da001d
Optimizer: Refactor and simplify outer to inner join conversion rule …
ghazalfamilyusa May 16, 2024
a269fbb
more tests
3pointer May 16, 2024
6612270
planner: column pruning should use a shallow copy slice (#53237)
winoros May 16, 2024
d650c27
rename some functions
3pointer May 16, 2024
cc127c1
ddl: change interface of lightning package wrappers (#53233)
lance6716 May 16, 2024
71821b7
*: fix query panic when query exceeded resource group quota limitatio…
crazycs520 May 16, 2024
a3e6d1c
statistics: split async load package (#53318)
hawkingrei May 16, 2024
e1a6b1d
*: check delete unique key's handle to handle corner case (#52975)
wjhuang2016 May 16, 2024
0875abd
ddl: do not reference owner_id for specific system DDL (#53332)
tangenta May 16, 2024
19a6436
test store changes
3pointer May 16, 2024
835ea04
backup: implement store based backup to solve the long tail issue (#5…
3pointer May 16, 2024
125829c
session: set sql security of `tidb_mdl_view` to 'invoker' (#53265)
tangenta May 16, 2024
2809439
table, executor: let `SplitIndexValue` work with clustered index tabl…
Defined2014 May 16, 2024
d8052e7
docs: design for extension auth plugin (#53182)
yzhan1 May 16, 2024
491740b
planner: move checks for noop functions in select statements to prepr…
henrybw May 17, 2024
180de9f
Merge branch 'master' into store_backup_part2
3pointer May 17, 2024
35e1bbc
update
3pointer May 17, 2024
1c4a9c6
br: separate the restore client (#53197)
Leavrth May 17, 2024
5d990c6
ddl: sync schema version using watch, notify sessions on owner node b…
D3Hunter May 17, 2024
9c89227
*: only output log when error is non-nil (#53350)
xhebox May 17, 2024
f3ca2db
fix build
3pointer May 17, 2024
9581442
tests: move `TestNoopFunctions` to `tests/integrationtest` (#53343)
henrybw May 17, 2024
6abe044
fix build
3pointer May 17, 2024
4d6715f
autoid_service: remove the force rebase on close behavior (#53315)
tiancaiamao May 17, 2024
68db9c0
infoschema: change datetime fields of ddl_jobs to datetime(6) (#53339)
tangenta May 17, 2024
2037a2f
ddl: do not reference owner_id for all system DDLs (#53355)
tangenta May 17, 2024
2ef3ac9
planner: skip fast-point-get for select-into-outfile statements (#53364)
qw4990 May 17, 2024
335af25
config: set max token limit to 1M (#53361)
djshow832 May 17, 2024
f5ac93e
executor: add a new column `is_global` into `information_schema.tidb_…
Defined2014 May 17, 2024
397a460
util/security: support client TLS verifies CommonName of server (#53358)
lance6716 May 20, 2024
61bf3d6
fix race
3pointer May 20, 2024
b1a19dc
add more cases
3pointer May 20, 2024
634b190
fix typo
3pointer May 20, 2024
68219b9
*: fix some typos in code comments (#53371)
lilin90 May 20, 2024
b7e32bc
planner: fix the wrong result caused by `year_col cmp out-of-range-ui…
qw4990 May 20, 2024
8ebe981
rename some function & refactor code for test
3pointer May 20, 2024
044f113
expression: fix default value of `GetDefaultCollationForUTF8MB4` of `…
lcwangchao May 20, 2024
dee4b45
ddl: remove `mock.Context` ref in `ddl/session/session_pool.go` (#53378)
lcwangchao May 20, 2024
2c76b1a
docs: update README.md (#53373)
DanRoscigno May 20, 2024
b3d740f
planner: stop pushing TopN down through Projection if it has undeterm…
qw4990 May 20, 2024
b69f81b
copr: add ballast mem to reduce the runtime overhead during execution…
you06 May 21, 2024
2e756fc
tests: speed up integration test (#53426)
Defined2014 May 21, 2024
35e4c84
docs: update readme to make heading and description match (#53434)
lilin90 May 21, 2024
a4f2d19
fix build
3pointer May 21, 2024
07508df
ddl: introduce `newReorgExprCtx` to replace `mock.Context` usage (#53…
lcwangchao May 21, 2024
dd92e5f
fix
3pointer May 21, 2024
aeafb18
lightning: when lightning exits while processing the task, it should …
zeminzhou May 21, 2024
5f0c7f6
doc: add Hash join v2 design doc (#53196)
windtalker May 21, 2024
67531d0
lightning: provide view for users when using conflict detection (#53289)
lyzx2001 May 21, 2024
6962b7c
fix leak
3pointer May 21, 2024
53bf76f
variable: remove text as a default type in the tidb_analyze_skip_colu…
hawkingrei May 21, 2024
def7c23
*: adaptively control analyze distsql concurrency (#53261)
hawkingrei May 21, 2024
61a2981
Optimizer: Fix null filtering logic for IN list (#53370)
ghazalfamilyusa May 21, 2024
0bbc5e0
update
3pointer May 22, 2024
d1b8c5b
ddl: refine fatal log when upgrade failed because of tidb_enable_dist…
tangenta May 22, 2024
3b784c6
Merge branch 'master' into store_backup_part2
3pointer May 22, 2024
0a861de
update
3pointer May 22, 2024
0afe54d
*: fix unstable assert for location (#53376)
lcwangchao May 22, 2024
6928519
planner: fix mpp final agg couldn't co-exist with other non-final mod…
AilinKid May 22, 2024
78e4db9
planner: make TestMppAggShouldAlignFinalMode test case stable (#53484)
AilinKid May 22, 2024
a6b4fca
table: simplify the locate partition args for `PartitionedTable` (#53…
lcwangchao May 22, 2024
e954bcf
*: not load unnecessay data during the non-lite-init-stats (#53399)
hawkingrei May 22, 2024
595cc15
br/cdcutil: don't skip failed changefeeds (#53482)
YuJuncen May 22, 2024
fe5858b
ranger: handle longlong overflow properly (#52365)
hawkingrei May 23, 2024
d6308af
planner: skip all system tables when collecting prediction columns (#…
Rustin170506 May 23, 2024
22dc064
lightning: update `parquet-go` to optimize the memory usage when impo…
zeminzhou May 23, 2024
322f87a
test: fix flaky test TestSyncJobSchemaVerLoop (#53507)
D3Hunter May 23, 2024
3004c07
copIterator: return context error to avoid return incorrect result on…
D3Hunter May 23, 2024
c973157
expression: Disallow conv fuction with hybrid type argement to be pus…
yibin87 May 23, 2024
3485857
ranger: handle float/integers overflow properly (#53501)
hawkingrei May 23, 2024
100aa05
sysvar: tidb_enable_column_tracking only takes effect when tidb_persi…
Rustin170506 May 23, 2024
1087388
executor: global index supports `admin [recover | cleanup] index` (#5…
Defined2014 May 24, 2024
4864408
OWNERS: Auto Sync OWNERS files from community membership (#53527)
ti-chi-bot May 24, 2024
81f4141
planner: fix panic error when subquery + always true predicate in whe…
elsa0520 May 24, 2024
027de7a
planner: remove some restrictions for global index (#53509)
Defined2014 May 24, 2024
9657f06
store: remove stores that have no region before balance (#52314)
xzhangxian1008 May 24, 2024
8f958de
ttl: const default ttl job interval (#53512)
ystaticy May 24, 2024
45318da
expression: fix the fsp of casting json as datetime/duration (#53535)
YangKeao May 25, 2024
0ac2ad0
infoschema: fix issue of information schema cache miss cause by schem…
crazycs520 May 27, 2024
d24fdb7
domain,infoschema: put the @@tidb_schema_cache_size variable into eff…
tiancaiamao May 27, 2024
c333f39
test: fix flaky test TestAlterAddConstraintStateChange3 (#53557)
D3Hunter May 27, 2024
df64c34
metrics: fix wrong value in the sync load QPS (#53559)
hawkingrei May 27, 2024
9164182
config: must set line terminator when use strict-format (#53444)
lance6716 May 27, 2024
e6d1899
*: upgrade failpoint version (#53563)
D3Hunter May 27, 2024
33abd63
use a more efficient way to locate the range tree for backup response
3pointer May 27, 2024
95f57ca
fix typo
3pointer May 27, 2024
153eb96
planner: make TestMppAggShouldAlignFinalMode stable (#53566)
AilinKid May 27, 2024
2300ac6
statistics: add comments and remove unnessary check (#53568)
Rustin170506 May 27, 2024
57d0b40
planner: move base logical plan to logical operator pkg. (#53293)
AilinKid May 27, 2024
c02a2b1
ddl,expression: introduce `CtxWithHandleTruncateErrLevel` to wrap a e…
lcwangchao May 28, 2024
f7cea6c
ddl: support create sequences with `BatchCreateTableWithInfo` (#53465)
tangenta May 28, 2024
d49a654
test: switch some failpoints to InjectCall (#53588)
D3Hunter May 28, 2024
7edf08e
executor: fix data race in topn (#53593)
xzhangxian1008 May 28, 2024
97aac23
util: move server.TokenLimiter into util (#53598)
hawkingrei May 28, 2024
68d1295
expression: fix wrong result when convert float to unsigned (#53590)
guo-shaoge May 28, 2024
bdb0545
statistics: make sure that get max table id when to init stats (#53606)
hawkingrei May 28, 2024
04cb62d
br: add more options for br sql client (#53039)
RidRisR May 28, 2024
d2d1257
planner: fix privilege for the view in the CTE and wrong error handl…
hawkingrei May 28, 2024
772db20
statistics: refine comments to FMSketch (#53601)
Rustin170506 May 28, 2024
bf704fd
domain: make the transaction from `initStatsCtx` blocking gc (#53602)
you06 May 28, 2024
4bf624b
ddl: introduce `newDefaultReorgDistSQLCtx` to replace mock context in…
lcwangchao May 28, 2024
4a51974
statistics: refactor write merge-global-stats (#53549)
hawkingrei May 28, 2024
dd372a7
BatchCreateTableWith jobs: concat query to support cdc (#53515)
GMHDBJD May 28, 2024
e560d03
ddl: remove `mock.Context` usage in `addIndexIngestWorker` (#53479)
lcwangchao May 28, 2024
afd2343
util: use runtime.Version() to get Go version (#53619)
dveeden May 29, 2024
6efce0f
executor: sync deletable columns to binlog when remove record (#53617)
tangenta May 29, 2024
c9c865c
test: fix flaky test TestParallelDDL (#53636)
D3Hunter May 29, 2024
da49e13
ddl: change finishedWritingNeedImport to importStarted (#53643)
lance6716 May 29, 2024
87999d1
planner: fix small regression caused by #53094 (#53646)
qw4990 May 29, 2024
2b68ccf
ddl: remove useless code in `adjustWorkerSize` (#53644)
lcwangchao May 29, 2024
c79716e
tests: add some test cases for global index (#53630)
Defined2014 May 29, 2024
fc2d0a6
ddl: split `sessionctx.Context` to sub contexts in `CopContextBase` (…
lcwangchao May 29, 2024
aee176b
statstics,ddl: fix FK table forgets to send CreateTable event (#53654)
hawkingrei May 29, 2024
22c41d0
table: memoize TableCommon.getCols (#53605)
ekexium May 29, 2024
afd8de1
lightning: release the requirement for NewTargetInfoGetterImpl (#53651)
lance6716 May 30, 2024
29bf008
localbackend: fix resource leak when err on new local backend (#53664)
D3Hunter May 30, 2024
04c66ee
ddl: decouple job scheduler from 'ddl' and make it run/exit as owner …
D3Hunter May 30, 2024
5e5326b
statistics: print the process of init stats (#53560)
hawkingrei May 30, 2024
e81a5bb
planner: fix DATA RACE caused by initializing PointGet stats (#53676)
qw4990 May 30, 2024
d860287
expression: fix the wrong behavior of `conv` function (#53681)
qw4990 May 30, 2024
9745a16
statistics: clean up dropped predicate columns stats usage (#53680)
Rustin170506 May 30, 2024
2038b1f
planner: bypass plan cache for cached PointGet plan (#53686)
qw4990 May 30, 2024
618e252
*: upgrade bazel's rules (#53596)
hawkingrei May 31, 2024
b137425
domain: fix unstable test TestRepairTable (#53688)
tiancaiamao May 31, 2024
35e09a2
br: Enhance stream restore logs (#53649)
dveeden May 31, 2024
fa66bba
go.mod: update PD client to fix the backoffer retry (#53718)
JmPotato May 31, 2024
90e1049
ddl: remove useless fields in `CopContextBase` (#53721)
lcwangchao May 31, 2024
3fdb963
domain: do not reset the data for infoschema v2 when full load (#53543)
tiancaiamao May 31, 2024
4670bf5
ddl: separate `sessionctx.Context` in `backfillCtx` to sub-contexts …
lcwangchao May 31, 2024
8483dc5
table: Update delta from column slice, instead of map (#53670)
ekexium Jun 1, 2024
2069651
parser: Accept `PASSWORD REQUIRE CURRENT DEFAULT` (#53306)
dveeden Jun 2, 2024
fc3132c
expression: JSON_SCHEMA_VALID() (#52780)
dveeden Jun 3, 2024
764ef4a
Merge branch 'master' into store_backup_part2
3pointer Jun 3, 2024
073f746
dumpling: check table-list types before dumping (#53683)
tangenta Jun 3, 2024
3d68bd2
expression: fail `ColumnSubstituteImpl` if creating function returns …
YangKeao Jun 3, 2024
6183610
grafana: use log2 for y-axis of uptime (#53749)
jackysp Jun 3, 2024
c6e6c15
lightning: turn off zstd decoding concurrency (#53589)
lance6716 Jun 3, 2024
5e6df4e
expression, *: Eval param get type with context (#53656)
YangKeao Jun 3, 2024
53d19dd
OWNERS_ALIAS: add approvers to dumpling (#53757)
tangenta Jun 3, 2024
44c9096
ddl: get latest old table ID before replace view (#53720)
tangenta Jun 3, 2024
f1ec74b
*: fix `insert on duplicate` and `replace into` with global index (#5…
Defined2014 Jun 3, 2024
d760a06
build(deps): bump github.com/prometheus/procfs from 0.13.0 to 0.15.1 …
dependabot[bot] Jun 3, 2024
7629a0d
meta: Add a GetDBMeta function to meta. (#53684)
asddongmen Jun 4, 2024
6b66e98
*: update tikv/client-go (#53698)
crazycs520 Jun 4, 2024
3e8ba69
statistics: add counter for sync load's backgroud tasks (#53769)
winoros Jun 4, 2024
66f97c2
txn: save unnecessary calls to newMemBuffer (#53715)
ekexium Jun 4, 2024
625516b
*: keep in-memory resident for model.TableInfo with special attribute…
tiancaiamao Jun 4, 2024
b23ac8e
ddl: fix ingest job panic when reset engine failed (#53754)
tangenta Jun 4, 2024
55d2e57
address comment
3pointer Jun 4, 2024
7230ed8
Merge branch 'master' into store_backup_part2
3pointer Jun 4, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
point-get: calculate the row level checksum on the real time to avoid…
… incorrect checksum caused by schema change (pingcap#52511)

close pingcap#52590
  • Loading branch information
3AceShowHand authored May 11, 2024
commit 6d92e7af94a2fe1dde6029ac2dbe32a2ba7b5dd8
12 changes: 10 additions & 2 deletions pkg/executor/batch_point_get.go
Original file line number Diff line number Diff line change
@@ -193,16 +193,24 @@ func (e *BatchPointGetExec) Next(ctx context.Context, req *chunk.Chunk) error {
if e.index >= len(e.values) {
return nil
}

schema := e.Schema()
sctx := e.BaseExecutor.Ctx()
start := e.index
for !req.IsFull() && e.index < len(e.values) {
handle, val := e.handles[e.index], e.values[e.index]
err := DecodeRowValToChunk(e.BaseExecutor.Ctx(), e.Schema(), e.tblInfo, handle, val, req, e.rowDecoder)
err := DecodeRowValToChunk(sctx, schema, e.tblInfo, handle, val, req, e.rowDecoder)
if err != nil {
return err
}
e.index++
}

err := table.FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex, e.Schema().Columns, e.columns, e.Ctx().GetExprCtx(), req)
err := fillRowChecksum(sctx, start, e.index, schema, e.tblInfo, e.values, e.handles, req, nil)
if err != nil {
return err
}
err = table.FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex, schema.Columns, e.columns, sctx.GetExprCtx(), req)
if err != nil {
return err
}
109 changes: 107 additions & 2 deletions pkg/executor/point_get.go
Original file line number Diff line number Diff line change
@@ -17,6 +17,8 @@ package executor
import (
"context"
"fmt"
"sort"
"strconv"

"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
@@ -387,19 +389,122 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error {
}
return nil
}
err = DecodeRowValToChunk(e.BaseExecutor.Ctx(), e.Schema(), e.tblInfo, e.handle, val, req, e.rowDecoder)

sctx := e.BaseExecutor.Ctx()
schema := e.Schema()
err = DecodeRowValToChunk(sctx, schema, e.tblInfo, e.handle, val, req, e.rowDecoder)
if err != nil {
return err
}

err = fillRowChecksum(sctx, 0, 1, schema, e.tblInfo, [][]byte{val}, []kv.Handle{e.handle}, req, nil)
if err != nil {
return err
}

err = table.FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex,
e.Schema().Columns, e.columns, e.Ctx().GetExprCtx(), req)
schema.Columns, e.columns, sctx.GetExprCtx(), req)
if err != nil {
return err
}
return nil
}

func shouldFillRowChecksum(schema *expression.Schema) (int, bool) {
for idx, col := range schema.Columns {
if col.ID == model.ExtraRowChecksumID {
return idx, true
}
}
return 0, false
}

func fillRowChecksum(
sctx sessionctx.Context,
start, end int,
schema *expression.Schema, tblInfo *model.TableInfo,
values [][]byte, handles []kv.Handle,
req *chunk.Chunk, buf []byte,
) error {
checksumColumnIndex, ok := shouldFillRowChecksum(schema)
if !ok {
return nil
}

var handleColIDs []int64
if tblInfo.PKIsHandle {
colInfo := tblInfo.GetPkColInfo()
handleColIDs = []int64{colInfo.ID}
} else if tblInfo.IsCommonHandle {
pkIdx := tables.FindPrimaryIndex(tblInfo)
for _, col := range pkIdx.Columns {
colInfo := tblInfo.Columns[col.Offset]
handleColIDs = append(handleColIDs, colInfo.ID)
}
}

columnFt := make(map[int64]*types.FieldType)
for idx := range tblInfo.Columns {
col := tblInfo.Columns[idx]
columnFt[col.ID] = &col.FieldType
}
tz := sctx.GetSessionVars().TimeZone
ft := []*types.FieldType{schema.Columns[checksumColumnIndex].GetType()}
checksumCols := chunk.NewChunkWithCapacity(ft, req.Capacity())
for i := start; i < end; i++ {
handle, val := handles[i], values[i]
if !rowcodec.IsNewFormat(val) {
checksumCols.AppendNull(0)
continue
}
datums, err := tablecodec.DecodeRowWithMapNew(val, columnFt, tz, nil)
if err != nil {
return err
}
datums, err = tablecodec.DecodeHandleToDatumMap(handle, handleColIDs, columnFt, tz, datums)
if err != nil {
return err
}
for _, col := range tblInfo.Columns {
// cannot found from the datums, which means the data is not stored, this
// may happen after `add column` executed, filling with the default value.
_, ok := datums[col.ID]
if !ok {
colInfo := getColInfoByID(tblInfo, col.ID)
d, err := table.GetColOriginDefaultValue(sctx.GetExprCtx(), colInfo)
if err != nil {
return err
}
datums[col.ID] = d
}
}

colData := make([]rowcodec.ColData, len(tblInfo.Columns))
for idx, col := range tblInfo.Columns {
d := datums[col.ID]
data := rowcodec.ColData{
ColumnInfo: col,
Datum: &d,
}
colData[idx] = data
}
row := rowcodec.RowData{
Cols: colData,
Data: buf,
}
if !sort.IsSorted(row) {
sort.Sort(row)
}
checksum, err := row.Checksum(tz)
if err != nil {
return err
}
checksumCols.AppendString(0, strconv.FormatUint(uint64(checksum), 10))
}
req.SetCol(checksumColumnIndex, checksumCols.Column(0))
return nil
}

func (e *PointGetExecutor) getAndLock(ctx context.Context, key kv.Key) (val []byte, err error) {
if e.Ctx().GetSessionVars().IsPessimisticReadConsistency() {
// Only Lock the existing keys in RC isolation.
2 changes: 1 addition & 1 deletion pkg/expression/integration_test/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@ go_test(
"main_test.go",
],
flaky = True,
shard_count = 25,
shard_count = 27,
deps = [
"//pkg/config",
"//pkg/domain",
96 changes: 77 additions & 19 deletions pkg/expression/integration_test/integration_test.go
Original file line number Diff line number Diff line change
@@ -2963,48 +2963,106 @@ PARTITION BY RANGE (c) (
tk.MustExec("set global tidb_enable_local_txn = off;")
}

func TestTiDBRowChecksumBuiltin(t *testing.T) {
store := testkit.CreateMockStore(t)

checksum := func(cols ...any) uint32 {
buf := make([]byte, 0, 64)
for _, col := range cols {
switch x := col.(type) {
case int:
buf = binary.LittleEndian.AppendUint64(buf, uint64(x))
case string:
buf = binary.LittleEndian.AppendUint32(buf, uint32(len(x)))
buf = append(buf, []byte(x)...)
}
func calculateChecksum(cols ...any) string {
buf := make([]byte, 0, 64)
for _, col := range cols {
switch x := col.(type) {
case int:
buf = binary.LittleEndian.AppendUint64(buf, uint64(x))
case string:
buf = binary.LittleEndian.AppendUint32(buf, uint32(len(x)))
buf = append(buf, []byte(x)...)
}
return crc32.ChecksumIEEE(buf)
}
checksum := crc32.ChecksumIEEE(buf)
return fmt.Sprintf("%d", checksum)
}

func TestTiDBRowChecksumBuiltinAfterDropColumn(t *testing.T) {
store := testkit.CreateMockStore(t)

tk := testkit.NewTestKit(t, store)
tk.MustExec("set global tidb_enable_row_level_checksum = 1")
tk.MustExec("use test")

tk.MustExec("create table t(a int primary key, b int, c int)")
tk.MustExec("insert into t values(1, 1, 1)")

oldChecksum := tk.MustQuery("select tidb_row_checksum() from t where a = 1").Rows()[0][0].(string)

tk.MustExec("alter table t drop column b")
newChecksum := tk.MustQuery("select tidb_row_checksum() from t where a = 1").Rows()[0][0].(string)

require.NotEqual(t, oldChecksum, newChecksum)
}

func TestTiDBRowChecksumBuiltinAfterAddColumn(t *testing.T) {
store := testkit.CreateMockStore(t)

tk := testkit.NewTestKit(t, store)
tk.MustExec("set global tidb_enable_row_level_checksum = 1")
tk.MustExec("use test")

tk.MustExec("create table t(a int primary key, b int)")
tk.MustExec("insert into t values(1, 1)")

oldChecksum := tk.MustQuery("select tidb_row_checksum() from t where a = 1").Rows()[0][0].(string)
expected := calculateChecksum(1, 1)
require.Equal(t, expected, oldChecksum)

tk.MustExec("alter table t add column c int default 1")
newChecksum := tk.MustQuery("select tidb_row_checksum() from t where a = 1").Rows()[0][0].(string)
expected = calculateChecksum(1, 1, 1)
require.Equal(t, expected, newChecksum)

require.NotEqual(t, oldChecksum, newChecksum)
}

func TestTiDBRowChecksumBuiltin(t *testing.T) {
store := testkit.CreateMockStore(t)

tk := testkit.NewTestKit(t, store)
tk.MustExec("set global tidb_enable_row_level_checksum = 1")
tk.MustExec("use test")
tk.MustExec("create table t (id int primary key, c int)")

// row with 2 checksums
// row with 1 checksum
tk.MustExec("insert into t values (1, 10)")
tk.MustExec("alter table t change column c c varchar(10)")
checksum1 := fmt.Sprintf("%d,%d", checksum(1, 10), checksum(1, "10"))
// row with 1 checksum
checksum1 := calculateChecksum(1, "10")
checksum11 := fmt.Sprintf("%d %v %v", 1, "10", checksum1)

tk.Session().GetSessionVars().EnableRowLevelChecksum = true
tk.MustExec("insert into t values (2, '20')")
checksum2 := fmt.Sprintf("%d", checksum(2, "20"))
checksum2 := calculateChecksum(2, "20")
checksum22 := fmt.Sprintf("%d %v %v", 2, checksum2, "20")

// row without checksum
tk.Session().GetSessionVars().EnableRowLevelChecksum = false
tk.MustExec("insert into t values (3, '30')")
checksum3 := "<nil>"
checksum3 := calculateChecksum(3, "30")
checksum33 := fmt.Sprintf("%v %d %v", checksum3, 3, "30")

// fast point-get
tk.MustQuery("select tidb_row_checksum() from t where id = 1").Check(testkit.Rows(checksum1))
tk.MustQuery("select id, c, tidb_row_checksum() from t where id = 1").Check(testkit.Rows(checksum11))

tk.MustQuery("select tidb_row_checksum() from t where id = 2").Check(testkit.Rows(checksum2))
tk.MustQuery("select id, tidb_row_checksum(), c from t where id = 2").Check(testkit.Rows(checksum22))

tk.MustQuery("select tidb_row_checksum() from t where id = 3").Check(testkit.Rows(checksum3))
tk.MustQuery("select tidb_row_checksum(), id, c from t where id = 3").Check(testkit.Rows(checksum33))

// fast batch-point-get
tk.MustQuery("select tidb_row_checksum() from t where id in (1, 2, 3)").Check(testkit.Rows(checksum1, checksum2, checksum3))

tk.MustQuery("select id, c, tidb_row_checksum() from t where id in (1, 2, 3)").
Check(testkit.Rows(
checksum11,
fmt.Sprintf("%d %v %v", 2, "20", checksum2),
fmt.Sprintf("%d %v %v", 3, "30", checksum3),
))

// non-fast point-get
tk.MustGetDBError("select length(tidb_row_checksum()) from t where id = 1", expression.ErrNotSupportedYet)
tk.MustGetDBError("select c from t where id = 1 and tidb_row_checksum() is not null", expression.ErrNotSupportedYet)
38 changes: 18 additions & 20 deletions pkg/tablecodec/tablecodec.go
Original file line number Diff line number Diff line change
@@ -543,26 +543,24 @@ func DecodeHandleToDatumMap(handle kv.Handle, handleColIDs []int64,
if row == nil {
row = make(map[int64]types.Datum, len(cols))
}
for id, ft := range cols {
for idx, hid := range handleColIDs {
if id != hid {
continue
}
if types.NeedRestoredData(ft) {
continue
}
d, err := decodeHandleToDatum(handle, ft, idx)
if err != nil {
return row, err
}
d, err = Unflatten(d, ft, loc)
if err != nil {
return row, err
}
if _, exists := row[id]; !exists {
row[id] = d
}
break
for idx, id := range handleColIDs {
ft, ok := cols[id]
if !ok {
continue
}
if types.NeedRestoredData(ft) {
continue
}
d, err := decodeHandleToDatum(handle, ft, idx)
if err != nil {
return row, err
}
d, err = Unflatten(d, ft, loc)
if err != nil {
return row, err
}
if _, exists := row[id]; !exists {
row[id] = d
}
}
return row, nil
6 changes: 1 addition & 5 deletions pkg/util/rowcodec/decoder.go
Original file line number Diff line number Diff line change
@@ -211,11 +211,7 @@ func (decoder *ChunkDecoder) DecodeToChunk(rowData []byte, handle kv.Handle, chk
continue
}
if col.ID == model.ExtraRowChecksumID {
if v := decoder.row.getChecksumInfo(); len(v) > 0 {
chk.AppendString(colIdx, v)
} else {
chk.AppendNull(colIdx)
}
chk.AppendNull(colIdx)
continue
}

12 changes: 0 additions & 12 deletions pkg/util/rowcodec/row.go
Original file line number Diff line number Diff line change
@@ -16,7 +16,6 @@ package rowcodec

import (
"encoding/binary"
"strconv"
)

const (
@@ -101,17 +100,6 @@ func (r *row) setChecksums(checksums ...uint32) {
}
}

func (r *row) getChecksumInfo() string {
var s string
if r.hasChecksum() {
s = strconv.FormatUint(uint64(r.checksum1), 10)
if r.hasExtraChecksum() {
s += "," + strconv.FormatUint(uint64(r.checksum2), 10)
}
}
return s
}

func (r *row) getData(i int) []byte {
var start, end uint32
if r.large() {