Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

json, expression: implicitly convert JSON string to time types #53363

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

json, expression: implicitly convert JSON string to time types #53363

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 17 additions & 7 deletions pkg/expression/builtin_cast.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -518,7 +518,7 @@ func (b *castJSONAsArrayFunctionSig) evalJSON(ctx EvalContext, row chunk.Row) (r
return types.CreateBinaryJSON(arrayVals), false, nil
}

// ConvertJSON2Tp returns a function that can convert JSON to the specified type.
// ConvertJSON2Tp converts JSON to the specified type.
func ConvertJSON2Tp(v types.BinaryJSON, targetType *types.FieldType) (any, error) {
convertFunc := convertJSON2Tp(targetType.EvalType())
if convertFunc == nil {
Expand Down Expand Up @@ -556,11 +556,21 @@ func convertJSON2Tp(evalType types.EvalType) func(*stmtctx.StatementContext, typ
return types.ConvertJSONToFloat(sc.TypeCtx(), item)
}
case types.ETDatetime:
return func(_ *stmtctx.StatementContext, item types.BinaryJSON, tp *types.FieldType) (any, error) {
if (tp.GetType() == mysql.TypeDatetime && item.TypeCode != types.JSONTypeCodeDatetime) || (tp.GetType() == mysql.TypeDate && item.TypeCode != types.JSONTypeCodeDate) {
return nil, ErrInvalidJSONForFuncIndex
return func(sc *stmtctx.StatementContext, item types.BinaryJSON, tp *types.FieldType) (any, error) {
var res types.Time
if item.TypeCode == types.JSONTypeCodeString {
var err error
// The user may insert a string which has a fractional part, so allow `MaxFsp` here.
res, err = types.ParseTime(sc.TypeCtx(), string(item.GetString()), tp.GetType(), tp.GetDecimal())
if err != nil {
return nil, err
}
} else {
if (tp.GetType() == mysql.TypeDatetime && item.TypeCode != types.JSONTypeCodeDatetime) || (tp.GetType() == mysql.TypeDate && item.TypeCode != types.JSONTypeCodeDate) {
return nil, ErrInvalidJSONForFuncIndex
}
res = item.GetTimeWithFsp(tp.GetDecimal())
}
res := item.GetTime()
res.SetType(tp.GetType())
if tp.GetType() == mysql.TypeDate {
// Truncate hh:mm:ss part if the type is Date.
Expand Down Expand Up @@ -1929,7 +1939,7 @@ func (b *builtinCastJSONAsTimeSig) evalTime(ctx EvalContext, row chunk.Row) (res

switch val.TypeCode {
case types.JSONTypeCodeDate, types.JSONTypeCodeDatetime, types.JSONTypeCodeTimestamp:
res = val.GetTime()
res = val.GetTimeWithFsp(b.tp.GetDecimal())
res.SetType(b.tp.GetType())
if b.tp.GetType() == mysql.TypeDate {
// Truncate hh:mm:ss part if the type is Date.
Expand Down Expand Up @@ -1991,7 +2001,7 @@ func (b *builtinCastJSONAsDurationSig) evalDuration(ctx EvalContext, row chunk.R

switch val.TypeCode {
case types.JSONTypeCodeDate, types.JSONTypeCodeDatetime, types.JSONTypeCodeTimestamp:
time := val.GetTime()
time := val.GetTimeWithFsp(b.tp.GetDecimal())
res, err = time.ConvertToDuration()
if err != nil {
return res, false, err
Expand Down
35 changes: 35 additions & 0 deletions pkg/expression/builtin_cast_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1691,6 +1691,41 @@ func TestCastArrayFunc(t *testing.T) {
false,
true,
},
{
[]any{"2024-10-24"},
[]any{types.NewTime(types.FromDate(2024, 10, 24, 0, 0, 0, 0), mysql.TypeDate, 0)},
types.NewFieldTypeBuilder().SetType(mysql.TypeDate).SetCharset(charset.CharsetBin).SetCollate(charset.CollationBin).SetArray(true).BuildP(),
true,
true,
},
{
[]any{"2024-13-24"},
nil,
types.NewFieldTypeBuilder().SetType(mysql.TypeDate).SetCharset(charset.CharsetBin).SetCollate(charset.CollationBin).SetArray(true).BuildP(),
false,
true,
},
{
[]any{"2024-10-24 10:24:10.241024"},
[]any{types.NewTime(types.FromDate(2024, 10, 24, 10, 24, 10, 241024), mysql.TypeDatetime, 6)},
types.NewFieldTypeBuilder().SetType(mysql.TypeDatetime).SetDecimal(6).SetCharset(charset.CharsetBin).SetCollate(charset.CollationBin).SetArray(true).BuildP(),
true,
true,
},
{
[]any{"2024-10-24 10:24:10.2410241024"},
[]any{types.NewTime(types.FromDate(2024, 10, 24, 10, 24, 10, 241024), mysql.TypeDatetime, 6)},
types.NewFieldTypeBuilder().SetType(mysql.TypeDatetime).SetDecimal(6).SetCharset(charset.CharsetBin).SetCollate(charset.CollationBin).SetArray(true).BuildP(),
true,
true,
},
{
[]any{"just random string"},
nil,
types.NewFieldTypeBuilder().SetType(mysql.TypeDatetime).SetCharset(charset.CharsetBin).SetCollate(charset.CollationBin).SetArray(true).BuildP(),
false,
true,
},
}
for _, tt := range tbl {
f, err := BuildCastFunctionWithCheck(ctx, datumsToConstants(types.MakeDatums(types.CreateBinaryJSON(tt.input)))[0], tt.tp, false)
Expand Down
4 changes: 2 additions & 2 deletions pkg/expression/builtin_cast_vec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -508,7 +508,7 @@ func (b *builtinCastJSONAsTimeSig) vecEvalTime(ctx EvalContext, input *chunk.Chu

switch val.TypeCode {
case types.JSONTypeCodeDate, types.JSONTypeCodeDatetime, types.JSONTypeCodeTimestamp:
tm := val.GetTime()
tm := val.GetTimeWithFsp(b.tp.GetDecimal())
times[i] = tm
times[i].SetType(b.tp.GetType())
if b.tp.GetType() == mysql.TypeDate {
Expand Down Expand Up @@ -1975,7 +1975,7 @@ func (b *builtinCastJSONAsDurationSig) vecEvalDuration(ctx EvalContext, input *c

switch val.TypeCode {
case types.JSONTypeCodeDate, types.JSONTypeCodeDatetime, types.JSONTypeCodeTimestamp:
time := val.GetTime()
time := val.GetTimeWithFsp(b.tp.GetDecimal())
d, err := time.ConvertToDuration()
if err != nil {
return err
Expand Down
12 changes: 10 additions & 2 deletions pkg/types/json_binary.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,8 +237,16 @@ func (bj BinaryJSON) GetOpaque() Opaque {
}
}

// GetTime gets the time value
// GetTime gets the time value with default fsp
//
// Deprecated: use GetTimeWithFsp instead. The `BinaryJSON` doesn't contain the fsp information, so the caller
// should always provide the fsp.
func (bj BinaryJSON) GetTime() Time {
return bj.GetTimeWithFsp(DefaultFsp)
}

// GetTimeWithFsp gets the time value with given fsp
func (bj BinaryJSON) GetTimeWithFsp(fsp int) Time {
coreTime := CoreTime(bj.GetUint64())

tp := mysql.TypeDate
Expand All @@ -248,7 +256,7 @@ func (bj BinaryJSON) GetTime() Time {
tp = mysql.TypeTimestamp
}

return NewTime(coreTime, tp, DefaultFsp)
return NewTime(coreTime, tp, fsp)
}

// GetDuration gets the duration value
Expand Down
12 changes: 12 additions & 0 deletions tests/integrationtest/r/expression/json.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -649,3 +649,15 @@ select 1 from t where cast(BINARY vc as json) = '1';
1
select 1 from t where cast(BINARY c as json) = '1';
1
drop table if exists t;
create table t (j json);
insert into t values (cast(cast("2024-10-24 11:11:11.12346" as datetime(6)) as json));
select cast(j as datetime(6)) from t;
cast(j as datetime(6))
2024-10-24 11:11:11.123460
select cast(j as datetime(3)) from t;
cast(j as datetime(3))
2024-10-24 11:11:11.123
select cast(j as datetime) from t;
cast(j as datetime)
2024-10-24 11:11:11
44 changes: 32 additions & 12 deletions tests/integrationtest/r/expression/multi_valued_index.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,11 +201,11 @@ Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
insert into t values ('[-1]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
insert into t values ('["1"]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
Error 1292 (22007): Incorrect time value: '1'
insert into t values ('["a"]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
Error 1292 (22007): Incorrect datetime value: 'a'
insert into t values ('["汉字"]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
Error 1292 (22007): Incorrect datetime value: '汉字'
insert into t values ('[1.2]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
insert into t values ('[1.0]');
Expand Down Expand Up @@ -247,11 +247,11 @@ Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
insert into t values ('[-1]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
insert into t values ('["1"]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
Error 1292 (22007): Incorrect time value: '1'
insert into t values ('["a"]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
Error 1292 (22007): Incorrect datetime value: 'a'
insert into t values ('["汉字"]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
Error 1292 (22007): Incorrect datetime value: '汉字'
insert into t values ('[1.2]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
insert into t values ('[1.0]');
Expand Down Expand Up @@ -423,11 +423,11 @@ Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
insert into t values ('[-1]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
insert into t values ('["1"]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
Error 1292 (22007): Incorrect time value: '1'
insert into t values ('["a"]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
Error 1292 (22007): Incorrect datetime value: 'a'
insert into t values ('["汉字"]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
Error 1292 (22007): Incorrect datetime value: '汉字'
insert into t values ('[1.2]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
insert into t values ('[1.0]');
Expand Down Expand Up @@ -469,11 +469,11 @@ Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
insert into t values ('[-1]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
insert into t values ('["1"]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
Error 1292 (22007): Incorrect time value: '1'
insert into t values ('["a"]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
Error 1292 (22007): Incorrect datetime value: 'a'
insert into t values ('["汉字"]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
Error 1292 (22007): Incorrect datetime value: '汉字'
insert into t values ('[1.2]');
Error 3903 (HY000): Invalid JSON value for CAST for expression index 'idx'
insert into t values ('[1.0]');
Expand Down Expand Up @@ -516,3 +516,23 @@ drop table t;
create table t (j json not null, str varchar(5) collate gbk_chinese_ci, KEY `idx` ((cast(`j` as unsigned array)),`str`));
insert into t values ('1', 'abcde');
drop table t;
drop table if exists t;
CREATE TABLE t (j JSON, UNIQUE INDEX idx_a ((CAST(j AS DATE ARRAY))) );
INSERT INTO t VALUES (JSON_ARRAY('2024-05-13', '2020-01-01'));
INSERT INTO t VALUES (JSON_ARRAY('2024-05-13', '2020-01-01'));
Error 1062 (23000): Duplicate entry '1847955789872889856' for key 't.idx_a'
INSERT INTO t VALUES (JSON_ARRAY('2024-05-13 12:12:12', '2020-01-01 11:11:11'));
Error 1062 (23000): Duplicate entry '1847955789872889856' for key 't.idx_a'
SELECT * FROM t;
j
["2024-05-13", "2020-01-01"]
drop table if exists t;
CREATE TABLE t (j JSON, UNIQUE INDEX idx_a ((CAST(j AS DATETIME(3) ARRAY))) );
INSERT INTO t VALUES (JSON_ARRAY('2024-05-13 11:11:11.111111'));
INSERT INTO t VALUES (JSON_ARRAY('2024-05-13 11:11:11.111222'));
Error 1062 (23000): Duplicate entry '1851923595735970200' for key 't.idx_a'
INSERT INTO t VALUES (JSON_ARRAY('2024-05-13 11:11:11.222222'));
SELECT * FROM t;
j
["2024-05-13 11:11:11.111111"]
["2024-05-13 11:11:11.222222"]
8 changes: 8 additions & 0 deletions tests/integrationtest/t/expression/json.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,3 +393,11 @@ select 1 from t where cast(vc as json) = '1';
select 1 from t where cast(c as json) = '1';
select 1 from t where cast(BINARY vc as json) = '1';
select 1 from t where cast(BINARY c as json) = '1';

# TestCastJSONToTimeWithCorrectFsp
drop table if exists t;
create table t (j json);
insert into t values (cast(cast("2024-10-24 11:11:11.12346" as datetime(6)) as json));
select cast(j as datetime(6)) from t;
select cast(j as datetime(3)) from t;
select cast(j as datetime) from t;
41 changes: 29 additions & 12 deletions tests/integrationtest/t/expression/multi_valued_index.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,11 +204,11 @@ create table t(a json, index idx((cast(a as date array))));
insert into t values ('[1,2,3]');
-- error 3903
insert into t values ('[-1]');
-- error 3903
-- error 1292
insert into t values ('["1"]');
-- error 3903
-- error 1292
insert into t values ('["a"]');
-- error 3903
-- error 1292
insert into t values ('["汉字"]');
-- error 3903
insert into t values ('[1.2]');
Expand Down Expand Up @@ -250,11 +250,11 @@ create table t(a json, index idx((cast(a as datetime array))));
insert into t values ('[1,2,3]');
-- error 3903
insert into t values ('[-1]');
-- error 3903
-- error 1292
insert into t values ('["1"]');
-- error 3903
-- error 1292
insert into t values ('["a"]');
-- error 3903
-- error 1292
insert into t values ('["汉字"]');
-- error 3903
insert into t values ('[1.2]');
Expand Down Expand Up @@ -426,11 +426,11 @@ create table t(a json, index idx((cast(a as date array))));
insert into t values ('[1,2,3]');
-- error 3903
insert into t values ('[-1]');
-- error 3903
-- error 1292
insert into t values ('["1"]');
-- error 3903
-- error 1292
insert into t values ('["a"]');
-- error 3903
-- error 1292
insert into t values ('["汉字"]');
-- error 3903
insert into t values ('[1.2]');
Expand Down Expand Up @@ -472,11 +472,11 @@ create table t(a json, index idx((cast(a as datetime array))));
insert into t values ('[1,2,3]');
-- error 3903
insert into t values ('[-1]');
-- error 3903
-- error 1292
insert into t values ('["1"]');
-- error 3903
-- error 1292
insert into t values ('["a"]');
-- error 3903
-- error 1292
insert into t values ('["汉字"]');
-- error 3903
insert into t values ('[1.2]');
Expand Down Expand Up @@ -523,3 +523,20 @@ drop table t;
create table t (j json not null, str varchar(5) collate gbk_chinese_ci, KEY `idx` ((cast(`j` as unsigned array)),`str`));
insert into t values ('1', 'abcde');
drop table t;

# TestImplicitCastTimeType
drop table if exists t;
CREATE TABLE t (j JSON, UNIQUE INDEX idx_a ((CAST(j AS DATE ARRAY))) );
INSERT INTO t VALUES (JSON_ARRAY('2024-05-13', '2020-01-01'));
-- error 1062
INSERT INTO t VALUES (JSON_ARRAY('2024-05-13', '2020-01-01'));
-- error 1062
INSERT INTO t VALUES (JSON_ARRAY('2024-05-13 12:12:12', '2020-01-01 11:11:11'));
SELECT * FROM t;
drop table if exists t;
CREATE TABLE t (j JSON, UNIQUE INDEX idx_a ((CAST(j AS DATETIME(3) ARRAY))) );
INSERT INTO t VALUES (JSON_ARRAY('2024-05-13 11:11:11.111111'));
-- error 1062
INSERT INTO t VALUES (JSON_ARRAY('2024-05-13 11:11:11.111222'));
INSERT INTO t VALUES (JSON_ARRAY('2024-05-13 11:11:11.222222'));
SELECT * FROM t;