From 2bf550ff9d20f25a564c699cfef23a21360fa106 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Fri, 19 Apr 2024 16:16:42 +0800 Subject: [PATCH 1/3] timezone default value should also consider tz. --- cdc/entry/mounter.go | 16 +++++++----- cdc/entry/mounter_test.go | 51 +++++++++++++++++++++++++++++++++------ 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/cdc/entry/mounter.go b/cdc/entry/mounter.go index dae83cc4320..a58e9f592a0 100644 --- a/cdc/entry/mounter.go +++ b/cdc/entry/mounter.go @@ -341,7 +341,7 @@ func parseJob(v []byte, startTs, CRTs uint64) (*timodel.Job, error) { } func datum2Column( - tableInfo *model.TableInfo, datums map[int64]types.Datum, + tableInfo *model.TableInfo, datums map[int64]types.Datum, tz *time.Location, ) ([]*model.ColumnData, []types.Datum, []*timodel.ColumnInfo, error) { cols := make([]*model.ColumnData, len(tableInfo.RowColumnsOffset)) rawCols := make([]types.Datum, len(tableInfo.RowColumnsOffset)) @@ -368,7 +368,7 @@ func datum2Column( if exist { colValue, size, warn, err = formatColVal(colDatum, colInfo) } else { - colDatum, colValue, size, warn, err = getDefaultOrZeroValue(colInfo) + colDatum, colValue, size, warn, err = getDefaultOrZeroValue(colInfo, tz) } if err != nil { return nil, nil, nil, errors.Trace(err) @@ -504,7 +504,7 @@ func (m *mounter) mountRowKVEntry(tableInfo *model.TableInfo, row *rowKVEntry, d if row.PreRowExist { // FIXME(leoppro): using pre table info to mounter pre column datum // the pre column and current column in one event may using different table info - preCols, preRawCols, columnInfos, err = datum2Column(tableInfo, row.PreRow) + preCols, preRawCols, columnInfos, err = datum2Column(tableInfo, row.PreRow, m.tz) if err != nil { return nil, rawRow, errors.Trace(err) } @@ -536,7 +536,7 @@ func (m *mounter) mountRowKVEntry(tableInfo *model.TableInfo, row *rowKVEntry, d currentChecksum uint32 ) if row.RowExist { - cols, rawCols, columnInfos, err = datum2Column(tableInfo, row.Row) + cols, rawCols, columnInfos, err = datum2Column(tableInfo, row.Row, m.tz) if err != nil { return nil, rawRow, errors.Trace(err) } @@ -698,7 +698,9 @@ func formatColVal(datum types.Datum, col *timodel.ColumnInfo) ( // https://github.com/golang/go/blob/go1.17.4/src/database/sql/driver/types.go#L236 // Supported type is: nil, basic type(Int, Int8,..., Float32, Float64, String), Slice(uint8), other types not support // TODO: Check default expr support -func getDefaultOrZeroValue(col *timodel.ColumnInfo) (types.Datum, any, int, string, error) { +func getDefaultOrZeroValue( + col *timodel.ColumnInfo, tz *time.Location, +) (types.Datum, any, int, string, error) { var ( d types.Datum err error @@ -711,7 +713,9 @@ func getDefaultOrZeroValue(col *timodel.ColumnInfo) (types.Datum, any, int, stri // Ref: https://github.com/pingcap/tidb/blob/d2c352980a43bb593db81fd1db996f47af596d91/table/column.go#L489 if col.GetOriginDefaultValue() != nil { datum := types.NewDatum(col.GetOriginDefaultValue()) - d, err = datum.ConvertTo(types.DefaultStmtNoWarningContext, &col.FieldType) + ctx := types.DefaultStmtNoWarningContext + ctx = ctx.WithLocation(tz) + d, err = datum.ConvertTo(ctx, &col.FieldType) if err != nil { return d, d.GetValue(), sizeOfDatum(d), "", errors.Trace(err) } diff --git a/cdc/entry/mounter_test.go b/cdc/entry/mounter_test.go index e6b6368eaea..1b190627439 100644 --- a/cdc/entry/mounter_test.go +++ b/cdc/entry/mounter_test.go @@ -873,8 +873,10 @@ func TestGetDefaultZeroValue(t *testing.T) { }, } + tz, err := util.GetTimezone(config.GetGlobalServerConfig().TZ) + require.NoError(t, err) for _, tc := range testCases { - _, val, _, _, _ := getDefaultOrZeroValue(&tc.ColInfo) + _, val, _, _, _ := getDefaultOrZeroValue(&tc.ColInfo, tz) require.Equal(t, tc.Res, val, tc.Name) } @@ -882,9 +884,9 @@ func TestGetDefaultZeroValue(t *testing.T) { OriginDefaultValue: "-3.14", // no float FieldType: *ftTypeNewDecimalNotNull, } - _, val, _, _, _ := getDefaultOrZeroValue(&colInfo) + _, val, _, _, _ := getDefaultOrZeroValue(&colInfo, tz) decimal := new(types.MyDecimal) - err := decimal.FromString([]byte("-3.14")) + err = decimal.FromString([]byte("-3.14")) require.NoError(t, err) require.Equal(t, decimal.String(), val, "mysql.TypeNewDecimal + notnull + default") @@ -892,7 +894,7 @@ func TestGetDefaultZeroValue(t *testing.T) { OriginDefaultValue: "2020-11-19 12:12:12", FieldType: *ftTypeTimestampNotNull, } - _, val, _, _, _ = getDefaultOrZeroValue(&colInfo) + _, val, _, _, _ = getDefaultOrZeroValue(&colInfo, tz) expected, err := types.ParseTimeFromFloatString( types.DefaultStmtNoWarningContext, "2020-11-19 12:12:12", colInfo.FieldType.GetType(), colInfo.FieldType.GetDecimal()) @@ -903,7 +905,7 @@ func TestGetDefaultZeroValue(t *testing.T) { OriginDefaultValue: "2020-11-19 12:12:12", FieldType: *ftTypeTimestampNull, } - _, val, _, _, _ = getDefaultOrZeroValue(&colInfo) + _, val, _, _, _ = getDefaultOrZeroValue(&colInfo, tz) expected, err = types.ParseTimeFromFloatString( types.DefaultStmtNoWarningContext, "2020-11-19 12:12:12", colInfo.FieldType.GetType(), colInfo.FieldType.GetDecimal()) @@ -914,7 +916,7 @@ func TestGetDefaultZeroValue(t *testing.T) { OriginDefaultValue: "e1", FieldType: *ftTypeEnumNotNull, } - _, val, _, _, _ = getDefaultOrZeroValue(&colInfo) + _, val, _, _, _ = getDefaultOrZeroValue(&colInfo, tz) expectedEnum, err := types.ParseEnumName(colInfo.FieldType.GetElems(), "e1", colInfo.FieldType.GetCollate()) require.NoError(t, err) require.Equal(t, expectedEnum.Value, val, "mysql.TypeEnum + notnull + default") @@ -923,7 +925,7 @@ func TestGetDefaultZeroValue(t *testing.T) { OriginDefaultValue: "1,e", FieldType: *ftTypeSetNotNull, } - _, val, _, _, _ = getDefaultOrZeroValue(&colInfo) + _, val, _, _, _ = getDefaultOrZeroValue(&colInfo, tz) expectedSet, err := types.ParseSetName(colInfo.FieldType.GetElems(), "1,e", colInfo.FieldType.GetCollate()) require.NoError(t, err) require.Equal(t, expectedSet.Value, val, "mysql.TypeSet + notnull + default") @@ -1102,6 +1104,37 @@ func TestE2ERowLevelChecksum(t *testing.T) { require.NoError(t, err) } +func TestTimezoneDefaultValue(t *testing.T) { + helper := NewSchemaTestHelper(t) + defer helper.Close() + + _ = helper.DDL2Event(`create table test.t(a int primary key)`) + insertEvent := helper.DML2Event(`insert into test.t values (1)`, "test", "t") + require.NotNil(t, insertEvent) + + tableInfo, ok := helper.schemaStorage.GetLastSnapshot().TableByName("test", "t") + require.True(t, ok) + + key, oldValue := helper.getLastKeyValue(tableInfo.ID) + + _ = helper.DDL2Event(`alter table test.t add column b timestamp default '2023-02-09 13:00:00'`) + ts := helper.schemaStorage.GetLastSnapshot().CurrentTs() + rawKV := &model.RawKVEntry{ + OpType: model.OpTypePut, + Key: key, + OldValue: oldValue, + StartTs: ts - 1, + CRTs: ts + 1, + } + polymorphicEvent := model.NewPolymorphicEvent(rawKV) + err := helper.mounter.DecodeEvent(context.Background(), polymorphicEvent) + require.NoError(t, err) + + event := polymorphicEvent.Row + require.NotNil(t, event) + require.Equal(t, "2023-02-09 13:00:00", event.PreColumns[1].Value.(string)) +} + func TestVerifyChecksumTime(t *testing.T) { replicaConfig := config.GetDefaultReplicaConfig() replicaConfig.Integrity.IntegrityCheckLevel = integrity.CheckLevelCorrectness @@ -1565,6 +1598,8 @@ func TestBuildTableInfo(t *testing.T) { ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin", }, } + tz, err := util.GetTimezone(config.GetGlobalServerConfig().TZ) + require.NoError(t, err) p := parser.New() for i, c := range cases { stmt, err := p.ParseOneStmt(c.origin, "", "") @@ -1572,7 +1607,7 @@ func TestBuildTableInfo(t *testing.T) { originTI, err := ddl.BuildTableInfoFromAST(stmt.(*ast.CreateTableStmt)) require.NoError(t, err) cdcTableInfo := model.WrapTableInfo(0, "test", 0, originTI) - colDatas, _, _, err := datum2Column(cdcTableInfo, map[int64]types.Datum{}) + colDatas, _, _, err := datum2Column(cdcTableInfo, map[int64]types.Datum{}, tz) require.NoError(t, err) e := model.RowChangedEvent{ TableInfo: cdcTableInfo, From 753f1db962140d9a84541e2e0ee2930dda807fe5 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Fri, 19 Apr 2024 17:16:34 +0800 Subject: [PATCH 2/3] fix test. --- cdc/entry/mounter.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cdc/entry/mounter.go b/cdc/entry/mounter.go index a58e9f592a0..0a9cb75cd72 100644 --- a/cdc/entry/mounter.go +++ b/cdc/entry/mounter.go @@ -713,12 +713,19 @@ func getDefaultOrZeroValue( // Ref: https://github.com/pingcap/tidb/blob/d2c352980a43bb593db81fd1db996f47af596d91/table/column.go#L489 if col.GetOriginDefaultValue() != nil { datum := types.NewDatum(col.GetOriginDefaultValue()) - ctx := types.DefaultStmtNoWarningContext - ctx = ctx.WithLocation(tz) - d, err = datum.ConvertTo(ctx, &col.FieldType) + d, err = datum.ConvertTo(types.DefaultStmtNoWarningContext, &col.FieldType) if err != nil { return d, d.GetValue(), sizeOfDatum(d), "", errors.Trace(err) } + switch col.GetType() { + case mysql.TypeTimestamp: + t := d.GetMysqlTime() + err = t.ConvertTimeZone(time.UTC, tz) + if err != nil { + return d, d.GetValue(), sizeOfDatum(d), "", errors.Trace(err) + } + d.SetMysqlTime(t) + } } else if !mysql.HasNotNullFlag(col.GetFlag()) { // NOTICE: NotNullCheck need do after OriginDefaultValue check, as when TiDB meet "amend + add column default xxx", // ref: https://github.com/pingcap/ticdc/issues/3929 From ce00302e313d3a46d6c1bf64125f9b7b83acc5fc Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Fri, 19 Apr 2024 18:11:25 +0800 Subject: [PATCH 3/3] fix ut --- cdc/entry/mounter_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdc/entry/mounter_test.go b/cdc/entry/mounter_test.go index 1b190627439..464830f0340 100644 --- a/cdc/entry/mounter_test.go +++ b/cdc/entry/mounter_test.go @@ -897,7 +897,7 @@ func TestGetDefaultZeroValue(t *testing.T) { _, val, _, _, _ = getDefaultOrZeroValue(&colInfo, tz) expected, err := types.ParseTimeFromFloatString( types.DefaultStmtNoWarningContext, - "2020-11-19 12:12:12", colInfo.FieldType.GetType(), colInfo.FieldType.GetDecimal()) + "2020-11-19 20:12:12", colInfo.FieldType.GetType(), colInfo.FieldType.GetDecimal()) require.NoError(t, err) require.Equal(t, expected.String(), val, "mysql.TypeTimestamp + notnull + default") @@ -908,7 +908,7 @@ func TestGetDefaultZeroValue(t *testing.T) { _, val, _, _, _ = getDefaultOrZeroValue(&colInfo, tz) expected, err = types.ParseTimeFromFloatString( types.DefaultStmtNoWarningContext, - "2020-11-19 12:12:12", colInfo.FieldType.GetType(), colInfo.FieldType.GetDecimal()) + "2020-11-19 20:12:12", colInfo.FieldType.GetType(), colInfo.FieldType.GetDecimal()) require.NoError(t, err) require.Equal(t, expected.String(), val, "mysql.TypeTimestamp + null + default")