Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

syncer(dm) : fix default collation with upstream in create table‘s columns #3769

Merged
merged 22 commits into from
Dec 17, 2021
Merged

syncer(dm) : fix default collation with upstream in create table‘s columns #3769

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
ee42f75
commit-message: add columns collation by show character set
WizardXiao Dec 7, 2021
b4cdec6
Merge branch 'master' of https://github.com/WizardXiao/ticdc into fix…
WizardXiao Dec 8, 2021
c809ba0
commit-message: add integration test
WizardXiao Dec 10, 2021
f82b248
Merge branch 'master' of https://github.com/WizardXiao/ticdc into fix…
WizardXiao Dec 10, 2021
a256072
commit-message: fix collation id which is greater than 255
WizardXiao Dec 13, 2021
fb4928d
Merge branch 'master' of https://github.com/WizardXiao/ticdc into fix…
WizardXiao Dec 13, 2021
8324c3b
commit-message: fix integration
WizardXiao Dec 13, 2021
6c3306c
commit-message: update import tidb version
WizardXiao Dec 13, 2021
e3dcb0e
commit-message: fix integration test sync_collation
WizardXiao Dec 13, 2021
6b74e37
commit-message: integration sync_collation add wait task
WizardXiao Dec 13, 2021
b78078d
commit-message: integration sync_collation add wait task
WizardXiao Dec 13, 2021
9fe5dd2
commit-message: fix comment bugs with out integration test
WizardXiao Dec 15, 2021
c3cd3b9
commit-message: fix confict with master
WizardXiao Dec 15, 2021
9656174
commit-message:
WizardXiao Dec 16, 2021
aacddb3
commit-message: Simplify integration test
WizardXiao Dec 16, 2021
07843a9
Merge branch 'master' of https://github.com/WizardXiao/ticdc into fix…
WizardXiao Dec 16, 2021
ae93b48
commit-message: fix collation check level
WizardXiao Dec 16, 2021
0f634b5
Merge branch 'master' of https://github.com/WizardXiao/ticdc into fix…
WizardXiao Dec 16, 2021
94b8949
commit-message: delete heartbeat in task config
WizardXiao Dec 17, 2021
be5e896
Merge branch 'master' of https://github.com/WizardXiao/ticdc into fix…
WizardXiao Dec 17, 2021
05cf2a5
Merge branch 'master' into fix-column-collation
ti-chi-bot Dec 17, 2021
eef6a1b
Merge branch 'master' into fix-column-collation
ti-chi-bot Dec 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions dm/pkg/binlog/event/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ func GetParserForStatusVars(statusVars []byte) (*parser.Parser, error) {
}

// GetServerCollationByStatusVars gets server collation by binlog statusVars.
func GetServerCollationByStatusVars(statusVars []byte) (string, error) {
func GetServerCollationByStatusVars(statusVars []byte, idAndCollationMap map[int]string) (string, error) {
vars, err := statusVarsToKV(statusVars)
b, ok := vars[QCharsetCode]

Expand All @@ -365,12 +365,15 @@ func GetServerCollationByStatusVars(statusVars []byte) (string, error) {
// only happen when this is a dummy event generated by DM
err = fmt.Errorf("Q_CHARSET_CODE not found in status_vars %v", statusVars)
}
// mysql default 'latin1_swedish_ci'
// mysql 5.7.22 default 'latin1_swedish_ci'
return "latin1_swedish_ci", err
}

// QCharsetCode 2-byte character_set_client + 2-byte collation_connection + 2-byte collation_server
// collation is less than 255 and we use the first byte.
return mysql.Collations[b[4]], err
r := bytes.NewReader(b[4:])
var v uint16
_ = binary.Read(r, binary.LittleEndian, &v)
return idAndCollationMap[int(v)], err
}

// if returned error is `io.EOF`, it means UnexpectedEOF because we handled expected `io.EOF` as success
Expand Down
52 changes: 29 additions & 23 deletions dm/pkg/utils/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -391,49 +391,55 @@ func GetServerUnixTS(ctx context.Context, db *sql.DB) (int64, error) {
return ts, err
}

// GetCharsetAndDefaultCollation gets charset and default collation map.
func GetCharsetAndDefaultCollation(ctx context.Context, db *sql.DB) (map[string]string, error) {
// GetCharsetAndCollationInfo gets charset and collation info.
func GetCharsetAndCollationInfo(ctx context.Context, db *sql.DB) (map[string]string, map[int]string, error) {
charsetAndDefaultCollation := make(map[string]string)
idAndCollationMap := make(map[int]string)

conn, err := db.Conn(ctx)
if err != nil {
return nil, terror.DBErrorAdapt(err, terror.ErrDBDriverError)
return nil, nil, terror.DBErrorAdapt(err, terror.ErrDBDriverError)
}
defer conn.Close()

// Show an example.
/*
mysql> SHOW CHARACTER SET;
+----------+---------------------------------+---------------------+--------+
| Charset | Description | Default collation | Maxlen |
+----------+---------------------------------+---------------------+--------+
| armscii8 | ARMSCII-8 Armenian | armscii8_general_ci | 1 |
| ascii | US ASCII | ascii_general_ci | 1 |
| big5 | Big5 Traditional Chinese | big5_chinese_ci | 2 |
| binary | Binary pseudo charset | binary | 1 |
| cp1250 | Windows Central European | cp1250_general_ci | 1 |
| cp1251 | Windows Cyrillic | cp1251_general_ci | 1 |
+----------+---------------------------------+---------------------+--------+
mysql> SELECT COLLATION_NAME,CHARACTER_SET_NAME,ID,IS_DEFAULT from INFORMATION_SCHEMA.COLLATIONS;
+----------------------------+--------------------+-----+------------+
| COLLATION_NAME | CHARACTER_SET_NAME | ID | IS_DEFAULT |
+----------------------------+--------------------+-----+------------+
| armscii8_general_ci | armscii8 | 32 | Yes |
| armscii8_bin | armscii8 | 64 | |
| ascii_general_ci | ascii | 11 | Yes |
| ascii_bin | ascii | 65 | |
| big5_chinese_ci | big5 | 1 | Yes |
| big5_bin | big5 | 84 | |
| binary | binary | 63 | Yes |
+----------------------------+--------------------+-----+------------+
*/

rows, err := conn.QueryContext(ctx, "SHOW CHARACTER SET")
rows, err := conn.QueryContext(ctx, "SELECT COLLATION_NAME,CHARACTER_SET_NAME,ID,IS_DEFAULT from INFORMATION_SCHEMA.COLLATIONS")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cc @okJiang we have extra requirements for privilege precheck of incremental phase 😂

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about we checking all INFORMATION_SCHEMA's SELECT privilege?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this query from information_schema.collations causes dead lock if there are other running tasks at dump phase? /cc @lichunzhu PTAL

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It won't cause a dead lock. But sometimes a query to information_schema could be unacceptably slow.

if err != nil {
return nil, terror.DBErrorAdapt(err, terror.ErrDBDriverError)
return nil, nil, terror.DBErrorAdapt(err, terror.ErrDBDriverError)
}

defer rows.Close()
for rows.Next() {
var charset, description, collation string
var maxlen int
if scanErr := rows.Scan(&charset, &description, &collation, &maxlen); scanErr != nil {
return nil, terror.DBErrorAdapt(scanErr, terror.ErrDBDriverError)
var collation, charset, isDefault string
var id int
if scanErr := rows.Scan(&collation, &charset, &id, &isDefault); scanErr != nil {
return nil, nil, terror.DBErrorAdapt(scanErr, terror.ErrDBDriverError)
}
idAndCollationMap[id] = strings.ToLower(collation)
if strings.ToLower(isDefault) == "yes" {
charsetAndDefaultCollation[strings.ToLower(charset)] = collation
}
charsetAndDefaultCollation[strings.ToLower(charset)] = collation
}

if err = rows.Close(); err != nil {
return nil, terror.DBErrorAdapt(rows.Err(), terror.ErrDBDriverError)
return nil, nil, terror.DBErrorAdapt(rows.Err(), terror.ErrDBDriverError)
WizardXiao marked this conversation as resolved.
Show resolved Hide resolved
}
return charsetAndDefaultCollation, err
return charsetAndDefaultCollation, idAndCollationMap, err
}

// GetSchemaList gets db schema list with `SHOW DATABASES`.
Expand Down
6 changes: 3 additions & 3 deletions dm/syncer/dbconn/upstream_db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,9 @@ func (conn *UpStreamConn) GetServerUnixTS(ctx context.Context) (int64, error) {
return utils.GetServerUnixTS(ctx, conn.BaseDB.DB)
}

// GetCharsetAndDefaultCollation returns charset and default collation map.
func (conn *UpStreamConn) GetCharsetAndDefaultCollation(ctx context.Context) (map[string]string, error) {
return utils.GetCharsetAndDefaultCollation(ctx, conn.BaseDB.DB)
// GetCharsetAndDefaultCollation returns charset and collation info.
func (conn *UpStreamConn) GetCharsetAndCollationInfo(ctx context.Context) (map[string]string, map[int]string, error) {
return utils.GetCharsetAndCollationInfo(ctx, conn.BaseDB.DB)
Ehco1996 marked this conversation as resolved.
Show resolved Hide resolved
}

// GetParser returns the parser with correct flag for upstream.
Expand Down
39 changes: 34 additions & 5 deletions dm/syncer/ddl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func (s *Syncer) genDDLInfo(qec *queryEventContext, sql string) (*ddlInfo, error
targetTables: targetTables,
}

adjustCollation(s.tctx, ddlInfo, qec.eventStatusVars, s.charsetAndDefaultCollation)
adjustCollation(s.tctx, ddlInfo, qec.eventStatusVars, s.charsetAndDefaultCollation, s.idAndCollationMap)
routedDDL, err := parserpkg.RenameDDLTable(ddlInfo.originStmt, ddlInfo.targetTables)
ddlInfo.routedDDL = routedDDL
return ddlInfo, err
Expand Down Expand Up @@ -196,7 +196,7 @@ func (s *Syncer) clearOnlineDDL(tctx *tcontext.Context, targetTable *filter.Tabl
}

// adjustCollation adds collation for create database and check create table.
func adjustCollation(tctx *tcontext.Context, ddlInfo *ddlInfo, statusVars []byte, charsetAndDefaultCollationMap map[string]string) {
func adjustCollation(tctx *tcontext.Context, ddlInfo *ddlInfo, statusVars []byte, charsetAndDefaultCollationMap map[string]string, idAndCollationMap map[int]string) {
switch createStmt := ddlInfo.originStmt.(type) {
case *ast.CreateTableStmt:
if createStmt.ReferTable != nil {
Expand All @@ -206,6 +206,7 @@ func adjustCollation(tctx *tcontext.Context, ddlInfo *ddlInfo, statusVars []byte
for _, tableOption := range createStmt.Options {
WizardXiao marked this conversation as resolved.
Show resolved Hide resolved
// already have 'Collation'
if tableOption.Tp == ast.TableOptionCollate {
adjustColumnsCollation(tctx, createStmt, charsetAndDefaultCollationMap)
return
}
if tableOption.Tp == ast.TableOptionCharset {
Expand All @@ -214,16 +215,19 @@ func adjustCollation(tctx *tcontext.Context, ddlInfo *ddlInfo, statusVars []byte
}
if justCharset == "" {
tctx.L().Warn("detect create table risk which use implicit charset and collation", zap.String("originSQL", ddlInfo.originDDL))
adjustColumnsCollation(tctx, createStmt, charsetAndDefaultCollationMap)
return
}
// just has charset, can add collation by charset and default collation map
collation, ok := charsetAndDefaultCollationMap[strings.ToLower(justCharset)]
if !ok {
tctx.L().Warn("not found charset default collation.", zap.String("originSQL", ddlInfo.originDDL), zap.String("charset", strings.ToLower(justCharset)))
adjustColumnsCollation(tctx, createStmt, charsetAndDefaultCollationMap)
return
}
tctx.L().Info("detect create table risk which use explicit charset and implicit collation, we will add collation by SHOW CHARACTER SET", zap.String("originSQL", ddlInfo.originDDL), zap.String("collation", collation))
tctx.L().Info("detect create table risk which use explicit charset and implicit collation, we will add collation by INFORMATION_SCHEMA.COLLATIONS", zap.String("originSQL", ddlInfo.originDDL), zap.String("collation", collation))
createStmt.Options = append(createStmt.Options, &ast.TableOption{Tp: ast.TableOptionCollate, StrValue: collation})
adjustColumnsCollation(tctx, createStmt, charsetAndDefaultCollationMap)

case *ast.CreateDatabaseStmt:
var justCharset, collation string
Expand All @@ -245,18 +249,43 @@ func adjustCollation(tctx *tcontext.Context, ddlInfo *ddlInfo, statusVars []byte
tctx.L().Warn("not found charset default collation.", zap.String("originSQL", ddlInfo.originDDL), zap.String("charset", strings.ToLower(justCharset)))
return
}
tctx.L().Info("detect create database risk which use explicit charset and implicit collation, we will add collation by SHOW CHARACTER SET", zap.String("originSQL", ddlInfo.originDDL), zap.String("collation", collation))
tctx.L().Info("detect create database risk which use explicit charset and implicit collation, we will add collation by INFORMATION_SCHEMA.COLLATIONS", zap.String("originSQL", ddlInfo.originDDL), zap.String("collation", collation))
} else {
// has no charset and collation
// add collation by server collation from binlog statusVars
collation, _ = event.GetServerCollationByStatusVars(statusVars)
collation, _ = event.GetServerCollationByStatusVars(statusVars, idAndCollationMap)
WizardXiao marked this conversation as resolved.
Show resolved Hide resolved
// add collation
tctx.L().Info("detect create database risk which use implicit charset and collation, we will add collation by binlog status_vars", zap.String("originSQL", ddlInfo.originDDL), zap.String("collation", collation))
}
createStmt.Options = append(createStmt.Options, &ast.DatabaseOption{Tp: ast.DatabaseOptionCollate, Value: collation})
}
}

// adjustColumnsCollation adds column's collation.
func adjustColumnsCollation(tctx *tcontext.Context, createStmt *ast.CreateTableStmt, charsetAndDefaultCollationMap map[string]string) {
WizardXiao marked this conversation as resolved.
Show resolved Hide resolved
for _, col := range createStmt.Cols {
for _, options := range col.Options {
// already have 'Collation'
if options.Tp == ast.ColumnOptionCollate {
continue
}
}
fieldType := col.Tp
if fieldType.Collate != "" {
continue
}
if fieldType.Charset != "" {
// just have charset
collation, ok := charsetAndDefaultCollationMap[strings.ToLower(fieldType.Charset)]
if !ok {
tctx.L().Warn("not found charset default collation for column.", zap.String("table", createStmt.Table.Name.String()), zap.String("column", col.Name.String()), zap.String("charset", strings.ToLower(fieldType.Charset)))
continue
}
fieldType.Collate = collation
}
}
}

type ddlInfo struct {
originDDL string
routedDDL string
Expand Down
39 changes: 35 additions & 4 deletions dm/syncer/ddl_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,7 @@ func (s *testDDLSuite) TestResolveDDLSQL(c *C) {
tctx: tctx,
}
statusVars := []byte{4, 0, 0, 0, 0, 46, 0}
syncer.idAndCollationMap = map[int]string{46: "utf8mb4_bin"}
for i, sql := range sqls {
qec := &queryEventContext{
eventContext: ec,
Expand Down Expand Up @@ -641,25 +642,51 @@ func (s *testDDLSuite) TestClearOnlineDDL(c *C) {
func (s *testDDLSuite) TestAdjustCollation(c *C) {
// duplicate with pkg/parser
sqls := []string{
"create database if not exists `test`",
"create database if not exists `test`",
"create table `test`.`t1` (id int) CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci",
"create table `test`.`t1` (id int) CHARSET=utf8mb4",
"create table `test`.`t1` (id int) COLLATE=utf8mb4_general_ci",
"create table `test`.`t1` (id int)",
"create database `test` CHARACTER SET=utf8mb4 COLLATE=utf8mb4_general_ci",
"create database `test` CHARACTER SET=utf8mb4",
"create database `test` COLLATE=utf8mb4_general_ci",
"create database if not exists `test`",
"create table `test`.`t1` (id int, name varchar(20) CHARACTER SET utf8mb4, work varchar(20))",
"create table `test`.`t1` (id int, name varchar(20), work varchar(20))",
"create table `test`.`t1` (id int, name varchar(20) COLLATE utf8mb4_general_ci, work varchar(20))",
"create table `test`.`t1` (id int, name varchar(20) COLLATE utf8mb4_general_ci, work varchar(20) CHARACTER SET utf8mb4)",
"create table `test`.`t1` (id int, name varchar(20) CHARACTER SET utf8mb4, work varchar(20)) COLLATE=utf8mb4_general_ci ",
"create table `test`.`t1` (id int, name varchar(20), work varchar(20)) COLLATE=utf8mb4_general_ci",
"create table `test`.`t1` (id int, name varchar(20) COLLATE utf8mb4_general_ci, work varchar(20)) COLLATE=utf8mb4_general_ci",
"create table `test`.`t1` (id int, name varchar(20) COLLATE utf8mb4_general_ci, work varchar(20) CHARACTER SET utf8mb4) COLLATE=utf8mb4_general_ci",
"create table `test`.`t1` (id int, name varchar(20) CHARACTER SET utf8mb4, work varchar(20)) CHARSET=utf8mb4 ",
"create table `test`.`t1` (id int, name varchar(20), work varchar(20)) CHARSET=utf8mb4",
"create table `test`.`t1` (id int, name varchar(20) COLLATE utf8mb4_general_ci, work varchar(20)) CHARSET=utf8mb4",
"create table `test`.`t1` (id int, name varchar(20) COLLATE utf8mb4_general_ci, work varchar(20) CHARACTER SET utf8mb4) CHARSET=utf8mb4",
lance6716 marked this conversation as resolved.
Show resolved Hide resolved
}

expectedSQLs := []string{
"CREATE DATABASE IF NOT EXISTS `test` COLLATE = utf8mb4_vi_0900_ai_ci",
"CREATE DATABASE IF NOT EXISTS `test` COLLATE = utf8mb4_bin",
"CREATE TABLE `test`.`t` (`id` INT) DEFAULT CHARACTER SET = UTF8MB4 DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
"CREATE TABLE `test`.`t` (`id` INT) DEFAULT CHARACTER SET = UTF8MB4 DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
"CREATE TABLE `test`.`t` (`id` INT) DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
"CREATE TABLE `test`.`t` (`id` INT)",
"CREATE DATABASE `test` CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci",
"CREATE DATABASE `test` CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci",
"CREATE DATABASE `test` COLLATE = utf8mb4_general_ci",
"CREATE DATABASE IF NOT EXISTS `test` COLLATE = utf8mb4_bin",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_general_ci,`work` VARCHAR(20))",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20),`work` VARCHAR(20))",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) COLLATE utf8mb4_general_ci,`work` VARCHAR(20))",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) COLLATE utf8mb4_general_ci,`work` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_general_ci)",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_general_ci,`work` VARCHAR(20)) DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20),`work` VARCHAR(20)) DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) COLLATE utf8mb4_general_ci,`work` VARCHAR(20)) DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) COLLATE utf8mb4_general_ci,`work` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_general_ci) DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_general_ci,`work` VARCHAR(20)) DEFAULT CHARACTER SET = UTF8MB4 DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20),`work` VARCHAR(20)) DEFAULT CHARACTER SET = UTF8MB4 DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) COLLATE utf8mb4_general_ci,`work` VARCHAR(20)) DEFAULT CHARACTER SET = UTF8MB4 DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
"CREATE TABLE `test`.`t` (`id` INT,`name` VARCHAR(20) COLLATE utf8mb4_general_ci,`work` VARCHAR(20) CHARACTER SET UTF8MB4 COLLATE utf8mb4_general_ci) DEFAULT CHARACTER SET = UTF8MB4 DEFAULT COLLATE = UTF8MB4_GENERAL_CI",
}

tctx := tcontext.Background().WithLogger(log.With(zap.String("test", "TestAdjustTableCollation")))
Expand All @@ -670,9 +697,13 @@ func (s *testDDLSuite) TestAdjustCollation(c *C) {
Schema: "test",
Name: "t",
}
statusVars := []byte{4, 0, 0, 0, 0, 46, 0}
statusVars := []byte{4, 0, 0, 0, 0, 21, 1}
charsetAndDefaultCollationMap := map[string]string{"utf8mb4": "utf8mb4_general_ci"}
idAndCollationMap := map[int]string{46: "utf8mb4_bin", 277: "utf8mb4_vi_0900_ai_ci"}
for i, sql := range sqls {
if i == 1 {
WizardXiao marked this conversation as resolved.
Show resolved Hide resolved
statusVars = []byte{4, 0, 0, 0, 0, 46, 0}
}
ddlInfo := &ddlInfo{
originDDL: sql,
routedDDL: sql,
Expand All @@ -683,7 +714,7 @@ func (s *testDDLSuite) TestAdjustCollation(c *C) {
c.Assert(err, IsNil)
c.Assert(stmt, NotNil)
ddlInfo.originStmt = stmt
adjustCollation(tctx, ddlInfo, statusVars, charsetAndDefaultCollationMap)
adjustCollation(tctx, ddlInfo, statusVars, charsetAndDefaultCollationMap, idAndCollationMap)
routedDDL, err := parserpkg.RenameDDLTable(ddlInfo.originStmt, ddlInfo.targetTables)
c.Assert(err, IsNil)
c.Assert(routedDDL, Equals, expectedSQLs[i])
Expand Down
3 changes: 2 additions & 1 deletion dm/syncer/syncer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ type Syncer struct {

relay relay.Process
charsetAndDefaultCollation map[string]string
idAndCollationMap map[int]string
}

// NewSyncer creates a new Syncer.
Expand Down Expand Up @@ -327,7 +328,7 @@ func (s *Syncer) Init(ctx context.Context) (err error) {
return terror.ErrSchemaTrackerInit.Delegate(err)
}

s.charsetAndDefaultCollation, err = s.fromDB.GetCharsetAndDefaultCollation(ctx)
s.charsetAndDefaultCollation, s.idAndCollationMap, err = s.fromDB.GetCharsetAndCollationInfo(ctx)
if err != nil {
return err
}
Expand Down
1 change: 1 addition & 0 deletions dm/tests/others_integration_1.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,4 @@ checkpoint_transaction
lightning_mode
downstream_diff_index
slow_relay_writer
sync_collation
4 changes: 4 additions & 0 deletions dm/tests/sync_collation/conf/dm-master.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Master Configuration.
master-addr = ":8261"
advertise-addr = "127.0.0.1:8261"
auto-compaction-retention = "3s"
Loading