Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

*: make drainer translator's SQL_MODE configurable #511

Merged
merged 4 commits into from
Apr 1, 2019
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
make everything same as before if no sql-mode in config file
  • Loading branch information
july2993 committed Apr 1, 2019
commit 61d408b3e82c8d1e39aabe38ec7947b4a23935f3
7 changes: 4 additions & 3 deletions cmd/drainer/drainer.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,10 @@ compressor = ""
# syncer Configuration.
[syncer]

# Assume the upstream sql-mode, will use the same sql-mode to parse DDL statment, and set the same sql-mode at downstream
# when db-type is mysql
sql-mode = "STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION"
# Assume the upstream sql-mode.
# If this is setted , will use the same sql-mode to parse DDL statment, and set the same sql-mode at downstream when db-type is mysql.
# If this is not setted, it will not set any sql-mode.
# sql-mode = "STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION"

# disable sync these schema
ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql"
Expand Down
14 changes: 7 additions & 7 deletions drainer/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ var (

// SyncerConfig is the Syncer's configuration.
type SyncerConfig struct {
StrSQLMode string `toml:"sql-mode" json:"sql-mode"`
StrSQLMode *string `toml:"sql-mode" json:"sql-mode"`
SQLMode mysql.SQLMode `toml:"-" json:"-"`
IgnoreSchemas string `toml:"ignore-schemas" json:"ignore-schemas"`
TxnBatch int `toml:"txn-batch" json:"txn-batch"`
Expand Down Expand Up @@ -86,9 +86,7 @@ type Config struct {
func NewConfig() *Config {
cfg := &Config{
EtcdTimeout: defaultEtcdTimeout,
SyncerCfg: &SyncerConfig{
StrSQLMode: mysql.DefaultSQLMode,
},
SyncerCfg: &SyncerConfig{},
}
cfg.FlagSet = flag.NewFlagSet("drainer", flag.ContinueOnError)
fs := cfg.FlagSet
Expand Down Expand Up @@ -164,9 +162,11 @@ func (cfg *Config) Parse(args []string) error {
return errors.Trace(err)
}

cfg.SyncerCfg.SQLMode, err = mysql.GetSQLMode(cfg.SyncerCfg.StrSQLMode)
if err != nil {
return errors.Annotate(err, "invalid config: `sql-mode` must be a valid SQL_MODE")
if cfg.SyncerCfg.StrSQLMode != nil {
cfg.SyncerCfg.SQLMode, err = mysql.GetSQLMode(*cfg.SyncerCfg.StrSQLMode)
if err != nil {
return errors.Annotate(err, "invalid config: `sql-mode` must be a valid SQL_MODE")
}
}

cfg.tls, err = cfg.Security.ToTLSConfig()
Expand Down
5 changes: 3 additions & 2 deletions drainer/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,9 @@ func (t *testDrainerSuite) TestConfig(c *C) {
c.Assert(cfg.SyncerCfg.TxnBatch, Equals, 1)
c.Assert(cfg.SyncerCfg.DestDBType, Equals, "mysql")
c.Assert(cfg.SyncerCfg.To.Host, Equals, "127.0.0.1")
c.Assert(cfg.SyncerCfg.StrSQLMode, Equals, mysql.DefaultSQLMode)
c.Assert(cfg.SyncerCfg.SQLMode, Equals, mysql.ModeStrictTransTables|mysql.ModeNoEngineSubstitution)
var strSQLMode *string
c.Assert(cfg.SyncerCfg.StrSQLMode, Equals, strSQLMode)
c.Assert(cfg.SyncerCfg.SQLMode, Equals, mysql.SQLMode(0))
}

func (t *testDrainerSuite) TestValidate(c *C) {
Expand Down
2 changes: 1 addition & 1 deletion drainer/executor/executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ type Executor interface {
}

// New returns the an Executor instance by given name
func New(name string, cfg *DBConfig, sqlMode string) (Executor, error) {
func New(name string, cfg *DBConfig, sqlMode *string) (Executor, error) {
switch name {
case "mysql", "tidb":
return newMysql(cfg, sqlMode)
Expand Down
2 changes: 1 addition & 1 deletion drainer/executor/mysql.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ type mysqlExecutor struct {
*baseError
}

func newMysql(cfg *DBConfig, sqlMode string) (Executor, error) {
func newMysql(cfg *DBConfig, sqlMode *string) (Executor, error) {
db, err := pkgsql.OpenDBWithSQLMode("mysql", cfg.Host, cfg.Port, cfg.User, cfg.Password, sqlMode)
if err != nil {
return nil, errors.Trace(err)
Expand Down
2 changes: 1 addition & 1 deletion drainer/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ func closeExecutors(executors ...executor.Executor) {
}
}

func createExecutors(destDBType string, cfg *executor.DBConfig, count int, sqlMODE string) ([]executor.Executor, error) {
func createExecutors(destDBType string, cfg *executor.DBConfig, count int, sqlMODE *string) ([]executor.Executor, error) {
executors := make([]executor.Executor, 0, count)
for i := 0; i < count; i++ {
executor, err := executor.New(destDBType, cfg, sqlMODE)
Expand Down
11 changes: 5 additions & 6 deletions pkg/sql/sql.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,24 +118,23 @@ func ExecuteTxnWithHistogram(db *sql.DB, sqls []string, args [][]interface{}, hi
}

// OpenDBWithSQLMode creates an instance of sql.DB.
func OpenDBWithSQLMode(proto string, host string, port int, username string, password string, sqlMode string) (*sql.DB, error) {
func OpenDBWithSQLMode(proto string, host string, port int, username string, password string, sqlMode *string) (*sql.DB, error) {
dbDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/?charset=utf8mb4,utf8&multiStatements=true", username, password, host, port)
if len(sqlMode) > 0 {
log.Info("set sql mode: ", sqlMode)
if sqlMode != nil {
// same as "set sql_mode = '<sqlMode>'"
dbDSN += "&sql_mode='" + url.QueryEscape(sqlMode) + "'"
dbDSN += "&sql_mode='" + url.QueryEscape(*sqlMode) + "'"
}
db, err := sql.Open(proto, dbDSN)
if err != nil {
return nil, errors.Trace(err)
return nil, errors.Annotatef(err, "dsn: %s", dbDSN)
}

return db, nil
}

// OpenDB creates an instance of sql.DB.
func OpenDB(proto string, host string, port int, username string, password string) (*sql.DB, error) {
return OpenDBWithSQLMode(proto, host, port, username, password, "")
return OpenDBWithSQLMode(proto, host, port, username, password, nil)
}

// IgnoreDDLError checks the error can be ignored or not.
Expand Down