From 0c09d6c9faa0b36b22697552f78f4eb958b35737 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Mon, 22 Jul 2019 14:56:28 +0800 Subject: [PATCH 01/11] add load toml check for pump and test whether it works --- pump/config.go | 29 +++++++++++++++++++++++++++-- pump/config_test.go | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/pump/config.go b/pump/config.go index b3a4932bb..b17aea9d3 100644 --- a/pump/config.go +++ b/pump/config.go @@ -80,6 +80,18 @@ type Config struct { Storage storage.Config `toml:"storage" json:"storage"` } +// The ErrConfigValidationFailed error is used so that external callers can do a type assertion +// to defer handling of this specific error when someone does not want strict type checking. +// This is needed only because logging hasn't been set up at the time we parse the config file. +// This should all be ripped out once strict config checking is made the default behavior. +type ErrConfigValidationFailed struct { + err string +} + +func (e *ErrConfigValidationFailed) Error() string { + return e.err +} + // NewConfig return an instance of configuration func NewConfig() *Config { cfg := &Config{ @@ -175,8 +187,21 @@ func (cfg *Config) Parse(arguments []string) error { } func (cfg *Config) configFromFile(path string) error { - _, err := toml.DecodeFile(path, cfg) - return errors.Trace(err) + metaData, err := toml.DecodeFile(path, cfg) + + // If any items in confFile file are not mapped into the Config struct, issue + // an error and stop the server from starting. + if err == nil { + if undecoded := metaData.Undecoded(); len(undecoded) > 0 { + var undecodedItems []string + for _, item := range undecoded { + undecodedItems = append(undecodedItems, item.String()) + } + err = &ErrConfigValidationFailed{fmt.Sprintf("config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", "))} + } + } + + return err } // validate checks whether the configuration is valid diff --git a/pump/config_test.go b/pump/config_test.go index 2d571a81f..8371d7ecd 100644 --- a/pump/config_test.go +++ b/pump/config_test.go @@ -84,7 +84,7 @@ func (s *testConfigSuite) TestConfigParsingEnvFlags(c *C) { func (s *testConfigSuite) TestConfigParsingFileFlags(c *C) { yc := struct { ListenAddr string `toml:"addr" json:"addr"` - AdvertiseAddr string `toml:"advertiser-addr" json:"advertise-addr"` + AdvertiseAddr string `toml:"advertise-addr" json:"advertise-addr"` EtcdURLs string `toml:"pd-urls" json:"pd-urls"` BinlogDir string `toml:"data-dir" json:"data-dir"` HeartbeatInterval uint `toml:"heartbeat-interval" json:"heartbeat-interval"` @@ -116,6 +116,43 @@ func (s *testConfigSuite) TestConfigParsingFileFlags(c *C) { validateConfig(c, cfg) } +func (s *testConfigSuite) TestConfigParsingFileWithInvalidArgs(c *C) { + yc := struct { + ListenAddr string `toml:"addr" json:"addr"` + AdvertiseAddr string `toml:"advertise-addr" json:"advertise-addr"` + EtcdURLs string `toml:"pd-urls" json:"pd-urls"` + BinlogDir string `toml:"data-dir" json:"data-dir"` + HeartbeatInterval uint `toml:"heartbeat-interval" json:"heartbeat-interval"` + UnrecognizedOptionTest bool `toml:"unrecognized-option-test" json:"unrecognized-option-test"` + }{ + "192.168.199.100:8260", + "192.168.199.100:8260", + "http://192.168.199.110:2379,http://hostname:2379", + "/tmp/pump", + 1500, + true, + } + + var buf bytes.Buffer + e := toml.NewEncoder(&buf) + err := e.Encode(yc) + c.Assert(err, IsNil) + + tmpfile := mustCreateCfgFile(c, buf.Bytes(), "pump_config") + defer os.Remove(tmpfile.Name()) + + args := []string{ + "--config", + tmpfile.Name(), + "-L", "debug", + } + + os.Clearenv() + cfg := NewConfig() + err = cfg.Parse(args) + c.Assert(err, ErrorMatches, ".*contained unknown configuration options.*") +} + func mustSuccess(c *C, err error) { c.Assert(err, IsNil) } From dafcb61634e8aa00e3c8d989dc322388164e7fa8 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Mon, 22 Jul 2019 16:42:14 +0800 Subject: [PATCH 02/11] add load toml check for drainer and test whether it works --- drainer/config.go | 17 +++++++++++-- drainer/config_test.go | 54 ++++++++++++++++++++++++++++++++++++++++++ pump/config.go | 14 +---------- 3 files changed, 70 insertions(+), 15 deletions(-) diff --git a/drainer/config.go b/drainer/config.go index 42e4a212f..fbc8ea578 100644 --- a/drainer/config.go +++ b/drainer/config.go @@ -220,8 +220,21 @@ func (c *SyncerConfig) adjustDoDBAndTable() { } func (cfg *Config) configFromFile(path string) error { - _, err := toml.DecodeFile(path, cfg) - return errors.Trace(err) + metaData, err := toml.DecodeFile(path, cfg) + + // If any items in confFile file are not mapped into the Config struct, issue + // an error and stop the server from starting. + if err == nil { + if undecoded := metaData.Undecoded(); len(undecoded) > 0 { + var undecodedItems []string + for _, item := range undecoded { + undecodedItems = append(undecodedItems, item.String()) + } + err = errors.New(fmt.Sprintf("config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", "))) + } + } + + return err } // validate checks whether the configuration is valid diff --git a/drainer/config_test.go b/drainer/config_test.go index 5ae65dd83..51c8904e6 100644 --- a/drainer/config_test.go +++ b/drainer/config_test.go @@ -14,6 +14,10 @@ package drainer import ( + "bytes" + "github.com/BurntSushi/toml" + "io/ioutil" + "os" "testing" "github.com/coreos/etcd/integration" @@ -118,3 +122,53 @@ func (t *testDrainerSuite) TestAdjustConfig(c *C) { c.Assert(cfg.ListenAddr, Equals, "http://0.0.0.0:8257") c.Assert(cfg.AdvertiseAddr, Equals, "http://192.168.15.12:8257") } + +func (t *testDrainerSuite) TestConfigParsingFileWithInvalidOptions(c *C) { + yc := struct { + DataDir string `toml:"data-dir" json:"data-dir"` + ListenAddr string `toml:"addr" json:"addr"` + AdvertiseAddr string `toml:"advertise-addr" json:"advertise-addr"` + UnrecognizedOptionTest bool `toml:"unrecognized-option-test" json:"unrecognized-option-test"` + }{ + "data.drainer", + "192.168.15.10:8257", + "192.168.15.10:8257", + true, + } + + var buf bytes.Buffer + e := toml.NewEncoder(&buf) + err := e.Encode(yc) + c.Assert(err, IsNil) + + tmpfile := mustCreateCfgFile(c, buf.Bytes(), "drainer_config") + defer os.Remove(tmpfile.Name()) + + args := []string{ + "--config", + tmpfile.Name(), + "-L", "debug", + } + + os.Clearenv() + cfg := NewConfig() + err = cfg.Parse(args) + c.Assert(err, ErrorMatches, ".*contained unknown configuration options:.*") +} + +func mustSuccess(c *C, err error) { + c.Assert(err, IsNil) +} + +func mustCreateCfgFile(c *C, b []byte, prefix string) *os.File { + tmpfile, err := ioutil.TempFile("", prefix) + mustSuccess(c, err) + + _, err = tmpfile.Write(b) + mustSuccess(c, err) + + err = tmpfile.Close() + mustSuccess(c, err) + + return tmpfile +} diff --git a/pump/config.go b/pump/config.go index b17aea9d3..729ea8f74 100644 --- a/pump/config.go +++ b/pump/config.go @@ -80,18 +80,6 @@ type Config struct { Storage storage.Config `toml:"storage" json:"storage"` } -// The ErrConfigValidationFailed error is used so that external callers can do a type assertion -// to defer handling of this specific error when someone does not want strict type checking. -// This is needed only because logging hasn't been set up at the time we parse the config file. -// This should all be ripped out once strict config checking is made the default behavior. -type ErrConfigValidationFailed struct { - err string -} - -func (e *ErrConfigValidationFailed) Error() string { - return e.err -} - // NewConfig return an instance of configuration func NewConfig() *Config { cfg := &Config{ @@ -197,7 +185,7 @@ func (cfg *Config) configFromFile(path string) error { for _, item := range undecoded { undecodedItems = append(undecodedItems, item.String()) } - err = &ErrConfigValidationFailed{fmt.Sprintf("config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", "))} + err = errors.New(fmt.Sprintf("config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", "))) } } From b6ee35b0de5c4e9a9bc8ac9b3ebebb663f69ee09 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Mon, 22 Jul 2019 19:05:59 +0800 Subject: [PATCH 03/11] move func createConfig to pkg/util/util.go and add test --- drainer/config_test.go | 25 +++++-------------------- pkg/util/util.go | 22 ++++++++++++++++++++++ pkg/util/util_test.go | 15 +++++++++++++++ pump/config_test.go | 26 ++++++++++---------------- 4 files changed, 52 insertions(+), 36 deletions(-) diff --git a/drainer/config_test.go b/drainer/config_test.go index 51c8904e6..0289a5510 100644 --- a/drainer/config_test.go +++ b/drainer/config_test.go @@ -16,7 +16,6 @@ package drainer import ( "bytes" "github.com/BurntSushi/toml" - "io/ioutil" "os" "testing" @@ -141,8 +140,11 @@ func (t *testDrainerSuite) TestConfigParsingFileWithInvalidOptions(c *C) { err := e.Encode(yc) c.Assert(err, IsNil) - tmpfile := mustCreateCfgFile(c, buf.Bytes(), "drainer_config") - defer os.Remove(tmpfile.Name()) + tmpfile, err := util.CreateCfgFile(buf.Bytes(), "drainer_config") + if tmpfile != nil && len(tmpfile.Name()) > 0 { + defer os.Remove(tmpfile.Name()) + } + c.Assert(err, IsNil) args := []string{ "--config", @@ -155,20 +157,3 @@ func (t *testDrainerSuite) TestConfigParsingFileWithInvalidOptions(c *C) { err = cfg.Parse(args) c.Assert(err, ErrorMatches, ".*contained unknown configuration options:.*") } - -func mustSuccess(c *C, err error) { - c.Assert(err, IsNil) -} - -func mustCreateCfgFile(c *C, b []byte, prefix string) *os.File { - tmpfile, err := ioutil.TempFile("", prefix) - mustSuccess(c, err) - - _, err = tmpfile.Write(b) - mustSuccess(c, err) - - err = tmpfile.Close() - mustSuccess(c, err) - - return tmpfile -} diff --git a/pkg/util/util.go b/pkg/util/util.go index cfd901a4b..831adfaef 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -16,7 +16,9 @@ package util import ( "context" "fmt" + "io/ioutil" "net" + "os" "time" "github.com/pingcap/errors" @@ -258,3 +260,23 @@ func AdjustDuration(v *time.Duration, defValue time.Duration) { *v = defValue } } + +// CreateCfgFile creates and writes b as config file to dir prefix +func CreateCfgFile(b []byte, prefix string) (*os.File, error) { + tmpfile, err := ioutil.TempFile("", prefix) + if err != nil { + return tmpfile, errors.Trace(err) + } + + _, err = tmpfile.Write(b) + if err != nil { + return tmpfile, errors.Trace(err) + } + + err = tmpfile.Close() + if err != nil { + return tmpfile, errors.Trace(err) + } + + return tmpfile, nil +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 7c73757c3..934224f3b 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -16,7 +16,9 @@ package util import ( "context" "errors" + "io/ioutil" "net" + "os" "testing" "time" @@ -110,6 +112,19 @@ func (s *utilSuite) TestStdLogger(c *C) { c.Assert(entrys[2].Message, Matches, ".*hola:Goodbye!.*") } +func (s *utilSuite) TestCreateCfgFile(c *C) { + var msg = []byte("hello world!") + filename := "TestCreateCfg" + tmpfile, err := CreateCfgFile(msg, filename) + if tmpfile != nil && len(tmpfile.Name()) > 0 { + defer os.Remove(tmpfile.Name()) + } + c.Assert(err, IsNil) + b, err := ioutil.ReadFile(tmpfile.Name()) + c.Assert(err, IsNil) + c.Assert(string(b), Equals, string(msg)) +} + type getAddrIPSuite struct{} var _ = Suite(&getAddrIPSuite{}) diff --git a/pump/config_test.go b/pump/config_test.go index 8371d7ecd..6f70b4964 100644 --- a/pump/config_test.go +++ b/pump/config_test.go @@ -15,7 +15,7 @@ package pump import ( "bytes" - "io/ioutil" + "github.com/pingcap/tidb-binlog/pkg/util" "os" "github.com/BurntSushi/toml" @@ -101,7 +101,12 @@ func (s *testConfigSuite) TestConfigParsingFileFlags(c *C) { err := e.Encode(yc) c.Assert(err, IsNil) - tmpfile := mustCreateCfgFile(c, buf.Bytes(), "pump_config") + tmpfile, err := util.CreateCfgFile(buf.Bytes(), "pump_config") + if tmpfile != nil && len(tmpfile.Name()) > 0 { + defer os.Remove(tmpfile.Name()) + } + c.Assert(err, IsNil) + defer os.Remove(tmpfile.Name()) args := []string{ @@ -138,7 +143,9 @@ func (s *testConfigSuite) TestConfigParsingFileWithInvalidArgs(c *C) { err := e.Encode(yc) c.Assert(err, IsNil) - tmpfile := mustCreateCfgFile(c, buf.Bytes(), "pump_config") + tmpfile, err := util.CreateCfgFile(buf.Bytes(), "pump_config") + c.Assert(err, IsNil) + defer os.Remove(tmpfile.Name()) args := []string{ @@ -157,19 +164,6 @@ func mustSuccess(c *C, err error) { c.Assert(err, IsNil) } -func mustCreateCfgFile(c *C, b []byte, prefix string) *os.File { - tmpfile, err := ioutil.TempFile("", prefix) - mustSuccess(c, err) - - _, err = tmpfile.Write(b) - mustSuccess(c, err) - - err = tmpfile.Close() - mustSuccess(c, err) - - return tmpfile -} - func validateConfig(c *C, cfg *Config) { vcfg := &Config{ ListenAddr: "http://192.168.199.100:8260", From 579056b78154be9f281f6c025cf3100567e0e6e0 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Tue, 23 Jul 2019 19:43:19 +0800 Subject: [PATCH 04/11] change errors.New to errors.Errorf, create tmp dir to storage tmp config --- drainer/config.go | 2 +- drainer/config_test.go | 5 +---- pkg/util/util.go | 4 ++-- pkg/util/util_test.go | 6 +----- pump/config.go | 2 +- pump/config_test.go | 16 ++++------------ 6 files changed, 10 insertions(+), 25 deletions(-) diff --git a/drainer/config.go b/drainer/config.go index 90426ab91..215b8f261 100644 --- a/drainer/config.go +++ b/drainer/config.go @@ -233,7 +233,7 @@ func (cfg *Config) configFromFile(path string) error { for _, item := range undecoded { undecodedItems = append(undecodedItems, item.String()) } - err = errors.New(fmt.Sprintf("config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", "))) + err = errors.Errorf("config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", ")) } } diff --git a/drainer/config_test.go b/drainer/config_test.go index 0289a5510..d1eed0e2b 100644 --- a/drainer/config_test.go +++ b/drainer/config_test.go @@ -140,10 +140,7 @@ func (t *testDrainerSuite) TestConfigParsingFileWithInvalidOptions(c *C) { err := e.Encode(yc) c.Assert(err, IsNil) - tmpfile, err := util.CreateCfgFile(buf.Bytes(), "drainer_config") - if tmpfile != nil && len(tmpfile.Name()) > 0 { - defer os.Remove(tmpfile.Name()) - } + tmpfile, err := util.CreateCfgFile(buf.Bytes(), c.MkDir(), "drainer_config_invalid") c.Assert(err, IsNil) args := []string{ diff --git a/pkg/util/util.go b/pkg/util/util.go index 831adfaef..4c4f321f7 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -262,8 +262,8 @@ func AdjustDuration(v *time.Duration, defValue time.Duration) { } // CreateCfgFile creates and writes b as config file to dir prefix -func CreateCfgFile(b []byte, prefix string) (*os.File, error) { - tmpfile, err := ioutil.TempFile("", prefix) +func CreateCfgFile(b []byte, dir, prefix string) (*os.File, error) { + tmpfile, err := ioutil.TempFile(dir, prefix) if err != nil { return tmpfile, errors.Trace(err) } diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 934224f3b..0c88499b6 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -18,7 +18,6 @@ import ( "errors" "io/ioutil" "net" - "os" "testing" "time" @@ -115,10 +114,7 @@ func (s *utilSuite) TestStdLogger(c *C) { func (s *utilSuite) TestCreateCfgFile(c *C) { var msg = []byte("hello world!") filename := "TestCreateCfg" - tmpfile, err := CreateCfgFile(msg, filename) - if tmpfile != nil && len(tmpfile.Name()) > 0 { - defer os.Remove(tmpfile.Name()) - } + tmpfile, err := CreateCfgFile(msg, c.MkDir(), filename) c.Assert(err, IsNil) b, err := ioutil.ReadFile(tmpfile.Name()) c.Assert(err, IsNil) diff --git a/pump/config.go b/pump/config.go index 34b79ba44..f743d12ec 100644 --- a/pump/config.go +++ b/pump/config.go @@ -185,7 +185,7 @@ func (cfg *Config) configFromFile(path string) error { for _, item := range undecoded { undecodedItems = append(undecodedItems, item.String()) } - err = errors.New(fmt.Sprintf("config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", "))) + err = errors.Errorf("config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", ")) } } diff --git a/pump/config_test.go b/pump/config_test.go index 6f70b4964..72531bb8f 100644 --- a/pump/config_test.go +++ b/pump/config_test.go @@ -15,11 +15,10 @@ package pump import ( "bytes" - "github.com/pingcap/tidb-binlog/pkg/util" - "os" - "github.com/BurntSushi/toml" . "github.com/pingcap/check" + "github.com/pingcap/tidb-binlog/pkg/util" + "os" ) var _ = Suite(&testConfigSuite{}) @@ -101,14 +100,9 @@ func (s *testConfigSuite) TestConfigParsingFileFlags(c *C) { err := e.Encode(yc) c.Assert(err, IsNil) - tmpfile, err := util.CreateCfgFile(buf.Bytes(), "pump_config") - if tmpfile != nil && len(tmpfile.Name()) > 0 { - defer os.Remove(tmpfile.Name()) - } + tmpfile, err := util.CreateCfgFile(buf.Bytes(), c.MkDir(), "pump_config") c.Assert(err, IsNil) - defer os.Remove(tmpfile.Name()) - args := []string{ "--config", tmpfile.Name(), @@ -143,11 +137,9 @@ func (s *testConfigSuite) TestConfigParsingFileWithInvalidArgs(c *C) { err := e.Encode(yc) c.Assert(err, IsNil) - tmpfile, err := util.CreateCfgFile(buf.Bytes(), "pump_config") + tmpfile, err := util.CreateCfgFile(buf.Bytes(), c.MkDir(), "pump_config_invalid") c.Assert(err, IsNil) - defer os.Remove(tmpfile.Name()) - args := []string{ "--config", tmpfile.Name(), From 255ad427ecbb142a42feb69ef6ee79e41a844554 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Wed, 24 Jul 2019 11:26:26 +0800 Subject: [PATCH 05/11] change CreateCfgFile to ioutil.WriteFile, add package name for errors for unrecognized option, make the unrecognized option test more strict, change error handle way in /*/config.go --- drainer/config.go | 15 ++++++++------- drainer/config_test.go | 9 ++++++--- pkg/util/util.go | 22 ---------------------- pkg/util/util_test.go | 11 ----------- pump/config.go | 15 ++++++++------- pump/config_test.go | 15 +++++++++------ 6 files changed, 31 insertions(+), 56 deletions(-) diff --git a/drainer/config.go b/drainer/config.go index 215b8f261..6659f62b5 100644 --- a/drainer/config.go +++ b/drainer/config.go @@ -227,14 +227,15 @@ func (cfg *Config) configFromFile(path string) error { // If any items in confFile file are not mapped into the Config struct, issue // an error and stop the server from starting. - if err == nil { - if undecoded := metaData.Undecoded(); len(undecoded) > 0 { - var undecodedItems []string - for _, item := range undecoded { - undecodedItems = append(undecodedItems, item.String()) - } - err = errors.Errorf("config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", ")) + if err != nil { + return err + } + if undecoded := metaData.Undecoded(); len(undecoded) > 0 { + var undecodedItems []string + for _, item := range undecoded { + undecodedItems = append(undecodedItems, item.String()) } + err = errors.Errorf("drainer config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", ")) } return err diff --git a/drainer/config_test.go b/drainer/config_test.go index d1eed0e2b..0e0a4840b 100644 --- a/drainer/config_test.go +++ b/drainer/config_test.go @@ -16,7 +16,9 @@ package drainer import ( "bytes" "github.com/BurntSushi/toml" + "io/ioutil" "os" + "path" "testing" "github.com/coreos/etcd/integration" @@ -140,17 +142,18 @@ func (t *testDrainerSuite) TestConfigParsingFileWithInvalidOptions(c *C) { err := e.Encode(yc) c.Assert(err, IsNil) - tmpfile, err := util.CreateCfgFile(buf.Bytes(), c.MkDir(), "drainer_config_invalid") + configFilename := path.Join(c.MkDir(), "drainer_config_invalid.toml") + err = ioutil.WriteFile(configFilename, buf.Bytes(), 0644) c.Assert(err, IsNil) args := []string{ "--config", - tmpfile.Name(), + configFilename, "-L", "debug", } os.Clearenv() cfg := NewConfig() err = cfg.Parse(args) - c.Assert(err, ErrorMatches, ".*contained unknown configuration options:.*") + c.Assert(err, ErrorMatches, ".*contained unknown configuration options: unrecognized-option-test.*") } diff --git a/pkg/util/util.go b/pkg/util/util.go index 4c4f321f7..cfd901a4b 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -16,9 +16,7 @@ package util import ( "context" "fmt" - "io/ioutil" "net" - "os" "time" "github.com/pingcap/errors" @@ -260,23 +258,3 @@ func AdjustDuration(v *time.Duration, defValue time.Duration) { *v = defValue } } - -// CreateCfgFile creates and writes b as config file to dir prefix -func CreateCfgFile(b []byte, dir, prefix string) (*os.File, error) { - tmpfile, err := ioutil.TempFile(dir, prefix) - if err != nil { - return tmpfile, errors.Trace(err) - } - - _, err = tmpfile.Write(b) - if err != nil { - return tmpfile, errors.Trace(err) - } - - err = tmpfile.Close() - if err != nil { - return tmpfile, errors.Trace(err) - } - - return tmpfile, nil -} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 0c88499b6..7c73757c3 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -16,7 +16,6 @@ package util import ( "context" "errors" - "io/ioutil" "net" "testing" "time" @@ -111,16 +110,6 @@ func (s *utilSuite) TestStdLogger(c *C) { c.Assert(entrys[2].Message, Matches, ".*hola:Goodbye!.*") } -func (s *utilSuite) TestCreateCfgFile(c *C) { - var msg = []byte("hello world!") - filename := "TestCreateCfg" - tmpfile, err := CreateCfgFile(msg, c.MkDir(), filename) - c.Assert(err, IsNil) - b, err := ioutil.ReadFile(tmpfile.Name()) - c.Assert(err, IsNil) - c.Assert(string(b), Equals, string(msg)) -} - type getAddrIPSuite struct{} var _ = Suite(&getAddrIPSuite{}) diff --git a/pump/config.go b/pump/config.go index f743d12ec..64d39308f 100644 --- a/pump/config.go +++ b/pump/config.go @@ -179,14 +179,15 @@ func (cfg *Config) configFromFile(path string) error { // If any items in confFile file are not mapped into the Config struct, issue // an error and stop the server from starting. - if err == nil { - if undecoded := metaData.Undecoded(); len(undecoded) > 0 { - var undecodedItems []string - for _, item := range undecoded { - undecodedItems = append(undecodedItems, item.String()) - } - err = errors.Errorf("config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", ")) + if err != nil { + return err + } + if undecoded := metaData.Undecoded(); len(undecoded) > 0 { + var undecodedItems []string + for _, item := range undecoded { + undecodedItems = append(undecodedItems, item.String()) } + err = errors.Errorf("pump config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", ")) } return err diff --git a/pump/config_test.go b/pump/config_test.go index 72531bb8f..69cdfb86a 100644 --- a/pump/config_test.go +++ b/pump/config_test.go @@ -17,8 +17,9 @@ import ( "bytes" "github.com/BurntSushi/toml" . "github.com/pingcap/check" - "github.com/pingcap/tidb-binlog/pkg/util" + "io/ioutil" "os" + "path" ) var _ = Suite(&testConfigSuite{}) @@ -100,12 +101,13 @@ func (s *testConfigSuite) TestConfigParsingFileFlags(c *C) { err := e.Encode(yc) c.Assert(err, IsNil) - tmpfile, err := util.CreateCfgFile(buf.Bytes(), c.MkDir(), "pump_config") + configFilename := path.Join(c.MkDir(), "pump_config.toml") + err = ioutil.WriteFile(configFilename, buf.Bytes(), 0644) c.Assert(err, IsNil) args := []string{ "--config", - tmpfile.Name(), + configFilename, "-L", "debug", } @@ -137,19 +139,20 @@ func (s *testConfigSuite) TestConfigParsingFileWithInvalidArgs(c *C) { err := e.Encode(yc) c.Assert(err, IsNil) - tmpfile, err := util.CreateCfgFile(buf.Bytes(), c.MkDir(), "pump_config_invalid") + configFilename := path.Join(c.MkDir(), "pump_config_invalid.toml") + err = ioutil.WriteFile(configFilename, buf.Bytes(), 0644) c.Assert(err, IsNil) args := []string{ "--config", - tmpfile.Name(), + configFilename, "-L", "debug", } os.Clearenv() cfg := NewConfig() err = cfg.Parse(args) - c.Assert(err, ErrorMatches, ".*contained unknown configuration options.*") + c.Assert(err, ErrorMatches, ".*contained unknown configuration options: unrecognized-option-test.*") } func mustSuccess(c *C, err error) { From 827732d7fa972bc5e7721de1ca575c162d6eca76 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Wed, 24 Jul 2019 13:13:14 +0800 Subject: [PATCH 06/11] remove os.ClearEnv, add configFromFile error handler for arbiter & reparo --- arbiter/config.go | 19 +++++++++++++++++-- arbiter/config_test.go | 35 +++++++++++++++++++++++++++++++++++ drainer/config_test.go | 2 -- pump/config_test.go | 2 -- reparo/config.go | 18 ++++++++++++++++-- reparo/config_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 109 insertions(+), 8 deletions(-) diff --git a/arbiter/config.go b/arbiter/config.go index 36644603a..ec26d3fc5 100644 --- a/arbiter/config.go +++ b/arbiter/config.go @@ -18,6 +18,7 @@ import ( "flag" "fmt" "os" + "strings" "github.com/BurntSushi/toml" "github.com/pingcap/errors" @@ -190,6 +191,20 @@ func (cfg *Config) adjustConfig() error { } func (cfg *Config) configFromFile(path string) error { - _, err := toml.DecodeFile(path, cfg) - return errors.Trace(err) + metaData, err := toml.DecodeFile(path, cfg) + + // If any items in confFile file are not mapped into the Config struct, issue + // an error and stop the server from starting. + if err != nil { + return err + } + if undecoded := metaData.Undecoded(); len(undecoded) > 0 { + var undecodedItems []string + for _, item := range undecoded { + undecodedItems = append(undecodedItems, item.String()) + } + err = errors.Errorf("arbiter config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", ")) + } + + return err } diff --git a/arbiter/config_test.go b/arbiter/config_test.go index 8b7907aed..0c4e7877a 100644 --- a/arbiter/config_test.go +++ b/arbiter/config_test.go @@ -14,7 +14,10 @@ package arbiter import ( + "bytes" "fmt" + "github.com/BurntSushi/toml" + "io/ioutil" "path" "runtime" "strings" @@ -105,6 +108,38 @@ func (t *TestConfigSuite) TestParseConfig(c *check.C) { c.Assert(strings.Contains(config.String(), listenAddr), check.IsTrue) } +func (t *TestConfigSuite) TestParseConfigFileWithInvalidArgs(c *check.C) { + yc := struct { + LogLevel string `toml:"log-level" json:"log-level"` + ListenAddr string `toml:"addr" json:"addr"` + LogFile string `toml:"log-file" json:"log-file"` + UnrecognizedOptionTest bool `toml:"unrecognized-option-test" json:"unrecognized-option-test"` + }{ + "debug", + "127.0.0.1:8251", + "/tmp/arbiter", + true, + } + + var buf bytes.Buffer + e := toml.NewEncoder(&buf) + err := e.Encode(yc) + c.Assert(err, check.IsNil) + + configFilename := path.Join(c.MkDir(), "arbiter_config_invalid.toml") + err = ioutil.WriteFile(configFilename, buf.Bytes(), 0644) + c.Assert(err, check.IsNil) + + args := []string{ + "--config", + configFilename, + } + + cfg := NewConfig() + err = cfg.Parse(args) + c.Assert(err, check.ErrorMatches, ".*contained unknown configuration options: unrecognized-option-test.*") +} + func getTemplateConfigFilePath() string { // we put the template config file in "cmd/arbiter/arbiter.toml" _, filename, _, _ := runtime.Caller(0) diff --git a/drainer/config_test.go b/drainer/config_test.go index 0e0a4840b..ae17846da 100644 --- a/drainer/config_test.go +++ b/drainer/config_test.go @@ -17,7 +17,6 @@ import ( "bytes" "github.com/BurntSushi/toml" "io/ioutil" - "os" "path" "testing" @@ -152,7 +151,6 @@ func (t *testDrainerSuite) TestConfigParsingFileWithInvalidOptions(c *C) { "-L", "debug", } - os.Clearenv() cfg := NewConfig() err = cfg.Parse(args) c.Assert(err, ErrorMatches, ".*contained unknown configuration options: unrecognized-option-test.*") diff --git a/pump/config_test.go b/pump/config_test.go index 69cdfb86a..667250687 100644 --- a/pump/config_test.go +++ b/pump/config_test.go @@ -111,7 +111,6 @@ func (s *testConfigSuite) TestConfigParsingFileFlags(c *C) { "-L", "debug", } - os.Clearenv() cfg := NewConfig() mustSuccess(c, cfg.Parse(args)) validateConfig(c, cfg) @@ -149,7 +148,6 @@ func (s *testConfigSuite) TestConfigParsingFileWithInvalidArgs(c *C) { "-L", "debug", } - os.Clearenv() cfg := NewConfig() err = cfg.Parse(args) c.Assert(err, ErrorMatches, ".*contained unknown configuration options: unrecognized-option-test.*") diff --git a/reparo/config.go b/reparo/config.go index 3de84f3da..b5be95994 100644 --- a/reparo/config.go +++ b/reparo/config.go @@ -168,8 +168,22 @@ func (c *Config) adjustDoDBAndTable() { } func (c *Config) configFromFile(path string) error { - _, err := toml.DecodeFile(path, c) - return errors.Trace(err) + metaData, err := toml.DecodeFile(path, c) + + // If any items in confFile file are not mapped into the Config struct, issue + // an error and stop the server from starting. + if err != nil { + return err + } + if undecoded := metaData.Undecoded(); len(undecoded) > 0 { + var undecodedItems []string + for _, item := range undecoded { + undecodedItems = append(undecodedItems, item.String()) + } + err = errors.Errorf("reparo config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", ")) + } + + return err } func (c *Config) validate() error { diff --git a/reparo/config_test.go b/reparo/config_test.go index 81efaa6a9..37f3be0a8 100644 --- a/reparo/config_test.go +++ b/reparo/config_test.go @@ -14,7 +14,10 @@ package reparo import ( + "bytes" "fmt" + "github.com/BurntSushi/toml" + "io/ioutil" "path" "runtime" @@ -72,6 +75,44 @@ func (s *testConfigSuite) TestAdjustDoDBAndTable(c *check.C) { c.Assert(config.DoDBs[1], check.Equals, "test2") } +func (s *testConfigSuite) TestParseConfigFileWithInvalidArgs(c *check.C) { + yc := struct { + Dir string `toml:"data-dir" json:"data-dir"` + StartDatetime string `toml:"start-datetime" json:"start-datetime"` + StopDatetime string `toml:"stop-datetime" json:"stop-datetime"` + StartTSO int64 `toml:"start-tso" json:"start-tso"` + StopTSO int64 `toml:"stop-tso" json:"stop-tso"` + LogFile string `toml:"log-file" json:"log-file"` + LogLevel string `toml:"log-level" json:"log-level"` + }{ + "/tmp/reparo", + "", + "", + 0, + 0, + "tmp/reparo/reparo.log", + "debug", + } + + var buf bytes.Buffer + e := toml.NewEncoder(&buf) + err := e.Encode(yc) + c.Assert(err, check.IsNil) + + configFilename := path.Join(c.MkDir(), "reparo_config_invalid.toml") + err = ioutil.WriteFile(configFilename, buf.Bytes(), 0644) + c.Assert(err, check.IsNil) + + args := []string{ + "--config", + configFilename, + } + + cfg := NewConfig() + err = cfg.Parse(args) + c.Assert(err, check.ErrorMatches, ".*contained unknown configuration options: unrecognized-option-test.*") +} + func getTemplateConfigFilePath() string { // we put the template config file in "cmd/reapro/reparo.toml" _, filename, _, _ := runtime.Caller(0) From 3766fa3438b227bf61f5e6c34b179f5844e17795 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Wed, 24 Jul 2019 13:49:13 +0800 Subject: [PATCH 07/11] fix bugs of pump & reparo's config_test --- pump/config_test.go | 1 + reparo/config_test.go | 16 +++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pump/config_test.go b/pump/config_test.go index 667250687..1c3668c10 100644 --- a/pump/config_test.go +++ b/pump/config_test.go @@ -75,6 +75,7 @@ func (s *testConfigSuite) TestConfigParsingEnvFlags(c *C) { os.Setenv("PUMP_ADDR", "192.168.199.200:9000") os.Setenv("PUMP_PD_URLS", "http://127.0.0.1:2379,http://localhost:2379") os.Setenv("PUMP_DATA_DIR", "/tmp/pump") + defer os.Clearenv() cfg := NewConfig() mustSuccess(c, cfg.Parse(args)) diff --git a/reparo/config_test.go b/reparo/config_test.go index 37f3be0a8..da755216b 100644 --- a/reparo/config_test.go +++ b/reparo/config_test.go @@ -77,13 +77,14 @@ func (s *testConfigSuite) TestAdjustDoDBAndTable(c *check.C) { func (s *testConfigSuite) TestParseConfigFileWithInvalidArgs(c *check.C) { yc := struct { - Dir string `toml:"data-dir" json:"data-dir"` - StartDatetime string `toml:"start-datetime" json:"start-datetime"` - StopDatetime string `toml:"stop-datetime" json:"stop-datetime"` - StartTSO int64 `toml:"start-tso" json:"start-tso"` - StopTSO int64 `toml:"stop-tso" json:"stop-tso"` - LogFile string `toml:"log-file" json:"log-file"` - LogLevel string `toml:"log-level" json:"log-level"` + Dir string `toml:"data-dir" json:"data-dir"` + StartDatetime string `toml:"start-datetime" json:"start-datetime"` + StopDatetime string `toml:"stop-datetime" json:"stop-datetime"` + StartTSO int64 `toml:"start-tso" json:"start-tso"` + StopTSO int64 `toml:"stop-tso" json:"stop-tso"` + LogFile string `toml:"log-file" json:"log-file"` + LogLevel string `toml:"log-level" json:"log-level"` + UnrecognizedOptionTest bool `toml:"unrecognized-option-test" json:"unrecognized-option-test"` }{ "/tmp/reparo", "", @@ -92,6 +93,7 @@ func (s *testConfigSuite) TestParseConfigFileWithInvalidArgs(c *check.C) { 0, "tmp/reparo/reparo.log", "debug", + true, } var buf bytes.Buffer From 6ab2712aa91c3353cc5ea53dbe20d12eb455d486 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Wed, 24 Jul 2019 15:43:38 +0800 Subject: [PATCH 08/11] simplify /*/config.go by using strict decode function in /pkg/util --- arbiter/config.go | 23 +++-------------------- drainer/config.go | 18 +----------------- pkg/util/util.go | 22 ++++++++++++++++++++++ pump/config.go | 18 +----------------- reparo/config.go | 19 ++----------------- 5 files changed, 29 insertions(+), 71 deletions(-) diff --git a/arbiter/config.go b/arbiter/config.go index ec26d3fc5..233af594a 100644 --- a/arbiter/config.go +++ b/arbiter/config.go @@ -17,15 +17,13 @@ import ( "encoding/json" "flag" "fmt" - "os" - "strings" - - "github.com/BurntSushi/toml" "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/pingcap/tidb-binlog/pkg/flags" + "github.com/pingcap/tidb-binlog/pkg/util" "github.com/pingcap/tidb-binlog/pkg/version" "go.uber.org/zap" + "os" ) const ( @@ -191,20 +189,5 @@ func (cfg *Config) adjustConfig() error { } func (cfg *Config) configFromFile(path string) error { - metaData, err := toml.DecodeFile(path, cfg) - - // If any items in confFile file are not mapped into the Config struct, issue - // an error and stop the server from starting. - if err != nil { - return err - } - if undecoded := metaData.Undecoded(); len(undecoded) > 0 { - var undecodedItems []string - for _, item := range undecoded { - undecodedItems = append(undecodedItems, item.String()) - } - err = errors.Errorf("arbiter config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", ")) - } - - return err + return util.StrictDecodeFile(path, "arbiter", cfg) } diff --git a/drainer/config.go b/drainer/config.go index 6659f62b5..32704366b 100644 --- a/drainer/config.go +++ b/drainer/config.go @@ -25,7 +25,6 @@ import ( "strings" "time" - "github.com/BurntSushi/toml" "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/pingcap/parser/mysql" @@ -223,22 +222,7 @@ func (c *SyncerConfig) adjustDoDBAndTable() { } func (cfg *Config) configFromFile(path string) error { - metaData, err := toml.DecodeFile(path, cfg) - - // If any items in confFile file are not mapped into the Config struct, issue - // an error and stop the server from starting. - if err != nil { - return err - } - if undecoded := metaData.Undecoded(); len(undecoded) > 0 { - var undecodedItems []string - for _, item := range undecoded { - undecodedItems = append(undecodedItems, item.String()) - } - err = errors.Errorf("drainer config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", ")) - } - - return err + return util.StrictDecodeFile(path, "drainer", cfg) } // validate checks whether the configuration is valid diff --git a/pkg/util/util.go b/pkg/util/util.go index cfd901a4b..b41a79548 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -16,7 +16,9 @@ package util import ( "context" "fmt" + "github.com/BurntSushi/toml" "net" + "strings" "time" "github.com/pingcap/errors" @@ -178,6 +180,26 @@ func RetryContext(ctx context.Context, retryCount int, sleepTime time.Duration, return err } +// StrictDecodeFile decodes the toml file strictly. If any item in confFile file is not mapped +// into the Config struct, issue an error and stop the server from starting. +func StrictDecodeFile(path, component string, cfg interface{}) error { + metaData, err := toml.DecodeFile(path, cfg) + if err != nil { + return err + } + + if undecoded := metaData.Undecoded(); len(undecoded) > 0 { + var undecodedItems []string + for _, item := range undecoded { + undecodedItems = append(undecodedItems, item.String()) + } + err = errors.Errorf("component %s's config file %s contained unknown configuration options: %s", + component, path, strings.Join(undecodedItems, ", ")) + } + + return err +} + // TryUntilSuccess retries the given function until error is nil or the context is done, // waiting for `waitInterval` time between retries. func TryUntilSuccess(ctx context.Context, waitInterval time.Duration, errMsg string, fn func() error) error { diff --git a/pump/config.go b/pump/config.go index 64d39308f..b9f29fed3 100644 --- a/pump/config.go +++ b/pump/config.go @@ -23,7 +23,6 @@ import ( "strings" "time" - "github.com/BurntSushi/toml" "github.com/pingcap/errors" "github.com/pingcap/tidb-binlog/pkg/flags" "github.com/pingcap/tidb-binlog/pkg/security" @@ -175,22 +174,7 @@ func (cfg *Config) Parse(arguments []string) error { } func (cfg *Config) configFromFile(path string) error { - metaData, err := toml.DecodeFile(path, cfg) - - // If any items in confFile file are not mapped into the Config struct, issue - // an error and stop the server from starting. - if err != nil { - return err - } - if undecoded := metaData.Undecoded(); len(undecoded) > 0 { - var undecodedItems []string - for _, item := range undecoded { - undecodedItems = append(undecodedItems, item.String()) - } - err = errors.Errorf("pump config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", ")) - } - - return err + return util.StrictDecodeFile(path, "pump", cfg) } // validate checks whether the configuration is valid diff --git a/reparo/config.go b/reparo/config.go index b5be95994..82fdef690 100644 --- a/reparo/config.go +++ b/reparo/config.go @@ -17,11 +17,11 @@ import ( "encoding/json" "flag" "fmt" + "github.com/pingcap/tidb-binlog/pkg/util" "os" "strings" "time" - "github.com/BurntSushi/toml" "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/pingcap/tidb-binlog/pkg/filter" @@ -168,22 +168,7 @@ func (c *Config) adjustDoDBAndTable() { } func (c *Config) configFromFile(path string) error { - metaData, err := toml.DecodeFile(path, c) - - // If any items in confFile file are not mapped into the Config struct, issue - // an error and stop the server from starting. - if err != nil { - return err - } - if undecoded := metaData.Undecoded(); len(undecoded) > 0 { - var undecodedItems []string - for _, item := range undecoded { - undecodedItems = append(undecodedItems, item.String()) - } - err = errors.Errorf("reparo config file %s contained unknown configuration options: %s", path, strings.Join(undecodedItems, ", ")) - } - - return err + return util.StrictDecodeFile(path, "reparo", c) } func (c *Config) validate() error { From 68461a92fcc790459468a69a9099e6add67669de Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Wed, 24 Jul 2019 15:50:00 +0800 Subject: [PATCH 09/11] change error to errors.Trace --- pkg/util/util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index b41a79548..8a315e7c4 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -185,7 +185,7 @@ func RetryContext(ctx context.Context, retryCount int, sleepTime time.Duration, func StrictDecodeFile(path, component string, cfg interface{}) error { metaData, err := toml.DecodeFile(path, cfg) if err != nil { - return err + return errors.Trace(err) } if undecoded := metaData.Undecoded(); len(undecoded) > 0 { @@ -197,7 +197,7 @@ func StrictDecodeFile(path, component string, cfg interface{}) error { component, path, strings.Join(undecodedItems, ", ")) } - return err + return errors.Trace(err) } // TryUntilSuccess retries the given function until error is nil or the context is done, From dffadec6bb7d61a11d1f020e5bcbb86c5aec47d2 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Wed, 24 Jul 2019 15:52:27 +0800 Subject: [PATCH 10/11] fix fmt error --- pkg/util/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index 8a315e7c4..6efe30b0c 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -194,7 +194,7 @@ func StrictDecodeFile(path, component string, cfg interface{}) error { undecodedItems = append(undecodedItems, item.String()) } err = errors.Errorf("component %s's config file %s contained unknown configuration options: %s", - component, path, strings.Join(undecodedItems, ", ")) + component, path, strings.Join(undecodedItems, ", ")) } return errors.Trace(err) From b47df28b74e2f59871b6d50f3e4f46f51847c867 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Wed, 24 Jul 2019 16:46:26 +0800 Subject: [PATCH 11/11] resort import packages order --- arbiter/config.go | 3 ++- arbiter/config_test.go | 2 +- drainer/config_test.go | 2 +- pkg/util/util.go | 2 +- pump/config_test.go | 5 +++-- reparo/config.go | 2 +- reparo/config_test.go | 2 +- 7 files changed, 10 insertions(+), 8 deletions(-) diff --git a/arbiter/config.go b/arbiter/config.go index 233af594a..e2f2a1ea3 100644 --- a/arbiter/config.go +++ b/arbiter/config.go @@ -17,13 +17,14 @@ import ( "encoding/json" "flag" "fmt" + "os" + "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/pingcap/tidb-binlog/pkg/flags" "github.com/pingcap/tidb-binlog/pkg/util" "github.com/pingcap/tidb-binlog/pkg/version" "go.uber.org/zap" - "os" ) const ( diff --git a/arbiter/config_test.go b/arbiter/config_test.go index 0c4e7877a..56bc7ac25 100644 --- a/arbiter/config_test.go +++ b/arbiter/config_test.go @@ -16,12 +16,12 @@ package arbiter import ( "bytes" "fmt" - "github.com/BurntSushi/toml" "io/ioutil" "path" "runtime" "strings" + "github.com/BurntSushi/toml" "github.com/pingcap/check" ) diff --git a/drainer/config_test.go b/drainer/config_test.go index ae17846da..4749b2843 100644 --- a/drainer/config_test.go +++ b/drainer/config_test.go @@ -15,11 +15,11 @@ package drainer import ( "bytes" - "github.com/BurntSushi/toml" "io/ioutil" "path" "testing" + "github.com/BurntSushi/toml" "github.com/coreos/etcd/integration" . "github.com/pingcap/check" "github.com/pingcap/parser/mysql" diff --git a/pkg/util/util.go b/pkg/util/util.go index 6efe30b0c..8cee45eda 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -16,11 +16,11 @@ package util import ( "context" "fmt" - "github.com/BurntSushi/toml" "net" "strings" "time" + "github.com/BurntSushi/toml" "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/pingcap/parser/model" diff --git a/pump/config_test.go b/pump/config_test.go index 1c3668c10..76dc6a6d4 100644 --- a/pump/config_test.go +++ b/pump/config_test.go @@ -15,11 +15,12 @@ package pump import ( "bytes" - "github.com/BurntSushi/toml" - . "github.com/pingcap/check" "io/ioutil" "os" "path" + + "github.com/BurntSushi/toml" + . "github.com/pingcap/check" ) var _ = Suite(&testConfigSuite{}) diff --git a/reparo/config.go b/reparo/config.go index 82fdef690..72652a190 100644 --- a/reparo/config.go +++ b/reparo/config.go @@ -17,7 +17,6 @@ import ( "encoding/json" "flag" "fmt" - "github.com/pingcap/tidb-binlog/pkg/util" "os" "strings" "time" @@ -26,6 +25,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/tidb-binlog/pkg/filter" "github.com/pingcap/tidb-binlog/pkg/flags" + "github.com/pingcap/tidb-binlog/pkg/util" "github.com/pingcap/tidb-binlog/pkg/version" "github.com/pingcap/tidb-binlog/reparo/syncer" "github.com/pingcap/tidb/store/tikv/oracle" diff --git a/reparo/config_test.go b/reparo/config_test.go index da755216b..e31c582d3 100644 --- a/reparo/config_test.go +++ b/reparo/config_test.go @@ -16,11 +16,11 @@ package reparo import ( "bytes" "fmt" - "github.com/BurntSushi/toml" "io/ioutil" "path" "runtime" + "github.com/BurntSushi/toml" "github.com/pingcap/check" "github.com/pingcap/tidb-binlog/pkg/filter" )