From 682cf94ef9d60693291cecb009bcb71e4dc0e43d Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 4 Jul 2022 12:26:14 +0800 Subject: [PATCH 01/15] binlog-filter: add new event type --- pkg/binlog-filter/filter.go | 84 ++++++++++++++++++++++++++++++-- pkg/binlog-filter/filter_test.go | 26 ++++++++++ 2 files changed, 105 insertions(+), 5 deletions(-) diff --git a/pkg/binlog-filter/filter.go b/pkg/binlog-filter/filter.go index 6ec81c0cf..f487f4746 100644 --- a/pkg/binlog-filter/filter.go +++ b/pkg/binlog-filter/filter.go @@ -64,9 +64,41 @@ const ( CreateView EventType = "create view" DropView EventType = "drop view" AlertTable EventType = "alter table" + + CreateSchema EventType = "create schema" // alias of CreateDatabase + DropSchema EventType = "drop schema" // alias of DropDatabase + AddColumn EventType = "add column" + DropColumn EventType = "drop column" + AddIndex EventType = "add index" // alias of CreateIndex + AddForeignKey EventType = "add foreign key" + DropForeignKey EventType = "drop foreign key" + ModifyColumn EventType = "modify column" + RebaseAutoID EventType = "rebase auto_increment" + SetDefaultValue EventType = "set default value" + ShardRowID EventType = "shard row id" + ModifyTableComment EventType = "modify table comment" + RenameIndex EventType = "rename index" + AddTablePartition EventType = "add table partition" + DropTablePartition EventType = "drop table partition" + TruncateTablePartition EventType = "truncate table partition" + ModifyTableCharsetAndCollate EventType = "modify table charset and collate" + ModifySchemaCharsetAndCollate EventType = "modify schema charset and collate" + RecoverTable EventType = "recover table" + LockTable EventType = "lock table" + UnlockTable EventType = "unlock table" + RepairTable EventType = "repair table" + SetTiFlashReplica EventType = "set tiflash replica" + UpdateTiFlashReplicaStatus EventType = "update tiflash replica status" + AddPrimaryKey EventType = "add primary key" + DropPrimaryKey EventType = "drop primary key" + CreateSequence EventType = "create sequence" + AlterSequence EventType = "alter sequence" + DropSequence EventType = "drop sequence" // if need, add more AlertTableOption = "alert table option" NullEvent EventType = "" + // InvalidEvent is used to indicate invalid event type + InvalidEvent EventType = "InvalidEvent" ) // ClassifyEvent classify event into dml/ddl @@ -75,12 +107,18 @@ func ClassifyEvent(event EventType) (EventType, error) { case InsertEvent, UpdateEvent, DeleteEvent: return dml, nil case CreateDatabase, DropDatabase, CreateTable, DropTable, TruncateTable, RenameTable, - CreateIndex, DropIndex, CreateView, DropView, AlertTable: + CreateIndex, DropIndex, CreateView, DropView, AlertTable, AddColumn, DropColumn, + AddForeignKey, DropForeignKey, ModifyColumn, RebaseAutoID, SetDefaultValue, + ShardRowID, ModifyTableComment, RenameIndex, AddTablePartition, DropTablePartition, + TruncateTablePartition, ModifyTableCharsetAndCollate, ModifySchemaCharsetAndCollate, + RecoverTable, LockTable, UnlockTable, RepairTable, SetTiFlashReplica, + UpdateTiFlashReplica, AddPrimaryKey, DropPrimaryKey, CreateSequence, + AlterSequence, DropSequence, CreateSchema, DropSchema, AddIndex: return ddl, nil case NullEvent: return NullEvent, nil default: - return NoneEvent, errors.NotValidf("event type %s", event) + return InvalidEvent, errors.NotValidf("event type %s", event) } } @@ -115,14 +153,50 @@ func (b *BinlogEventRule) Valid() error { return errors.Errorf("action of binlog event rule %+v should not be empty", b) } - // TODO: check validity of dml/ddl event. for i := range b.Events { - b.Events[i] = EventType(strings.ToLower(string(b.Events[i]))) + et, err := b.toEvent(string(b.Events[i])) + if err != nil { + return errors.Annotatef(ErrInvalidEventType, "event type %s", b.Events[i]) + } + b.Events[i] = et } - return nil } +// ErrInvalidEventType is returned when event type is not valid. +var ErrInvalidEventType = errors.New("event type not found") + +// toEvent converts event type string to EventType and check if it is valid. +func (b *BinlogEventRule) toEvent(es string) (EventType, error) { + event := EventType(strings.ToLower(es)) + switch event { + case dml, ddl, AllEvent, AllDDL, AllDML, + NoneEvent, NoneDDL, NoneDML, + InsertEvent, UpdateEvent, DeleteEvent, + CreateDatabase, DropDatabase, CreateTable, + DropTable, TruncateTable, RenameTable, + CreateIndex, DropIndex, CreateView, DropView, + AlertTable, AddColumn, DropColumn, + AddForeignKey, DropForeignKey, + ModifyColumn, RebaseAutoID, SetDefaultValue, + ShardRowID, ModifyTableComment, RenameIndex, + AddTablePartition, DropTablePartition, TruncateTablePartition, + ModifyTableCharsetAndCollate, ModifySchemaCharsetAndCollate, + RecoverTable, LockTable, UnlockTable, RepairTable, + SetTiFlashReplica, UpdateTiFlashReplica, AddPrimaryKey, DropPrimaryKey, + CreateSequence, AlterSequence, DropSequence: + return event, nil + case CreateSchema: // alias of CreateDatabase + return CreateDatabase, nil + case DropSchema: // alias of DropDatabase + return DropDatabase, nil + case AddIndex: // alias of CreateIndex + return CreateIndex, nil + default: + return InvalidEvent, ErrInvalidEventType + } +} + // BinlogEvent filters binlog events by given rules type BinlogEvent struct { selector.Selector diff --git a/pkg/binlog-filter/filter_test.go b/pkg/binlog-filter/filter_test.go index 69db26eb6..b375d2cdc 100644 --- a/pkg/binlog-filter/filter_test.go +++ b/pkg/binlog-filter/filter_test.go @@ -245,3 +245,29 @@ func (t *testFilterSuite) TestGlobalFilter(c *C) { c.Assert(action, Equals, cs.action) } } + +func (t *testFilterSuit) TestToEvent(c *C) { + cases := []struct { + eventStr string + event EventType + err error + }{ + {"", NullEvent, nil}, + {"insert", InsertEvent, nil}, + {"update", UpdateEvent, nil}, + {"delete", DeleteEvent, nil}, + {"create", CreateEvent, nil}, + {"create schema", CreateDatabase, nil}, + {"drop schema", DropDatabase, nil}, + {"add index", CreateIndex, nil}, + {"xxx", InvalidEvent, ErrInvalidEvent}, + {"I don't know", InvalidEvent, ErrInvalidEvent}, + {"add column", AddColumn, nil}, + {"drop column", DropColumn, nil}, + } + for _, cs := range cases { + event, err := ToEvent(cs.eventStr) + c.Assert(event, Equals, cs.event) + c.Assert(err, Equals, cs.event) + } +} From 19f585d9216c94b06fd6fce223dd05bdfb66f046 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 4 Jul 2022 12:28:23 +0800 Subject: [PATCH 02/15] fix error --- pkg/binlog-filter/filter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/binlog-filter/filter_test.go b/pkg/binlog-filter/filter_test.go index b375d2cdc..3f89bb2ad 100644 --- a/pkg/binlog-filter/filter_test.go +++ b/pkg/binlog-filter/filter_test.go @@ -268,6 +268,6 @@ func (t *testFilterSuit) TestToEvent(c *C) { for _, cs := range cases { event, err := ToEvent(cs.eventStr) c.Assert(event, Equals, cs.event) - c.Assert(err, Equals, cs.event) + c.Assert(err, Equals, cs.err) } } From 05c429783ba1b2bab697031872139d6cf5067393 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 4 Jul 2022 12:40:10 +0800 Subject: [PATCH 03/15] fix error --- pkg/binlog-filter/filter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/binlog-filter/filter_test.go b/pkg/binlog-filter/filter_test.go index 3f89bb2ad..3fa3e886b 100644 --- a/pkg/binlog-filter/filter_test.go +++ b/pkg/binlog-filter/filter_test.go @@ -246,7 +246,7 @@ func (t *testFilterSuite) TestGlobalFilter(c *C) { } } -func (t *testFilterSuit) TestToEvent(c *C) { +func (t *testFilterSuite) TestToEvent(c *C) { cases := []struct { eventStr string event EventType From e5834c558a540c3c19d6878d1c45028c8ffe5ce8 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 4 Jul 2022 12:44:01 +0800 Subject: [PATCH 04/15] fix error --- pkg/binlog-filter/filter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/binlog-filter/filter.go b/pkg/binlog-filter/filter.go index f487f4746..baa797651 100644 --- a/pkg/binlog-filter/filter.go +++ b/pkg/binlog-filter/filter.go @@ -112,7 +112,7 @@ func ClassifyEvent(event EventType) (EventType, error) { ShardRowID, ModifyTableComment, RenameIndex, AddTablePartition, DropTablePartition, TruncateTablePartition, ModifyTableCharsetAndCollate, ModifySchemaCharsetAndCollate, RecoverTable, LockTable, UnlockTable, RepairTable, SetTiFlashReplica, - UpdateTiFlashReplica, AddPrimaryKey, DropPrimaryKey, CreateSequence, + UpdateTiFlashReplicaStatus, AddPrimaryKey, DropPrimaryKey, CreateSequence, AlterSequence, DropSequence, CreateSchema, DropSchema, AddIndex: return ddl, nil case NullEvent: @@ -183,7 +183,7 @@ func (b *BinlogEventRule) toEvent(es string) (EventType, error) { AddTablePartition, DropTablePartition, TruncateTablePartition, ModifyTableCharsetAndCollate, ModifySchemaCharsetAndCollate, RecoverTable, LockTable, UnlockTable, RepairTable, - SetTiFlashReplica, UpdateTiFlashReplica, AddPrimaryKey, DropPrimaryKey, + SetTiFlashReplica, UpdateTiFlashReplicaStatus, AddPrimaryKey, DropPrimaryKey, CreateSequence, AlterSequence, DropSequence: return event, nil case CreateSchema: // alias of CreateDatabase From 0d3f1131304413b315b53d44381ecf8e60343b55 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 4 Jul 2022 13:26:32 +0800 Subject: [PATCH 05/15] add test cases --- pkg/binlog-filter/filter_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/binlog-filter/filter_test.go b/pkg/binlog-filter/filter_test.go index 3fa3e886b..7507bce95 100644 --- a/pkg/binlog-filter/filter_test.go +++ b/pkg/binlog-filter/filter_test.go @@ -256,10 +256,13 @@ func (t *testFilterSuite) TestToEvent(c *C) { {"insert", InsertEvent, nil}, {"update", UpdateEvent, nil}, {"delete", DeleteEvent, nil}, - {"create", CreateEvent, nil}, + {"create", InvalidEvent, ErrInvalidEvent}, {"create schema", CreateDatabase, nil}, + {"create database", CreateDatabase, nil}, {"drop schema", DropDatabase, nil}, + {"drop database", DropDatabase, nil}, {"add index", CreateIndex, nil}, + {"create index", CreateIndex, nil}, {"xxx", InvalidEvent, ErrInvalidEvent}, {"I don't know", InvalidEvent, ErrInvalidEvent}, {"add column", AddColumn, nil}, From 3c191930686059d558f4b76cea3fb23c1a9ae252 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 4 Jul 2022 13:36:09 +0800 Subject: [PATCH 06/15] fix error --- pkg/binlog-filter/filter_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/binlog-filter/filter_test.go b/pkg/binlog-filter/filter_test.go index 7507bce95..b27877e9c 100644 --- a/pkg/binlog-filter/filter_test.go +++ b/pkg/binlog-filter/filter_test.go @@ -256,15 +256,15 @@ func (t *testFilterSuite) TestToEvent(c *C) { {"insert", InsertEvent, nil}, {"update", UpdateEvent, nil}, {"delete", DeleteEvent, nil}, - {"create", InvalidEvent, ErrInvalidEvent}, + {"create", InvalidEvent, ErrInvalidEventType}, {"create schema", CreateDatabase, nil}, {"create database", CreateDatabase, nil}, {"drop schema", DropDatabase, nil}, {"drop database", DropDatabase, nil}, {"add index", CreateIndex, nil}, {"create index", CreateIndex, nil}, - {"xxx", InvalidEvent, ErrInvalidEvent}, - {"I don't know", InvalidEvent, ErrInvalidEvent}, + {"xxx", InvalidEvent, ErrInvalidEventType}, + {"I don't know", InvalidEvent, ErrInvalidEventType}, {"add column", AddColumn, nil}, {"drop column", DropColumn, nil}, } From b794c00cf64cd97843a65a2e894206b62d395ca1 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 4 Jul 2022 14:13:56 +0800 Subject: [PATCH 07/15] fix error --- pkg/binlog-filter/filter.go | 2 +- pkg/binlog-filter/filter_test.go | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/binlog-filter/filter.go b/pkg/binlog-filter/filter.go index baa797651..b56d162af 100644 --- a/pkg/binlog-filter/filter.go +++ b/pkg/binlog-filter/filter.go @@ -170,7 +170,7 @@ var ErrInvalidEventType = errors.New("event type not found") func (b *BinlogEventRule) toEvent(es string) (EventType, error) { event := EventType(strings.ToLower(es)) switch event { - case dml, ddl, AllEvent, AllDDL, AllDML, + case AllEvent, AllDDL, AllDML, NullEvent, NoneEvent, NoneDDL, NoneDML, InsertEvent, UpdateEvent, DeleteEvent, CreateDatabase, DropDatabase, CreateTable, diff --git a/pkg/binlog-filter/filter_test.go b/pkg/binlog-filter/filter_test.go index b27877e9c..787345075 100644 --- a/pkg/binlog-filter/filter_test.go +++ b/pkg/binlog-filter/filter_test.go @@ -14,10 +14,10 @@ package filter import ( - "testing" - + "fmt" . "github.com/pingcap/check" selector "github.com/pingcap/tidb-tools/pkg/table-rule-selector" + "testing" ) func TestClient(t *testing.T) { @@ -268,9 +268,11 @@ func (t *testFilterSuite) TestToEvent(c *C) { {"add column", AddColumn, nil}, {"drop column", DropColumn, nil}, } + br := &BinlogEventRule{} + for _, cs := range cases { - event, err := ToEvent(cs.eventStr) - c.Assert(event, Equals, cs.event) - c.Assert(err, Equals, cs.err) + event, err := br.toEvent(cs.eventStr) + c.Assert(cs.event, Equals, event) + c.Assert(cs.err, Equals, err) } } From 135630bce5f0fb453b3d03c85832a697a8a1a693 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 4 Jul 2022 14:17:21 +0800 Subject: [PATCH 08/15] fix error --- pkg/binlog-filter/filter_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/binlog-filter/filter_test.go b/pkg/binlog-filter/filter_test.go index 787345075..4c37c3145 100644 --- a/pkg/binlog-filter/filter_test.go +++ b/pkg/binlog-filter/filter_test.go @@ -14,7 +14,6 @@ package filter import ( - "fmt" . "github.com/pingcap/check" selector "github.com/pingcap/tidb-tools/pkg/table-rule-selector" "testing" From 8434f612caf4fde7432af6f3bd8a5e7206705352 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Tue, 5 Jul 2022 22:01:50 +0800 Subject: [PATCH 09/15] remove useless type --- pkg/binlog-filter/filter.go | 41 +++++++++++--------------------- pkg/binlog-filter/filter_test.go | 1 + 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/pkg/binlog-filter/filter.go b/pkg/binlog-filter/filter.go index b56d162af..69b88041a 100644 --- a/pkg/binlog-filter/filter.go +++ b/pkg/binlog-filter/filter.go @@ -70,12 +70,8 @@ const ( AddColumn EventType = "add column" DropColumn EventType = "drop column" AddIndex EventType = "add index" // alias of CreateIndex - AddForeignKey EventType = "add foreign key" - DropForeignKey EventType = "drop foreign key" ModifyColumn EventType = "modify column" - RebaseAutoID EventType = "rebase auto_increment" SetDefaultValue EventType = "set default value" - ShardRowID EventType = "shard row id" ModifyTableComment EventType = "modify table comment" RenameIndex EventType = "rename index" AddTablePartition EventType = "add table partition" @@ -84,16 +80,9 @@ const ( ModifyTableCharsetAndCollate EventType = "modify table charset and collate" ModifySchemaCharsetAndCollate EventType = "modify schema charset and collate" RecoverTable EventType = "recover table" - LockTable EventType = "lock table" - UnlockTable EventType = "unlock table" - RepairTable EventType = "repair table" - SetTiFlashReplica EventType = "set tiflash replica" UpdateTiFlashReplicaStatus EventType = "update tiflash replica status" AddPrimaryKey EventType = "add primary key" DropPrimaryKey EventType = "drop primary key" - CreateSequence EventType = "create sequence" - AlterSequence EventType = "alter sequence" - DropSequence EventType = "drop sequence" // if need, add more AlertTableOption = "alert table option" NullEvent EventType = "" @@ -106,14 +95,15 @@ func ClassifyEvent(event EventType) (EventType, error) { switch event { case InsertEvent, UpdateEvent, DeleteEvent: return dml, nil - case CreateDatabase, DropDatabase, CreateTable, DropTable, TruncateTable, RenameTable, - CreateIndex, DropIndex, CreateView, DropView, AlertTable, AddColumn, DropColumn, - AddForeignKey, DropForeignKey, ModifyColumn, RebaseAutoID, SetDefaultValue, - ShardRowID, ModifyTableComment, RenameIndex, AddTablePartition, DropTablePartition, - TruncateTablePartition, ModifyTableCharsetAndCollate, ModifySchemaCharsetAndCollate, - RecoverTable, LockTable, UnlockTable, RepairTable, SetTiFlashReplica, - UpdateTiFlashReplicaStatus, AddPrimaryKey, DropPrimaryKey, CreateSequence, - AlterSequence, DropSequence, CreateSchema, DropSchema, AddIndex: + case CreateDatabase, DropDatabase, CreateTable, + DropTable, TruncateTable, RenameTable, + CreateIndex, DropIndex, CreateView, + DropView, AlertTable, AddColumn, + DropColumn, ModifyColumn, SetDefaultValue, + ModifyTableComment, RenameIndex, AddTablePartition, + DropTablePartition, TruncateTablePartition, ModifyTableCharsetAndCollate, + ModifySchemaCharsetAndCollate, RecoverTable, UpdateTiFlashReplicaStatus, + AddPrimaryKey, DropPrimaryKey, CreateSchema, DropSchema, AddIndex: return ddl, nil case NullEvent: return NullEvent, nil @@ -177,14 +167,11 @@ func (b *BinlogEventRule) toEvent(es string) (EventType, error) { DropTable, TruncateTable, RenameTable, CreateIndex, DropIndex, CreateView, DropView, AlertTable, AddColumn, DropColumn, - AddForeignKey, DropForeignKey, - ModifyColumn, RebaseAutoID, SetDefaultValue, - ShardRowID, ModifyTableComment, RenameIndex, - AddTablePartition, DropTablePartition, TruncateTablePartition, - ModifyTableCharsetAndCollate, ModifySchemaCharsetAndCollate, - RecoverTable, LockTable, UnlockTable, RepairTable, - SetTiFlashReplica, UpdateTiFlashReplicaStatus, AddPrimaryKey, DropPrimaryKey, - CreateSequence, AlterSequence, DropSequence: + ModifyColumn, SetDefaultValue, ModifyTableComment, + RenameIndex, AddTablePartition, DropTablePartition, + TruncateTablePartition, ModifyTableCharsetAndCollate, + ModifySchemaCharsetAndCollate, RecoverTable, UpdateTiFlashReplicaStatus, + AddPrimaryKey, DropPrimaryKey: return event, nil case CreateSchema: // alias of CreateDatabase return CreateDatabase, nil diff --git a/pkg/binlog-filter/filter_test.go b/pkg/binlog-filter/filter_test.go index 4c37c3145..c8ec787ea 100644 --- a/pkg/binlog-filter/filter_test.go +++ b/pkg/binlog-filter/filter_test.go @@ -124,6 +124,7 @@ func (t *testFilterSuite) TestFilter(c *C) { // mismatched action, err := filter.Filter("xxx_a", "", InsertEvent, "") + c.Assert(err, IsNil) c.Assert(action, Equals, Do) // invalid rule From 382ce21018a52f0e39ef6ffa08a509d53ed939fd Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Tue, 5 Jul 2022 22:02:47 +0800 Subject: [PATCH 10/15] address comments --- pkg/binlog-filter/filter_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/binlog-filter/filter_test.go b/pkg/binlog-filter/filter_test.go index c8ec787ea..063d4ff55 100644 --- a/pkg/binlog-filter/filter_test.go +++ b/pkg/binlog-filter/filter_test.go @@ -14,9 +14,10 @@ package filter import ( + "testing" + . "github.com/pingcap/check" selector "github.com/pingcap/tidb-tools/pkg/table-rule-selector" - "testing" ) func TestClient(t *testing.T) { From f43cb13f3a429ca22c71efdfd27a6b1e25241835 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Wed, 13 Jul 2022 10:37:26 +0800 Subject: [PATCH 11/15] remove some evnt types --- pkg/binlog-filter/filter.go | 40 +++++++------------------------- pkg/binlog-filter/filter_test.go | 2 -- pkg/binlog-filter/util.go | 2 +- 3 files changed, 9 insertions(+), 35 deletions(-) diff --git a/pkg/binlog-filter/filter.go b/pkg/binlog-filter/filter.go index 69b88041a..08bd591dd 100644 --- a/pkg/binlog-filter/filter.go +++ b/pkg/binlog-filter/filter.go @@ -63,26 +63,11 @@ const ( DropIndex EventType = "drop index" CreateView EventType = "create view" DropView EventType = "drop view" - AlertTable EventType = "alter table" - - CreateSchema EventType = "create schema" // alias of CreateDatabase - DropSchema EventType = "drop schema" // alias of DropDatabase - AddColumn EventType = "add column" - DropColumn EventType = "drop column" - AddIndex EventType = "add index" // alias of CreateIndex - ModifyColumn EventType = "modify column" - SetDefaultValue EventType = "set default value" - ModifyTableComment EventType = "modify table comment" - RenameIndex EventType = "rename index" - AddTablePartition EventType = "add table partition" - DropTablePartition EventType = "drop table partition" - TruncateTablePartition EventType = "truncate table partition" - ModifyTableCharsetAndCollate EventType = "modify table charset and collate" - ModifySchemaCharsetAndCollate EventType = "modify schema charset and collate" - RecoverTable EventType = "recover table" - UpdateTiFlashReplicaStatus EventType = "update tiflash replica status" - AddPrimaryKey EventType = "add primary key" - DropPrimaryKey EventType = "drop primary key" + AlterTable EventType = "alter table" + + CreateSchema EventType = "create schema" // alias of CreateDatabase + DropSchema EventType = "drop schema" // alias of DropDatabase + AddIndex EventType = "add index" // alias of CreateIndex // if need, add more AlertTableOption = "alert table option" NullEvent EventType = "" @@ -98,12 +83,8 @@ func ClassifyEvent(event EventType) (EventType, error) { case CreateDatabase, DropDatabase, CreateTable, DropTable, TruncateTable, RenameTable, CreateIndex, DropIndex, CreateView, - DropView, AlertTable, AddColumn, - DropColumn, ModifyColumn, SetDefaultValue, - ModifyTableComment, RenameIndex, AddTablePartition, - DropTablePartition, TruncateTablePartition, ModifyTableCharsetAndCollate, - ModifySchemaCharsetAndCollate, RecoverTable, UpdateTiFlashReplicaStatus, - AddPrimaryKey, DropPrimaryKey, CreateSchema, DropSchema, AddIndex: + DropView, AlterTable, + CreateSchema, DropSchema, AddIndex: return ddl, nil case NullEvent: return NullEvent, nil @@ -166,12 +147,7 @@ func (b *BinlogEventRule) toEvent(es string) (EventType, error) { CreateDatabase, DropDatabase, CreateTable, DropTable, TruncateTable, RenameTable, CreateIndex, DropIndex, CreateView, DropView, - AlertTable, AddColumn, DropColumn, - ModifyColumn, SetDefaultValue, ModifyTableComment, - RenameIndex, AddTablePartition, DropTablePartition, - TruncateTablePartition, ModifyTableCharsetAndCollate, - ModifySchemaCharsetAndCollate, RecoverTable, UpdateTiFlashReplicaStatus, - AddPrimaryKey, DropPrimaryKey: + AlterTable: return event, nil case CreateSchema: // alias of CreateDatabase return CreateDatabase, nil diff --git a/pkg/binlog-filter/filter_test.go b/pkg/binlog-filter/filter_test.go index 063d4ff55..81f928c91 100644 --- a/pkg/binlog-filter/filter_test.go +++ b/pkg/binlog-filter/filter_test.go @@ -266,8 +266,6 @@ func (t *testFilterSuite) TestToEvent(c *C) { {"create index", CreateIndex, nil}, {"xxx", InvalidEvent, ErrInvalidEventType}, {"I don't know", InvalidEvent, ErrInvalidEventType}, - {"add column", AddColumn, nil}, - {"drop column", DropColumn, nil}, } br := &BinlogEventRule{} diff --git a/pkg/binlog-filter/util.go b/pkg/binlog-filter/util.go index b60919e26..4bec641d3 100644 --- a/pkg/binlog-filter/util.go +++ b/pkg/binlog-filter/util.go @@ -38,7 +38,7 @@ func AstToDDLEvent(node ast.StmtNode) EventType { case *ast.DropIndexStmt: return DropIndex case *ast.AlterTableStmt: - return AlertTable + return AlterTable case *ast.CreateViewStmt: return CreateView } From 04423edf8071773a047df8f95f76ee990709cdea Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Wed, 13 Jul 2022 10:38:33 +0800 Subject: [PATCH 12/15] refine type string --- pkg/binlog-filter/filter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/binlog-filter/filter.go b/pkg/binlog-filter/filter.go index 08bd591dd..e972125e6 100644 --- a/pkg/binlog-filter/filter.go +++ b/pkg/binlog-filter/filter.go @@ -72,7 +72,7 @@ const ( NullEvent EventType = "" // InvalidEvent is used to indicate invalid event type - InvalidEvent EventType = "InvalidEvent" + InvalidEvent EventType = "invalid event" ) // ClassifyEvent classify event into dml/ddl From 1bcdfb3ee43c9a5829a088c51f7c4d709e5396a9 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Wed, 13 Jul 2022 11:19:52 +0800 Subject: [PATCH 13/15] add alter database type --- pkg/binlog-filter/filter.go | 33 ++++------------------ pkg/binlog-filter/filter_test.go | 7 +++-- pkg/binlog-filter/util.go | 48 +++++++++++++++++++++++++++++++- 3 files changed, 57 insertions(+), 31 deletions(-) diff --git a/pkg/binlog-filter/filter.go b/pkg/binlog-filter/filter.go index e972125e6..5695f55f7 100644 --- a/pkg/binlog-filter/filter.go +++ b/pkg/binlog-filter/filter.go @@ -65,9 +65,11 @@ const ( DropView EventType = "drop view" AlterTable EventType = "alter table" - CreateSchema EventType = "create schema" // alias of CreateDatabase - DropSchema EventType = "drop schema" // alias of DropDatabase - AddIndex EventType = "add index" // alias of CreateIndex + CreateSchema EventType = "create schema" // alias of CreateDatabase + DropSchema EventType = "drop schema" // alias of DropDatabase + AddIndex EventType = "add index" // alias of CreateIndex + AlterDatabase EventType = "alter database" + AlterSchema EventType = "alter schema" // alias of AlterDatabase // if need, add more AlertTableOption = "alert table option" NullEvent EventType = "" @@ -125,7 +127,7 @@ func (b *BinlogEventRule) Valid() error { } for i := range b.Events { - et, err := b.toEvent(string(b.Events[i])) + et, err := toEventType(string(b.Events[i])) if err != nil { return errors.Annotatef(ErrInvalidEventType, "event type %s", b.Events[i]) } @@ -137,29 +139,6 @@ func (b *BinlogEventRule) Valid() error { // ErrInvalidEventType is returned when event type is not valid. var ErrInvalidEventType = errors.New("event type not found") -// toEvent converts event type string to EventType and check if it is valid. -func (b *BinlogEventRule) toEvent(es string) (EventType, error) { - event := EventType(strings.ToLower(es)) - switch event { - case AllEvent, AllDDL, AllDML, NullEvent, - NoneEvent, NoneDDL, NoneDML, - InsertEvent, UpdateEvent, DeleteEvent, - CreateDatabase, DropDatabase, CreateTable, - DropTable, TruncateTable, RenameTable, - CreateIndex, DropIndex, CreateView, DropView, - AlterTable: - return event, nil - case CreateSchema: // alias of CreateDatabase - return CreateDatabase, nil - case DropSchema: // alias of DropDatabase - return DropDatabase, nil - case AddIndex: // alias of CreateIndex - return CreateIndex, nil - default: - return InvalidEvent, ErrInvalidEventType - } -} - // BinlogEvent filters binlog events by given rules type BinlogEvent struct { selector.Selector diff --git a/pkg/binlog-filter/filter_test.go b/pkg/binlog-filter/filter_test.go index 81f928c91..5ed87bfac 100644 --- a/pkg/binlog-filter/filter_test.go +++ b/pkg/binlog-filter/filter_test.go @@ -247,7 +247,7 @@ func (t *testFilterSuite) TestGlobalFilter(c *C) { } } -func (t *testFilterSuite) TestToEvent(c *C) { +func (t *testFilterSuite) TestToEventType(c *C) { cases := []struct { eventStr string event EventType @@ -262,15 +262,16 @@ func (t *testFilterSuite) TestToEvent(c *C) { {"create database", CreateDatabase, nil}, {"drop schema", DropDatabase, nil}, {"drop database", DropDatabase, nil}, + {"alter database", AlterDatabase, nil}, + {"alter schema", AlterDatabase, nil}, {"add index", CreateIndex, nil}, {"create index", CreateIndex, nil}, {"xxx", InvalidEvent, ErrInvalidEventType}, {"I don't know", InvalidEvent, ErrInvalidEventType}, } - br := &BinlogEventRule{} for _, cs := range cases { - event, err := br.toEvent(cs.eventStr) + event, err := toEventType(cs.eventStr) c.Assert(cs.event, Equals, event) c.Assert(cs.err, Equals, err) } diff --git a/pkg/binlog-filter/util.go b/pkg/binlog-filter/util.go index 4bec641d3..92892997e 100644 --- a/pkg/binlog-filter/util.go +++ b/pkg/binlog-filter/util.go @@ -13,7 +13,11 @@ package filter -import "github.com/pingcap/tidb/parser/ast" +import ( + "strings" + + "github.com/pingcap/tidb/parser/ast" +) // AstToDDLEvent returns filter.DDLEvent func AstToDDLEvent(node ast.StmtNode) EventType { @@ -41,7 +45,49 @@ func AstToDDLEvent(node ast.StmtNode) EventType { return AlterTable case *ast.CreateViewStmt: return CreateView + case *ast.AlterDatabaseStmt: + return AlterDatabase } return NullEvent } + +// toEventType converts event type string to EventType and check if it is valid. +func toEventType(es string) (EventType, error) { + event := EventType(strings.ToLower(es)) + switch event { + case AllEvent, + AllDDL, + AllDML, + NullEvent, + NoneEvent, + NoneDDL, + NoneDML, + InsertEvent, + UpdateEvent, + DeleteEvent, + CreateDatabase, + DropDatabase, + CreateTable, + DropTable, + TruncateTable, + RenameTable, + CreateIndex, + DropIndex, + CreateView, + DropView, + AlterTable, + AlterDatabase: + return event, nil + case CreateSchema: // alias of CreateDatabase + return CreateDatabase, nil + case DropSchema: // alias of DropDatabase + return DropDatabase, nil + case AddIndex: // alias of CreateIndex + return CreateIndex, nil + case AlterSchema: + return AlterDatabase, nil + default: + return InvalidEvent, ErrInvalidEventType + } +} From 3b34fdb85e4e861f465a66f0b69c39de273f84ad Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Wed, 13 Jul 2022 13:53:51 +0800 Subject: [PATCH 14/15] address comments --- pkg/binlog-filter/filter.go | 21 +++++++++------------ pkg/binlog-filter/filter_test.go | 15 +++++++++++---- pkg/binlog-filter/util.go | 7 ++++--- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/pkg/binlog-filter/filter.go b/pkg/binlog-filter/filter.go index 5695f55f7..58d6e4813 100644 --- a/pkg/binlog-filter/filter.go +++ b/pkg/binlog-filter/filter.go @@ -55,6 +55,7 @@ const ( CreateDatabase EventType = "create database" DropDatabase EventType = "drop database" + AlterDatabase EventType = "alter database" CreateTable EventType = "create table" DropTable EventType = "drop table" TruncateTable EventType = "truncate table" @@ -65,16 +66,15 @@ const ( DropView EventType = "drop view" AlterTable EventType = "alter table" - CreateSchema EventType = "create schema" // alias of CreateDatabase - DropSchema EventType = "drop schema" // alias of DropDatabase - AddIndex EventType = "add index" // alias of CreateIndex - AlterDatabase EventType = "alter database" - AlterSchema EventType = "alter schema" // alias of AlterDatabase + CreateSchema EventType = "create schema" // alias of CreateDatabase + DropSchema EventType = "drop schema" // alias of DropDatabase + AlterSchema EventType = "alter schema" // alias of AlterDatabase + AddIndex EventType = "add index" // alias of CreateIndex // if need, add more AlertTableOption = "alert table option" + // NullEvent is used to represents unsupported ddl event type when we + // convert a ast.StmtNode or a string to EventType. NullEvent EventType = "" - // InvalidEvent is used to indicate invalid event type - InvalidEvent EventType = "invalid event" ) // ClassifyEvent classify event into dml/ddl @@ -91,7 +91,7 @@ func ClassifyEvent(event EventType) (EventType, error) { case NullEvent: return NullEvent, nil default: - return InvalidEvent, errors.NotValidf("event type %s", event) + return NullEvent, errors.NotValidf("event type %s", event) } } @@ -129,16 +129,13 @@ func (b *BinlogEventRule) Valid() error { for i := range b.Events { et, err := toEventType(string(b.Events[i])) if err != nil { - return errors.Annotatef(ErrInvalidEventType, "event type %s", b.Events[i]) + return errors.NotValidf("event type %s", b.Events[i]) } b.Events[i] = et } return nil } -// ErrInvalidEventType is returned when event type is not valid. -var ErrInvalidEventType = errors.New("event type not found") - // BinlogEvent filters binlog events by given rules type BinlogEvent struct { selector.Selector diff --git a/pkg/binlog-filter/filter_test.go b/pkg/binlog-filter/filter_test.go index 5ed87bfac..2f7fbf153 100644 --- a/pkg/binlog-filter/filter_test.go +++ b/pkg/binlog-filter/filter_test.go @@ -17,6 +17,7 @@ import ( "testing" . "github.com/pingcap/check" + "github.com/pingcap/errors" selector "github.com/pingcap/tidb-tools/pkg/table-rule-selector" ) @@ -255,24 +256,30 @@ func (t *testFilterSuite) TestToEventType(c *C) { }{ {"", NullEvent, nil}, {"insert", InsertEvent, nil}, + {"Insert", InsertEvent, nil}, {"update", UpdateEvent, nil}, + {"UPDATE", UpdateEvent, nil}, {"delete", DeleteEvent, nil}, - {"create", InvalidEvent, ErrInvalidEventType}, + {"create", NullEvent, errors.NotValidf("event type %s", "create")}, {"create schema", CreateDatabase, nil}, + {"create SCHEMA", CreateDatabase, nil}, {"create database", CreateDatabase, nil}, {"drop schema", DropDatabase, nil}, + {"drop Schema", DropDatabase, nil}, {"drop database", DropDatabase, nil}, {"alter database", AlterDatabase, nil}, {"alter schema", AlterDatabase, nil}, {"add index", CreateIndex, nil}, {"create index", CreateIndex, nil}, - {"xxx", InvalidEvent, ErrInvalidEventType}, - {"I don't know", InvalidEvent, ErrInvalidEventType}, + {"xxx", NullEvent, errors.NotValidf("event type %s", "xxx")}, + {"I don't know", NullEvent, errors.NotValidf("event type %s", "I don't know")}, } for _, cs := range cases { event, err := toEventType(cs.eventStr) c.Assert(cs.event, Equals, event) - c.Assert(cs.err, Equals, err) + if err != nil { + c.Assert(cs.err.Error(), Equals, err.Error()) + } } } diff --git a/pkg/binlog-filter/util.go b/pkg/binlog-filter/util.go index 92892997e..939750a82 100644 --- a/pkg/binlog-filter/util.go +++ b/pkg/binlog-filter/util.go @@ -16,6 +16,7 @@ package filter import ( "strings" + "github.com/pingcap/errors" "github.com/pingcap/tidb/parser/ast" ) @@ -68,6 +69,7 @@ func toEventType(es string) (EventType, error) { DeleteEvent, CreateDatabase, DropDatabase, + AlterDatabase, CreateTable, DropTable, TruncateTable, @@ -76,8 +78,7 @@ func toEventType(es string) (EventType, error) { DropIndex, CreateView, DropView, - AlterTable, - AlterDatabase: + AlterTable: return event, nil case CreateSchema: // alias of CreateDatabase return CreateDatabase, nil @@ -88,6 +89,6 @@ func toEventType(es string) (EventType, error) { case AlterSchema: return AlterDatabase, nil default: - return InvalidEvent, ErrInvalidEventType + return NullEvent, errors.NotValidf("event type %s", es) } } From a741961f7620e939f3534d810468d677c14154f6 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Wed, 13 Jul 2022 14:21:31 +0800 Subject: [PATCH 15/15] address comments --- pkg/binlog-filter/filter_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/binlog-filter/filter_test.go b/pkg/binlog-filter/filter_test.go index 2f7fbf153..1cc004723 100644 --- a/pkg/binlog-filter/filter_test.go +++ b/pkg/binlog-filter/filter_test.go @@ -280,6 +280,8 @@ func (t *testFilterSuite) TestToEventType(c *C) { c.Assert(cs.event, Equals, event) if err != nil { c.Assert(cs.err.Error(), Equals, err.Error()) + } else { + c.Assert(cs.err, IsNil) } } }