Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cherry pick table filters to 3.1 #372

Merged
merged 4 commits into from
Jul 9, 2020
Merged

Cherry pick table filters to 3.1 #372

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d1d05c4
table-filter: implement the new table filter (#341)
kennytm May 15, 2020
83b16c0
table-filter: added filter.All() to conveniently make a match-all fil…
kennytm May 30, 2020
32b6440
filter,table-filter: replace blacklist/whitelist by blocklist/allowli…
kennytm Jun 18, 2020
4e30420
filter: fix build error
kennytm Jul 8, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 17 additions & 64 deletions pkg/filter/filter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@
package filter

import (
"fmt"
"regexp"
"strings"
"sync"

"github.com/pingcap/errors"
tfilter "github.com/pingcap/tidb-tools/pkg/table-filter"
selector "github.com/pingcap/tidb-tools/pkg/table-rule-selector"
)

Expand All @@ -33,26 +33,7 @@ const (
)

// Table represents a table.
type Table struct {
Schema string `toml:"db-name" json:"db-name" yaml:"db-name"`
Name string `toml:"tbl-name" json:"tbl-name" yaml:"tbl-name"`
}

// Clone clones a new filter.Table
func (t *Table) Clone() *Table {
return &Table{
Schema: t.Schema,
Name: t.Name,
}
}

// String implements the fmt.Stringer interface.
func (t *Table) String() string {
if len(t.Name) > 0 {
return fmt.Sprintf("`%s`.`%s`", t.Schema, t.Name)
}
return fmt.Sprintf("`%s`", t.Schema)
}
type Table = tfilter.Table

type cache struct {
sync.RWMutex
Expand All @@ -74,37 +55,9 @@ func (c *cache) set(key string, action ActionType) {
}

// Rules contains Filter rules.
type Rules struct {
DoTables []*Table `json:"do-tables" toml:"do-tables" yaml:"do-tables"`
DoDBs []string `json:"do-dbs" toml:"do-dbs" yaml:"do-dbs"`

IgnoreTables []*Table `json:"ignore-tables" toml:"ignore-tables" yaml:"ignore-tables"`
IgnoreDBs []string `json:"ignore-dbs" toml:"ignore-dbs" yaml:"ignore-dbs"`
}

// ToLower convert all entries to lowercase
func (r *Rules) ToLower() {
if r == nil {
return
}

for _, table := range r.DoTables {
table.Name = strings.ToLower(table.Name)
table.Schema = strings.ToLower(table.Schema)
}
for _, table := range r.IgnoreTables {
table.Name = strings.ToLower(table.Name)
table.Schema = strings.ToLower(table.Schema)
}
for i, db := range r.IgnoreDBs {
r.IgnoreDBs[i] = strings.ToLower(db)
}
for i, db := range r.DoDBs {
r.DoDBs[i] = strings.ToLower(db)
}
}
type Rules = tfilter.MySQLReplicationRules

// Filter implements whitelist and blacklist filters.
// Filter implements table filter in the style of MySQL replication rules.
type Filter struct {
selector.Selector
patternMap map[string]*regexp.Regexp
Expand Down Expand Up @@ -148,7 +101,7 @@ const (
type nodeEndRule struct {
kind int
r *regexp.Regexp
isWhiteList bool
isAllowList bool
}

// initRules initialize the rules to regex expr or trie node.
Expand Down Expand Up @@ -215,17 +168,17 @@ func (f *Filter) initOneRegex(originStr string) error {
return nil
}

func (f *Filter) initSchemaRule(dbStr string, isWhiteList bool) error {
func (f *Filter) initSchemaRule(dbStr string, isAllowList bool) error {
if strings.HasPrefix(dbStr, "~") {
return f.initOneRegex(dbStr[1:])
}
return f.Selector.Insert(dbStr, "", &nodeEndRule{
kind: dbRule,
isWhiteList: isWhiteList,
isAllowList: isAllowList,
}, selector.Append)
}

func (f *Filter) initTableRule(dbStr, tableStr string, isWhiteList bool) error {
func (f *Filter) initTableRule(dbStr, tableStr string, isAllowList bool) error {
dbIsRegex := strings.HasPrefix(dbStr, "~")
tblIsRegex := strings.HasPrefix(tableStr, "~")
if dbIsRegex && tblIsRegex {
Expand All @@ -244,7 +197,7 @@ func (f *Filter) initTableRule(dbStr, tableStr string, isWhiteList bool) error {
}
err = f.Selector.Insert(tableStr, "", &nodeEndRule{
kind: tblRuleOnlyTblPart,
isWhiteList: isWhiteList,
isAllowList: isAllowList,
}, selector.Append)
if err != nil {
return err
Expand All @@ -257,15 +210,15 @@ func (f *Filter) initTableRule(dbStr, tableStr string, isWhiteList bool) error {
err = f.Selector.Insert(dbStr, "", &nodeEndRule{
kind: tblRuleOnlyDBPart,
r: f.patternMap[tableStr[1:]],
isWhiteList: isWhiteList,
isAllowList: isAllowList,
}, selector.Append)
if err != nil {
return err
}
} else {
err := f.Selector.Insert(dbStr, tableStr, &nodeEndRule{
kind: tblRuleFull,
isWhiteList: isWhiteList,
isAllowList: isAllowList,
}, selector.Append)
if err != nil {
return err
Expand Down Expand Up @@ -364,7 +317,7 @@ func (f *Filter) filterOnTables(tb *Table) bool {
return len(f.rules.DoTables) == 0
}

func (f *Filter) matchDB(patternDBS []string, a string, isWhiteListCheck bool) bool {
func (f *Filter) matchDB(patternDBS []string, a string, isAllowListCheck bool) bool {
for _, b := range patternDBS {
isRegex := strings.HasPrefix(b, "~")
if isRegex && f.matchString(b[1:], a) {
Expand All @@ -374,14 +327,14 @@ func (f *Filter) matchDB(patternDBS []string, a string, isWhiteListCheck bool) b
ruleSet := f.Selector.Match(a, "")
for _, r := range ruleSet {
rule := r.(*nodeEndRule)
if rule.kind == dbRule && rule.isWhiteList == isWhiteListCheck {
if rule.kind == dbRule && rule.isAllowList == isAllowListCheck {
return true
}
}
return false
}

func (f *Filter) matchTable(patternTBS []*Table, tb *Table, isWhiteListCheck bool) bool {
func (f *Filter) matchTable(patternTBS []*Table, tb *Table, isAllowListCheck bool) bool {
for _, ptb := range patternTBS {
dbIsRegex, tblIsRegex := strings.HasPrefix(ptb.Schema, "~"), strings.HasPrefix(ptb.Name, "~")
if dbIsRegex && tblIsRegex {
Expand All @@ -395,22 +348,22 @@ func (f *Filter) matchTable(patternTBS []*Table, tb *Table, isWhiteListCheck boo
ruleSet := f.Selector.Match(tb.Name, "")
for _, r := range ruleSet {
rule := r.(*nodeEndRule)
if rule.kind == tblRuleOnlyTblPart && rule.isWhiteList == isWhiteListCheck {
if rule.kind == tblRuleOnlyTblPart && rule.isAllowList == isAllowListCheck {
return true
}
}
}
ruleSet := f.Selector.Match(tb.Schema, "")
for _, r := range ruleSet {
rule := r.(*nodeEndRule)
if rule.kind == tblRuleOnlyDBPart && rule.isWhiteList == isWhiteListCheck && rule.r.MatchString(tb.Name) {
if rule.kind == tblRuleOnlyDBPart && rule.isAllowList == isAllowListCheck && rule.r.MatchString(tb.Name) {
return true
}
}
ruleSet = f.Selector.Match(tb.Schema, tb.Name)
for _, r := range ruleSet {
rule := r.(*nodeEndRule)
if rule.kind == tblRuleFull && rule.isWhiteList == isWhiteListCheck {
if rule.kind == tblRuleFull && rule.isAllowList == isAllowListCheck {
return true
}
}
Expand Down
Loading