diff --git a/Dockerfile b/Dockerfile index 46d7d05db347e..c6fca6bc22b75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # limitations under the License. # Builder image -FROM golang:1.13-alpine as builder +FROM golang:1.16-alpine as builder RUN apk add --no-cache \ wget \ diff --git a/Makefile b/Makefile index dd0e5f521acab..d09ed06f2c528 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,7 @@ check-static: tools/bin/golangci-lint --enable=unused \ --enable=structcheck \ --enable=deadcode \ + --enable=gosimple \ $$($(PACKAGE_DIRECTORIES)) check-slow:tools/bin/gometalinter tools/bin/gosec @@ -238,11 +239,13 @@ tools/bin/unconvert: tools/check/go.mod cd tools/check; \ $(GO) build -o ../bin/unconvert github.com/mdempsky/unconvert -tools/bin/failpoint-ctl: go.mod - $(GO) build -o $@ github.com/pingcap/failpoint/failpoint-ctl +tools/bin/failpoint-ctl: tools/check/go.mod + cd tools/check; \ + $(GO) build -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl -tools/bin/errdoc-gen: go.mod - $(GO) build -o $@ github.com/pingcap/errors/errdoc-gen +tools/bin/errdoc-gen: tools/check/go.mod + cd tools/check; \ + $(GO) build -o ../bin/errdoc-gen github.com/pingcap/errors/errdoc-gen tools/bin/golangci-lint: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ./tools/bin v1.29.0 diff --git a/README.md b/README.md index f3c07bf0b1ad8..b4fcc35237280 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,6 @@ In addition, you may enjoy following: - [@PingCAP](https://twitter.com/PingCAP) on Twitter - Question tagged [#tidb on StackOverflow](https://stackoverflow.com/questions/tagged/tidb) - The PingCAP Team [English Blog](https://en.pingcap.com/blog) and [Chinese Blog](https://pingcap.com/blog-cn/) -- [TiDB Monthly](https://pingcap.com/weekly/) For support, please contact [PingCAP](http://bit.ly/contact_us_via_github). diff --git a/bindinfo/bind_test.go b/bindinfo/bind_test.go index 1eb7e1478b2f9..4175ddce77eb9 100644 --- a/bindinfo/bind_test.go +++ b/bindinfo/bind_test.go @@ -35,6 +35,7 @@ import ( "github.com/pingcap/tidb/metrics" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/session/txninfo" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/store/tikv/mockstore/cluster" "github.com/pingcap/tidb/util" @@ -70,6 +71,10 @@ type mockSessionManager struct { PS []*util.ProcessInfo } +func (msm *mockSessionManager) ShowTxnList() []*txninfo.TxnInfo { + panic("unimplemented!") +} + func (msm *mockSessionManager) ShowProcessList() map[uint64]*util.ProcessInfo { ret := make(map[uint64]*util.ProcessInfo) for _, item := range msm.PS { @@ -151,7 +156,8 @@ func normalizeWithDefaultDB(c *C, sql, db string) (string, string) { testParser := parser.New() stmt, err := testParser.ParseOneStmt(sql, "", "") c.Assert(err, IsNil) - return parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(stmt, "test", "")) + normalized, digest := parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(stmt, "test", "")) + return normalized, digest.String() } func (s *testSuite) TestBindParse(c *C) { @@ -177,7 +183,7 @@ func (s *testSuite) TestBindParse(c *C) { c.Check(bindHandle.Size(), Equals, 1) sql, hash := parser.NormalizeDigest("select * from test . t") - bindData := bindHandle.GetBindRecord(hash, sql, "test") + bindData := bindHandle.GetBindRecord(hash.String(), sql, "test") c.Check(bindData, NotNil) c.Check(bindData.OriginalSQL, Equals, "select * from `test` . `t`") bind := bindData.Bindings[0] @@ -651,7 +657,7 @@ func (s *testSuite) TestBindingSymbolList(c *C) { // Normalize sql, hash := parser.NormalizeDigest("select a, b from test . t where a = 1 limit 0, 1") - bindData := s.domain.BindHandle().GetBindRecord(hash, sql, "test") + bindData := s.domain.BindHandle().GetBindRecord(hash.String(), sql, "test") c.Assert(bindData, NotNil) c.Check(bindData.OriginalSQL, Equals, "select `a` , `b` from `test` . `t` where `a` = ? limit ...") bind := bindData.Bindings[0] @@ -771,7 +777,7 @@ func (s *testSuite) TestErrorBind(c *C) { c.Assert(err, IsNil, Commentf("err %v", err)) sql, hash := parser.NormalizeDigest("select * from test . t where i > ?") - bindData := s.domain.BindHandle().GetBindRecord(hash, sql, "test") + bindData := s.domain.BindHandle().GetBindRecord(hash.String(), sql, "test") c.Check(bindData, NotNil) c.Check(bindData.OriginalSQL, Equals, "select * from `test` . `t` where `i` > ?") bind := bindData.Bindings[0] diff --git a/bindinfo/handle.go b/bindinfo/handle.go index 6111910395d55..2281af3c88bd3 100644 --- a/bindinfo/handle.go +++ b/bindinfo/handle.go @@ -210,7 +210,7 @@ func (h *BindHandle) CreateBindRecord(sctx sessionctx.Context, record *BindRecor } sqlDigest := parser.DigestNormalized(record.OriginalSQL) - h.setBindRecord(sqlDigest, record) + h.setBindRecord(sqlDigest.String(), record) }() // Lock mysql.bind_info to synchronize with CreateBindRecord / AddBindRecord / DropBindRecord on other tidb instances. @@ -256,7 +256,7 @@ func (h *BindHandle) AddBindRecord(sctx sessionctx.Context, record *BindRecord) } record.Db = strings.ToLower(record.Db) - oldRecord := h.GetBindRecord(parser.DigestNormalized(record.OriginalSQL), record.OriginalSQL, record.Db) + oldRecord := h.GetBindRecord(parser.DigestNormalized(record.OriginalSQL).String(), record.OriginalSQL, record.Db) var duplicateBinding *Binding if oldRecord != nil { binding := oldRecord.FindBinding(record.Bindings[0].ID) @@ -294,7 +294,7 @@ func (h *BindHandle) AddBindRecord(sctx sessionctx.Context, record *BindRecord) return } - h.appendBindRecord(parser.DigestNormalized(record.OriginalSQL), record) + h.appendBindRecord(parser.DigestNormalized(record.OriginalSQL).String(), record) }() // Lock mysql.bind_info to synchronize with CreateBindRecord / AddBindRecord / DropBindRecord on other tidb instances. @@ -367,7 +367,7 @@ func (h *BindHandle) DropBindRecord(originalSQL, db string, binding *Binding) (e if binding != nil { record.Bindings = append(record.Bindings, *binding) } - h.removeBindRecord(parser.DigestNormalized(originalSQL), record) + h.removeBindRecord(parser.DigestNormalized(originalSQL).String(), record) }() // Lock mysql.bind_info to synchronize with CreateBindRecord / AddBindRecord / DropBindRecord on other tidb instances. @@ -515,7 +515,7 @@ func (h *BindHandle) newBindRecord(row chunk.Row) (string, *BindRecord, error) { defer h.sctx.Unlock() h.sctx.GetSessionVars().CurrentDB = bindRecord.Db err := bindRecord.prepareHints(h.sctx.Context) - return hash, bindRecord, err + return hash.String(), bindRecord, err } // setBindRecord sets the BindRecord to the cache, if there already exists a BindRecord, @@ -624,7 +624,7 @@ func (h *BindHandle) CaptureBaselines() { } dbName := utilparser.GetDefaultDB(stmt, bindableStmt.Schema) normalizedSQL, digest := parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(stmt, dbName, bindableStmt.Query)) - if r := h.GetBindRecord(digest, normalizedSQL, dbName); r != nil && r.HasUsingBinding() { + if r := h.GetBindRecord(digest.String(), normalizedSQL, dbName); r != nil && r.HasUsingBinding() { continue } bindSQL := GenerateBindSQL(context.TODO(), stmt, bindableStmt.PlanHint, true, dbName) diff --git a/bindinfo/session_handle.go b/bindinfo/session_handle.go index 2604d5b563f52..6b54aa9118f77 100644 --- a/bindinfo/session_handle.go +++ b/bindinfo/session_handle.go @@ -60,7 +60,7 @@ func (h *SessionHandle) CreateBindRecord(sctx sessionctx.Context, record *BindRe } // update the BindMeta to the cache. - h.appendBindRecord(parser.DigestNormalized(record.OriginalSQL), record) + h.appendBindRecord(parser.DigestNormalized(record.OriginalSQL).String(), record) return nil } @@ -78,14 +78,14 @@ func (h *SessionHandle) DropBindRecord(originalSQL, db string, binding *Binding) } else { newRecord = record } - h.ch.setBindRecord(parser.DigestNormalized(record.OriginalSQL), newRecord) + h.ch.setBindRecord(parser.DigestNormalized(record.OriginalSQL).String(), newRecord) updateMetrics(metrics.ScopeSession, oldRecord, newRecord, false) return nil } // GetBindRecord return the BindMeta of the (normdOrigSQL,db) if BindMeta exist. func (h *SessionHandle) GetBindRecord(normdOrigSQL, db string) *BindRecord { - hash := parser.DigestNormalized(normdOrigSQL) + hash := parser.DigestNormalized(normdOrigSQL).String() bindRecords := h.ch[hash] for _, bindRecord := range bindRecords { if bindRecord.OriginalSQL == normdOrigSQL { diff --git a/cmd/benchfilesort/main.go b/cmd/benchfilesort/main.go index 1a580f5ca14d6..943968a0ea046 100644 --- a/cmd/benchfilesort/main.go +++ b/cmd/benchfilesort/main.go @@ -17,7 +17,6 @@ import ( "encoding/binary" "flag" "fmt" - "io/ioutil" "math/rand" "os" "path/filepath" @@ -332,7 +331,7 @@ func driveRunCmd() { for i := 0; i < keySize; i++ { byDesc[i] = false } - dir, err = ioutil.TempDir(tmpDir, "benchfilesort_test") + dir, err = os.MkdirTemp(tmpDir, "benchfilesort_test") terror.MustNil(err) fs, err = fsBuilder.SetSC(sc).SetSchema(keySize, valSize).SetBuf(bufSize).SetWorkers(nWorkers).SetDesc(byDesc).SetDir(dir).Build() terror.MustNil(err) diff --git a/cmd/benchkv/main.go b/cmd/benchkv/main.go index 23bfd44a7476a..e8c3cd1b941a6 100644 --- a/cmd/benchkv/main.go +++ b/cmd/benchkv/main.go @@ -17,7 +17,7 @@ import ( "context" "flag" "fmt" - "io/ioutil" + "io" "net/http" _ "net/http/pprof" "sync" @@ -129,7 +129,7 @@ func main() { terror.MustNil(err) defer terror.Call(resp.Body.Close) - text, err1 := ioutil.ReadAll(resp.Body) + text, err1 := io.ReadAll(resp.Body) terror.Log(errors.Trace(err1)) fmt.Println(string(text)) diff --git a/cmd/ddltest/ddl_test.go b/cmd/ddltest/ddl_test.go index 52009b10de142..36922638950a8 100644 --- a/cmd/ddltest/ddl_test.go +++ b/cmd/ddltest/ddl_test.go @@ -143,7 +143,7 @@ func (s *TestDDLSuite) SetUpSuite(c *C) { s.procs = make([]*server, *serverNum) // Set server restart retry count. - s.retryCount = 5 + s.retryCount = 20 createLogFiles(c, *serverNum) err = s.startServers() diff --git a/cmd/explaintest/main.go b/cmd/explaintest/main.go index fa5265f7af871..ae4fa18d5a461 100644 --- a/cmd/explaintest/main.go +++ b/cmd/explaintest/main.go @@ -19,7 +19,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "net/http" "os" "os/exec" @@ -30,7 +29,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/pingcap/parser/ast" - "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/util/logutil" @@ -164,7 +162,7 @@ LOOP: } func (t *tester) loadQueries() ([]query, error) { - data, err := ioutil.ReadFile(t.testFileName()) + data, err := os.ReadFile(t.testFileName()) if err != nil { return nil, err } @@ -428,12 +426,12 @@ func (t *tester) create(tableName string, qText string) error { return err } - js, err := ioutil.ReadAll(resp.Body) + js, err := io.ReadAll(resp.Body) if err != nil { return err } - return ioutil.WriteFile(t.statsFileName(tableName), js, 0644) + return os.WriteFile(t.statsFileName(tableName), js, 0644) } func (t *tester) commit() error { @@ -532,7 +530,7 @@ func (t *tester) flushResult() error { if !record { return nil } - return ioutil.WriteFile(t.resultFileName(), t.buf.Bytes(), 0644) + return os.WriteFile(t.resultFileName(), t.buf.Bytes(), 0644) } func (t *tester) statsFileName(tableName string) string { @@ -551,7 +549,7 @@ func (t *tester) resultFileName() string { func loadAllTests() ([]string, error) { // tests must be in t folder - files, err := ioutil.ReadDir("./t") + files, err := os.ReadDir("./t") if err != nil { return nil, err } @@ -663,8 +661,6 @@ func main() { log.Fatal(fmt.Sprintf("%s failed", sql), zap.Error(err)) } } - // Wait global variables to reload. - time.Sleep(domain.GlobalVariableCacheExpiry) if _, err = mdb.Exec("set sql_mode='STRICT_TRANS_TABLES'"); err != nil { log.Fatal("set sql_mode='STRICT_TRANS_TABLES' failed", zap.Error(err)) diff --git a/cmd/explaintest/r/cte.result b/cmd/explaintest/r/cte.result new file mode 100644 index 0000000000000..af8209d6c9e48 --- /dev/null +++ b/cmd/explaintest/r/cte.result @@ -0,0 +1,149 @@ +use test; +drop table if exists tbl_0; +create table tbl_0(a int); +with recursive cte_0 (col_10,col_11,col_12) AS ( select 1, 2,3 from tbl_0 UNION select col_10 + 1,col_11 + 1,col_12 + 1 from cte_0 where col_10 < 10 ) select * from cte_0; +drop table if exists tbl_1; +CREATE TABLE `tbl_1` ( +`col_5` decimal(47,21) NOT NULL DEFAULT '5308.880000000000000000000', +`col_6` enum('Alice','Bob','Charlie','David') DEFAULT NULL, +`col_7` float NOT NULL, +`col_8` bigint NOT NULL DEFAULT '-688199144806783096', +`col_9` varchar(248) NOT NULL, +PRIMARY KEY (`col_5`,`col_7`,`col_9`,`col_8`), +UNIQUE KEY `idx_4` (`col_8`), +UNIQUE KEY `idx_7` (`col_5`,`col_7`,`col_8`), +UNIQUE KEY `idx_9` (`col_9`,`col_8`), +UNIQUE KEY `idx_3` (`col_9`(3),`col_8`), +UNIQUE KEY `idx_8` (`col_7`,`col_6`,`col_8`,`col_5`), +KEY `idx_5` (`col_7`), +KEY `idx_6` (`col_7`), +KEY `idx_10` (`col_9`,`col_5`), +KEY `idx_11` (`col_5`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +/*!50100 PARTITION BY HASH (`col_8`) +PARTITIONS 4 */; +with recursive cte_1 (col_13,col_14,col_15,col_16,col_17) AS ( with recursive cte_2 (col_18,col_19,col_20,col_21,col_22,col_23,col_24) AS ( select 1, 2,col_8,4,5,6,7 from tbl_1 ) select col_19,col_18,col_22,col_23,col_21 from cte_2 UNION ALL select col_13 + 1,col_14 + 1,col_15 + 1,col_16 + 1,col_17 + 1 from cte_1 where col_13 < 10 ) select * from cte_1; +with recursive cte_256 (col_969,col_970,col_971) AS ( with recursive cte_257 (col_972,col_973,col_974,col_975) AS ( select 1, 2,col_8,4 from tbl_1 UNION select col_972 + 1,col_973 + 1,col_974 + 1,col_975 + 1 from cte_257 where col_972 < 10 ) select col_975,col_974,col_973 from cte_257 UNION DISTINCT select col_969 + 1,col_970 + 1,col_971 + 1 from cte_256 where col_969 < 10 ) select * from cte_256; +drop table if exists tbl_2, tbl_3; +create table tbl_2 ( col_4 char(246) collate utf8_unicode_ci not null , col_5 char(253) collate utf8mb4_unicode_ci ) ; +create table tbl_3 ( col_6 char(207) collate utf8mb4_unicode_ci , col_7 int unsigned not null ) ; +insert into tbl_2 values ( "0",null ) ; +insert into tbl_2 values ( "1","0" ) ; +insert into tbl_2 values ( "1","1" ) ; +insert into tbl_2 values ( "0","0" ) ; +insert into tbl_2 values ( "0","1" ) ; +insert into tbl_3 values ( "1",0 ) ; +insert into tbl_3 values ( "1",1 ) ; +insert into tbl_3 values ( "0",0 ) ; +insert into tbl_3 values ( "0",1 ) ; +with recursive tbl_2 (col_64,col_65,col_66,col_67) AS ( select 1, col_6,col_6,4 from tbl_3 UNION DISTINCT select col_64 + 1,concat(col_65, 1),col_66 + 1,concat(col_67, 1) from tbl_2 where col_64 < 5 ) select * from tbl_2 order by col_64; +drop table if exists tbl_3, tbl_4; +create table tbl_3 ( col_6 int not null , col_7 char(95) collate utf8_general_ci ) ; +create table tbl_4 ( col_8 char collate utf8_unicode_ci , col_9 char collate utf8mb4_bin ) ; +insert into tbl_3 values ( 0,"1" ) ; +insert into tbl_4 values ( "1","0" ) ; +with recursive cte_2245 (col_8692,col_8693) AS ( select 1, col_7 from tbl_3 UNION select col_8692 + 1,concat(col_8693, 1) from cte_2245 where col_8692 < 5 ) , cte_2246 (col_8694,col_8695,col_8696,col_8697) AS ( with recursive cte_2247 (col_8698,col_8699,col_8700,col_8701) AS ( select 1, cast("2" as char(20)),3,col_8 from tbl_4 ) select col_8698,col_8699,col_8700,col_8701 from cte_2247 UNION select col_8694 + 1,col_8695 + 1,col_8696 + 1,col_8697 + 1 from cte_2246 where col_8694 < 5 ) select * from cte_2245,cte_2246 order by col_8692,col_8693,col_8696,col_8695,col_8697,col_8694; +with recursive cte2 as (select 1 as col_1, 2 as col_2) select c1.col_1, c2.col_2 from cte2 as c1, cte2 as c2 where c2.col_2 = 1; +with recursive cte (c1) as (select 1), cte1 (c2) as (select 1 union select c1 + 1 from cte, cte1) select * from cte, cte1; +with recursive tbl_0 (col_943,col_944,col_945,col_946,col_947) AS ( with recursive tbl_0 (col_948,col_949,col_950,col_951,col_952) AS ( select 1, 2,3,4,5 UNION ALL select col_948 + 1,col_949 + 1,col_950 + 1,col_951 + 1,col_952 + 1 from tbl_0 where col_948 < 5 ) select col_948,col_949,col_951,col_950,col_952 from tbl_0 UNION ALL select col_943 + 1,col_944 + 1,col_945 + 1,col_946 + 1,col_947 + 1 from tbl_0 where col_943 < 5 ) select * from tbl_0; +Error 1054: Unknown column 'col_943' in 'where clause' +with recursive cte1 (c1, c2) as (select 1, '1' union select concat(c1, 1), c2 + 1 from cte1 where c1 < 100) select * from cte1; +with recursive cte_8 (col_51,col_52,col_53,col_54) AS ( with recursive cte_9 (col_55,col_56,col_57,col_58) AS ( select 1, 2,3,4 UNION ALL select col_55 + 1,col_56 + 1,col_57 + 1,col_58 + 1 from cte_9 where col_55 < 5 ) select col_55,col_57,col_56,col_58 from cte_9 UNION DISTINCT select col_51 + 1,col_52 + 1,col_53 + 1,col_54 + 1 from cte_8 where col_51 < 5 ) select * from cte_8; +with recursive qn as (select 1 from dual union all select 1 from dual) select * from qn; +with recursive qn as (select 1 as a from dual group by a union all select a+1 from qn where a<3) select * from qn; +with recursive qn as ((select 1 as a from dual order by a) union all select a+1 from qn where a<3) select * from qn; +drop table if exists employees; +CREATE TABLE employees ( +ID INT PRIMARY KEY, +NAME VARCHAR(100), +MANAGER_ID INT, +INDEX (MANAGER_ID), +FOREIGN KEY (MANAGER_ID) REFERENCES employees(ID) +); +INSERT INTO employees VALUES +(333, "Yasmina", NULL), +(198, "John", 333), +(692, "Tarek", 333), +(29, "Pedro", 198), +(4610, "Sarah", 29), +(72, "Pierre", 29), +(123, "Adil", 692); +WITH RECURSIVE employees_extended AS (SELECT ID, NAME, MANAGER_ID, CAST(ID AS CHAR(200)) AS PATH FROM employees WHERE NAME='Pierre' UNION ALL SELECT S.ID, S.NAME, S.MANAGER_ID, CONCAT(M.PATH, ",", S.ID) FROM employees_extended M JOIN employees S ON M.MANAGER_ID=S.ID) SELECT * FROM employees_extended; +with recursive cte (c1) as (select 1), cte1 (c2) as (select 1 union select c1 + 1 from cte where c1 < 10) select * from cte where c1 < 5; +with recursive cte_581 (col_2343,col_2344,col_2345) AS ( select 1, '2',cast('3' as char(20))) , cte_582 (col_2346,col_2347,col_2348) AS ( select 1, 2, 3) select * from cte_581 as cte_as_583,cte_582 as cte_as_584,cte_582 as cte_as_585 order by cte_as_583.col_2343,cte_as_585.col_2348,cte_as_584.col_2346,cte_as_584.col_2348,cte_as_583.col_2344,cte_as_584.col_2347,cte_as_585.col_2346,cte_as_585.col_2347,cte_as_583.col_2345; +with recursive tbl_3 (col_19,col_20,col_21,col_22) AS ( select 1, 2,3,4 UNION select col_19 + 1,col_20 + 1,col_21 + 1,concat(col_22, 1) from tbl_3 where col_19 < 5 ) , cte_4 (col_23,col_24,col_25,col_26) AS ( select 1, 2,cast("3" as char(20)),4 UNION DISTINCT select col_23 + 1,col_24 + 1,concat(col_25, 1),col_26 + 1 from cte_4 where col_23 < 5 ) select * from tbl_3 as cte_as_3,cte_4 as cte_as_4,tbl_3 as cte_as_5 order by cte_as_3.col_19,cte_as_4.col_23,cte_as_4.col_25,cte_as_4.col_24,cte_as_4.col_26,cte_as_3.col_20,cte_as_5.col_22,cte_as_3.col_21,cte_as_5.col_20,cte_as_3.col_22,cte_as_5.col_19,cte_as_5.col_21; +with cte1 (c1) as (select 1) select * from cte1 as b, cte1 as a; +WITH RECURSIVE qn AS +( +select 1 +union all +select 3, 0 from qn +) +select * from qn; +Error 1222: The used SELECT statements have a different number of columns +with recursive cte1 as (select 1 union all (select 1 from cte1 limit 10)) select * from cte1; +Error 1235: This version of TiDB doesn't yet support 'ORDER BY / LIMIT / SELECT DISTINCT in recursive query block of Common Table Expression' +with recursive qn as (select 123 as a union all select null from qn where a is not null) select * from qn; +with recursive q (b) as (select 1, 1 union all select 1, 1 from q) select b from q; +Error 1353: In definition of view, derived table or common table expression, SELECT list and column names list have different column counts +drop table if exists t1; +create table t1(a int); +insert into t1 values(1); +insert into t1 values(2); +SELECT * +FROM +t1 dt +WHERE +EXISTS( +WITH RECURSIVE qn AS (SELECT a*0 AS b UNION ALL SELECT b+1 FROM qn WHERE b=0) +SELECT * FROM qn WHERE b=a +); +a +1 +drop table if exists t1; +create table t1 (a int); +insert into t1 values (1); +SELECT (WITH qn AS (SELECT 10*a as a FROM t1), +qn2 AS (SELECT 3*a AS b FROM qn) SELECT * from qn2 LIMIT 1) +FROM t1; +(WITH qn AS (SELECT 10*a as a FROM t1), +qn2 AS (SELECT 3*a AS b FROM qn) +30 +select (with qn as (select "with") select * from qn) as scal_subq +from dual; +scal_subq +with +drop table if exists t1; +create table t1 (a int); + insert into t1 values(1), (2), (3); +with q as (select * from t1) +select /*+ merge(q) no_merge(q1) */ * from q, q q1 where q.a=1 and q1.a=2; +drop table if exists t1; + create table t1 (a int, b int); +with qn as (select a, b from t1) select b from qn group by a; +drop table if exists t1; +create table t1(a int); +insert into t1 values(1); +insert into t1 values(2); +SELECT * +FROM +t1 dt +WHERE +EXISTS( +WITH RECURSIVE qn AS (SELECT a*0+1 AS b UNION ALL SELECT b+1 FROM qn WHERE b=0) +SELECT * FROM qn WHERE b=1 +); +a +1 +2 +drop table if exists tbl_1; +CREATE TABLE `tbl_1` ( +`col_2` char(65) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, +`col_3` int(11) NOT NULL +); +with recursive cte_8932 (col_34891,col_34892) AS ( with recursive cte_8932 (col_34893,col_34894,col_34895) AS ( with tbl_1 (col_34896,col_34897,col_34898,col_34899) AS ( select 1, "2",3,col_3 from tbl_1 ) select cte_as_8958.col_34896,cte_as_8958.col_34898,cte_as_8958.col_34899 from tbl_1 as cte_as_8958 UNION DISTINCT select col_34893 + 1,concat(col_34894, 1),col_34895 + 1 from cte_8932 where col_34893 < 5 ) select cte_as_8959.col_34893,cte_as_8959.col_34895 from cte_8932 as cte_as_8959 ) select * from cte_8932 as cte_as_8960 order by cte_as_8960.col_34891,cte_as_8960.col_34892; +drop table if exists t1; +create table t1(c1 bigint unsigned); +insert into t1 values(0); +with recursive cte1 as (select c1 - 1 c1 from t1 union all select c1 - 1 c1 from cte1 where c1 != 0) select * from cte1 dt1, cte1 dt2; +Error 1690: BIGINT UNSIGNED value is out of range in '(test.t1.c1 - 1)' diff --git a/cmd/explaintest/r/explain_cte.result b/cmd/explaintest/r/explain_cte.result new file mode 100644 index 0000000000000..36eda33222fd6 --- /dev/null +++ b/cmd/explaintest/r/explain_cte.result @@ -0,0 +1,204 @@ +use test; +drop table if exists t1, t2; +create table t1 (c1 int primary key, c2 int, index c2 (c2)); +create table t2 (c1 int unique, c2 int); +insert into t1 values(1, 0), (2, 1); +insert into t2 values(1, 0), (2, 1); +explain with cte(a) as (select 1) select * from cte; +id estRows task access object operator info +CTEFullScan_8 1.00 root CTE:cte data:CTE_0 +CTE_0 1.00 root Non-Recursive CTE +└─Projection_6(Seed Part) 1.00 root 1->Column#1 + └─TableDual_7 1.00 root rows:1 +explain with cte(a) as (select c1 from t1) select * from cte; +id estRows task access object operator info +CTEFullScan_11 1.00 root CTE:cte data:CTE_0 +CTE_0 1.00 root Non-Recursive CTE +└─TableReader_8(Seed Part) 10000.00 root data:TableFullScan_7 + └─TableFullScan_7 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +explain with cte(a,b,c,d) as (select * from t1, t2) select * from cte; +id estRows task access object operator info +CTEFullScan_18 1.00 root CTE:cte data:CTE_0 +CTE_0 1.00 root Non-Recursive CTE +└─HashJoin_10(Seed Part) 100000000.00 root CARTESIAN inner join + ├─TableReader_17(Build) 10000.00 root data:TableFullScan_16 + │ └─TableFullScan_16 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo + └─TableReader_13(Probe) 10000.00 root data:TableFullScan_12 + └─TableFullScan_12 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +explain with recursive cte(a) as (select 1 union select a+1 from cte where a < 10) select * from cte; +id estRows task access object operator info +CTEFullScan_17 1.00 root CTE:cte data:CTE_0 +CTE_0 1.00 root Recursive CTE +├─Projection_12(Seed Part) 1.00 root 1->Column#2 +│ └─TableDual_13 1.00 root rows:1 +└─Projection_14(Recursive Part) 0.80 root cast(plus(Column#3, 1), bigint(1) BINARY)->Column#5 + └─Selection_15 0.80 root lt(Column#3, 10) + └─CTETable_16 1.00 root Scan on CTE_0 +explain with recursive cte(a) as (select c2 from t1 union select a+1 from cte where a < 10) select * from cte; +id estRows task access object operator info +CTEFullScan_20 1.00 root CTE:cte data:CTE_0 +CTE_0 1.00 root Recursive CTE +├─TableReader_14(Seed Part) 10000.00 root data:TableFullScan_13 +│ └─TableFullScan_13 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Projection_17(Recursive Part) 0.80 root cast(plus(test.t1.c2, 1), int(11))->test.t1.c2 + └─Selection_18 0.80 root lt(test.t1.c2, 10) + └─CTETable_19 1.00 root Scan on CTE_0 +explain with cte(a) as (with recursive cte1(a) as (select 1 union select a + 1 from cte1 where a < 10) select * from cte1) select * from cte; +id estRows task access object operator info +CTEFullScan_21 1.00 root CTE:cte data:CTE_0 +CTE_0 1.00 root Non-Recursive CTE +└─CTEFullScan_20(Seed Part) 1.00 root CTE:cte1 data:CTE_1 +CTE_1 1.00 root Recursive CTE +├─Projection_15(Seed Part) 1.00 root 1->Column#2 +│ └─TableDual_16 1.00 root rows:1 +└─Projection_17(Recursive Part) 0.80 root cast(plus(Column#3, 1), bigint(1) BINARY)->Column#5 + └─Selection_18 0.80 root lt(Column#3, 10) + └─CTETable_19 1.00 root Scan on CTE_1 +explain with recursive cte(a) as (select 1 union select a+1 from cte where a < 10) select * from cte t1, cte t2; +id estRows task access object operator info +HashJoin_15 1.00 root CARTESIAN inner join +├─CTEFullScan_23(Build) 1.00 root CTE:t2 data:CTE_0 +└─CTEFullScan_22(Probe) 1.00 root CTE:t1 data:CTE_0 +CTE_0 1.00 root Recursive CTE +├─Projection_17(Seed Part) 1.00 root 1->Column#2 +│ └─TableDual_18 1.00 root rows:1 +└─Projection_19(Recursive Part) 0.80 root cast(plus(Column#3, 1), bigint(1) BINARY)->Column#5 + └─Selection_20 0.80 root lt(Column#3, 10) + └─CTETable_21 1.00 root Scan on CTE_0 +explain with cte(a) as (with recursive cte1(a) as (select 1 union select a + 1 from cte1 where a < 10) select * from cte1) select * from cte t1, cte t2; +id estRows task access object operator info +HashJoin_17 1.00 root CARTESIAN inner join +├─CTEFullScan_27(Build) 1.00 root CTE:t2 data:CTE_0 +└─CTEFullScan_26(Probe) 1.00 root CTE:t1 data:CTE_0 +CTE_0 1.00 root Non-Recursive CTE +└─CTEFullScan_25(Seed Part) 1.00 root CTE:cte1 data:CTE_1 +CTE_1 1.00 root Recursive CTE +├─Projection_20(Seed Part) 1.00 root 1->Column#2 +│ └─TableDual_21 1.00 root rows:1 +└─Projection_22(Recursive Part) 0.80 root cast(plus(Column#3, 1), bigint(1) BINARY)->Column#5 + └─Selection_23 0.80 root lt(Column#3, 10) + └─CTETable_24 1.00 root Scan on CTE_1 +explain with recursive cte1(a) as (select 1 union select a+1 from cte1 where a < 10), cte2(a) as (select c2 from t1 union select a+1 from cte2 where a < 10) select * from cte1, cte2; +id estRows task access object operator info +HashJoin_23 1.00 root CARTESIAN inner join +├─CTEFullScan_39(Build) 1.00 root CTE:cte2 data:CTE_1 +└─CTEFullScan_30(Probe) 1.00 root CTE:cte1 data:CTE_0 +CTE_1 1.00 root Recursive CTE +├─TableReader_33(Seed Part) 10000.00 root data:TableFullScan_32 +│ └─TableFullScan_32 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Projection_36(Recursive Part) 0.80 root cast(plus(test.t1.c2, 1), int(11))->test.t1.c2 + └─Selection_37 0.80 root lt(test.t1.c2, 10) + └─CTETable_38 1.00 root Scan on CTE_1 +CTE_0 1.00 root Recursive CTE +├─Projection_25(Seed Part) 1.00 root 1->Column#2 +│ └─TableDual_26 1.00 root rows:1 +└─Projection_27(Recursive Part) 0.80 root cast(plus(Column#3, 1), bigint(1) BINARY)->Column#5 + └─Selection_28 0.80 root lt(Column#3, 10) + └─CTETable_29 1.00 root Scan on CTE_0 +explain with q(a,b) as (select * from t1) select /*+ merge(q) no_merge(q1) */ * from q, q q1 where q.a=1 and q1.a=2; +id estRows task access object operator info +HashJoin_12 0.64 root CARTESIAN inner join +├─Selection_21(Build) 0.80 root eq(test.t1.c1, 2) +│ └─CTEFullScan_22 1.00 root CTE:q1 data:CTE_0 +└─Selection_14(Probe) 0.80 root eq(test.t1.c1, 1) + └─CTEFullScan_20 1.00 root CTE:q data:CTE_0 +CTE_0 1.00 root Non-Recursive CTE +└─TableReader_17(Seed Part) 10000.00 root data:TableFullScan_16 + └─TableFullScan_16 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +explain with recursive cte(a,b) as (select 1, concat('a', 1) union select a+1, concat(b, 1) from cte where a < 5) select * from cte; +id estRows task access object operator info +CTEFullScan_17 1.00 root CTE:cte data:CTE_0 +CTE_0 1.00 root Recursive CTE +├─Projection_12(Seed Part) 1.00 root 1->Column#3, a1->Column#4 +│ └─TableDual_13 1.00 root rows:1 +└─Projection_14(Recursive Part) 0.80 root cast(plus(Column#5, 1), bigint(1) BINARY)->Column#9, cast(concat(Column#6, 1), var_string(21))->Column#10 + └─Selection_15 0.80 root lt(Column#5, 5) + └─CTETable_16 1.00 root Scan on CTE_0 +explain select * from t1 dt where exists(with recursive qn as (select c1*0+1 as b union all select b+1 from qn where b=0) select * from qn where b=1); +id estRows task access object operator info +Apply_19 10000.00 root CARTESIAN semi join +├─TableReader_21(Build) 10000.00 root data:TableFullScan_20 +│ └─TableFullScan_20 10000.00 cop[tikv] table:dt keep order:false, stats:pseudo +└─Selection_24(Probe) 0.80 root eq(Column#8, 1) + └─CTEFullScan_30 1.00 root CTE:qn data:CTE_0 +CTE_0 1.00 root Recursive CTE +├─Projection_25(Seed Part) 1.00 root plus(mul(test.t1.c1, 0), 1)->Column#4 +│ └─TableDual_26 1.00 root rows:1 +└─Projection_27(Recursive Part) 0.80 root plus(Column#5, 1)->Column#7 + └─Selection_28 0.80 root eq(Column#5, 0) + └─CTETable_29 1.00 root Scan on CTE_0 +explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 1) select * from cte1; +id estRows task access object operator info +CTEFullScan_19 1.00 root CTE:cte1 data:CTE_0 +CTE_0 1.00 root Recursive CTE, limit(offset:0, count:1) +├─TableReader_14(Seed Part) 10000.00 root data:TableFullScan_13 +│ └─TableFullScan_13 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Projection_17(Recursive Part) 1.00 root cast(plus(test.t1.c1, 1), int(11))->test.t1.c1 + └─CTETable_18 1.00 root Scan on CTE_0 +explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 100 offset 100) select * from cte1; +id estRows task access object operator info +CTEFullScan_19 1.00 root CTE:cte1 data:CTE_0 +CTE_0 1.00 root Recursive CTE, limit(offset:100, count:100) +├─TableReader_14(Seed Part) 10000.00 root data:TableFullScan_13 +│ └─TableFullScan_13 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Projection_17(Recursive Part) 1.00 root cast(plus(test.t1.c1, 1), int(11))->test.t1.c1 + └─CTETable_18 1.00 root Scan on CTE_0 +explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 0 offset 0) select * from cte1; +id estRows task access object operator info +CTEFullScan_19 1.00 root CTE:cte1 data:CTE_0 +CTE_0 1.00 root Recursive CTE, limit(offset:0, count:0) +├─TableReader_14(Seed Part) 10000.00 root data:TableFullScan_13 +│ └─TableFullScan_13 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Projection_17(Recursive Part) 1.00 root cast(plus(test.t1.c1, 1), int(11))->test.t1.c1 + └─CTETable_18 1.00 root Scan on CTE_0 +explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 1) select * from cte1 dt1 join cte1 dt2 on dt1.c1 = dt2.c1; +id estRows task access object operator info +HashJoin_18 0.64 root inner join, equal:[eq(test.t1.c1, test.t1.c1)] +├─Selection_29(Build) 0.80 root not(isnull(test.t1.c1)) +│ └─CTEFullScan_30 1.00 root CTE:dt2 data:CTE_0 +└─Selection_20(Probe) 0.80 root not(isnull(test.t1.c1)) + └─CTEFullScan_28 1.00 root CTE:dt1 data:CTE_0 +CTE_0 1.00 root Recursive CTE, limit(offset:0, count:1) +├─TableReader_23(Seed Part) 10000.00 root data:TableFullScan_22 +│ └─TableFullScan_22 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Projection_26(Recursive Part) 1.00 root cast(plus(test.t1.c1, 1), int(11))->test.t1.c1 + └─CTETable_27 1.00 root Scan on CTE_0 +explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 0 offset 0) select * from cte1 dt1 join cte1 dt2 on dt1.c1 = dt2.c1; +id estRows task access object operator info +HashJoin_18 0.64 root inner join, equal:[eq(test.t1.c1, test.t1.c1)] +├─Selection_29(Build) 0.80 root not(isnull(test.t1.c1)) +│ └─CTEFullScan_30 1.00 root CTE:dt2 data:CTE_0 +└─Selection_20(Probe) 0.80 root not(isnull(test.t1.c1)) + └─CTEFullScan_28 1.00 root CTE:dt1 data:CTE_0 +CTE_0 1.00 root Recursive CTE, limit(offset:0, count:0) +├─TableReader_23(Seed Part) 10000.00 root data:TableFullScan_22 +│ └─TableFullScan_22 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Projection_26(Recursive Part) 1.00 root cast(plus(test.t1.c1, 1), int(11))->test.t1.c1 + └─CTETable_27 1.00 root Scan on CTE_0 +explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 1) select * from cte1; +id estRows task access object operator info +CTEFullScan_34 1.00 root CTE:cte1 data:CTE_0 +CTE_0 1.00 root Non-Recursive CTE +└─Limit_21(Seed Part) 1.00 root offset:0, count:1 + └─HashAgg_22 1.00 root group by:Column#11, funcs:firstrow(Column#11)->Column#11 + └─Union_23 20000.00 root + ├─TableReader_26 10000.00 root data:TableFullScan_25 + │ └─TableFullScan_25 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo + └─IndexReader_33 10000.00 root index:IndexFullScan_32 + └─IndexFullScan_32 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo +explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 100 offset 100) select * from cte1; +id estRows task access object operator info +CTEFullScan_34 1.00 root CTE:cte1 data:CTE_0 +CTE_0 1.00 root Non-Recursive CTE +└─Limit_21(Seed Part) 100.00 root offset:100, count:100 + └─HashAgg_22 200.00 root group by:Column#11, funcs:firstrow(Column#11)->Column#11 + └─Union_23 20000.00 root + ├─TableReader_26 10000.00 root data:TableFullScan_25 + │ └─TableFullScan_25 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo + └─IndexReader_33 10000.00 root index:IndexFullScan_32 + └─IndexFullScan_32 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo +explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 0 offset 0) select * from cte1; +id estRows task access object operator info +CTEFullScan_18 1.00 root CTE:cte1 data:CTE_0 +CTE_0 1.00 root Non-Recursive CTE +└─TableDual_17(Seed Part) 0.00 root rows:0 diff --git a/cmd/explaintest/r/explain_easy.result b/cmd/explaintest/r/explain_easy.result index a33fe5a791bbc..c2c61bf79b58e 100644 --- a/cmd/explaintest/r/explain_easy.result +++ b/cmd/explaintest/r/explain_easy.result @@ -108,9 +108,9 @@ HashJoin 9990.00 root inner join, equal:[eq(test.t1.c1, test.t2.c2)] └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo explain format = 'brief' select (select count(1) k from t1 s where s.c1 = t1.c1 having k != 0) from t1; id estRows task access object operator info -Projection 10000.00 root ifnull(Column#10, 0)->Column#10 -└─MergeJoin 10000.00 root left outer join, left key:test.t1.c1, right key:test.t1.c1 - ├─Projection(Build) 8000.00 root 1->Column#10, test.t1.c1 +Projection 12500.00 root ifnull(Column#10, 0)->Column#10 +└─MergeJoin 12500.00 root left outer join, left key:test.t1.c1, right key:test.t1.c1 + ├─Projection(Build) 10000.00 root 1->Column#10, test.t1.c1 │ └─TableReader 10000.00 root data:TableFullScan │ └─TableFullScan 10000.00 cop[tikv] table:s keep order:true, stats:pseudo └─TableReader(Probe) 10000.00 root data:TableFullScan @@ -124,11 +124,12 @@ Projection 10000.00 root eq(test.t1.c2, test.t2.c2)->Column#11 └─Apply 10000.00 root CARTESIAN left outer join ├─TableReader(Build) 10000.00 root data:TableFullScan │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo - └─Projection(Probe) 1.00 root test.t2.c1, test.t2.c2 - └─IndexLookUp 1.00 root limit embedded(offset:0, count:1) - ├─Limit(Build) 1.00 cop[tikv] offset:0, count:1 - │ └─IndexRangeScan 1.00 cop[tikv] table:t2, index:c1(c1) range: decided by [eq(test.t1.c1, test.t2.c1)], keep order:true, stats:pseudo - └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t2 keep order:false, stats:pseudo + └─Limit(Probe) 1.00 root offset:0, count:1 + └─Projection 1.00 root test.t2.c1, test.t2.c2 + └─IndexLookUp 1.00 root + ├─Limit(Build) 1.00 cop[tikv] offset:0, count:1 + │ └─IndexRangeScan 1.00 cop[tikv] table:t2, index:c1(c1) range: decided by [eq(test.t1.c1, test.t2.c1)], keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t2 keep order:false, stats:pseudo explain format = 'brief' select * from t1 order by c1 desc limit 1; id estRows task access object operator info Limit 1.00 root offset:0, count:1 @@ -194,31 +195,32 @@ test t4 1 expr_idx 1 NULL NULL (`a` + `b` + 1) 2 YES NO explain format = 'brief' select count(1) from (select count(1) from (select * from t1 where c3 = 100) k) k2; id estRows task access object operator info StreamAgg 1.00 root funcs:count(1)->Column#5 -└─StreamAgg 1.00 root funcs:firstrow(Column#9)->Column#7 +└─StreamAgg 1.00 root funcs:count(Column#9)->Column#7 └─TableReader 1.00 root data:StreamAgg - └─StreamAgg 1.00 cop[tikv] funcs:firstrow(1)->Column#9 + └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#9 └─Selection 10.00 cop[tikv] eq(test.t1.c3, 100) └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo explain format = 'brief' select 1 from (select count(c2), count(c3) from t1) k; id estRows task access object operator info Projection 1.00 root 1->Column#6 -└─StreamAgg 1.00 root funcs:firstrow(Column#14)->Column#9 +└─StreamAgg 1.00 root funcs:count(Column#14)->Column#9 └─TableReader 1.00 root data:StreamAgg - └─StreamAgg 1.00 cop[tikv] funcs:firstrow(1)->Column#14 + └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#14 └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo explain format = 'brief' select count(1) from (select max(c2), count(c3) as m from t1) k; id estRows task access object operator info StreamAgg 1.00 root funcs:count(1)->Column#6 -└─StreamAgg 1.00 root funcs:firstrow(Column#13)->Column#8 +└─StreamAgg 1.00 root funcs:count(Column#13)->Column#8 └─TableReader 1.00 root data:StreamAgg - └─StreamAgg 1.00 cop[tikv] funcs:firstrow(1)->Column#13 + └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#13 └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo explain format = 'brief' select count(1) from (select count(c2) from t1 group by c3) k; id estRows task access object operator info StreamAgg 1.00 root funcs:count(1)->Column#5 -└─HashAgg 8000.00 root group by:test.t1.c3, funcs:firstrow(1)->Column#7 - └─TableReader 10000.00 root data:TableFullScan - └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─HashAgg 8000.00 root group by:test.t1.c3, funcs:count(Column#9)->Column#7 + └─TableReader 8000.00 root data:HashAgg + └─HashAgg 8000.00 cop[tikv] group by:test.t1.c3, funcs:count(1)->Column#9 + └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo set @@session.tidb_opt_insubq_to_join_and_agg=0; explain format = 'brief' select sum(t1.c1 in (select c1 from t2)) from t1; id estRows task access object operator info @@ -498,7 +500,7 @@ PRIMARY KEY (`id`) explain format = 'brief' SELECT COUNT(1) FROM (SELECT COALESCE(b.region_name, '不详') region_name, SUM(a.registration_num) registration_num FROM (SELECT stat_date, show_date, region_id, 0 registration_num FROM test01 WHERE period = 1 AND stat_date >= 20191202 AND stat_date <= 20191202 UNION ALL SELECT stat_date, show_date, region_id, registration_num registration_num FROM test01 WHERE period = 1 AND stat_date >= 20191202 AND stat_date <= 20191202) a LEFT JOIN test02 b ON a.region_id = b.id WHERE registration_num > 0 AND a.stat_date >= '20191202' AND a.stat_date <= '20191202' GROUP BY a.stat_date , a.show_date , COALESCE(b.region_name, '不详') ) JLS; id estRows task access object operator info StreamAgg 1.00 root funcs:count(1)->Column#22 -└─HashAgg 8000.00 root group by:Column#32, Column#33, Column#34, funcs:firstrow(1)->Column#31 +└─HashAgg 8000.00 root group by:Column#32, Column#33, Column#34, funcs:count(1)->Column#31 └─Projection 10000.01 root Column#14, Column#15, coalesce(test.test02.region_name, 不详)->Column#34 └─HashJoin 10000.01 root left outer join, equal:[eq(Column#16, test.test02.id)] ├─TableReader(Build) 10000.00 root data:TableFullScan diff --git a/cmd/explaintest/r/explain_easy_stats.result b/cmd/explaintest/r/explain_easy_stats.result index e76b0d4d233a6..8305dfa929e2b 100644 --- a/cmd/explaintest/r/explain_easy_stats.result +++ b/cmd/explaintest/r/explain_easy_stats.result @@ -106,11 +106,12 @@ Projection 1999.00 root eq(test.t1.c2, test.t2.c2)->Column#11 └─Apply 1999.00 root CARTESIAN left outer join ├─TableReader(Build) 1999.00 root data:TableFullScan │ └─TableFullScan 1999.00 cop[tikv] table:t1 keep order:false - └─Projection(Probe) 1.00 root test.t2.c1, test.t2.c2 - └─IndexLookUp 1.00 root limit embedded(offset:0, count:1) - ├─Limit(Build) 1.00 cop[tikv] offset:0, count:1 - │ └─IndexRangeScan 1.25 cop[tikv] table:t2, index:c1(c1) range: decided by [eq(test.t1.c1, test.t2.c1)], keep order:true - └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t2 keep order:false, stats:pseudo + └─Limit(Probe) 1.00 root offset:0, count:1 + └─Projection 1.00 root test.t2.c1, test.t2.c2 + └─IndexLookUp 1.00 root + ├─Limit(Build) 1.00 cop[tikv] offset:0, count:1 + │ └─IndexRangeScan 1.25 cop[tikv] table:t2, index:c1(c1) range: decided by [eq(test.t1.c1, test.t2.c1)], keep order:true + └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t2 keep order:false, stats:pseudo explain format = 'brief' select * from t1 order by c1 desc limit 1; id estRows task access object operator info Limit 1.00 root offset:0, count:1 diff --git a/cmd/explaintest/r/explain_generate_column_substitute.result b/cmd/explaintest/r/explain_generate_column_substitute.result index 821988170dd87..2442479202c1a 100644 --- a/cmd/explaintest/r/explain_generate_column_substitute.result +++ b/cmd/explaintest/r/explain_generate_column_substitute.result @@ -389,3 +389,28 @@ explain format = 'brief' select c0 from t0; id estRows task access object operator info TableReader 10000.00 root data:TableFullScan └─TableFullScan 10000.00 cop[tikv] table:t0 keep order:false, stats:pseudo + -- TableRead +drop table if exists tbl1; +create table tbl1 (id int unsigned not null auto_increment primary key, s int, index((md5(s)))); +insert into tbl1 (id) select null; + insert into tbl1 (id) select null from tbl1; + insert into tbl1 (id) select null from tbl1; + insert into tbl1 (id) select null from tbl1; + insert into tbl1 (id) select null from tbl1; + insert into tbl1 (id) select null from tbl1; + insert into tbl1 (id) select null from tbl1; + insert into tbl1 (id) select null from tbl1; + insert into tbl1 (id) select null from tbl1; + insert into tbl1 (id) select null from tbl1; + insert into tbl1 (id) select null from tbl1; + insert into tbl1 (id) select null from tbl1; +update tbl1 set s=id%32; +explain format = 'brief' select count(*) from tbl1 where md5(s) like '02e74f10e0327ad868d138f2b4fdd6f%'; +id estRows task access object operator info +StreamAgg 1.00 root funcs:count(Column#6)->Column#4 +└─IndexReader 1.00 root index:StreamAgg + └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#6 + └─IndexRangeScan 250.00 cop[tikv] table:tbl1, index:expression_index(md5(`s`)) range:["02e74f10e0327ad868d138f2b4fdd6f","02e74f10e0327ad868d138f2b4fdd6g"), keep order:false, stats:pseudo +select count(*) from tbl1 use index() where md5(s) like '02e74f10e0327ad868d138f2b4fdd6f%'; +count(*) +64 diff --git a/cmd/explaintest/r/partition_pruning.result b/cmd/explaintest/r/partition_pruning.result index b3c3a8b3169b9..2e7d5dfcf0989 100644 --- a/cmd/explaintest/r/partition_pruning.result +++ b/cmd/explaintest/r/partition_pruning.result @@ -206,50 +206,43 @@ a 1 explain format = 'brief' SELECT * FROM t1 WHERE a = 1; id estRows task access object operator info -TableReader 1.00 root partition:p1 data:TableRangeScan -└─TableRangeScan 1.00 cop[tikv] table:t1 range:[1,1], keep order:false, stats:pseudo +Point_Get 1.00 root table:t1, partition:p1 handle:1 SELECT * FROM t1 WHERE a = 2 order by a; a 2 explain format = 'brief' SELECT * FROM t1 WHERE a = 2; id estRows task access object operator info -TableReader 1.00 root partition:p2 data:TableRangeScan -└─TableRangeScan 1.00 cop[tikv] table:t1 range:[2,2], keep order:false, stats:pseudo +Point_Get 1.00 root table:t1, partition:p2 handle:2 SELECT * FROM t1 WHERE a = 3 order by a; a 3 explain format = 'brief' SELECT * FROM t1 WHERE a = 3; id estRows task access object operator info -TableReader 1.00 root partition:p3 data:TableRangeScan -└─TableRangeScan 1.00 cop[tikv] table:t1 range:[3,3], keep order:false, stats:pseudo +Point_Get 1.00 root table:t1, partition:p3 handle:3 SELECT * FROM t1 WHERE a = 4 order by a; a 4 explain format = 'brief' SELECT * FROM t1 WHERE a = 4; id estRows task access object operator info -TableReader 1.00 root partition:p4 data:TableRangeScan -└─TableRangeScan 1.00 cop[tikv] table:t1 range:[4,4], keep order:false, stats:pseudo +Point_Get 1.00 root table:t1, partition:p4 handle:4 SELECT * FROM t1 WHERE a = 5 order by a; a 5 explain format = 'brief' SELECT * FROM t1 WHERE a = 5; id estRows task access object operator info -TableReader 1.00 root partition:p5 data:TableRangeScan -└─TableRangeScan 1.00 cop[tikv] table:t1 range:[5,5], keep order:false, stats:pseudo +Point_Get 1.00 root table:t1, partition:p5 handle:5 SELECT * FROM t1 WHERE a = 6 order by a; a 6 explain format = 'brief' SELECT * FROM t1 WHERE a = 6; id estRows task access object operator info -TableReader 1.00 root partition:max data:TableRangeScan -└─TableRangeScan 1.00 cop[tikv] table:t1 range:[6,6], keep order:false, stats:pseudo +Point_Get 1.00 root table:t1, partition:max handle:6 SELECT * FROM t1 WHERE a = 7 order by a; a 7 explain format = 'brief' SELECT * FROM t1 WHERE a = 7; id estRows task access object operator info -TableReader 1.00 root partition:max data:TableRangeScan -└─TableRangeScan 1.00 cop[tikv] table:t1 range:[7,7], keep order:false, stats:pseudo +Point_Get 1.00 root table:t1, partition:max handle:7 SELECT * FROM t1 WHERE a >= 1 order by a; a 1 @@ -544,43 +537,37 @@ a 1 explain format = 'brief' SELECT * FROM t1 WHERE a = 1; id estRows task access object operator info -TableReader 1.00 root partition:p1 data:TableRangeScan -└─TableRangeScan 1.00 cop[tikv] table:t1 range:[1,1], keep order:false, stats:pseudo +Point_Get 1.00 root table:t1, partition:p1 handle:1 SELECT * FROM t1 WHERE a = 2; a 2 explain format = 'brief' SELECT * FROM t1 WHERE a = 2; id estRows task access object operator info -TableReader 1.00 root partition:p2 data:TableRangeScan -└─TableRangeScan 1.00 cop[tikv] table:t1 range:[2,2], keep order:false, stats:pseudo +Point_Get 1.00 root table:t1, partition:p2 handle:2 SELECT * FROM t1 WHERE a = 3; a 3 explain format = 'brief' SELECT * FROM t1 WHERE a = 3; id estRows task access object operator info -TableReader 1.00 root partition:p3 data:TableRangeScan -└─TableRangeScan 1.00 cop[tikv] table:t1 range:[3,3], keep order:false, stats:pseudo +Point_Get 1.00 root table:t1, partition:p3 handle:3 SELECT * FROM t1 WHERE a = 4; a 4 explain format = 'brief' SELECT * FROM t1 WHERE a = 4; id estRows task access object operator info -TableReader 1.00 root partition:p4 data:TableRangeScan -└─TableRangeScan 1.00 cop[tikv] table:t1 range:[4,4], keep order:false, stats:pseudo +Point_Get 1.00 root table:t1, partition:p4 handle:4 SELECT * FROM t1 WHERE a = 5; a 5 explain format = 'brief' SELECT * FROM t1 WHERE a = 5; id estRows task access object operator info -TableReader 1.00 root partition:max data:TableRangeScan -└─TableRangeScan 1.00 cop[tikv] table:t1 range:[5,5], keep order:false, stats:pseudo +Point_Get 1.00 root table:t1, partition:max handle:5 SELECT * FROM t1 WHERE a = 6; a 6 explain format = 'brief' SELECT * FROM t1 WHERE a = 6; id estRows task access object operator info -TableReader 1.00 root partition:max data:TableRangeScan -└─TableRangeScan 1.00 cop[tikv] table:t1 range:[6,6], keep order:false, stats:pseudo +Point_Get 1.00 root table:t1, partition:max handle:6 SELECT * FROM t1 WHERE a >= 1 order by a; a 1 @@ -1795,9 +1782,7 @@ TableReader 3323.33 root partition:all data:Selection └─TableFullScan 10000.00 cop[tikv] table:t7 keep order:false, stats:pseudo explain format = 'brief' select * from t7 where a = 90; id estRows task access object operator info -TableReader 10.00 root partition:dual data:Selection -└─Selection 10.00 cop[tikv] eq(test.t7.a, 90) - └─TableFullScan 10000.00 cop[tikv] table:t7 keep order:false, stats:pseudo +TableDual 0.00 root rows:0 explain format = 'brief' select * from t7 where a > 90; id estRows task access object operator info TableReader 3333.33 root partition:dual data:Selection @@ -1919,9 +1904,7 @@ TableReader 3323.33 root partition:all data:Selection └─TableFullScan 10000.00 cop[tikv] table:t7 keep order:false, stats:pseudo explain format = 'brief' select * from t7 where a = 90; id estRows task access object operator info -TableReader 10.00 root partition:dual data:Selection -└─Selection 10.00 cop[tikv] eq(test.t7.a, 90) - └─TableFullScan 10000.00 cop[tikv] table:t7 keep order:false, stats:pseudo +TableDual 0.00 root rows:0 explain format = 'brief' select * from t7 where a > 90; id estRows task access object operator info TableReader 3333.33 root partition:dual data:Selection diff --git a/cmd/explaintest/r/select.result b/cmd/explaintest/r/select.result index 0836e2461a2a3..41369bffcfbbf 100644 --- a/cmd/explaintest/r/select.result +++ b/cmd/explaintest/r/select.result @@ -468,7 +468,7 @@ PRIMARY KEY (`id`) explain format = 'brief' select row_number() over( partition by i ) - x as rnk from t; id estRows task access object operator info Projection 10000.00 root minus(Column#5, test.t.x)->Column#7 -└─Window 10000.00 root row_number()->Column#5 over(partition by test.t.i) +└─Window 10000.00 root row_number()->Column#5 over(partition by test.t.i rows between current row and current row) └─Sort 10000.00 root test.t.i └─TableReader 10000.00 root data:TableRangeScan └─TableRangeScan 10000.00 cop[tikv] table:t range:[0,+inf], keep order:false, stats:pseudo diff --git a/cmd/explaintest/r/subquery.result b/cmd/explaintest/r/subquery.result index 0b054d45ba2a3..84bac87bb1d23 100644 --- a/cmd/explaintest/r/subquery.result +++ b/cmd/explaintest/r/subquery.result @@ -22,9 +22,9 @@ Projection 5.00 root Column#22 ├─TableReader(Build) 5.00 root data:TableFullScan │ └─TableFullScan 5.00 cop[tikv] table:t keep order:false └─StreamAgg(Probe) 1.00 root funcs:count(1)->Column#21 - └─IndexJoin 0.50 root inner join, inner:TableReader, outer key:test.t.a, inner key:test.t.a, equal cond:eq(test.t.a, test.t.a) - ├─IndexReader(Build) 1.00 root index:IndexRangeScan - │ └─IndexRangeScan 1.00 cop[tikv] table:s, index:idx(b, c, d) range: decided by [eq(test.t.b, 1) eq(test.t.c, 1) eq(test.t.d, test.t.a)], keep order:false + └─IndexJoin 0.22 root inner join, inner:TableReader, outer key:test.t.a, inner key:test.t.a, equal cond:eq(test.t.a, test.t.a) + ├─IndexReader(Build) 0.45 root index:IndexRangeScan + │ └─IndexRangeScan 0.45 cop[tikv] table:s, index:idx(b, c, d) range: decided by [eq(test.t.b, 1) eq(test.t.c, 1) eq(test.t.d, test.t.a)], keep order:false └─TableReader(Probe) 1.00 root data:TableRangeScan └─TableRangeScan 1.00 cop[tikv] table:t1 range: decided by [test.t.a], keep order:false drop table if exists t; @@ -41,3 +41,8 @@ HashJoin 7992.00 root inner join, equal:[eq(test.t.b, test.t.b) eq(test.t.c, te └─TableReader(Probe) 9970.03 root data:Selection └─Selection 9970.03 cop[tikv] not(isnull(test.t.a)), not(isnull(test.t.b)), not(isnull(test.t.c)) └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +drop table if exists t1, t2; +create table t1(a int(11)); +create table t2(a decimal(40,20) unsigned, b decimal(40,20)); +select count(*) as x from t1 group by a having x not in (select a from t2 where x = t2.b); +x diff --git a/cmd/explaintest/t/cte.test b/cmd/explaintest/t/cte.test new file mode 100644 index 0000000000000..b45695e641135 --- /dev/null +++ b/cmd/explaintest/t/cte.test @@ -0,0 +1,170 @@ +use test; +# case 1 +drop table if exists tbl_0; +create table tbl_0(a int); +with recursive cte_0 (col_10,col_11,col_12) AS ( select 1, 2,3 from tbl_0 UNION select col_10 + 1,col_11 + 1,col_12 + 1 from cte_0 where col_10 < 10 ) select * from cte_0; +# case 2 +drop table if exists tbl_1; +CREATE TABLE `tbl_1` ( + `col_5` decimal(47,21) NOT NULL DEFAULT '5308.880000000000000000000', + `col_6` enum('Alice','Bob','Charlie','David') DEFAULT NULL, + `col_7` float NOT NULL, + `col_8` bigint NOT NULL DEFAULT '-688199144806783096', + `col_9` varchar(248) NOT NULL, + PRIMARY KEY (`col_5`,`col_7`,`col_9`,`col_8`), + UNIQUE KEY `idx_4` (`col_8`), + UNIQUE KEY `idx_7` (`col_5`,`col_7`,`col_8`), + UNIQUE KEY `idx_9` (`col_9`,`col_8`), + UNIQUE KEY `idx_3` (`col_9`(3),`col_8`), + UNIQUE KEY `idx_8` (`col_7`,`col_6`,`col_8`,`col_5`), + KEY `idx_5` (`col_7`), + KEY `idx_6` (`col_7`), + KEY `idx_10` (`col_9`,`col_5`), + KEY `idx_11` (`col_5`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +/*!50100 PARTITION BY HASH (`col_8`) +PARTITIONS 4 */; +with recursive cte_1 (col_13,col_14,col_15,col_16,col_17) AS ( with recursive cte_2 (col_18,col_19,col_20,col_21,col_22,col_23,col_24) AS ( select 1, 2,col_8,4,5,6,7 from tbl_1 ) select col_19,col_18,col_22,col_23,col_21 from cte_2 UNION ALL select col_13 + 1,col_14 + 1,col_15 + 1,col_16 + 1,col_17 + 1 from cte_1 where col_13 < 10 ) select * from cte_1; +# case 3 +with recursive cte_256 (col_969,col_970,col_971) AS ( with recursive cte_257 (col_972,col_973,col_974,col_975) AS ( select 1, 2,col_8,4 from tbl_1 UNION select col_972 + 1,col_973 + 1,col_974 + 1,col_975 + 1 from cte_257 where col_972 < 10 ) select col_975,col_974,col_973 from cte_257 UNION DISTINCT select col_969 + 1,col_970 + 1,col_971 + 1 from cte_256 where col_969 < 10 ) select * from cte_256; +# case 4 +drop table if exists tbl_2, tbl_3; +create table tbl_2 ( col_4 char(246) collate utf8_unicode_ci not null , col_5 char(253) collate utf8mb4_unicode_ci ) ; +create table tbl_3 ( col_6 char(207) collate utf8mb4_unicode_ci , col_7 int unsigned not null ) ; +insert into tbl_2 values ( "0",null ) ; +insert into tbl_2 values ( "1","0" ) ; +insert into tbl_2 values ( "1","1" ) ; +insert into tbl_2 values ( "0","0" ) ; +insert into tbl_2 values ( "0","1" ) ; +insert into tbl_3 values ( "1",0 ) ; +insert into tbl_3 values ( "1",1 ) ; +insert into tbl_3 values ( "0",0 ) ; +insert into tbl_3 values ( "0",1 ) ; +with recursive tbl_2 (col_64,col_65,col_66,col_67) AS ( select 1, col_6,col_6,4 from tbl_3 UNION DISTINCT select col_64 + 1,concat(col_65, 1),col_66 + 1,concat(col_67, 1) from tbl_2 where col_64 < 5 ) select * from tbl_2 order by col_64; +# case 5 +drop table if exists tbl_3, tbl_4; +create table tbl_3 ( col_6 int not null , col_7 char(95) collate utf8_general_ci ) ; +create table tbl_4 ( col_8 char collate utf8_unicode_ci , col_9 char collate utf8mb4_bin ) ; +insert into tbl_3 values ( 0,"1" ) ; +insert into tbl_4 values ( "1","0" ) ; +with recursive cte_2245 (col_8692,col_8693) AS ( select 1, col_7 from tbl_3 UNION select col_8692 + 1,concat(col_8693, 1) from cte_2245 where col_8692 < 5 ) , cte_2246 (col_8694,col_8695,col_8696,col_8697) AS ( with recursive cte_2247 (col_8698,col_8699,col_8700,col_8701) AS ( select 1, cast("2" as char(20)),3,col_8 from tbl_4 ) select col_8698,col_8699,col_8700,col_8701 from cte_2247 UNION select col_8694 + 1,col_8695 + 1,col_8696 + 1,col_8697 + 1 from cte_2246 where col_8694 < 5 ) select * from cte_2245,cte_2246 order by col_8692,col_8693,col_8696,col_8695,col_8697,col_8694; +# case 6 +with recursive cte2 as (select 1 as col_1, 2 as col_2) select c1.col_1, c2.col_2 from cte2 as c1, cte2 as c2 where c2.col_2 = 1; +# case 7 +with recursive cte (c1) as (select 1), cte1 (c2) as (select 1 union select c1 + 1 from cte, cte1) select * from cte, cte1; +# case 8 +--error 1054 +with recursive tbl_0 (col_943,col_944,col_945,col_946,col_947) AS ( with recursive tbl_0 (col_948,col_949,col_950,col_951,col_952) AS ( select 1, 2,3,4,5 UNION ALL select col_948 + 1,col_949 + 1,col_950 + 1,col_951 + 1,col_952 + 1 from tbl_0 where col_948 < 5 ) select col_948,col_949,col_951,col_950,col_952 from tbl_0 UNION ALL select col_943 + 1,col_944 + 1,col_945 + 1,col_946 + 1,col_947 + 1 from tbl_0 where col_943 < 5 ) select * from tbl_0; +# case 9 +with recursive cte1 (c1, c2) as (select 1, '1' union select concat(c1, 1), c2 + 1 from cte1 where c1 < 100) select * from cte1; +# case 10 +with recursive cte_8 (col_51,col_52,col_53,col_54) AS ( with recursive cte_9 (col_55,col_56,col_57,col_58) AS ( select 1, 2,3,4 UNION ALL select col_55 + 1,col_56 + 1,col_57 + 1,col_58 + 1 from cte_9 where col_55 < 5 ) select col_55,col_57,col_56,col_58 from cte_9 UNION DISTINCT select col_51 + 1,col_52 + 1,col_53 + 1,col_54 + 1 from cte_8 where col_51 < 5 ) select * from cte_8; +# case 11 +with recursive qn as (select 1 from dual union all select 1 from dual) select * from qn; +# case 12 +with recursive qn as (select 1 as a from dual group by a union all select a+1 from qn where a<3) select * from qn; +# case 13 +with recursive qn as ((select 1 as a from dual order by a) union all select a+1 from qn where a<3) select * from qn; +# case 14 +drop table if exists employees; +CREATE TABLE employees ( +ID INT PRIMARY KEY, +NAME VARCHAR(100), +MANAGER_ID INT, +INDEX (MANAGER_ID), +FOREIGN KEY (MANAGER_ID) REFERENCES employees(ID) +); +INSERT INTO employees VALUES +(333, "Yasmina", NULL), +(198, "John", 333), +(692, "Tarek", 333), +(29, "Pedro", 198), +(4610, "Sarah", 29), +(72, "Pierre", 29), +(123, "Adil", 692); +WITH RECURSIVE employees_extended AS (SELECT ID, NAME, MANAGER_ID, CAST(ID AS CHAR(200)) AS PATH FROM employees WHERE NAME='Pierre' UNION ALL SELECT S.ID, S.NAME, S.MANAGER_ID, CONCAT(M.PATH, ",", S.ID) FROM employees_extended M JOIN employees S ON M.MANAGER_ID=S.ID) SELECT * FROM employees_extended; +# case 15 +with recursive cte (c1) as (select 1), cte1 (c2) as (select 1 union select c1 + 1 from cte where c1 < 10) select * from cte where c1 < 5; +# case 16 +with recursive cte_581 (col_2343,col_2344,col_2345) AS ( select 1, '2',cast('3' as char(20))) , cte_582 (col_2346,col_2347,col_2348) AS ( select 1, 2, 3) select * from cte_581 as cte_as_583,cte_582 as cte_as_584,cte_582 as cte_as_585 order by cte_as_583.col_2343,cte_as_585.col_2348,cte_as_584.col_2346,cte_as_584.col_2348,cte_as_583.col_2344,cte_as_584.col_2347,cte_as_585.col_2346,cte_as_585.col_2347,cte_as_583.col_2345; +# case 17 +with recursive tbl_3 (col_19,col_20,col_21,col_22) AS ( select 1, 2,3,4 UNION select col_19 + 1,col_20 + 1,col_21 + 1,concat(col_22, 1) from tbl_3 where col_19 < 5 ) , cte_4 (col_23,col_24,col_25,col_26) AS ( select 1, 2,cast("3" as char(20)),4 UNION DISTINCT select col_23 + 1,col_24 + 1,concat(col_25, 1),col_26 + 1 from cte_4 where col_23 < 5 ) select * from tbl_3 as cte_as_3,cte_4 as cte_as_4,tbl_3 as cte_as_5 order by cte_as_3.col_19,cte_as_4.col_23,cte_as_4.col_25,cte_as_4.col_24,cte_as_4.col_26,cte_as_3.col_20,cte_as_5.col_22,cte_as_3.col_21,cte_as_5.col_20,cte_as_3.col_22,cte_as_5.col_19,cte_as_5.col_21; +# case 18 +with cte1 (c1) as (select 1) select * from cte1 as b, cte1 as a; +# case 19 +--error 1222 +WITH RECURSIVE qn AS +( +select 1 +union all +select 3, 0 from qn +) +select * from qn; +# case 20 +--error 1235 +with recursive cte1 as (select 1 union all (select 1 from cte1 limit 10)) select * from cte1; +# case 21 +# TODO: uncomment this case after we support limit +# with recursive cte1 as (select 1 union all select 1 from cte1 limit 10) select * from cte1; +# case 22 +with recursive qn as (select 123 as a union all select null from qn where a is not null) select * from qn; +# case 23 +--error 1353 +with recursive q (b) as (select 1, 1 union all select 1, 1 from q) select b from q; +# case 24 +drop table if exists t1; +create table t1(a int); +insert into t1 values(1); +insert into t1 values(2); +SELECT * +FROM +t1 dt +WHERE +EXISTS( + WITH RECURSIVE qn AS (SELECT a*0 AS b UNION ALL SELECT b+1 FROM qn WHERE b=0) + SELECT * FROM qn WHERE b=a + ); +# case 25 +drop table if exists t1; +create table t1 (a int); +insert into t1 values (1); +SELECT (WITH qn AS (SELECT 10*a as a FROM t1), + qn2 AS (SELECT 3*a AS b FROM qn) SELECT * from qn2 LIMIT 1) +FROM t1; +# case 26 +select (with qn as (select "with") select * from qn) as scal_subq +from dual; +# case 27 +drop table if exists t1; +create table t1 (a int); insert into t1 values(1), (2), (3); +with q as (select * from t1) +select /*+ merge(q) no_merge(q1) */ * from q, q q1 where q.a=1 and q1.a=2; +# case 28 +drop table if exists t1; create table t1 (a int, b int); +with qn as (select a, b from t1) select b from qn group by a; +# case 29 +drop table if exists t1; +create table t1(a int); +insert into t1 values(1); +insert into t1 values(2); +SELECT * +FROM +t1 dt +WHERE +EXISTS( + WITH RECURSIVE qn AS (SELECT a*0+1 AS b UNION ALL SELECT b+1 FROM qn WHERE b=0) + SELECT * FROM qn WHERE b=1 + ); +# case 30 +drop table if exists tbl_1; +CREATE TABLE `tbl_1` ( + `col_2` char(65) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, + `col_3` int(11) NOT NULL +); +with recursive cte_8932 (col_34891,col_34892) AS ( with recursive cte_8932 (col_34893,col_34894,col_34895) AS ( with tbl_1 (col_34896,col_34897,col_34898,col_34899) AS ( select 1, "2",3,col_3 from tbl_1 ) select cte_as_8958.col_34896,cte_as_8958.col_34898,cte_as_8958.col_34899 from tbl_1 as cte_as_8958 UNION DISTINCT select col_34893 + 1,concat(col_34894, 1),col_34895 + 1 from cte_8932 where col_34893 < 5 ) select cte_as_8959.col_34893,cte_as_8959.col_34895 from cte_8932 as cte_as_8959 ) select * from cte_8932 as cte_as_8960 order by cte_as_8960.col_34891,cte_as_8960.col_34892; +# case 31 +drop table if exists t1; +create table t1(c1 bigint unsigned); +insert into t1 values(0); +--error 1690 +with recursive cte1 as (select c1 - 1 c1 from t1 union all select c1 - 1 c1 from cte1 where c1 != 0) select * from cte1 dt1, cte1 dt2; diff --git a/cmd/explaintest/t/explain_cte.test b/cmd/explaintest/t/explain_cte.test new file mode 100644 index 0000000000000..c657ad5c68898 --- /dev/null +++ b/cmd/explaintest/t/explain_cte.test @@ -0,0 +1,44 @@ +use test; +drop table if exists t1, t2; +create table t1 (c1 int primary key, c2 int, index c2 (c2)); +create table t2 (c1 int unique, c2 int); +insert into t1 values(1, 0), (2, 1); +insert into t2 values(1, 0), (2, 1); + +# simple cte +explain with cte(a) as (select 1) select * from cte; +explain with cte(a) as (select c1 from t1) select * from cte; +explain with cte(a,b,c,d) as (select * from t1, t2) select * from cte; + +# recursive cte +explain with recursive cte(a) as (select 1 union select a+1 from cte where a < 10) select * from cte; +explain with recursive cte(a) as (select c2 from t1 union select a+1 from cte where a < 10) select * from cte; + +# nested cte +explain with cte(a) as (with recursive cte1(a) as (select 1 union select a + 1 from cte1 where a < 10) select * from cte1) select * from cte; + +# cte with join +explain with recursive cte(a) as (select 1 union select a+1 from cte where a < 10) select * from cte t1, cte t2; +explain with cte(a) as (with recursive cte1(a) as (select 1 union select a + 1 from cte1 where a < 10) select * from cte1) select * from cte t1, cte t2; + +# multiple cte +explain with recursive cte1(a) as (select 1 union select a+1 from cte1 where a < 10), cte2(a) as (select c2 from t1 union select a+1 from cte2 where a < 10) select * from cte1, cte2; + +# other +explain with q(a,b) as (select * from t1) select /*+ merge(q) no_merge(q1) */ * from q, q q1 where q.a=1 and q1.a=2; +# explain with cte(a,b) as (select * from t1) select (select 1 from cte limit 1) from cte; +explain with recursive cte(a,b) as (select 1, concat('a', 1) union select a+1, concat(b, 1) from cte where a < 5) select * from cte; +explain select * from t1 dt where exists(with recursive qn as (select c1*0+1 as b union all select b+1 from qn where b=0) select * from qn where b=1); + +# recursive limit +explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 1) select * from cte1; +explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 100 offset 100) select * from cte1; +explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 0 offset 0) select * from cte1; + +explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 1) select * from cte1 dt1 join cte1 dt2 on dt1.c1 = dt2.c1; +explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 0 offset 0) select * from cte1 dt1 join cte1 dt2 on dt1.c1 = dt2.c1; + +# non-recursive limit +explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 1) select * from cte1; +explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 100 offset 100) select * from cte1; +explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 0 offset 0) select * from cte1; diff --git a/cmd/explaintest/t/explain_generate_column_substitute.test b/cmd/explaintest/t/explain_generate_column_substitute.test index 78096628c293d..395ed2311fda6 100644 --- a/cmd/explaintest/t/explain_generate_column_substitute.test +++ b/cmd/explaintest/t/explain_generate_column_substitute.test @@ -174,3 +174,11 @@ explain format = 'brief' select c0 from t0; -- TableRead drop table if exists t0; create table t0(c0 double, c1 float as (c0) unique); explain format = 'brief' select c0 from t0; -- TableRead + +drop table if exists tbl1; +create table tbl1 (id int unsigned not null auto_increment primary key, s int, index((md5(s)))); +insert into tbl1 (id) select null; insert into tbl1 (id) select null from tbl1; insert into tbl1 (id) select null from tbl1; insert into tbl1 (id) select null from tbl1; insert into tbl1 (id) select null from tbl1; insert into tbl1 (id) select null from tbl1; insert into tbl1 (id) select null from tbl1; insert into tbl1 (id) select null from tbl1; insert into tbl1 (id) select null from tbl1; insert into tbl1 (id) select null from tbl1; insert into tbl1 (id) select null from tbl1; insert into tbl1 (id) select null from tbl1; +update tbl1 set s=id%32; +explain format = 'brief' select count(*) from tbl1 where md5(s) like '02e74f10e0327ad868d138f2b4fdd6f%'; +select count(*) from tbl1 use index() where md5(s) like '02e74f10e0327ad868d138f2b4fdd6f%'; + diff --git a/cmd/explaintest/t/subquery.test b/cmd/explaintest/t/subquery.test index 8d9bd730b767c..6a3aa13e7e95a 100644 --- a/cmd/explaintest/t/subquery.test +++ b/cmd/explaintest/t/subquery.test @@ -15,3 +15,8 @@ explain format = 'brief' select t.c in (select count(*) from t s use index(idx), drop table if exists t; create table t(a int, b int, c int); explain format = 'brief' select a from t t1 where t1.a = (select max(t2.a) from t t2 where t1.b=t2.b and t1.c=t2.b); + +drop table if exists t1, t2; +create table t1(a int(11)); +create table t2(a decimal(40,20) unsigned, b decimal(40,20)); +select count(*) as x from t1 group by a having x not in (select a from t2 where x = t2.b); diff --git a/cmd/importcheck/importcheck.go b/cmd/importcheck/importcheck.go index e668705230e35..ee4042e052760 100644 --- a/cmd/importcheck/importcheck.go +++ b/cmd/importcheck/importcheck.go @@ -20,7 +20,6 @@ import ( "go/ast" "go/parser" "go/token" - "io/ioutil" "os" "path/filepath" "strings" @@ -60,7 +59,7 @@ func run() error { } func checkFile(path string) error { - src, err := ioutil.ReadFile(path) + src, err := os.ReadFile(path) if err != nil { return err } diff --git a/cmd/importer/stats.go b/cmd/importer/stats.go index 2f258462b93dc..08c52b61a775a 100644 --- a/cmd/importer/stats.go +++ b/cmd/importer/stats.go @@ -15,8 +15,8 @@ package main import ( "encoding/json" - "io/ioutil" "math/rand" + "os" "time" "github.com/pingcap/errors" @@ -29,7 +29,7 @@ import ( ) func loadStats(tblInfo *model.TableInfo, path string) (*stats.Table, error) { - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { return nil, errors.Trace(err) } diff --git a/config/config.go b/config/config.go index 7d6560b5d6783..3b4bda54436bd 100644 --- a/config/config.go +++ b/config/config.go @@ -60,9 +60,9 @@ const ( DefHost = "0.0.0.0" // DefStatusHost is the default status host of TiDB DefStatusHost = "0.0.0.0" - // Def TableColumnCountLimit is limit of the number of columns in a table + // DefTableColumnCountLimit is limit of the number of columns in a table DefTableColumnCountLimit = 1017 - // Def TableColumnCountLimit is maximum limitation of the number of columns in a table + // DefMaxOfTableColumnCountLimit is maximum limitation of the number of columns in a table DefMaxOfTableColumnCountLimit = 4096 ) @@ -73,7 +73,7 @@ var ( "tikv": true, "unistore": true, } - // checkTableBeforeDrop enable to execute `admin check table` before `drop table`. + // CheckTableBeforeDrop enable to execute `admin check table` before `drop table`. CheckTableBeforeDrop = false // checkBeforeDropLDFlag is a go build flag. checkBeforeDropLDFlag = "None" @@ -198,7 +198,6 @@ func (c *Config) getTiKVConfig() *tikvcfg.Config { return &tikvcfg.Config{ CommitterConcurrency: c.Performance.CommitterConcurrency, MaxTxnTTL: c.Performance.MaxTxnTTL, - ServerMemoryQuota: defTiKVCfg.ServerMemoryQuota, TiKVClient: c.TiKVClient, Security: c.Security.ClusterSecurity(), PDClient: c.PDClient, @@ -298,7 +297,7 @@ func (b *nullableBool) UnmarshalJSON(data []byte) error { type Log struct { // Log level. Level string `toml:"level" json:"level"` - // Log format. one of json, text, or console. + // Log format, one of json or text. Format string `toml:"format" json:"format"` // Disable automatic timestamps in output. Deprecated: use EnableTimestamp instead. DisableTimestamp nullableBool `toml:"disable-timestamp" json:"disable-timestamp"` @@ -382,7 +381,6 @@ func (e *ErrConfigValidationFailed) Error() string { "TiDB manual to make sure this option has not been deprecated and removed from your TiDB "+ "version if the option does not appear to be a typo", e.confFile, strings.Join( e.UndecodedItems, ", ")) - } // ClusterSecurity returns Security info for cluster @@ -495,13 +493,15 @@ type Binlog struct { // PessimisticTxn is the config for pessimistic transaction. type PessimisticTxn struct { // The max count of retry for a single statement in a pessimistic transaction. - MaxRetryCount uint `toml:"max-retry-count" json:"max-retry-count"` + MaxRetryCount uint `toml:"max-retry-count" json:"max-retry-count"` + DeadlockHistoryCapacity uint `toml:"deadlock-history-capacity" json:"deadlock-history-capacity"` } // DefaultPessimisticTxn returns the default configuration for PessimisticTxn func DefaultPessimisticTxn() PessimisticTxn { return PessimisticTxn{ - MaxRetryCount: 256, + MaxRetryCount: 256, + DeadlockHistoryCapacity: 10, } } diff --git a/config/config.toml.example b/config/config.toml.example index 6db0b5c517277..1cc2674d01727 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -136,7 +136,7 @@ enable-enum-length-limit = true # Log level: debug, info, warn, error, fatal. level = "info" -# Log format, one of json, text, console. +# Log format, one of json or text. format = "text" # Enable automatic timestamps in log output, if not set, it will be defaulted to true. diff --git a/config/config_util.go b/config/config_util.go index 0bf3374e001d7..35df9421097a0 100644 --- a/config/config_util.go +++ b/config/config_util.go @@ -17,7 +17,6 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" "os" "path/filepath" "reflect" @@ -110,7 +109,7 @@ func atomicWriteConfig(c *Config, confPath string) (err error) { return err } tmpConfPath := filepath.Join(os.TempDir(), fmt.Sprintf("tmp_conf_%v.toml", time.Now().Format("20060102150405"))) - if err := ioutil.WriteFile(tmpConfPath, []byte(content), 0666); err != nil { + if err := os.WriteFile(tmpConfPath, []byte(content), 0666); err != nil { return errors.Trace(err) } return errors.Trace(os.Rename(tmpConfPath, confPath)) diff --git a/config/config_util_test.go b/config/config_util_test.go index 7972fcf706000..e83c1e2046072 100644 --- a/config/config_util_test.go +++ b/config/config_util_test.go @@ -16,7 +16,6 @@ package config import ( "encoding/json" "fmt" - "io/ioutil" "os" "path/filepath" "reflect" @@ -100,7 +99,7 @@ func (s *testConfigSuite) TestAtomicWriteConfig(c *C) { conf.Performance.PseudoEstimateRatio = 3.45 c.Assert(atomicWriteConfig(conf, confPath), IsNil) - content, err := ioutil.ReadFile(confPath) + content, err := os.ReadFile(confPath) c.Assert(err, IsNil) dconf, err := decodeConfig(string(content)) c.Assert(err, IsNil) @@ -113,7 +112,7 @@ func (s *testConfigSuite) TestAtomicWriteConfig(c *C) { conf.Performance.PseudoEstimateRatio = 54.3 c.Assert(atomicWriteConfig(conf, confPath), IsNil) - content, err = ioutil.ReadFile(confPath) + content, err = os.ReadFile(confPath) c.Assert(err, IsNil) dconf, err = decodeConfig(string(content)) c.Assert(err, IsNil) diff --git a/ddl/backfilling.go b/ddl/backfilling.go index 56512eec6ab65..8a5b7a837f552 100644 --- a/ddl/backfilling.go +++ b/ddl/backfilling.go @@ -20,7 +20,6 @@ import ( "strconv" "sync/atomic" "time" - "unsafe" "github.com/pingcap/errors" "github.com/pingcap/failpoint" @@ -32,8 +31,9 @@ import ( "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" + "github.com/pingcap/tidb/store/copr" + "github.com/pingcap/tidb/store/driver/backoff" "github.com/pingcap/tidb/store/tikv" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util" @@ -331,9 +331,9 @@ func splitTableRanges(t table.PhysicalTable, store kv.Storage, startKey, endKey } maxSleep := 10000 // ms - bo := tikv.NewBackofferWithVars(context.Background(), maxSleep, nil) - tikvRange := *(*tikvstore.KeyRange)(unsafe.Pointer(&kvRange)) - ranges, err := tikv.SplitRegionRanges(bo, s.GetRegionCache(), []tikvstore.KeyRange{tikvRange}) + bo := backoff.NewBackofferWithVars(context.Background(), maxSleep, nil) + rc := copr.NewRegionCache(s.GetRegionCache()) + ranges, err := rc.SplitRegionRanges(bo, []kv.KeyRange{kvRange}) if err != nil { return nil, errors.Trace(err) } @@ -341,8 +341,7 @@ func splitTableRanges(t table.PhysicalTable, store kv.Storage, startKey, endKey errMsg := fmt.Sprintf("cannot find region in range [%s, %s]", startKey.String(), endKey.String()) return nil, errors.Trace(errInvalidSplitRegionRanges.GenWithStackByArgs(errMsg)) } - res := *(*[]kv.KeyRange)(unsafe.Pointer(&ranges)) - return res, nil + return ranges, nil } func (w *worker) waitTaskResults(workers []*backfillWorker, taskCnt int, @@ -677,7 +676,7 @@ func iterateSnapshotRows(store kv.Storage, priority int, t table.Table, version ver := kv.Version{Ver: version} snap := store.GetSnapshot(ver) - snap.SetOption(tikvstore.Priority, priority) + snap.SetOption(kv.Priority, priority) it, err := snap.Iter(firstKey, upperBound) if err != nil { diff --git a/ddl/column.go b/ddl/column.go index 18c23b4d9c45a..ff3fb5c0a39ce 100644 --- a/ddl/column.go +++ b/ddl/column.go @@ -37,12 +37,11 @@ import ( "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/stmtctx" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" - "github.com/pingcap/tidb/types/json" "github.com/pingcap/tidb/util" + "github.com/pingcap/tidb/util/collate" "github.com/pingcap/tidb/util/logutil" decoder "github.com/pingcap/tidb/util/rowDecoder" "github.com/pingcap/tidb/util/sqlexec" @@ -714,6 +713,10 @@ func needChangeColumnData(oldCol, newCol *model.ColumnInfo) bool { return needTruncationOrToggleSign() } + if convertBetweenCharAndVarchar(oldCol.Tp, newCol.Tp) { + return true + } + // Deal with the different type. switch oldCol.Tp { case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob: @@ -736,6 +739,15 @@ func needChangeColumnData(oldCol, newCol *model.ColumnInfo) bool { return true } +// Column type conversion between varchar to char need reorganization because +// 1. varchar -> char: char type is stored with the padding removed. All the indexes need to be rewritten. +// 2. char -> varchar: the index value encoding of secondary index on clustered primary key tables is different. +// These secondary indexes need to be rewritten. +func convertBetweenCharAndVarchar(oldCol, newCol byte) bool { + return (types.IsTypeVarchar(oldCol) && newCol == mysql.TypeString) || + (oldCol == mysql.TypeString && types.IsTypeVarchar(newCol) && collate.NewCollationEnabled()) +} + func isElemsChangedToModifyColumn(oldElems, newElems []string) bool { if len(newElems) < len(oldElems) { return true @@ -838,7 +850,7 @@ func (w *worker) onModifyColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in newColName := model.NewCIStr(genChangingColumnUniqueName(tblInfo, oldCol)) if mysql.HasPriKeyFlag(oldCol.Flag) { job.State = model.JobStateCancelled - msg := "tidb_enable_change_column_type is true and this column has primary key flag" + msg := "this column has primary key flag" return ver, errUnsupportedModifyColumn.GenWithStackByArgs(msg) } @@ -849,12 +861,15 @@ func (w *worker) onModifyColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in // Since column type change is implemented as adding a new column then substituting the old one. // Case exists when update-where statement fetch a NULL for not-null column without any default data, // it will errors. - // So we set zero original default value here to prevent this error. besides, in insert & update records, + // So we set zero original default value here to prevent this error. Besides, in insert & update records, // we have already implement using the casted value of relative column to insert rather than the origin // default value. - originDefVal, err := generateOriginDefaultValue(jobParam.newCol) - if err != nil { - return ver, errors.Trace(err) + originDefVal := oldCol.GetOriginDefaultValue() + if originDefVal == nil { + originDefVal, err = generateOriginDefaultValue(jobParam.newCol) + if err != nil { + return ver, errors.Trace(err) + } } if err = jobParam.changingCol.SetOriginDefaultValue(originDefVal); err != nil { return ver, errors.Trace(err) @@ -872,8 +887,13 @@ func (w *worker) onModifyColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in newIdxInfo := idxInfo.Clone() newIdxInfo.Name = model.NewCIStr(genChangingIndexUniqueName(tblInfo, idxInfo)) newIdxInfo.ID = allocateIndexID(tblInfo) - newIdxInfo.Columns[offsets[i]].Name = newColName - newIdxInfo.Columns[offsets[i]].Offset = jobParam.changingCol.Offset + newIdxChangingCol := newIdxInfo.Columns[offsets[i]] + newIdxChangingCol.Name = newColName + newIdxChangingCol.Offset = jobParam.changingCol.Offset + canPrefix := types.IsTypePrefixable(jobParam.changingCol.Tp) + if !canPrefix || (canPrefix && jobParam.changingCol.Flen < newIdxChangingCol.Length) { + newIdxChangingCol.Length = types.UnspecifiedLength + } jobParam.changingIdxs = append(jobParam.changingIdxs, newIdxInfo) } tblInfo.Indices = append(tblInfo.Indices, jobParam.changingIdxs...) @@ -1024,17 +1044,17 @@ func (w *worker) doModifyColumnTypeWithData( // If timeout, we should return, check for the owner and re-wait job done. return ver, nil } - if needRollbackData(err) { - if err1 := t.RemoveDDLReorgHandle(job, reorgInfo.elements); err1 != nil { - logutil.BgLogger().Warn("[ddl] run modify column job failed, RemoveDDLReorgHandle failed, can't convert job to rollback", - zap.String("job", job.String()), zap.Error(err1)) - return ver, errors.Trace(err) - } - logutil.BgLogger().Warn("[ddl] run modify column job failed, convert job to rollback", zap.String("job", job.String()), zap.Error(err)) - // When encounter these error above, we change the job to rolling back job directly. - job.State = model.JobStateRollingback + if kv.IsTxnRetryableError(err) { + // Clean up the channel of notifyCancelReorgJob. Make sure it can't affect other jobs. + w.reorgCtx.cleanNotifyReorgCancel() return ver, errors.Trace(err) } + if err1 := t.RemoveDDLReorgHandle(job, reorgInfo.elements); err1 != nil { + logutil.BgLogger().Warn("[ddl] run modify column job failed, RemoveDDLReorgHandle failed, can't convert job to rollback", + zap.String("job", job.String()), zap.Error(err1)) + } + logutil.BgLogger().Warn("[ddl] run modify column job failed, convert job to rollback", zap.String("job", job.String()), zap.Error(err)) + job.State = model.JobStateRollingback // Clean up the channel of notifyCancelReorgJob. Make sure it can't affect other jobs. w.reorgCtx.cleanNotifyReorgCancel() return ver, errors.Trace(err) @@ -1059,6 +1079,13 @@ func (w *worker) doModifyColumnTypeWithData( changingColumnUniqueName := changingCol.Name changingCol.Name = colName changingCol.ChangeStateInfo = nil + // After changing the column, the column's type is change, so it needs to set OriginDefaultValue back + // so that there is no error in getting the default value from OriginDefaultValue. + // Besides, nil data that was not backfilled in the "add column" is backfilled after the column is changed. + // So it can set OriginDefaultValue to nil. + if err = changingCol.SetOriginDefaultValue(nil); err != nil { + return ver, errors.Trace(err) + } tblInfo.Indices = tblInfo.Indices[:len(tblInfo.Indices)-len(changingIdxs)] // Adjust table column offset. if err = adjustColumnInfoInModifyColumn(job, tblInfo, changingCol, oldCol, pos, changingColumnUniqueName.L); err != nil { @@ -1083,14 +1110,6 @@ func (w *worker) doModifyColumnTypeWithData( return ver, errors.Trace(err) } -// needRollbackData indicates whether it needs to rollback data when specific error occurs. -func needRollbackData(err error) bool { - return kv.ErrKeyExists.Equal(err) || errCancelledDDLJob.Equal(err) || errCantDecodeRecord.Equal(err) || - types.ErrOverflow.Equal(err) || types.ErrDataTooLong.Equal(err) || types.ErrTruncated.Equal(err) || - json.ErrInvalidJSONText.Equal(err) || types.ErrBadNumber.Equal(err) || types.ErrInvalidYear.Equal(err) || - types.ErrWrongValue.Equal(err) -} - // BuildElements is exported for testing. func BuildElements(changingCol *model.ColumnInfo, changingIdxs []*model.IndexInfo) []*meta.Element { elements := make([]*meta.Element, 0, len(changingIdxs)+1) @@ -1106,8 +1125,24 @@ func (w *worker) updatePhysicalTableRow(t table.PhysicalTable, oldColInfo, colIn return w.writePhysicalTableRecord(t.(table.PhysicalTable), typeUpdateColumnWorker, nil, oldColInfo, colInfo, reorgInfo) } +// TestReorgGoroutineRunning is only used in test to indicate the reorg goroutine has been started. +var TestReorgGoroutineRunning = make(chan interface{}) + // updateColumnAndIndexes handles the modify column reorganization state for a table. func (w *worker) updateColumnAndIndexes(t table.Table, oldCol, col *model.ColumnInfo, idxes []*model.IndexInfo, reorgInfo *reorgInfo) error { + failpoint.Inject("mockInfiniteReorgLogic", func(val failpoint.Value) { + if val.(bool) { + a := new(interface{}) + TestReorgGoroutineRunning <- a + for { + time.Sleep(30 * time.Millisecond) + if w.reorgCtx.isReorgCanceled() { + // Job is cancelled. So it can't be done. + failpoint.Return(errCancelledDDLJob) + } + } + } + }) // TODO: Support partition tables. if bytes.Equal(reorgInfo.currElement.TypeKey, meta.ColumnElementKey) { err := w.updatePhysicalTableRow(t.(table.PhysicalTable), oldCol, col, reorgInfo) @@ -1346,7 +1381,7 @@ func (w *updateColumnWorker) BackfillDataInTxn(handleRange reorgBackfillTask) (t errInTxn = kv.RunInNewTxn(context.Background(), w.sessCtx.GetStore(), true, func(ctx context.Context, txn kv.Transaction) error { taskCtx.addedCount = 0 taskCtx.scanCount = 0 - txn.SetOption(tikvstore.Priority, w.priority) + txn.SetOption(kv.Priority, w.priority) rowRecords, nextKey, taskDone, err := w.fetchRowColVals(txn, handleRange) if err != nil { diff --git a/ddl/column_change_test.go b/ddl/column_change_test.go index 94e8787a2bdc4..6bd5a94f7235e 100644 --- a/ddl/column_change_test.go +++ b/ddl/column_change_test.go @@ -47,15 +47,18 @@ type testColumnChangeSuite struct { func (s *testColumnChangeSuite) SetUpSuite(c *C) { SetWaitTimeWhenErrorOccurred(1 * time.Microsecond) s.store = testCreateStore(c, "test_column_change") - s.dbInfo = &model.DBInfo{ - Name: model.NewCIStr("test_column_change"), - ID: 1, - } - err := kv.RunInNewTxn(context.Background(), s.store, true, func(ctx context.Context, txn kv.Transaction) error { - t := meta.NewMeta(txn) - return errors.Trace(t.CreateDatabase(s.dbInfo)) - }) - c.Check(err, IsNil) + d := testNewDDLAndStart( + context.Background(), + c, + WithStore(s.store), + WithLease(testLease), + ) + defer func() { + err := d.Stop() + c.Assert(err, IsNil) + }() + s.dbInfo = testSchemaInfo(c, d, "test_index_change") + testCreateSchema(c, testNewContext(d), d, s.dbInfo) } func (s *testColumnChangeSuite) TearDownSuite(c *C) { diff --git a/ddl/column_test.go b/ddl/column_test.go index 862fb4aa04c59..793c7bec45b2f 100644 --- a/ddl/column_test.go +++ b/ddl/column_test.go @@ -54,8 +54,7 @@ func (s *testColumnSuite) SetUpSuite(c *C) { s.dbInfo = testSchemaInfo(c, d, "test_column") testCreateSchema(c, testNewContext(d), d, s.dbInfo) - err := d.Stop() - c.Assert(err, IsNil) + c.Assert(d.Stop(), IsNil) } func (s *testColumnSuite) TearDownSuite(c *C) { @@ -1160,6 +1159,7 @@ func (s *testColumnSuite) TestModifyColumn(c *C) { WithLease(testLease), ) ctx := testNewContext(d) + defer func() { err := d.Stop() c.Assert(err, IsNil) @@ -1170,17 +1170,19 @@ func (s *testColumnSuite) TestModifyColumn(c *C) { err error }{ {"int", "bigint", nil}, - {"int", "int unsigned", errUnsupportedModifyColumn.GenWithStackByArgs("can't change unsigned integer to signed or vice versa, and tidb_enable_change_column_type is false")}, + {"int", "int unsigned", nil}, + {"varchar(10)", "text", nil}, + {"varbinary(10)", "blob", nil}, {"text", "blob", errUnsupportedModifyCharset.GenWithStackByArgs("charset from utf8mb4 to binary")}, - {"varchar(10)", "varchar(8)", errUnsupportedModifyColumn.GenWithStackByArgs("length 8 is less than origin 10, and tidb_enable_change_column_type is false")}, + {"varchar(10)", "varchar(8)", nil}, {"varchar(10)", "varchar(11)", nil}, {"varchar(10) character set utf8 collate utf8_bin", "varchar(10) character set utf8", nil}, - {"decimal(2,1)", "decimal(3,2)", errUnsupportedModifyColumn.GenWithStackByArgs("decimal change from decimal(2, 1) to decimal(3, 2), and tidb_enable_change_column_type is false")}, - {"decimal(2,1)", "decimal(2,2)", errUnsupportedModifyColumn.GenWithStackByArgs("decimal change from decimal(2, 1) to decimal(2, 2), and tidb_enable_change_column_type is false")}, + {"decimal(2,1)", "decimal(3,2)", nil}, + {"decimal(2,1)", "decimal(2,2)", nil}, {"decimal(2,1)", "decimal(2,1)", nil}, - {"decimal(2,1)", "int", errUnsupportedModifyColumn.GenWithStackByArgs("type int(11) not match origin decimal(2,1), and tidb_enable_change_column_type is false")}, - {"decimal", "int", errUnsupportedModifyColumn.GenWithStackByArgs("type int(11) not match origin decimal(10,0), and tidb_enable_change_column_type is false")}, - {"decimal(2,1)", "bigint", errUnsupportedModifyColumn.GenWithStackByArgs("type bigint(20) not match origin decimal(2,1), and tidb_enable_change_column_type is false")}, + {"decimal(2,1)", "int", nil}, + {"decimal", "int", nil}, + {"decimal(2,1)", "bigint", nil}, } for _, tt := range tests { ftA := s.colDefStrToFieldType(c, tt.origin) diff --git a/ddl/column_type_change_test.go b/ddl/column_type_change_test.go index 992631b4bd97b..d0ca4f10c35a6 100644 --- a/ddl/column_type_change_test.go +++ b/ddl/column_type_change_test.go @@ -16,6 +16,10 @@ package ddl_test import ( "context" "errors" + "fmt" + "strconv" + "strings" + "sync" "time" . "github.com/pingcap/check" @@ -36,17 +40,23 @@ import ( "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/tablecodec" + "github.com/pingcap/tidb/util/collate" "github.com/pingcap/tidb/util/dbterror" "github.com/pingcap/tidb/util/testkit" ) var _ = SerialSuites(&testColumnTypeChangeSuite{}) +var _ = SerialSuites(&testCTCSerialSuiteWrapper{&testColumnTypeChangeSuite{}}) type testColumnTypeChangeSuite struct { store kv.Storage dom *domain.Domain } +type testCTCSerialSuiteWrapper struct { + *testColumnTypeChangeSuite +} + func (s *testColumnTypeChangeSuite) SetUpSuite(c *C) { var err error ddl.SetWaitTimeWhenErrorOccurred(1 * time.Microsecond) @@ -64,11 +74,6 @@ func (s *testColumnTypeChangeSuite) TearDownSuite(c *C) { func (s *testColumnTypeChangeSuite) TestColumnTypeChangeBetweenInteger(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - // Enable column change variable. - tk.Se.GetSessionVars().EnableChangeColumnType = true - defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = false - }() // Modify column from null to not null. tk.MustExec("drop table if exists t") @@ -136,11 +141,6 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeStateBetweenInteger(c *C tk.MustExec("drop table if exists t") tk.MustExec("create table t (c1 int, c2 int)") tk.MustExec("insert into t(c1, c2) values (1, 1)") - // Enable column change variable. - tk.Se.GetSessionVars().EnableChangeColumnType = true - defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = false - }() // use new session to check meta in callback function. internalTK := testkit.NewTestKit(c, s.store) @@ -211,11 +211,6 @@ func (s *testColumnTypeChangeSuite) TestRollbackColumnTypeChangeBetweenInteger(c tk.MustExec("drop table if exists t") tk.MustExec("create table t (c1 bigint, c2 bigint)") tk.MustExec("insert into t(c1, c2) values (1, 1)") - // Enable column change variable. - tk.Se.GetSessionVars().EnableChangeColumnType = true - defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = false - }() tbl := testGetTableByName(c, tk.Se, "test", "t") c.Assert(tbl, NotNil) @@ -298,11 +293,6 @@ func init() { func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromIntegerToOthers(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - // Enable column change variable. - tk.Se.GetSessionVars().EnableChangeColumnType = true - defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = false - }() prepare := func(tk *testkit.TestKit) { tk.MustExec("drop table if exists t") @@ -317,10 +307,14 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromIntegerToOthers(c *C // integer to string prepare(tk) - tk.MustGetErrCode("alter table t modify a varchar(10)", mysql.ErrUnsupportedDDLOperation) + tk.MustExec("alter table t modify a varchar(10)") + modifiedColumn := getModifyColumn(c, tk.Se, "test", "t", "a", false) + c.Assert(modifiedColumn, NotNil) + c.Assert(modifiedColumn.Tp, Equals, parser_mysql.TypeVarchar) + tk.MustQuery("select a from t").Check(testkit.Rows("1")) tk.MustExec("alter table t modify b char(10)") - modifiedColumn := getModifyColumn(c, tk.Se, "test", "t", "b", false) + modifiedColumn = getModifyColumn(c, tk.Se, "test", "t", "b", false) c.Assert(modifiedColumn, NotNil) c.Assert(modifiedColumn.Tp, Equals, parser_mysql.TypeString) tk.MustQuery("select b from t").Check(testkit.Rows("11")) @@ -331,7 +325,11 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromIntegerToOthers(c *C c.Assert(modifiedColumn.Tp, Equals, parser_mysql.TypeString) tk.MustQuery("select c from t").Check(testkit.Rows("111\x00\x00\x00\x00\x00\x00\x00")) - tk.MustGetErrCode("alter table t modify d varbinary(10)", mysql.ErrUnsupportedDDLOperation) + tk.MustExec("alter table t modify d varbinary(10)") + modifiedColumn = getModifyColumn(c, tk.Se, "test", "t", "d", false) + c.Assert(modifiedColumn, NotNil) + c.Assert(modifiedColumn.Tp, Equals, parser_mysql.TypeVarchar) + tk.MustQuery("select d from t").Check(testkit.Rows("1111")) tk.MustExec("alter table t modify e blob(10)") modifiedColumn = getModifyColumn(c, tk.Se, "test", "t", "e", false) @@ -445,46 +443,52 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromIntegerToOthers(c *C func (s *testColumnTypeChangeSuite) TestColumnTypeChangeBetweenVarcharAndNonVarchar(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - + collate.SetNewCollationEnabledForTest(true) + defer collate.SetNewCollationEnabledForTest(false) tk.MustExec("drop database if exists col_type_change_char;") tk.MustExec("create database col_type_change_char;") tk.MustExec("use col_type_change_char;") - tk.MustExec("create table t(a char(10), b varchar(10));") - tk.MustExec("insert into t values ('aaa ', 'bbb ');") - tk.MustExec("alter table t change column a a char(10);") - tk.MustExec("alter table t change column b b varchar(10);") - tk.MustGetErrCode("alter table t change column a a varchar(10);", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t change column b b char(10);", mysql.ErrUnsupportedDDLOperation) - tk.MustExec("alter table t add index idx_a(a);") - tk.MustExec("alter table t add index idx_b(b);") - tk.MustGetErrCode("alter table t change column a a varchar(10);", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t change column b b char(10);", mysql.ErrUnsupportedDDLOperation) + // https://github.com/pingcap/tidb/issues/23624 + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(b varchar(10));") + tk.MustExec("insert into t values ('aaa ');") + tk.MustExec("alter table t change column b b char(10);") + tk.MustExec("alter table t add index idx(b);") + tk.MustExec("alter table t change column b b varchar(10);") + tk.MustQuery("select b from t use index(idx);").Check(testkit.Rows("aaa")) + tk.MustQuery("select b from t ignore index(idx);").Check(testkit.Rows("aaa")) tk.MustExec("admin check table t;") + + // https://github.com/pingcap/tidb/pull/23688#issuecomment-810166597 + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a varchar(10));") + tk.MustExec("insert into t values ('aaa ');") + tk.MustQuery("select a from t;").Check(testkit.Rows("aaa ")) + tk.MustExec("alter table t modify column a char(10);") + tk.MustQuery("select a from t;").Check(testkit.Rows("aaa")) } func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromStringToOthers(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - // Enable column change variable. - tk.Se.GetSessionVars().EnableChangeColumnType = true // Set time zone to UTC. originalTz := tk.Se.GetSessionVars().TimeZone tk.Se.GetSessionVars().TimeZone = time.UTC defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = false tk.Se.GetSessionVars().TimeZone = originalTz }() // Init string date type table. reset := func(tk *testkit.TestKit) { tk.MustExec("drop table if exists t") - // FIXME(tangenta): not support changing from varchar/varbinary to other types. tk.MustExec(` create table t ( c char(8), + vc varchar(8), bny binary(8), + vbny varbinary(8), bb blob, txt text, e enum('123', '2020-07-15 18:32:17.888', 'str', '{"k1": "value"}'), @@ -496,147 +500,181 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromStringToOthers(c *C) // To numeric data types. // tinyint reset(tk) - tk.MustExec("insert into t values ('123', '123', '123', '123', '123', '123')") + tk.MustExec("insert into t values ('123', '123', '123', '123', '123', '123', '123', '123')") tk.MustExec("alter table t modify c tinyint") + tk.MustExec("alter table t modify vc tinyint") tk.MustExec("alter table t modify bny tinyint") + tk.MustExec("alter table t modify vbny tinyint") tk.MustExec("alter table t modify bb tinyint") tk.MustExec("alter table t modify txt tinyint") tk.MustExec("alter table t modify e tinyint") tk.MustExec("alter table t modify s tinyint") - tk.MustQuery("select * from t").Check(testkit.Rows("123 123 123 123 1 1")) + tk.MustQuery("select * from t").Check(testkit.Rows("123 123 123 123 123 123 1 1")) // int reset(tk) - tk.MustExec("insert into t values ('17305', '17305', '17305', '17305', '123', '123')") + tk.MustExec("insert into t values ('17305', '17305', '17305', '17305', '17305', '17305', '123', '123')") tk.MustExec("alter table t modify c int") + tk.MustExec("alter table t modify vc int") tk.MustExec("alter table t modify bny int") + tk.MustExec("alter table t modify vbny int") tk.MustExec("alter table t modify bb int") tk.MustExec("alter table t modify txt int") tk.MustExec("alter table t modify e int") tk.MustExec("alter table t modify s int") - tk.MustQuery("select * from t").Check(testkit.Rows("17305 17305 17305 17305 1 1")) + tk.MustQuery("select * from t").Check(testkit.Rows("17305 17305 17305 17305 17305 17305 1 1")) // bigint reset(tk) - tk.MustExec("insert into t values ('17305867', '17305867', '17305867', '17305867', '123', '123')") + tk.MustExec("insert into t values ('17305867', '17305867', '17305867', '17305867', '17305867', '17305867', '123', '123')") tk.MustExec("alter table t modify c bigint") + tk.MustExec("alter table t modify vc bigint") tk.MustExec("alter table t modify bny bigint") + tk.MustExec("alter table t modify vbny bigint") tk.MustExec("alter table t modify bb bigint") tk.MustExec("alter table t modify txt bigint") tk.MustExec("alter table t modify e bigint") tk.MustExec("alter table t modify s bigint") - tk.MustQuery("select * from t").Check(testkit.Rows("17305867 17305867 17305867 17305867 1 1")) + tk.MustQuery("select * from t").Check(testkit.Rows("17305867 17305867 17305867 17305867 17305867 17305867 1 1")) // bit reset(tk) - tk.MustExec("insert into t values ('1', '1', '1', '1', '123', '123')") + tk.MustExec("insert into t values ('1', '1', '1', '1', '1', '1', '123', '123')") tk.MustGetErrCode("alter table t modify c bit", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify vc bit", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify bny bit", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify vbny bit", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify bb bit", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify txt bit", mysql.ErrUnsupportedDDLOperation) tk.MustExec("alter table t modify e bit") tk.MustExec("alter table t modify s bit") - tk.MustQuery("select * from t").Check(testkit.Rows("1 1\x00\x00\x00\x00\x00\x00\x00 1 1 \x01 \x01")) + tk.MustQuery("select * from t").Check(testkit.Rows("1 1 1\x00\x00\x00\x00\x00\x00\x00 1 1 1 \x01 \x01")) // decimal reset(tk) - tk.MustExec("insert into t values ('123.45', '123.45', '123.45', '123.45', '123', '123')") + tk.MustExec("insert into t values ('123.45', '123.45', '123.45', '123.45', '123.45', '123.45', '123', '123')") tk.MustExec("alter table t modify c decimal(7, 4)") + tk.MustExec("alter table t modify vc decimal(7, 4)") tk.MustExec("alter table t modify bny decimal(7, 4)") + tk.MustExec("alter table t modify vbny decimal(7, 4)") tk.MustExec("alter table t modify bb decimal(7, 4)") tk.MustExec("alter table t modify txt decimal(7, 4)") tk.MustExec("alter table t modify e decimal(7, 4)") tk.MustExec("alter table t modify s decimal(7, 4)") - tk.MustQuery("select * from t").Check(testkit.Rows("123.4500 123.4500 123.4500 123.4500 1.0000 1.0000")) + tk.MustQuery("select * from t").Check(testkit.Rows("123.4500 123.4500 123.4500 123.4500 123.4500 123.4500 1.0000 1.0000")) // double reset(tk) - tk.MustExec("insert into t values ('123.45', '123.45', '123.45', '123.45', '123', '123')") + tk.MustExec("insert into t values ('123.45', '123.45', '123.45', '123.45', '123.45', '123.45', '123', '123')") tk.MustExec("alter table t modify c double(7, 4)") + tk.MustExec("alter table t modify vc double(7, 4)") tk.MustExec("alter table t modify bny double(7, 4)") + tk.MustExec("alter table t modify vbny double(7, 4)") tk.MustExec("alter table t modify bb double(7, 4)") tk.MustExec("alter table t modify txt double(7, 4)") tk.MustExec("alter table t modify e double(7, 4)") tk.MustExec("alter table t modify s double(7, 4)") - tk.MustQuery("select * from t").Check(testkit.Rows("123.45 123.45 123.45 123.45 1 1")) + tk.MustQuery("select * from t").Check(testkit.Rows("123.45 123.45 123.45 123.45 123.45 123.45 1 1")) // To date and time data types. // date reset(tk) - tk.MustExec("insert into t values ('20200826', '20200826', '2020-08-26', '08-26 19:35:41', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") + tk.MustExec("insert into t values ('20200826', '2008261', '20200826', '200826', '2020-08-26', '08-26 19:35:41', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") tk.MustExec("alter table t modify c date") + tk.MustExec("alter table t modify vc date") tk.MustExec("alter table t modify bny date") + tk.MustExec("alter table t modify vbny date") tk.MustExec("alter table t modify bb date") // Alter text '08-26 19:35:41' to date will error. (same as mysql does) tk.MustGetErrCode("alter table t modify txt date", mysql.ErrTruncatedWrongValue) tk.MustGetErrCode("alter table t modify e date", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify s date", mysql.ErrUnsupportedDDLOperation) - tk.MustQuery("select * from t").Check(testkit.Rows("2020-08-26 2020-08-26 2020-08-26 08-26 19:35:41 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) + tk.MustQuery("select * from t").Check(testkit.Rows("2020-08-26 2020-08-26 2020-08-26 2020-08-26 2020-08-26 08-26 19:35:41 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) // time reset(tk) - tk.MustExec("insert into t values ('19:35:41', '19:35:41', '19:35:41.45678', '19:35:41.45678', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") + tk.MustExec("insert into t values ('19:35:41', '19:35:41', '19:35:41', '19:35:41', '19:35:41.45678', '19:35:41.45678', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") tk.MustExec("alter table t modify c time") + tk.MustExec("alter table t modify vc time") tk.MustExec("alter table t modify bny time") + tk.MustExec("alter table t modify vbny time") tk.MustExec("alter table t modify bb time") tk.MustExec("alter table t modify txt time") tk.MustGetErrCode("alter table t modify e time", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify s time", mysql.ErrUnsupportedDDLOperation) - tk.MustQuery("select * from t").Check(testkit.Rows("19:35:41 19:35:41 19:35:41 19:35:41 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) + tk.MustQuery("select * from t").Check(testkit.Rows("19:35:41 19:35:41 19:35:41 19:35:41 19:35:41 19:35:41 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) // datetime reset(tk) tk.MustExec("alter table t modify c char(23)") + tk.MustExec("alter table t modify vc varchar(23)") tk.MustExec("alter table t modify bny binary(23)") - tk.MustExec("insert into t values ('2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") + tk.MustExec("alter table t modify vbny varbinary(23)") + tk.MustExec("insert into t values ('2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") tk.MustExec("alter table t modify c datetime") + tk.MustExec("alter table t modify vc datetime") tk.MustExec("alter table t modify bny datetime") + tk.MustExec("alter table t modify vbny datetime") tk.MustExec("alter table t modify bb datetime") tk.MustExec("alter table t modify txt datetime") tk.MustGetErrCode("alter table t modify e datetime", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify s datetime", mysql.ErrUnsupportedDDLOperation) - tk.MustQuery("select * from t").Check(testkit.Rows("2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) + tk.MustQuery("select * from t").Check(testkit.Rows("2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) // timestamp reset(tk) tk.MustExec("alter table t modify c char(23)") + tk.MustExec("alter table t modify vc varchar(23)") tk.MustExec("alter table t modify bny binary(23)") - tk.MustExec("insert into t values ('2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") + tk.MustExec("alter table t modify vbny varbinary(23)") + tk.MustExec("insert into t values ('2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") tk.MustExec("alter table t modify c timestamp") + tk.MustExec("alter table t modify vc timestamp") tk.MustExec("alter table t modify bny timestamp") + tk.MustExec("alter table t modify vbny timestamp") tk.MustExec("alter table t modify bb timestamp") tk.MustExec("alter table t modify txt timestamp") tk.MustGetErrCode("alter table t modify e timestamp", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify s timestamp", mysql.ErrUnsupportedDDLOperation) - tk.MustQuery("select * from t").Check(testkit.Rows("2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) + tk.MustQuery("select * from t").Check(testkit.Rows("2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) // year reset(tk) - tk.MustExec("insert into t values ('2020', '2', '20', '99', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") + tk.MustExec("insert into t values ('2020', '91', '2', '2020', '20', '99', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") tk.MustExec("alter table t modify c year") + tk.MustExec("alter table t modify vc year") tk.MustExec("alter table t modify bny year") + tk.MustExec("alter table t modify vbny year") tk.MustExec("alter table t modify bb year") tk.MustExec("alter table t modify txt year") tk.MustExec("alter table t modify e year") tk.MustExec("alter table t modify s year") - tk.MustQuery("select * from t").Check(testkit.Rows("2020 2002 2020 1999 2002 2002")) + tk.MustQuery("select * from t").Check(testkit.Rows("2020 1991 2002 2020 2020 1999 2002 2002")) // To json data type. reset(tk) tk.MustExec("alter table t modify c char(15)") + tk.MustExec("alter table t modify vc varchar(15)") tk.MustExec("alter table t modify bny binary(15)") - tk.MustExec("insert into t values ('{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}')") + tk.MustExec("alter table t modify vbny varbinary(15)") + tk.MustExec("insert into t values ('{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}')") tk.MustExec("alter table t modify c json") + tk.MustExec("alter table t modify vc json") tk.MustExec("alter table t modify bny json") + tk.MustExec("alter table t modify vbny json") tk.MustExec("alter table t modify bb json") tk.MustExec("alter table t modify txt json") tk.MustExec("alter table t modify e json") tk.MustExec("alter table t modify s json") - tk.MustQuery("select * from t").Check(testkit.Rows("{\"k1\": \"value\"} {\"k1\": \"value\"} {\"k1\": \"value\"} {\"k1\": \"value\"} \"{\\\"k1\\\": \\\"value\\\"}\" \"{\\\"k1\\\": \\\"value\\\"}\"")) + tk.MustQuery("select * from t").Check(testkit.Rows("{\"k1\": \"value\"} {\"k1\": \"value\"} {\"k1\": \"value\"} {\"k1\": \"value\"} {\"k1\": \"value\"} {\"k1\": \"value\"} \"{\\\"k1\\\": \\\"value\\\"}\" \"{\\\"k1\\\": \\\"value\\\"}\"")) reset(tk) - tk.MustExec("insert into t values ('123x', 'abc', 'timestamp', 'date', '123', '123')") + tk.MustExec("insert into t values ('123x', 'x123', 'abc', 'datetime', 'timestamp', 'date', '123', '123')") tk.MustGetErrCode("alter table t modify c int", mysql.ErrTruncatedWrongValue) + tk.MustGetErrCode("alter table t modify vc smallint", mysql.ErrTruncatedWrongValue) + tk.MustGetErrCode("alter table t modify bny bigint", mysql.ErrTruncatedWrongValue) + tk.MustGetErrCode("alter table t modify vbny datetime", mysql.ErrTruncatedWrongValue) + tk.MustGetErrCode("alter table t modify bb timestamp", mysql.ErrTruncatedWrongValue) tk.MustGetErrCode("alter table t modify txt date", mysql.ErrTruncatedWrongValue) reset(tk) - tk.MustExec("alter table t add vc char(20)") + tk.MustExec("alter table t modify vc varchar(20)") tk.MustExec("insert into t(c, vc) values ('1x', '20200915110836')") tk.MustGetErrCode("alter table t modify c year", mysql.ErrTruncatedWrongValue) @@ -649,7 +687,6 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromStringToOthers(c *C) // Both error but different error message. // MySQL will get "ERROR 3140 (22032): Invalid JSON text: "The document root must not be followed by other values." at position 1 in value for column '#sql-5b_42.c'." error. reset(tk) - tk.MustExec("alter table t add vc char(20)") tk.MustExec("alter table t modify c char(15)") tk.MustExec("insert into t(c) values ('{\"k1\": \"value\"')") tk.MustGetErrCode("alter table t modify c json", mysql.ErrInvalidJSONText) @@ -662,14 +699,11 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromStringToOthers(c *C) func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromNumericToOthers(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - // Enable column change variable. - tk.Se.GetSessionVars().EnableChangeColumnType = true // Set time zone to UTC. originalTz := tk.Se.GetSessionVars().TimeZone tk.Se.GetSessionVars().TimeZone = time.UTC defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = false tk.Se.GetSessionVars().TimeZone = originalTz }() @@ -752,20 +786,20 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromNumericToOthers(c *C // MySQL will get "ERROR 1406 (22001): Data truncation: Data too long for column 'f64' at row 1". tk.MustExec("alter table t modify f64 char(20)") tk.MustExec("alter table t modify b char(20)") - tk.MustQuery("select * from t").Check(testkit.Rows("-258.1234500 333.33 2000000.20000002 323232323.32323235 -111.111115 -222222222222.22223 \x15")) + tk.MustQuery("select * from t").Check(testkit.Rows("-258.1234500 333.33 2000000.20000002 323232323.32323235 -111.111115 -222222222222.22223 21")) // varchar reset(tk) tk.MustExec("insert into t values (-258.12345, 333.33, 2000000.20000002, 323232323.3232323232, -111.11111111, -222222222222.222222222222222, b'10101')") - tk.MustGetErrCode("alter table t modify d varchar(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify n varchar(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify r varchar(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify db varchar(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustExec("alter table t modify d varchar(30)") + tk.MustExec("alter table t modify n varchar(30)") + tk.MustExec("alter table t modify r varchar(30)") + tk.MustExec("alter table t modify db varchar(30)") // MySQL will get "-111.111" rather than "-111.111115" at TiDB. - tk.MustGetErrCode("alter table t modify f32 varchar(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustExec("alter table t modify f32 varchar(30)") // MySQL will get "ERROR 1406 (22001): Data truncation: Data too long for column 'f64' at row 1". - tk.MustGetErrCode("alter table t modify f64 varchar(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify b varchar(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustExec("alter table t modify f64 varchar(30)") + tk.MustExec("alter table t modify b varchar(30)") tk.MustQuery("select * from t").Check(testkit.Rows("-258.1234500 333.33 2000000.20000002 323232323.32323235 -111.111115 -222222222222.22223 \x15")) // binary @@ -779,21 +813,21 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromNumericToOthers(c *C tk.MustGetErrCode("alter table t modify f32 binary(10)", mysql.ErrDataTooLong) tk.MustGetErrCode("alter table t modify f64 binary(10)", mysql.ErrDataTooLong) tk.MustExec("alter table t modify b binary(10)") - tk.MustQuery("select * from t").Check(testkit.Rows("-258.1234500 333.33\x00\x00\x00\x00 2000000.20000002 323232323.32323235 -111.111115 -222222222222.22223 \x15\x00\x00\x00\x00\x00\x00\x00\x00\x00")) + tk.MustQuery("select * from t").Check(testkit.Rows("-258.1234500 333.33\x00\x00\x00\x00 2000000.20000002 323232323.32323235 -111.111115 -222222222222.22223 21\x00\x00\x00\x00\x00\x00\x00\x00")) // varbinary reset(tk) tk.MustExec("insert into t values (-258.12345, 333.33, 2000000.20000002, 323232323.3232323232, -111.11111111, -222222222222.222222222222222, b'10101')") - tk.MustGetErrCode("alter table t modify d varbinary(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify n varbinary(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify r varbinary(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify db varbinary(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustExec("alter table t modify d varbinary(30)") + tk.MustExec("alter table t modify n varbinary(30)") + tk.MustExec("alter table t modify r varbinary(30)") + tk.MustExec("alter table t modify db varbinary(30)") // MySQL will get "-111.111" rather than "-111.111115" at TiDB. - tk.MustGetErrCode("alter table t modify f32 varbinary(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustExec("alter table t modify f32 varbinary(30)") // MySQL will get "ERROR 1406 (22001): Data truncation: Data too long for column 'f64' at row 1". - tk.MustGetErrCode("alter table t modify f64 varbinary(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify b varbinary(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustQuery("select * from t").Check(testkit.Rows("-258.1234500 333.33 2000000.20000002 323232323.32323235 -111.111115 -222222222222.22223 \x15")) + tk.MustExec("alter table t modify f64 varbinary(30)") + tk.MustExec("alter table t modify b varbinary(30)") + tk.MustQuery("select * from t").Check(testkit.Rows("-258.1234500 333.33 2000000.20000002 323232323.32323235 -111.111115 -222222222222.22223 21")) // blob reset(tk) @@ -850,15 +884,15 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromNumericToOthers(c *C reset(tk) tk.MustExec("insert into t values (200805.11, 307.333, 20200805.11111111, 20200805111307.11111111, 200805111307.11111111, 20200805111307.11111111, b'10101')") // MySQL will get "ERROR 1292 (22001) Data truncation: Incorrect datetime value: '200805.1100000' for column 'd' at row 1". - tk.MustExec("alter table t modify d datetime") + tk.MustGetErrCode("alter table t modify d datetime", mysql.ErrUnsupportedDDLOperation) // MySQL will get "ERROR 1292 (22001) Data truncation: Incorrect datetime value: '307.33' for column 'n' at row 1". - tk.MustExec("alter table t modify n datetime") + tk.MustGetErrCode("alter table t modify n datetime", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify r datetime", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify db datetime", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify f32 datetime", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify f64 datetime", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify b datetime", mysql.ErrUnsupportedDDLOperation) - tk.MustQuery("select * from t").Check(testkit.Rows("2020-08-05 00:00:00 2000-03-07 00:00:00 20200805.11111111 20200805111307.11 200805100000 20200805111307.11 \x15")) + tk.MustQuery("select * from t").Check(testkit.Rows("200805.1100000 307.33 20200805.11111111 20200805111307.11 200805100000 20200805111307.11 \x15")) // time reset(tk) tk.MustExec("insert into t values (200805.11, 307.333, 20200805.11111111, 20200805111307.11111111, 200805111307.11111111, 20200805111307.11111111, b'10101')") @@ -874,28 +908,28 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromNumericToOthers(c *C reset(tk) tk.MustExec("insert into t values (200805.11, 307.333, 20200805.11111111, 20200805111307.11111111, 200805111307.11111111, 20200805111307.11111111, b'10101')") // MySQL will get "ERROR 1292 (22001) Data truncation: Incorrect date value: '200805.1100000' for column 'd' at row 1". - tk.MustExec("alter table t modify d date") + tk.MustGetErrCode("alter table t modify d date", mysql.ErrUnsupportedDDLOperation) // MySQL will get "ERROR 1292 (22001) Data truncation: Incorrect date value: '307.33' for column 'n' at row 1". - tk.MustExec("alter table t modify n date") + tk.MustGetErrCode("alter table t modify n date", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify r date", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify db date", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify f32 date", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify f64 date", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify b date", mysql.ErrUnsupportedDDLOperation) - tk.MustQuery("select * from t").Check(testkit.Rows("2020-08-05 2000-03-07 20200805.11111111 20200805111307.11 200805100000 20200805111307.11 \x15")) + tk.MustQuery("select * from t").Check(testkit.Rows("200805.1100000 307.33 20200805.11111111 20200805111307.11 200805100000 20200805111307.11 \x15")) // timestamp reset(tk) tk.MustExec("insert into t values (200805.11, 307.333, 20200805.11111111, 20200805111307.11111111, 200805111307.11111111, 20200805111307.11111111, b'10101')") // MySQL will get "ERROR 1292 (22001) Data truncation: Incorrect datetime value: '200805.1100000' for column 'd' at row 1". - tk.MustExec("alter table t modify d timestamp") + tk.MustGetErrCode("alter table t modify d timestamp", mysql.ErrUnsupportedDDLOperation) // MySQL will get "ERROR 1292 (22001) Data truncation: Incorrect datetime value: '307.33' for column 'n' at row 1". - tk.MustExec("alter table t modify n timestamp") + tk.MustGetErrCode("alter table t modify n timestamp", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify r timestamp", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify db timestamp", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify f32 timestamp", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify f64 timestamp", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify b timestamp", mysql.ErrUnsupportedDDLOperation) - tk.MustQuery("select * from t").Check(testkit.Rows("2020-08-05 00:00:00 2000-03-07 00:00:00 20200805.11111111 20200805111307.11 200805100000 20200805111307.11 \x15")) + tk.MustQuery("select * from t").Check(testkit.Rows("200805.1100000 307.33 20200805.11111111 20200805111307.11 200805100000 20200805111307.11 \x15")) // year reset(tk) tk.MustExec("insert into t values (200805.11, 307.333, 2.55555, 98.1111111, 2154.00001, 20200805111307.11111111, b'10101')") @@ -928,10 +962,6 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromNumericToOthers(c *C func (s *testColumnTypeChangeSuite) TestColumnTypeChangeIgnoreDisplayLength(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - tk.Se.GetSessionVars().EnableChangeColumnType = true - defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = false - }() originalHook := s.dom.DDL().GetHook() defer s.dom.DDL().(ddl.DDLForTest).SetHook(originalHook) @@ -975,14 +1005,11 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeIgnoreDisplayLength(c *C func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromDateTimeTypeToOthers(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - // Enable column change variable. - tk.Se.GetSessionVars().EnableChangeColumnType = true // Set time zone to UTC. originalTz := tk.Se.GetSessionVars().TimeZone tk.Se.GetSessionVars().TimeZone = time.UTC defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = false tk.Se.GetSessionVars().TimeZone = originalTz }() @@ -1070,11 +1097,11 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromDateTimeTypeToOthers // varchar reset(tk) tk.MustExec("insert into t values ('2020-10-30', '19:38:25.001', 20201030082133.455555, 20201030082133.455555, 2020)") - tk.MustGetErrCode("alter table t modify d varchar(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify t varchar(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify dt varchar(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify tmp varchar(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify y varchar(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustExec("alter table t modify d varchar(30)") + tk.MustExec("alter table t modify t varchar(30)") + tk.MustExec("alter table t modify dt varchar(30)") + tk.MustExec("alter table t modify tmp varchar(30)") + tk.MustExec("alter table t modify y varchar(30)") tk.MustQuery("select * from t").Check(testkit.Rows("2020-10-30 19:38:25.001 2020-10-30 08:21:33.455555 2020-10-30 08:21:33.455555 2020")) // binary @@ -1094,11 +1121,11 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromDateTimeTypeToOthers // varbinary reset(tk) tk.MustExec("insert into t values ('2020-10-30', '19:38:25.001', 20201030082133.455555, 20201030082133.455555, 2020)") - tk.MustGetErrCode("alter table t modify d varbinary(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify t varbinary(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify dt varbinary(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify tmp varbinary(30)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify y varbinary(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustExec("alter table t modify d varbinary(30)") + tk.MustExec("alter table t modify t varbinary(30)") + tk.MustExec("alter table t modify dt varbinary(30)") + tk.MustExec("alter table t modify tmp varbinary(30)") + tk.MustExec("alter table t modify y varbinary(30)") tk.MustQuery("select * from t").Check(testkit.Rows("2020-10-30 19:38:25.001 2020-10-30 08:21:33.455555 2020-10-30 08:21:33.455555 2020")) // text @@ -1155,14 +1182,11 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromDateTimeTypeToOthers func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromJsonToOthers(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - // Enable column change variable. - tk.Se.GetSessionVars().EnableChangeColumnType = true // Set time zone to UTC. originalTz := tk.Se.GetSessionVars().TimeZone tk.Se.GetSessionVars().TimeZone = time.UTC defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = false tk.Se.GetSessionVars().TimeZone = originalTz }() @@ -1336,15 +1360,15 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromJsonToOthers(c *C) { // varchar reset(tk) tk.MustExec("insert into t values ('{\"obj\": 100}', '[-1, 0, 1]', 'null', 'true', 'false', '-22', '22', '323232323.3232323232', '\"json string\"')") - tk.MustGetErrCode("alter table t modify obj varchar(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify arr varchar(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify nil varchar(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify t varchar(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify f varchar(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify i varchar(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify ui varchar(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify f64 varchar(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify str varchar(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustExec("alter table t modify obj varchar(20)") + tk.MustExec("alter table t modify arr varchar(20)") + tk.MustExec("alter table t modify nil varchar(20)") + tk.MustExec("alter table t modify t varchar(20)") + tk.MustExec("alter table t modify f varchar(20)") + tk.MustExec("alter table t modify i varchar(20)") + tk.MustExec("alter table t modify ui varchar(20)") + tk.MustExec("alter table t modify f64 varchar(20)") + tk.MustExec("alter table t modify str varchar(20)") tk.MustQuery("select * from t").Check(testkit.Rows("{\"obj\": 100} [-1, 0, 1] null true false -22 22 323232323.32323235 \"json string\"")) // binary @@ -1372,15 +1396,15 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromJsonToOthers(c *C) { // varbinary reset(tk) tk.MustExec("insert into t values ('{\"obj\": 100}', '[-1, 0, 1]', 'null', 'true', 'false', '-22', '22', '323232323.3232323232', '\"json string\"')") - tk.MustGetErrCode("alter table t modify obj varbinary(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify arr varbinary(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify nil varbinary(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify t varbinary(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify f varbinary(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify i varbinary(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify ui varbinary(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify f64 varbinary(20)", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify str varbinary(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustExec("alter table t modify obj varbinary(20)") + tk.MustExec("alter table t modify arr varbinary(20)") + tk.MustExec("alter table t modify nil varbinary(20)") + tk.MustExec("alter table t modify t varbinary(20)") + tk.MustExec("alter table t modify f varbinary(20)") + tk.MustExec("alter table t modify i varbinary(20)") + tk.MustExec("alter table t modify ui varbinary(20)") + tk.MustExec("alter table t modify f64 varbinary(20)") + tk.MustExec("alter table t modify str varbinary(20)") tk.MustQuery("select * from t").Check(testkit.Rows("{\"obj\": 100} [-1, 0, 1] null true false -22 22 323232323.32323235 \"json string\"")) // blob @@ -1526,15 +1550,30 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromJsonToOthers(c *C) { tk.MustQuery("select * from t").Check(testkit.Rows("{\"obj\": 100} [-1, 0, 1] 0 2001 0 2020 1991 2009 2020")) } +func (s *testColumnTypeChangeSuite) TestUpdateDataAfterChangeTimestampToDate(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t, t1") + tk.MustExec("create table t (col timestamp default '1971-06-09' not null, col1 int default 1, unique key(col1));") + tk.MustExec("alter table t modify column col date not null;") + tk.MustExec("update t set col = '2002-12-31';") + // point get + tk.MustExec("update t set col = '2002-12-31' where col1 = 1;") + + // Make sure the original default value isn't rewritten. + tk.MustExec("create table t1 (col timestamp default '1971-06-09' not null, col1 int default 1, unique key(col1));") + tk.MustExec("insert into t1 value('2001-01-01', 1);") + tk.MustExec("alter table t1 add column col2 timestamp default '2020-06-02' not null;") + tk.MustExec("alter table t1 modify column col2 date not null;") + tk.MustExec("update t1 set col = '2002-11-22';") + // point get + tk.MustExec("update t1 set col = '2002-12-31' where col1 = 1;") +} + // TestRowFormat is used to close issue #21391, the encoded row in column type change should be aware of the new row format. func (s *testColumnTypeChangeSuite) TestRowFormat(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - // Enable column change variable. - tk.Se.GetSessionVars().EnableChangeColumnType = true - defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = false - }() tk.MustExec("drop table if exists t") tk.MustExec("create table t (id int primary key, v varchar(10))") tk.MustExec("insert into t values (1, \"123\");") @@ -1551,39 +1590,6 @@ func (s *testColumnTypeChangeSuite) TestRowFormat(c *C) { tk.MustExec("drop table if exists t") } -// Close issue #17530 -func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFlenErrorMsg(c *C) { - tk := testkit.NewTestKit(c, s.store) - tk.MustExec("use test") - - tk.MustExec("drop table if exists t") - tk.MustExec("create table t(a int4)") - _, err := tk.Exec("alter table t MODIFY COLUMN a tinyint(11)") - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: length 4 is less than origin 11, and tidb_enable_change_column_type is false") - - tk.MustExec("drop table if exists t") - tk.MustExec("create table t (a decimal(20))") - tk.MustExec("insert into t values (12345678901234567890)") - _, err = tk.Exec("alter table t modify column a bigint") - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: type bigint(20) not match origin decimal(20,0), and tidb_enable_change_column_type is false") - - tk.MustExec("drop table if exists t") - tk.MustExec("create table a (a bigint(2))") - tk.MustExec("insert into a values(123456),(789123)") - _, err = tk.Exec("alter table a modify column a tinyint") - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: length 4 is less than origin 20, and tidb_enable_change_column_type is false") - - tk.MustExec("drop table if exists t") - tk.MustExec("CREATE TABLE t ( id int not null primary key auto_increment, token varchar(512) NOT NULL DEFAULT '', index (token))") - tk.MustExec("INSERT INTO t VALUES (NULL, 'aa')") - _, err = tk.Exec("ALTER TABLE t CHANGE COLUMN token token varchar(255) DEFAULT '' NOT NULL") - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: length 255 is less than origin 512, and tidb_enable_change_column_type is false") -} - // Close issue #22395 // Background: // Since the changing column is implemented as adding a new column and substitute the old one when it finished. @@ -1593,17 +1599,12 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFlenErrorMsg(c *C) { func (s *testColumnTypeChangeSuite) TestChangingColOriginDefaultValue(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - // Enable column change variable. - tk.Se.GetSessionVars().EnableChangeColumnType = true - defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = false - }() tk1 := testkit.NewTestKit(c, s.store) tk1.MustExec("use test") tk.MustExec("drop table if exists t") - tk.MustExec("create table t(a int, b int)") + tk.MustExec("create table t(a int, b int not null, unique key(a))") tk.MustExec("insert into t values(1, 1)") tk.MustExec("insert into t values(2, 2)") @@ -1614,7 +1615,8 @@ func (s *testColumnTypeChangeSuite) TestChangingColOriginDefaultValue(c *C) { once bool checkErr error ) - hook.OnJobRunBeforeExported = func(job *model.Job) { + i := 0 + hook.OnJobUpdatedExported = func(job *model.Job) { if checkErr != nil { return } @@ -1636,7 +1638,8 @@ func (s *testColumnTypeChangeSuite) TestChangingColOriginDefaultValue(c *C) { } // For writable column: // Insert/ Update should set the column with the casted-related column value. - _, err := tk1.Exec("insert into t values(3, 3)") + sql := fmt.Sprintf("insert into t values(%d, %d)", i+3, i+3) + _, err := tk1.Exec(sql) if err != nil { checkErr = err return @@ -1656,13 +1659,96 @@ func (s *testColumnTypeChangeSuite) TestChangingColOriginDefaultValue(c *C) { return } } + i++ } } s.dom.DDL().(ddl.DDLForTest).SetHook(hook) tk.MustExec("alter table t modify column b tinyint NOT NULL") s.dom.DDL().(ddl.DDLForTest).SetHook(originalHook) + c.Assert(checkErr, IsNil) + // Since getReorgInfo will stagnate StateWriteReorganization for a ddl round, so insert should exec 3 times. + tk.MustQuery("select * from t order by a").Check(testkit.Rows("1 -1", "2 -2", "3 3", "4 4", "5 5")) + tk.MustExec("drop table if exists t") +} + +func (s *testColumnTypeChangeSuite) TestChangingColOriginDefaultValueAfterAddCol(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + tk1 := testkit.NewTestKit(c, s.store) + tk1.MustExec("use test") + + tk.MustExec(fmt.Sprintf("set time_zone = 'UTC'")) + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int, b int not null, unique key(a))") + tk.MustExec("insert into t values(1, 1)") + tk.MustExec("insert into t values(2, 2)") + tk.MustExec("alter table t add column c timestamp default '1971-06-09' not null") + + tbl := testGetTableByName(c, tk.Se, "test", "t") + originalHook := s.dom.DDL().GetHook() + hook := &ddl.TestDDLCallback{Do: s.dom} + var ( + once bool + checkErr error + ) + i := 0 + hook.OnJobRunBeforeExported = func(job *model.Job) { + if checkErr != nil { + return + } + if tbl.Meta().ID != job.TableID { + return + } + if job.SchemaState == model.StateWriteOnly || job.SchemaState == model.StateWriteReorganization { + if !once { + once = true + tbl := testGetTableByName(c, tk1.Se, "test", "t") + if len(tbl.WritableCols()) != 4 { + checkErr = errors.New("assert the writable column number error") + return + } + if tbl.WritableCols()[3].OriginDefaultValue.(string) != "1971-06-09 00:00:00" { + checkErr = errors.New("assert the write only column origin default value error") + return + } + } + // For writable column: + // Insert / Update should set the column with the casted-related column value. + sql := fmt.Sprintf("insert into t values(%d, %d, '2021-06-06 12:13:14')", i+3, i+3) + _, err := tk1.Exec(sql) + if err != nil { + checkErr = err + return + } + if job.SchemaState == model.StateWriteOnly { + // The casted value will be inserted into changing column too. + // for point get + _, err := tk1.Exec("update t set b = -1 where a = 1") + if err != nil { + checkErr = err + return + } + } else { + // The casted value will be inserted into changing column too. + // for point get + _, err := tk1.Exec("update t set b = -2 where a = 2") + if err != nil { + checkErr = err + return + } + } + } + i++ + } + + s.dom.DDL().(ddl.DDLForTest).SetHook(hook) + tk.MustExec("alter table t modify column c date NOT NULL") + s.dom.DDL().(ddl.DDLForTest).SetHook(originalHook) + c.Assert(checkErr, IsNil) // Since getReorgInfo will stagnate StateWriteReorganization for a ddl round, so insert should exec 3 times. - tk.MustQuery("select * from t order by a").Check(testkit.Rows("1 -1", "2 -2", "3 3", "3 3", "3 3")) + tk.MustQuery("select * from t order by a").Check( + testkit.Rows("1 -1 1971-06-09", "2 -2 1971-06-09", "5 5 2021-06-06", "6 6 2021-06-06", "7 7 2021-06-06")) tk.MustExec("drop table if exists t") } @@ -1705,7 +1791,6 @@ func (s *testColumnTypeChangeSuite) TestChangingAttributeOfColumnWithFK(c *C) { func (s *testColumnTypeChangeSuite) TestAlterPrimaryKeyToNull(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - tk.Se.GetSessionVars().EnableChangeColumnType = true tk.MustExec("drop table if exists t, t1") tk.MustExec("create table t(a int not null, b int not null, primary key(a, b));") @@ -1716,6 +1801,26 @@ func (s *testColumnTypeChangeSuite) TestAlterPrimaryKeyToNull(c *C) { tk.MustGetErrCode("alter table t change column a a bigint null;", mysql.ErrPrimaryCantHaveNull) } +// Close https://github.com/pingcap/tidb/issues/24839. +func (s testColumnTypeChangeSuite) TestChangeUnsignedIntToDatetime(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test;") + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int(10) unsigned default null, b bigint unsigned, c tinyint unsigned);") + tk.MustExec("insert into t values (1, 1, 1);") + tk.MustGetErrCode("alter table t modify column a datetime;", mysql.ErrTruncatedWrongValue) + tk.MustGetErrCode("alter table t modify column b datetime;", mysql.ErrTruncatedWrongValue) + tk.MustGetErrCode("alter table t modify column c datetime;", mysql.ErrTruncatedWrongValue) + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int(10) unsigned default null, b bigint unsigned, c tinyint unsigned);") + tk.MustExec("insert into t values (4294967295, 18446744073709551615, 255);") + tk.MustGetErrCode("alter table t modify column a datetime;", mysql.ErrTruncatedWrongValue) + tk.MustGetErrCode("alter table t modify column b datetime;", mysql.ErrTruncatedWrongValue) + tk.MustGetErrCode("alter table t modify column c datetime;", mysql.ErrTruncatedWrongValue) +} + // Close issue #23202 func (s *testColumnTypeChangeSuite) TestDDLExitWhenCancelMeetPanic(c *C) { tk := testkit.NewTestKit(c, s.store) @@ -1771,8 +1876,6 @@ func (s *testColumnTypeChangeSuite) TestDDLExitWhenCancelMeetPanic(c *C) { func (s *testColumnTypeChangeSuite) TestChangeIntToBitWillPanicInBackfillIndexes(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - // Enable column change variable. - tk.Se.GetSessionVars().EnableChangeColumnType = true tk.MustExec("drop table if exists t") tk.MustExec("CREATE TABLE `t` (" + @@ -1797,3 +1900,149 @@ func (s *testColumnTypeChangeSuite) TestChangeIntToBitWillPanicInBackfillIndexes ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin")) tk.MustQuery("select * from t").Check(testkit.Rows("\x13 1 1.00", "\x11 2 2.00")) } + +// Close issue #24584 +func (s *testColumnTypeChangeSuite) TestCancelCTCInReorgStateWillCauseGoroutineLeak(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + failpoint.Enable("github.com/pingcap/tidb/ddl/mockInfiniteReorgLogic", `return(true)`) + defer func() { + failpoint.Disable("github.com/pingcap/tidb/ddl/mockInfiniteReorgLogic") + }() + + // set ddl hook + originalHook := s.dom.DDL().GetHook() + defer s.dom.DDL().(ddl.DDLForTest).SetHook(originalHook) + + tk.MustExec("drop table if exists ctc_goroutine_leak") + tk.MustExec("create table ctc_goroutine_leak (a int)") + tk.MustExec("insert into ctc_goroutine_leak values(1),(2),(3)") + tbl := testGetTableByName(c, tk.Se, "test", "ctc_goroutine_leak") + + hook := &ddl.TestDDLCallback{} + var jobID int64 + hook.OnJobRunBeforeExported = func(job *model.Job) { + if jobID != 0 { + return + } + if tbl.Meta().ID != job.TableID { + return + } + if job.Query == "alter table ctc_goroutine_leak modify column a tinyint" { + jobID = job.ID + } + } + s.dom.DDL().(ddl.DDLForTest).SetHook(hook) + + tk1 := testkit.NewTestKit(c, s.store) + tk1.MustExec("use test") + var ( + wg = sync.WaitGroup{} + alterErr error + ) + wg.Add(1) + go func() { + defer wg.Done() + // This ddl will be hang over in the failpoint loop, waiting for outside cancel. + _, alterErr = tk1.Exec("alter table ctc_goroutine_leak modify column a tinyint") + }() + <-ddl.TestReorgGoroutineRunning + tk.MustExec("admin cancel ddl jobs " + strconv.Itoa(int(jobID))) + wg.Wait() + c.Assert(alterErr.Error(), Equals, "[ddl:8214]Cancelled DDL job") +} + +// Close issue #24971, #24973, #24974 +func (s *testColumnTypeChangeSuite) TestCTCShouldCastTheDefaultValue(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int)") + tk.MustExec("insert into t values(1)") + tk.MustExec("alter table t add column b bit(51) default 1512687856625472") // virtual fill the column data + tk.MustGetErrCode("alter table t modify column b decimal(30,18)", mysql.ErrDataOutOfRange) // because 1512687856625472 is out of range. + + tk.MustExec("drop table if exists t") + tk.MustExec("create table tbl_1 (col int)") + tk.MustExec("insert into tbl_1 values (9790)") + tk.MustExec("alter table tbl_1 add column col1 blob(6) collate binary not null") + tk.MustQuery("select col1 from tbl_1").Check(testkit.Rows("")) + tk.MustGetErrCode("alter table tbl_1 change column col1 col2 int", mysql.ErrTruncatedWrongValue) + tk.MustQuery("select col1 from tbl_1").Check(testkit.Rows("")) + + tk.MustExec("drop table if exists t") + tk.MustExec("create table tbl(col_214 decimal(30,8))") + tk.MustExec("replace into tbl values (89687.448)") + tk.MustExec("alter table tbl add column col_279 binary(197) collate binary default 'RAWTdm' not null") + tk.MustQuery("select col_279 from tbl").Check(testkit.Rows("RAWTdm\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00")) + tk.MustGetErrCode("alter table tbl change column col_279 col_287 int", mysql.ErrTruncatedWrongValue) + tk.MustQuery("select col_279 from tbl").Check(testkit.Rows("RAWTdm\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00")) +} + +// Close issue #25037 +// 1: for default value of binary of create-table, it should append the \0 as the suffix to meet flen. +// 2: when cast the bit to binary, we should consider to convert it to uint then cast uint to string, rather than taking the bit to string directly. +func (s *testColumnTypeChangeSuite) TestCTCCastBitToBinary(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + // For point 1: + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a binary(10) default 't')") + tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n `a` binary(10) DEFAULT 't\\0\\0\\0\\0\\0\\0\\0\\0\\0'\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin")) + + // For point 2 with binary: + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a bit(13) not null) collate utf8mb4_general_ci") + tk.MustExec("insert into t values ( 4047 )") + tk.MustExec("alter table t change column a a binary(248) collate binary default 't'") + tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n `a` binary(248) DEFAULT 't\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0'\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci")) + tk.MustQuery("select * from t").Check(testkit.Rows("4047\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00")) + + // For point 2 with varbinary: + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a bit(13) not null) collate utf8mb4_general_ci") + tk.MustExec("insert into t values ( 4047 )") + tk.MustExec("alter table t change column a a varbinary(248) collate binary default 't'") + tk.MustQuery("show create table t").Check(testkit.Rows("t CREATE TABLE `t` (\n `a` varbinary(248) DEFAULT 't'\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci")) + tk.MustQuery("select * from t").Check(testkit.Rows("4047")) +} + +func (s *testColumnTypeChangeSuite) TestChangePrefixedIndexColumnToNonPrefixOne(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test;") + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a text, unique index idx(a(2)));") + tk.MustExec("alter table t modify column a int;") + showCreateTable := tk.MustQuery("show create table t").Rows()[0][1].(string) + c.Assert(strings.Contains(showCreateTable, "UNIQUE KEY `idx` (`a`)"), IsTrue, + Commentf("%s", showCreateTable)) + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a char(255), unique index idx(a(2)));") + tk.MustExec("alter table t modify column a float;") + showCreateTable = tk.MustQuery("show create table t").Rows()[0][1].(string) + c.Assert(strings.Contains(showCreateTable, "UNIQUE KEY `idx` (`a`)"), IsTrue, + Commentf("%s", showCreateTable)) + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a char(255), b text, unique index idx(a(2), b(10)));") + tk.MustExec("alter table t modify column b int;") + showCreateTable = tk.MustQuery("show create table t").Rows()[0][1].(string) + c.Assert(strings.Contains(showCreateTable, "UNIQUE KEY `idx` (`a`(2),`b`)"), IsTrue, + Commentf("%s", showCreateTable)) + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(a char(250), unique key idx(a(10)));") + tk.MustExec("alter table t modify a char(9);") + showCreateTable = tk.MustQuery("show create table t").Rows()[0][1].(string) + c.Assert(strings.Contains(showCreateTable, "UNIQUE KEY `idx` (`a`)"), IsTrue, + Commentf("%s", showCreateTable)) + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(a varchar(700), key(a(700)));") + tk.MustGetErrCode("alter table t change column a a tinytext;", mysql.ErrBlobKeyWithoutLength) +} diff --git a/ddl/db_change_test.go b/ddl/db_change_test.go index 7c3a0f9ad970f..dcc5cb23418fb 100644 --- a/ddl/db_change_test.go +++ b/ddl/db_change_test.go @@ -494,10 +494,10 @@ func (s *testStateChangeSuite) TestAppendEnum(c *C) { c.Assert(err.Error(), Equals, "[types:1265]Data truncated for column 'c2' at row 1") failAlterTableSQL1 := "alter table t change c2 c2 enum('N') DEFAULT 'N'" _, err = s.se.Execute(context.Background(), failAlterTableSQL1) - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: the number of enum column's elements is less than the original: 2, and tidb_enable_change_column_type is false") + c.Assert(err, IsNil) failAlterTableSQL2 := "alter table t change c2 c2 int default 0" _, err = s.se.Execute(context.Background(), failAlterTableSQL2) - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: type int(11) not match origin enum('N','Y'), and tidb_enable_change_column_type is false") + c.Assert(err, IsNil) alterTableSQL := "alter table t change c2 c2 enum('N','Y','A') DEFAULT 'A'" _, err = s.se.Execute(context.Background(), alterTableSQL) c.Assert(err, IsNil) @@ -576,11 +576,6 @@ func (s *serialTestStateChangeSuite) TestWriteReorgForModifyColumnWithUniqIdx(c // TestWriteReorgForModifyColumnWithPKIsHandle tests whether the correct columns is used in PhysicalIndexScan's ToPB function. func (s *serialTestStateChangeSuite) TestWriteReorgForModifyColumnWithPKIsHandle(c *C) { modifyColumnSQL := "alter table tt change column c cc tinyint unsigned not null default 1 first" - enableChangeColumnType := s.se.GetSessionVars().EnableChangeColumnType - s.se.GetSessionVars().EnableChangeColumnType = true - defer func() { - s.se.GetSessionVars().EnableChangeColumnType = enableChangeColumnType - }() _, err := s.se.Execute(context.Background(), "use test_db_state") c.Assert(err, IsNil) @@ -638,12 +633,6 @@ func (s *serialTestStateChangeSuite) TestDeleteOnlyForModifyColumnWithoutDefault } func (s *serialTestStateChangeSuite) testModifyColumn(c *C, state model.SchemaState, modifyColumnSQL string, idx idxType) { - enableChangeColumnType := s.se.GetSessionVars().EnableChangeColumnType - s.se.GetSessionVars().EnableChangeColumnType = true - defer func() { - s.se.GetSessionVars().EnableChangeColumnType = enableChangeColumnType - }() - _, err := s.se.Execute(context.Background(), "use test_db_state") c.Assert(err, IsNil) switch idx { @@ -1057,19 +1046,11 @@ func (s *testStateChangeSuite) TestParallelAlterModifyColumn(c *C) { } func (s *testStateChangeSuite) TestParallelAddGeneratedColumnAndAlterModifyColumn(c *C) { - _, err := s.se.Execute(context.Background(), "set global tidb_enable_change_column_type = 1") - c.Assert(err, IsNil) - defer func() { - _, err = s.se.Execute(context.Background(), "set global tidb_enable_change_column_type = 0") - c.Assert(err, IsNil) - }() - domain.GetDomain(s.se).GetGlobalVarsCache().Disable() - sql1 := "ALTER TABLE t ADD COLUMN f INT GENERATED ALWAYS AS(a+1);" sql2 := "ALTER TABLE t MODIFY COLUMN a tinyint;" f := func(c *C, err1, err2 error) { c.Assert(err1, IsNil) - c.Assert(err2.Error(), Equals, "[ddl:8200]Unsupported modify column: tidb_enable_change_column_type is true, oldCol is a dependent column 'a' for generated column") + c.Assert(err2.Error(), Equals, "[ddl:8200]Unsupported modify column: oldCol is a dependent column 'a' for generated column") _, err := s.se.Execute(context.Background(), "select * from t") c.Assert(err, IsNil) } @@ -1077,19 +1058,11 @@ func (s *testStateChangeSuite) TestParallelAddGeneratedColumnAndAlterModifyColum } func (s *testStateChangeSuite) TestParallelAlterModifyColumnAndAddPK(c *C) { - _, err := s.se.Execute(context.Background(), "set global tidb_enable_change_column_type = 1") - c.Assert(err, IsNil) - defer func() { - _, err = s.se.Execute(context.Background(), "set global tidb_enable_change_column_type = 0") - c.Assert(err, IsNil) - }() - domain.GetDomain(s.se).GetGlobalVarsCache().Disable() - sql1 := "ALTER TABLE t ADD PRIMARY KEY (b) NONCLUSTERED;" sql2 := "ALTER TABLE t MODIFY COLUMN b tinyint;" f := func(c *C, err1, err2 error) { c.Assert(err1, IsNil) - c.Assert(err2.Error(), Equals, "[ddl:8200]Unsupported modify column: tidb_enable_change_column_type is true and this column has primary key flag") + c.Assert(err2.Error(), Equals, "[ddl:8200]Unsupported modify column: this column has primary key flag") _, err := s.se.Execute(context.Background(), "select * from t") c.Assert(err, IsNil) } @@ -1742,12 +1715,6 @@ func (s *serialTestStateChangeSuite) TestModifyColumnTypeArgs(c *C) { tk.MustExec("drop table if exists t_modify_column_args") tk.MustExec("create table t_modify_column_args(a int, unique(a))") - enableChangeColumnType := tk.Se.GetSessionVars().EnableChangeColumnType - tk.Se.GetSessionVars().EnableChangeColumnType = true - defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = enableChangeColumnType - }() - _, err := tk.Exec("alter table t_modify_column_args modify column a tinyint") c.Assert(err, NotNil) // error goes like `mock update version and tableInfo error,jobID=xx` diff --git a/ddl/db_integration_test.go b/ddl/db_integration_test.go index fe6bca7dc4563..d91158f991be2 100644 --- a/ddl/db_integration_test.go +++ b/ddl/db_integration_test.go @@ -41,7 +41,6 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/store/tikv/mockstore/cluster" - "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" @@ -58,8 +57,6 @@ var _ = Suite(&testIntegrationSuite3{&testIntegrationSuite{}}) var _ = Suite(&testIntegrationSuite4{&testIntegrationSuite{}}) var _ = Suite(&testIntegrationSuite5{&testIntegrationSuite{}}) var _ = Suite(&testIntegrationSuite6{&testIntegrationSuite{}}) -var _ = SerialSuites(&testIntegrationSuite7{&testIntegrationSuite{}}) -var _ = SerialSuites(&testIntegrationSuite8{&testIntegrationSuite{}}) type testIntegrationSuite struct { lease time.Duration @@ -128,8 +125,6 @@ type testIntegrationSuite4 struct{ *testIntegrationSuite } type testIntegrationSuite5 struct{ *testIntegrationSuite } type testIntegrationSuite6 struct{ *testIntegrationSuite } type testIntegrationSuite7 struct{ *testIntegrationSuite } -type testIntegrationSuite8 struct{ *testIntegrationSuite } -type testIntegrationSuite9 struct{ *testIntegrationSuite } func (s *testIntegrationSuite5) TestNoZeroDateMode(c *C) { tk := testkit.NewTestKit(c, s.store) @@ -224,9 +219,10 @@ func (s *testIntegrationSuite2) TestCreateTableWithKeyWord(c *C) { c.Assert(err, IsNil) } -func (s *testIntegrationSuite1) TestUniqueKeyNullValue(c *C) { +func (s *testIntegrationSuite6) TestUniqueKeyNullValue(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("USE test") + tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int primary key, b varchar(255))") tk.MustExec("insert into t values(1, NULL)") @@ -309,7 +305,7 @@ func (s *testIntegrationSuite2) TestIssue19229(c *C) { tk.MustExec("drop table sett") } -func (s *testIntegrationSuite1) TestIndexLength(c *C) { +func (s *testIntegrationSuite7) TestIndexLength(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("create table idx_len(a int(0), b timestamp(0), c datetime(0), d time(0), f float(0), g decimal(0))") @@ -385,7 +381,7 @@ func (s *testIntegrationSuite1) TestIssue4432(c *C) { tk.MustExec("drop table tx") } -func (s *testIntegrationSuite1) TestIssue5092(c *C) { +func (s *testIntegrationSuite7) TestIssue5092(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -642,6 +638,9 @@ func (s *testIntegrationSuite3) TestTableDDLWithFloatType(c *C) { } func (s *testIntegrationSuite1) TestTableDDLWithTimeType(c *C) { + if israce.RaceEnabled { + c.Skip("skip race test") + } tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -956,22 +955,22 @@ func (s *testIntegrationSuite5) TestModifyColumnOption(c *C) { tk.MustExec("create table t2 (b char, c int)") assertErrCode("alter table t2 modify column c int references t1(a)", errMsg) _, err := tk.Exec("alter table t1 change a a varchar(16)") - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: type varchar(16) not match origin int(11), and tidb_enable_change_column_type is false") + c.Assert(err, IsNil) _, err = tk.Exec("alter table t1 change a a varchar(10)") - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: type varchar(10) not match origin int(11), and tidb_enable_change_column_type is false") + c.Assert(err, IsNil) _, err = tk.Exec("alter table t1 change a a datetime") - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: type datetime not match origin int(11), and tidb_enable_change_column_type is false") + c.Assert(err, IsNil) _, err = tk.Exec("alter table t1 change a a int(11) unsigned") - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: can't change unsigned integer to signed or vice versa, and tidb_enable_change_column_type is false") + c.Assert(err, IsNil) _, err = tk.Exec("alter table t2 change b b int(11) unsigned") - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: type int(11) not match origin char(1), and tidb_enable_change_column_type is false") + c.Assert(err, IsNil) } func (s *testIntegrationSuite4) TestIndexOnMultipleGeneratedColumn(c *C) { tk := testkit.NewTestKit(c, s.store) - tk.MustExec("create database if not exists test") tk.MustExec("use test") + tk.MustExec("drop table if exists t") tk.MustExec("create table t (a int, b int as (a + 1), c int as (b + 1))") tk.MustExec("insert into t (a) values (1)") @@ -980,42 +979,72 @@ func (s *testIntegrationSuite4) TestIndexOnMultipleGeneratedColumn(c *C) { res := tk.MustQuery("select * from t use index(idx) where c > 1") tk.MustQuery("select * from t ignore index(idx) where c > 1").Check(res.Rows()) tk.MustExec("admin check table t") +} + +func (s *testIntegrationSuite4) TestIndexOnMultipleGeneratedColumn1(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database if not exists test") + tk.MustExec("use test") tk.MustExec("drop table if exists t") tk.MustExec("create table t (a int, b int as (a + 1), c int as (b + 1), d int as (c + 1))") tk.MustExec("insert into t (a) values (1)") tk.MustExec("create index idx on t (d)") tk.MustQuery("select * from t where d > 2").Check(testkit.Rows("1 2 3 4")) - res = tk.MustQuery("select * from t use index(idx) where d > 2") + res := tk.MustQuery("select * from t use index(idx) where d > 2") tk.MustQuery("select * from t ignore index(idx) where d > 2").Check(res.Rows()) tk.MustExec("admin check table t") +} + +func (s *testIntegrationSuite4) TestIndexOnMultipleGeneratedColumn2(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database if not exists test") + tk.MustExec("use test") tk.MustExec("drop table if exists t") tk.MustExec("create table t (a bigint, b decimal as (a+1), c varchar(20) as (b*2), d float as (a*23+b-1+length(c)))") tk.MustExec("insert into t (a) values (1)") tk.MustExec("create index idx on t (d)") tk.MustQuery("select * from t where d > 2").Check(testkit.Rows("1 2 4 25")) - res = tk.MustQuery("select * from t use index(idx) where d > 2") + res := tk.MustQuery("select * from t use index(idx) where d > 2") tk.MustQuery("select * from t ignore index(idx) where d > 2").Check(res.Rows()) tk.MustExec("admin check table t") +} + +func (s *testIntegrationSuite4) TestIndexOnMultipleGeneratedColumn3(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database if not exists test") + tk.MustExec("use test") tk.MustExec("drop table if exists t") tk.MustExec("create table t (a varchar(10), b float as (length(a)+123), c varchar(20) as (right(a, 2)), d float as (b+b-7+1-3+3*ASCII(c)))") tk.MustExec("insert into t (a) values ('adorable')") tk.MustExec("create index idx on t (d)") tk.MustQuery("select * from t where d > 2").Check(testkit.Rows("adorable 131 le 577")) // 131+131-7+1-3+3*108 - res = tk.MustQuery("select * from t use index(idx) where d > 2") + res := tk.MustQuery("select * from t use index(idx) where d > 2") tk.MustQuery("select * from t ignore index(idx) where d > 2").Check(res.Rows()) tk.MustExec("admin check table t") +} + +func (s *testIntegrationSuite4) TestIndexOnMultipleGeneratedColumn4(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database if not exists test") + tk.MustExec("use test") tk.MustExec("drop table if exists t") tk.MustExec("create table t (a bigint, b decimal as (a), c int(10) as (a+b), d float as (a+b+c), e decimal as (a+b+c+d))") tk.MustExec("insert into t (a) values (1)") tk.MustExec("create index idx on t (d)") tk.MustQuery("select * from t where d > 2").Check(testkit.Rows("1 1 2 4 8")) - res = tk.MustQuery("select * from t use index(idx) where d > 2") + res := tk.MustQuery("select * from t use index(idx) where d > 2") tk.MustQuery("select * from t ignore index(idx) where d > 2").Check(res.Rows()) tk.MustExec("admin check table t") +} + +func (s *testIntegrationSuite4) TestIndexOnMultipleGeneratedColumn5(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database if not exists test") + tk.MustExec("use test") tk.MustExec("drop table if exists t") tk.MustExec("create table t(a bigint, b bigint as (a+1) virtual, c bigint as (b+1) virtual)") @@ -1025,12 +1054,12 @@ func (s *testIntegrationSuite4) TestIndexOnMultipleGeneratedColumn(c *C) { tk.MustExec("alter table t add column(d bigint as (c+1) virtual)") tk.MustExec("alter table t add index idx_d(d)") tk.MustQuery("select * from t where d > 2").Check(testkit.Rows("1 2 3 4")) - res = tk.MustQuery("select * from t use index(idx_d) where d > 2") + res := tk.MustQuery("select * from t use index(idx_d) where d > 2") tk.MustQuery("select * from t ignore index(idx_d) where d > 2").Check(res.Rows()) tk.MustExec("admin check table t") } -func (s *testIntegrationSuite2) TestCaseInsensitiveCharsetAndCollate(c *C) { +func (s *testIntegrationSuite6) TestCaseInsensitiveCharsetAndCollate(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("create database if not exists test_charset_collate") @@ -1204,7 +1233,7 @@ func (s *testIntegrationSuite5) TestBackwardCompatibility(c *C) { // Split the table. tableStart := tablecodec.GenTableRecordPrefix(tbl.Meta().ID) - s.cluster.SplitKeys(tableStart, tableStart.PrefixNext(), 100) + s.cluster.SplitKeys(tableStart, tableStart.PrefixNext(), 10) unique := false indexName := model.NewCIStr("idx_b") @@ -1246,7 +1275,6 @@ func (s *testIntegrationSuite5) TestBackwardCompatibility(c *C) { historyJob, err := getHistoryDDLJob(s.store, job.ID) c.Assert(err, IsNil) if historyJob == nil { - continue } c.Assert(historyJob.Error, IsNil) @@ -1260,41 +1288,6 @@ func (s *testIntegrationSuite5) TestBackwardCompatibility(c *C) { tk.MustExec("admin check index t idx_b") } -func (s *testIntegrationSuite3) TestMultiRegionGetTableEndHandle(c *C) { - tk := testkit.NewTestKit(c, s.store) - tk.MustExec("drop database if exists test_get_endhandle") - tk.MustExec("create database test_get_endhandle") - tk.MustExec("use test_get_endhandle") - - tk.MustExec("create table t(a bigint PRIMARY KEY nonclustered, b int)") - for i := 0; i < 1000; i++ { - tk.MustExec(fmt.Sprintf("insert into t values(%v, %v)", i, i)) - } - - // Get table ID for split. - dom := domain.GetDomain(tk.Se) - is := dom.InfoSchema() - tbl, err := is.TableByName(model.NewCIStr("test_get_endhandle"), model.NewCIStr("t")) - c.Assert(err, IsNil) - tblID := tbl.Meta().ID - - d := s.dom.DDL() - testCtx := newTestMaxTableRowIDContext(c, d, tbl) - - // Split the table. - tableStart := tablecodec.GenTableRecordPrefix(tblID) - s.cluster.SplitKeys(tableStart, tableStart.PrefixNext(), 100) - - maxHandle, emptyTable := getMaxTableHandle(testCtx, s.store) - c.Assert(emptyTable, IsFalse) - c.Assert(maxHandle, Equals, kv.IntHandle(1000)) - - tk.MustExec("insert into t values(10000, 1000)") - maxHandle, emptyTable = getMaxTableHandle(testCtx, s.store) - c.Assert(emptyTable, IsFalse) - c.Assert(maxHandle, Equals, kv.IntHandle(1001)) -} - type testMaxTableRowIDContext struct { c *C d ddl.DDL @@ -1313,7 +1306,7 @@ func getMaxTableHandle(ctx *testMaxTableRowIDContext, store kv.Storage) (kv.Hand c := ctx.c d := ctx.d tbl := ctx.tbl - curVer, err := store.CurrentVersion(oracle.GlobalTxnScope) + curVer, err := store.CurrentVersion(kv.GlobalTxnScope) c.Assert(err, IsNil) maxHandle, emptyTable, err := d.GetTableMaxHandle(curVer.Ver, tbl.(table.PhysicalTable)) c.Assert(err, IsNil) @@ -1341,6 +1334,9 @@ func getHistoryDDLJob(store kv.Storage, id int64) (*model.Job, error) { } func (s *testIntegrationSuite6) TestCreateTableTooLarge(c *C) { + if israce.RaceEnabled { + c.Skip("skip race test") + } tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -1362,7 +1358,7 @@ func (s *testIntegrationSuite6) TestCreateTableTooLarge(c *C) { atomic.StoreUint32(&config.GetGlobalConfig().TableColumnCountLimit, originLimit) } -func (s *testIntegrationSuite8) TestCreateTableTooManyIndexes(c *C) { +func (s *testSerialDBSuite1) TestCreateTableTooManyIndexes(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -1491,7 +1487,7 @@ func (s *testIntegrationSuite6) TestAddColumnTooMany(c *C) { tk.MustGetErrCode(alterSQL, errno.ErrTooManyFields) } -func (s *testIntegrationSuite8) TestCreateTooManyIndexes(c *C) { +func (s *testSerialDBSuite1) TestCreateTooManyIndexes(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") count := config.GetGlobalConfig().IndexLimit - 1 @@ -1513,7 +1509,7 @@ func (s *testIntegrationSuite8) TestCreateTooManyIndexes(c *C) { tk.MustGetErrCode(alterSQL, errno.ErrTooManyKeys) } -func (s *testIntegrationSuite8) TestCreateSecondaryIndexInCluster(c *C) { +func (s *testSerialDBSuite1) TestCreateSecondaryIndexInCluster(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -2243,7 +2239,7 @@ func (s *testIntegrationSuite3) TestParserIssue284(c *C) { tk.MustExec("drop table test.t_parser_issue_284_2") } -func (s *testIntegrationSuite7) TestAddExpressionIndex(c *C) { +func (s *testSerialDBSuite1) TestAddExpressionIndex(c *C) { config.UpdateGlobal(func(conf *config.Config) { conf.Experimental.AllowsExpressionIndex = true }) @@ -2309,7 +2305,7 @@ func (s *testIntegrationSuite7) TestAddExpressionIndex(c *C) { tk.MustGetErrMsg("create table t(a int, key ((a+1)));", "[ddl:8200]Unsupported creating expression index without allow-expression-index in config") } -func (s *testIntegrationSuite7) TestCreateExpressionIndexError(c *C) { +func (s *testSerialDBSuite1) TestCreateExpressionIndexError(c *C) { defer config.RestoreFunc()() config.UpdateGlobal(func(conf *config.Config) { conf.Experimental.AllowsExpressionIndex = true @@ -2352,7 +2348,7 @@ func (s *testIntegrationSuite7) TestCreateExpressionIndexError(c *C) { tk.MustGetErrCode("CREATE TABLE t1 (col1 INT, PRIMARY KEY ((ABS(col1))) NONCLUSTERED);", errno.ErrFunctionalIndexPrimaryKey) } -func (s *testIntegrationSuite7) TestAddExpressionIndexOnPartition(c *C) { +func (s *testSerialDBSuite1) TestAddExpressionIndexOnPartition(c *C) { config.UpdateGlobal(func(conf *config.Config) { conf.Experimental.AllowsExpressionIndex = true }) @@ -2586,27 +2582,27 @@ func (s *testIntegrationSuite5) TestDropColumnsWithMultiIndex(c *C) { tk.MustQuery(query).Check(testkit.Rows()) } -func (s *testIntegrationSuite5) TestDropLastVisibleColumn(c *C) { +func (s *testSerialDBSuite) TestDropLastVisibleColumnOrColumns(c *C) { + defer config.RestoreFunc() + config.UpdateGlobal(func(conf *config.Config) { + conf.Experimental.AllowsExpressionIndex = true + }) tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test_db") tk.MustExec("create table t_drop_last_column(x int, key((1+1)))") - defer tk.MustExec("drop table if exists t_drop_last_column") _, err := tk.Exec("alter table t_drop_last_column drop column x") c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "[ddl:1113]A table must have at least 1 column") -} - -func (s *testIntegrationSuite5) TestDropLastVisibleColumns(c *C) { - tk := testkit.NewTestKit(c, s.store) - tk.MustExec("use test_db") + // for visible columns tk.MustExec("create table t_drop_last_columns(x int, y int, key((1+1)))") - defer tk.MustExec("drop table if exists t_drop_last_columns") - _, err := tk.Exec("alter table t_drop_last_columns drop column x, drop column y") + _, err = tk.Exec("alter table t_drop_last_columns drop column x, drop column y") c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "[ddl:1113]A table must have at least 1 column") + + tk.MustExec("drop table if exists t_drop_last_column, t_drop_last_columns") } -func (s *testIntegrationSuite7) TestAutoIncrementTableOption(c *C) { +func (s *testSerialDBSuite1) TestAutoIncrementTableOption(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("drop database if exists test_auto_inc_table_opt;") tk.MustExec("create database test_auto_inc_table_opt;") @@ -2698,7 +2694,7 @@ func (s *testIntegrationSuite3) TestStrictDoubleTypeCheck(c *C) { tk.MustExec(sql) } -func (s *testIntegrationSuite7) TestDuplicateErrorMessage(c *C) { +func (s *testSerialDBSuite) TestDuplicateErrorMessage(c *C) { defer collate.SetNewCollationEnabledForTest(false) tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -2773,3 +2769,28 @@ func (s *testIntegrationSuite3) TestIssue21835(c *C) { _, err := tk.Exec("create table t( col decimal(1,2) not null default 0);") c.Assert(err.Error(), Equals, "[types:1427]For float(M,D), double(M,D) or decimal(M,D), M must be >= D (column 'col').") } + +func (s *testIntegrationSuite3) TestCreateTemporaryTable(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t;") + + // Grammar error. + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustGetErrCode("create global temporary table t(a double(0, 0))", errno.ErrParse) + tk.MustGetErrCode("create temporary table t(id int) on commit delete rows", errno.ErrParse) + tk.MustGetErrCode("create temporary table t(id int) on commit preserve rows", errno.ErrParse) + tk.MustGetErrCode("create table t(id int) on commit delete rows", errno.ErrParse) + tk.MustGetErrCode("create table t(id int) on commit preserve rows", errno.ErrParse) + + // Not support yet. + tk.MustGetErrCode("create global temporary table t (id int) on commit preserve rows", errno.ErrUnsupportedDDLOperation) + // Engine type can only be 'memory' or empty for now. + tk.MustGetErrCode("create global temporary table t (id int) engine = 'innodb' on commit delete rows", errno.ErrUnsupportedDDLOperation) + // Follow the behaviour of the old version TiDB: parse and ignore the 'temporary' keyword. + tk.MustGetErrCode("create temporary table t(id int)", errno.ErrNotSupportedYet) + + tk.MustExec("set @@tidb_enable_noop_functions = 1") + tk.MustExec("create temporary table t (id int)") + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning 1105 local TEMPORARY TABLE is not supported yet, TEMPORARY will be parsed but ignored")) +} diff --git a/ddl/db_partition_test.go b/ddl/db_partition_test.go index b52644c151fd4..04e69ee89dd82 100644 --- a/ddl/db_partition_test.go +++ b/ddl/db_partition_test.go @@ -45,6 +45,7 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/admin" "github.com/pingcap/tidb/util/collate" + "github.com/pingcap/tidb/util/israce" "github.com/pingcap/tidb/util/mock" "github.com/pingcap/tidb/util/testkit" ) @@ -359,7 +360,7 @@ func (s *testIntegrationSuite2) TestCreateTableWithHashPartition(c *C) { tk.MustExec("create table t4 (a int, b int) partition by hash(floor(a-b)) partitions 10") } -func (s *testIntegrationSuite7) TestCreateTableWithRangeColumnPartition(c *C) { +func (s *testSerialDBSuite1) TestCreateTableWithRangeColumnPartition(c *C) { collate.SetNewCollationEnabledForTest(true) defer collate.SetNewCollationEnabledForTest(false) tk := testkit.NewTestKit(c, s.store) @@ -593,6 +594,9 @@ create table log_message_1 ( } func (s *testIntegrationSuite1) TestDisableTablePartition(c *C) { + if israce.RaceEnabled { + c.Skip("skip race test") + } tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test;") for _, v := range []string{"'AUTO'", "'OFF'", "0", "'ON'"} { @@ -1603,7 +1607,7 @@ func (s *testIntegrationSuite5) TestMultiPartitionDropAndTruncate(c *C) { result.Check(testkit.Rows(`2010`)) } -func (s *testIntegrationSuite7) TestDropPartitionWithGlobalIndex(c *C) { +func (s *testSerialDBSuite1) TestDropPartitionWithGlobalIndex(c *C) { config.UpdateGlobal(func(conf *config.Config) { conf.EnableGlobalIndex = true }) @@ -1641,7 +1645,7 @@ func (s *testIntegrationSuite7) TestDropPartitionWithGlobalIndex(c *C) { }) } -func (s *testIntegrationSuite7) TestAlterTableExchangePartition(c *C) { +func (s *testSerialDBSuite1) TestAlterTableExchangePartition(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists e") @@ -2076,7 +2080,7 @@ func (s *testIntegrationSuite4) TestExchangePartitionTableCompatiable(c *C) { c.Assert(err, IsNil) } -func (s *testIntegrationSuite7) TestExchangePartitionExpressIndex(c *C) { +func (s *testSerialDBSuite1) TestExchangePartitionExpressIndex(c *C) { config.UpdateGlobal(func(conf *config.Config) { conf.Experimental.AllowsExpressionIndex = true }) @@ -3205,7 +3209,7 @@ func (s *testIntegrationSuite3) TestUnsupportedPartitionManagementDDLs(c *C) { c.Assert(err, ErrorMatches, ".*alter table partition is unsupported") } -func (s *testIntegrationSuite7) TestCommitWhenSchemaChange(c *C) { +func (s *testSerialDBSuite1) TestCommitWhenSchemaChange(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec(`create table schema_change (a int, b timestamp) @@ -3270,7 +3274,7 @@ func (s *testIntegrationSuite7) TestCommitWhenSchemaChange(c *C) { tk.MustQuery("select * from nt").Check(testkit.Rows()) } -func (s *testIntegrationSuite7) TestCreatePartitionTableWithWrongType(c *C) { +func (s *testSerialDBSuite1) TestCreatePartitionTableWithWrongType(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -3311,7 +3315,7 @@ func (s *testIntegrationSuite7) TestCreatePartitionTableWithWrongType(c *C) { c.Assert(err, NotNil) } -func (s *testIntegrationSuite7) TestAddPartitionForTableWithWrongType(c *C) { +func (s *testSerialDBSuite1) TestAddPartitionForTableWithWrongType(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop tables if exists t_int, t_char, t_date") @@ -3361,7 +3365,7 @@ func (s *testIntegrationSuite7) TestAddPartitionForTableWithWrongType(c *C) { c.Assert(ddl.ErrWrongTypeColumnValue.Equal(err), IsTrue) } -func (s *testIntegrationSuite7) TestPartitionListWithTimeType(c *C) { +func (s *testSerialDBSuite1) TestPartitionListWithTimeType(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("use test;") tk.MustExec("set @@session.tidb_enable_list_partition = ON") @@ -3370,7 +3374,7 @@ func (s *testIntegrationSuite7) TestPartitionListWithTimeType(c *C) { tk.MustQuery(`select * from t_list1 partition (p0);`).Check(testkit.Rows("2018-02-03")) } -func (s *testIntegrationSuite7) TestPartitionListWithNewCollation(c *C) { +func (s *testSerialDBSuite1) TestPartitionListWithNewCollation(c *C) { collate.SetNewCollationEnabledForTest(true) defer collate.SetNewCollationEnabledForTest(false) tk := testkit.NewTestKitWithInit(c, s.store) @@ -3387,8 +3391,9 @@ func (s *testIntegrationSuite7) TestPartitionListWithNewCollation(c *C) { c.Assert(strings.Contains(str, "partition:p0"), IsTrue) } -func (s *testIntegrationSuite7) TestAddTableWithPartition(c *C) { +func (s *testSerialDBSuite1) TestAddTableWithPartition(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("set tidb_enable_global_temporary_table=true") tk.MustExec("use test;") tk.MustExec("drop table if exists global_partition_table;") tk.MustGetErrCode("create global temporary table global_partition_table (a int, b int) partition by hash(a) partitions 3 ON COMMIT DELETE ROWS;", errno.ErrPartitionNoTemporary) diff --git a/ddl/db_test.go b/ddl/db_test.go index e865de39d3248..0082322b53680 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -43,6 +43,7 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/meta/autoid" + "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" @@ -76,8 +77,9 @@ var _ = Suite(&testDBSuite4{&testDBSuite{}}) var _ = Suite(&testDBSuite5{&testDBSuite{}}) var _ = Suite(&testDBSuite6{&testDBSuite{}}) var _ = Suite(&testDBSuite7{&testDBSuite{}}) -var _ = SerialSuites(&testSerialDBSuite{&testDBSuite{}}) var _ = Suite(&testDBSuite8{&testDBSuite{}}) +var _ = SerialSuites(&testSerialDBSuite{&testDBSuite{}}) +var _ = SerialSuites(&testSerialDBSuite1{&testDBSuite{}}) const defaultBatchSize = 1024 const defaultReorgBatchSize = 256 @@ -90,6 +92,7 @@ type testDBSuite struct { s session.Session lease time.Duration autoIDStep int64 + ctx sessionctx.Context } func setUpSuite(s *testDBSuite, c *C) { @@ -114,6 +117,7 @@ func setUpSuite(s *testDBSuite, c *C) { c.Assert(err, IsNil) s.s, err = session.CreateSession4Test(s.store) c.Assert(err, IsNil) + s.ctx = s.s.(sessionctx.Context) _, err = s.s.Execute(context.Background(), "create database test_db") c.Assert(err, IsNil) @@ -145,8 +149,9 @@ type testDBSuite4 struct{ *testDBSuite } type testDBSuite5 struct{ *testDBSuite } type testDBSuite6 struct{ *testDBSuite } type testDBSuite7 struct{ *testDBSuite } -type testSerialDBSuite struct{ *testDBSuite } type testDBSuite8 struct{ *testDBSuite } +type testSerialDBSuite struct{ *testDBSuite } +type testSerialDBSuite1 struct{ *testDBSuite } func testAddIndexWithPK(tk *testkit.TestKit) { tk.MustExec("drop table if exists test_add_index_with_pk") @@ -287,7 +292,7 @@ func backgroundExec(s kv.Storage, sql string, done chan error) { } // TestAddPrimaryKeyRollback1 is used to test scenarios that will roll back when a duplicate primary key is encountered. -func (s *testDBSuite5) TestAddPrimaryKeyRollback1(c *C) { +func (s *testDBSuite8) TestAddPrimaryKeyRollback1(c *C) { hasNullValsInKey := false idxName := "PRIMARY" addIdxSQL := "alter table t1 add primary key c3_index (c3);" @@ -296,7 +301,7 @@ func (s *testDBSuite5) TestAddPrimaryKeyRollback1(c *C) { } // TestAddPrimaryKeyRollback2 is used to test scenarios that will roll back when a null primary key is encountered. -func (s *testDBSuite1) TestAddPrimaryKeyRollback2(c *C) { +func (s *testDBSuite8) TestAddPrimaryKeyRollback2(c *C) { hasNullValsInKey := true idxName := "PRIMARY" addIdxSQL := "alter table t1 add primary key c3_index (c3);" @@ -444,7 +449,7 @@ LOOP: tk.MustExec("drop table t1") } -func (s *testDBSuite5) TestCancelAddPrimaryKey(c *C) { +func (s *testDBSuite8) TestCancelAddPrimaryKey(c *C) { idxName := "primary" addIdxSQL := "alter table t1 add primary key idx_c2 (c2);" testCancelAddIndex(c, s.store, s.dom.DDL(), s.lease, idxName, addIdxSQL, "", s.dom) @@ -460,7 +465,7 @@ func (s *testDBSuite5) TestCancelAddPrimaryKey(c *C) { tk.MustExec("drop table t1") } -func (s *testDBSuite3) TestCancelAddIndex(c *C) { +func (s *testDBSuite7) TestCancelAddIndex(c *C) { idxName := "c3_index " addIdxSQL := "create unique index c3_index on t1 (c3)" testCancelAddIndex(c, s.store, s.dom.DDL(), s.lease, idxName, addIdxSQL, "", s.dom) @@ -1072,7 +1077,7 @@ func (s *testDBSuite6) TestAddMultiColumnsIndexClusterIndex(c *C) { tk.MustExec("admin check table t;") } -func (s *testDBSuite1) TestAddPrimaryKey1(c *C) { +func (s *testDBSuite6) TestAddPrimaryKey1(c *C) { testAddIndex(c, s.store, s.lease, testPlain, "create table test_add_index (c1 bigint, c2 bigint, c3 bigint, unique key(c1))", "primary") } @@ -1105,7 +1110,7 @@ func (s *testDBSuite4) TestAddPrimaryKey4(c *C) { partition p4 values less than maxvalue)`, "primary") } -func (s *testDBSuite1) TestAddIndex1(c *C) { +func (s *testDBSuite6) TestAddIndex1(c *C) { testAddIndex(c, s.store, s.lease, testPlain, "create table test_add_index (c1 bigint, c2 bigint, c3 bigint, primary key(c1))", "") } @@ -1127,7 +1132,7 @@ func (s *testDBSuite3) TestAddIndex3(c *C) { partition by hash (c1) partitions 4;`, "") } -func (s *testDBSuite4) TestAddIndex4(c *C) { +func (s *testDBSuite8) TestAddIndex4(c *C) { testAddIndex(c, s.store, s.lease, testPartition, `create table test_add_index (c1 bigint, c2 bigint, c3 bigint, primary key(c1)) partition by range columns (c1) ( @@ -2035,7 +2040,7 @@ func (s *testDBSuite5) TestCreateIndexType(c *C) { tk.MustExec(sql) } -func (s *testDBSuite1) TestColumn(c *C) { +func (s *testDBSuite6) TestColumn(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use " + s.schemaName) tk.MustExec("create table t2 (c1 int, c2 int, c3 int)") @@ -2280,10 +2285,8 @@ func (s *testDBSuite6) TestDropColumn(c *C) { testddlutil.ExecMultiSQLInGoroutine(c, s.store, "drop_col_db", []string{"insert into t2 set c1 = 1, c2 = 1, c3 = 1, c4 = 1"}, dmlDone) } for i := 0; i < num; i++ { - select { - case err := <-ddlDone: - c.Assert(err, IsNil, Commentf("err:%v", errors.ErrorStack(err))) - } + err := <-ddlDone + c.Assert(err, IsNil, Commentf("err:%v", errors.ErrorStack(err))) } // Test for drop partition table column. @@ -2365,7 +2368,7 @@ func (s *testDBSuite4) TestChangeColumn(c *C) { sql = "alter table t4 change c2 a bigint not null;" tk.MustGetErrCode(sql, mysql.WarnDataTruncated) sql = "alter table t3 modify en enum('a', 'z', 'b', 'c') not null default 'a'" - tk.MustGetErrCode(sql, errno.ErrUnsupportedDDLOperation) + tk.MustExec(sql) // Rename to an existing column. s.mustExec(tk, c, "alter table t3 add column a bigint") sql = "alter table t3 change aa a bigint" @@ -3009,7 +3012,7 @@ func (s *testDBSuite2) TestTableForeignKey(c *C) { tk.MustExec("create table t3 (a int, b int);") failSQL = "alter table t1 add foreign key (c) REFERENCES t3(a);" tk.MustGetErrCode(failSQL, errno.ErrKeyColumnDoesNotExits) - // test oreign key not match error + // test origin key not match error failSQL = "alter table t1 add foreign key (a) REFERENCES t3(a, b);" tk.MustGetErrCode(failSQL, errno.ErrWrongFkDef) // Test drop column with foreign key. @@ -3028,7 +3031,26 @@ func (s *testDBSuite2) TestTableForeignKey(c *C) { tk.MustExec("drop table if exists t1,t2,t3,t4;") } -func (s *testDBSuite3) TestFKOnGeneratedColumns(c *C) { +func (s *testDBSuite2) TestTemporaryTableForeignKey(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1 (a int, b int);") + tk.MustExec("drop table if exists t1_tmp;") + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("create global temporary table t1_tmp (a int, b int) on commit delete rows;") + // test add foreign key. + tk.MustExec("drop table if exists t2;") + tk.MustExec("create table t2 (a int, b int);") + failSQL := "alter table t1_tmp add foreign key (c) REFERENCES t2(a);" + tk.MustGetErrCode(failSQL, mysql.ErrCannotAddForeign) + // Test drop column with foreign key. + failSQL = "create global temporary table t3 (c int,d int,foreign key (d) references t1 (b)) on commit delete rows;" + tk.MustGetErrCode(failSQL, mysql.ErrCannotAddForeign) + tk.MustExec("drop table if exists t1,t2,t3,t1_tmp;") +} + +func (s *testDBSuite8) TestFKOnGeneratedColumns(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") // test add foreign key to generated column @@ -3492,6 +3514,37 @@ out: tk.MustExec("drop table tnn") } +func (s *testDBSuite3) TestVirtualColumnDDL(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("use test") + tk.MustExec("drop table if exists test_gv_ddl") + tk.MustExec(`create global temporary table test_gv_ddl(a int, b int as (a+8) virtual, c int as (b + 2) stored) on commit delete rows;`) + defer tk.MustExec("drop table if exists test_gv_ddl") + is := tk.Se.(sessionctx.Context).GetInfoSchema().(infoschema.InfoSchema) + table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("test_gv_ddl")) + c.Assert(err, IsNil) + testCases := []struct { + generatedExprString string + generatedStored bool + }{ + {"", false}, + {"`a` + 8", false}, + {"`b` + 2", true}, + } + for i, column := range table.Meta().Columns { + c.Assert(column.GeneratedExprString, Equals, testCases[i].generatedExprString) + c.Assert(column.GeneratedStored, Equals, testCases[i].generatedStored) + } + result := tk.MustQuery(`DESC test_gv_ddl`) + result.Check(testkit.Rows(`a int(11) YES `, `b int(11) YES VIRTUAL GENERATED`, `c int(11) YES STORED GENERATED`)) + tk.MustExec("begin;") + tk.MustExec("insert into test_gv_ddl values (1, default, default)") + tk.MustQuery("select * from test_gv_ddl").Check(testkit.Rows("1 9 11")) + _, err = tk.Exec("commit") + c.Assert(err, IsNil) +} + func (s *testDBSuite3) TestGeneratedColumnDDL(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -3948,12 +4001,6 @@ func (s *testSerialDBSuite) TestModifyColumnnReorgInfo(c *C) { // Make sure the count of regions more than backfill workers. tk.MustQuery("split table t1 between (0) and (8192) regions 8;").Check(testkit.Rows("8 1")) - enableChangeColumnType := tk.Se.GetSessionVars().EnableChangeColumnType - tk.Se.GetSessionVars().EnableChangeColumnType = true - defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = enableChangeColumnType - }() - tbl := s.testGetTable(c, "t1") originalHook := s.dom.DDL().GetHook() defer s.dom.DDL().(ddl.DDLForTest).SetHook(originalHook) @@ -4054,11 +4101,8 @@ func (s *testSerialDBSuite) TestModifyColumnnReorgInfo(c *C) { func (s *testSerialDBSuite) TestModifyColumnNullToNotNullWithChangingVal2(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) - enableChangeColumnType := tk.Se.GetSessionVars().EnableChangeColumnType - tk.Se.GetSessionVars().EnableChangeColumnType = true c.Assert(failpoint.Enable("github.com/pingcap/tidb/ddl/mockInsertValueAfterCheckNull", `return("insert into test.tt values (NULL, NULL)")`), IsNil) defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = enableChangeColumnType err := failpoint.Disable("github.com/pingcap/tidb/ddl/mockInsertValueAfterCheckNull") c.Assert(err, IsNil) }() @@ -4087,7 +4131,6 @@ func (s *testSerialDBSuite) TestModifyColumnNullToNotNullWithChangingVal(c *C) { func (s *testSerialDBSuite) TestModifyColumnBetweenStringTypes(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) - tk.Se.GetSessionVars().EnableChangeColumnType = true // varchar to varchar tk.MustExec("drop table if exists tt;") @@ -4100,9 +4143,11 @@ func (s *testSerialDBSuite) TestModifyColumnBetweenStringTypes(c *C) { tk.MustGetErrMsg("alter table tt change a a varchar(4);", "[types:1406]Data Too Long, field len 4, data len 5") tk.MustExec("alter table tt change a a varchar(100);") - tk.MustExec("drop table if exists tt;") - tk.MustExec("create table tt (a char(10));") - tk.MustExec("insert into tt values ('111'),('10000');") + // varchar to char + tk.MustExec("alter table tt change a a char(10);") + c2 = getModifyColumn(c, s.s.(sessionctx.Context), "test", "tt", "a", false) + c.Assert(c2.FieldType.Tp, Equals, mysql.TypeString) + c.Assert(c2.FieldType.Flen, Equals, 10) tk.MustQuery("select * from tt").Check(testkit.Rows("111", "10000")) tk.MustGetErrMsg("alter table tt change a a char(4);", "[types:1406]Data Too Long, field len 4, data len 5") @@ -4167,14 +4212,6 @@ func testModifyColumnNullToNotNull(c *C, s *testDBSuite, enableChangeColumnType s.mustExec(tk, c, "drop table if exists t1") s.mustExec(tk, c, "create table t1 (c1 int, c2 int);") - if enableChangeColumnType { - enableChangeColumnType := tk.Se.GetSessionVars().EnableChangeColumnType - tk.Se.GetSessionVars().EnableChangeColumnType = true - defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = enableChangeColumnType - }() - } - tbl := s.testGetTable(c, "t1") getModifyColumn(c, s.s.(sessionctx.Context), s.schemaName, "t1", "c2", false) @@ -4801,52 +4838,9 @@ func (s *testSerialDBSuite) TestModifyColumnCharset(c *C) { } -func (s *testDBSuite1) TestModifyColumnTime(c *C) { - limit := variable.GetDDLErrorCountLimit() - variable.SetDDLErrorCountLimit(3) - - tk := testkit.NewTestKit(c, s.store) - tk.MustExec("use test_db") - enableChangeColumnType := tk.Se.GetSessionVars().EnableChangeColumnType - tk.Se.GetSessionVars().EnableChangeColumnType = true - - // Set time zone to UTC. - originalTz := tk.Se.GetSessionVars().TimeZone - tk.Se.GetSessionVars().TimeZone = time.UTC - defer func() { - variable.SetDDLErrorCountLimit(limit) - tk.Se.GetSessionVars().EnableChangeColumnType = enableChangeColumnType - tk.Se.GetSessionVars().TimeZone = originalTz - }() - - now := time.Now().UTC() - now = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) - timeToDate1 := now.Format("2006-01-02") - timeToDate2 := now.AddDate(0, 0, 30).Format("2006-01-02") - - timeToDatetime1 := now.Add(20 * time.Hour).Add(12 * time.Second).Format("2006-01-02 15:04:05") - timeToDatetime2 := now.Add(20 * time.Hour).Format("2006-01-02 15:04:05") - timeToDatetime3 := now.Add(12 * time.Second).Format("2006-01-02 15:04:05") - timeToDatetime4 := now.AddDate(0, 0, 30).Add(20 * time.Hour).Add(12 * time.Second).Format("2006-01-02 15:04:05") - timeToDatetime5 := now.AddDate(0, 0, 30).Add(20 * time.Hour).Format("2006-01-02 15:04:05") - - timeToTimestamp1 := now.Add(20 * time.Hour).Add(12 * time.Second).Format("2006-01-02 15:04:05") - timeToTimestamp2 := now.Add(20 * time.Hour).Format("2006-01-02 15:04:05") - timeToTimestamp3 := now.Add(12 * time.Second).Format("2006-01-02 15:04:05") - timeToTimestamp4 := now.AddDate(0, 0, 30).Add(20 * time.Hour).Add(12 * time.Second).Format("2006-01-02 15:04:05") - timeToTimestamp5 := now.AddDate(0, 0, 30).Add(20 * time.Hour).Format("2006-01-02 15:04:05") +func (s *testDBSuite1) TestModifyColumnTime_TimeToYear(c *C) { currentYear := strconv.Itoa(time.Now().Year()) - - // 1. In conversion between date/time, fraction parts are taken into account - // Refer to doc: https://dev.mysql.com/doc/refman/5.7/en/date-and-time-type-conversion.html - // 2. Failed tests are commentd to pass unit-test - tests := []struct { - from string - value string - to string - expect string - err uint16 - }{ + tests := []testModifyColumnTimeCase{ // time to year, it's reasonable to return current year and discard the time (even if MySQL may get data out of range error). {"time", `"30 20:00:12"`, "year", currentYear, 0}, {"time", `"30 20:00"`, "year", currentYear, 0}, @@ -4862,7 +4856,16 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"time", `"20:00:12.498"`, "year", currentYear, 0}, {"time", `"200012.498"`, "year", currentYear, 0}, {"time", `200012.498`, "year", currentYear, 0}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_TimeToDate(c *C) { + now := time.Now().UTC() + now = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) + timeToDate1 := now.Format("2006-01-02") + timeToDate2 := now.AddDate(0, 0, 30).Format("2006-01-02") + tests := []testModifyColumnTimeCase{ // time to date {"time", `"30 20:00:12"`, "date", timeToDate2, 0}, {"time", `"30 20:00"`, "date", timeToDate2, 0}, @@ -4878,7 +4881,19 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"time", `"20:00:12.498"`, "date", timeToDate1, 0}, {"time", `"200012.498"`, "date", timeToDate1, 0}, {"time", `200012.498`, "date", timeToDate1, 0}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_TimeToDatetime(c *C) { + now := time.Now().UTC() + now = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) + timeToDatetime1 := now.Add(20 * time.Hour).Add(12 * time.Second).Format("2006-01-02 15:04:05") + timeToDatetime2 := now.Add(20 * time.Hour).Format("2006-01-02 15:04:05") + timeToDatetime3 := now.Add(12 * time.Second).Format("2006-01-02 15:04:05") + timeToDatetime4 := now.AddDate(0, 0, 30).Add(20 * time.Hour).Add(12 * time.Second).Format("2006-01-02 15:04:05") + timeToDatetime5 := now.AddDate(0, 0, 30).Add(20 * time.Hour).Format("2006-01-02 15:04:05") + tests := []testModifyColumnTimeCase{ // time to datetime {"time", `"30 20:00:12"`, "datetime", timeToDatetime4, 0}, {"time", `"30 20:00"`, "datetime", timeToDatetime5, 0}, @@ -4894,7 +4909,19 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"time", `"20:00:12.498"`, "datetime", timeToDatetime1, 0}, {"time", `"200012.498"`, "datetime", timeToDatetime1, 0}, {"time", `200012.498`, "datetime", timeToDatetime1, 0}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_TimeToTimestamp(c *C) { + now := time.Now().UTC() + now = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) + timeToTimestamp1 := now.Add(20 * time.Hour).Add(12 * time.Second).Format("2006-01-02 15:04:05") + timeToTimestamp2 := now.Add(20 * time.Hour).Format("2006-01-02 15:04:05") + timeToTimestamp3 := now.Add(12 * time.Second).Format("2006-01-02 15:04:05") + timeToTimestamp4 := now.AddDate(0, 0, 30).Add(20 * time.Hour).Add(12 * time.Second).Format("2006-01-02 15:04:05") + timeToTimestamp5 := now.AddDate(0, 0, 30).Add(20 * time.Hour).Format("2006-01-02 15:04:05") + tests := []testModifyColumnTimeCase{ // time to timestamp {"time", `"30 20:00:12"`, "timestamp", timeToTimestamp4, 0}, {"time", `"30 20:00"`, "timestamp", timeToTimestamp5, 0}, @@ -4910,7 +4937,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"time", `"20:00:12.498"`, "timestamp", timeToTimestamp1, 0}, {"time", `"200012.498"`, "timestamp", timeToTimestamp1, 0}, {"time", `200012.498`, "timestamp", timeToTimestamp1, 0}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite7) TestModifyColumnTime_DateToTime(c *C) { + tests := []testModifyColumnTimeCase{ // date to time {"date", `"2019-01-02"`, "time", "00:00:00", 0}, {"date", `"19-01-02"`, "time", "00:00:00", 0}, @@ -4918,7 +4950,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"date", `"190102"`, "time", "00:00:00", 0}, {"date", `20190102`, "time", "00:00:00", 0}, {"date", `190102`, "time", "00:00:00", 0}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_DateToYear(c *C) { + tests := []testModifyColumnTimeCase{ // date to year {"date", `"2019-01-02"`, "year", "2019", 0}, {"date", `"19-01-02"`, "year", "2019", 0}, @@ -4926,7 +4963,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"date", `"190102"`, "year", "2019", 0}, {"date", `20190102`, "year", "2019", 0}, {"date", `190102`, "year", "2019", 0}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_DateToDatetime(c *C) { + tests := []testModifyColumnTimeCase{ // date to datetime {"date", `"2019-01-02"`, "datetime", "2019-01-02 00:00:00", 0}, {"date", `"19-01-02"`, "datetime", "2019-01-02 00:00:00", 0}, @@ -4934,7 +4976,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"date", `"190102"`, "datetime", "2019-01-02 00:00:00", 0}, {"date", `20190102`, "datetime", "2019-01-02 00:00:00", 0}, {"date", `190102`, "datetime", "2019-01-02 00:00:00", 0}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_DateToTimestamp(c *C) { + tests := []testModifyColumnTimeCase{ // date to timestamp {"date", `"2019-01-02"`, "timestamp", "2019-01-02 00:00:00", 0}, {"date", `"19-01-02"`, "timestamp", "2019-01-02 00:00:00", 0}, @@ -4942,7 +4989,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"date", `"190102"`, "timestamp", "2019-01-02 00:00:00", 0}, {"date", `20190102`, "timestamp", "2019-01-02 00:00:00", 0}, {"date", `190102`, "timestamp", "2019-01-02 00:00:00", 0}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_TimestampToYear(c *C) { + tests := []testModifyColumnTimeCase{ // timestamp to year {"timestamp", `"2006-01-02 15:04:05"`, "year", "2006", 0}, {"timestamp", `"06-01-02 15:04:05"`, "year", "2006", 0}, @@ -4951,7 +5003,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"timestamp", `20060102150405`, "year", "2006", 0}, {"timestamp", `060102150405`, "year", "2006", 0}, {"timestamp", `"2006-01-02 23:59:59.506"`, "year", "2006", 0}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_TimestampToTime(c *C) { + tests := []testModifyColumnTimeCase{ // timestamp to time {"timestamp", `"2006-01-02 15:04:05"`, "time", "15:04:05", 0}, {"timestamp", `"06-01-02 15:04:05"`, "time", "15:04:05", 0}, @@ -4960,7 +5017,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"timestamp", `20060102150405`, "time", "15:04:05", 0}, {"timestamp", `060102150405`, "time", "15:04:05", 0}, {"timestamp", `"2006-01-02 23:59:59.506"`, "time", "00:00:00", 0}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_TimestampToDate(c *C) { + tests := []testModifyColumnTimeCase{ // timestamp to date {"timestamp", `"2006-01-02 15:04:05"`, "date", "2006-01-02", 0}, {"timestamp", `"06-01-02 15:04:05"`, "date", "2006-01-02", 0}, @@ -4969,7 +5031,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"timestamp", `20060102150405`, "date", "2006-01-02", 0}, {"timestamp", `060102150405`, "date", "2006-01-02", 0}, {"timestamp", `"2006-01-02 23:59:59.506"`, "date", "2006-01-03", 0}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_TimestampToDatetime(c *C) { + tests := []testModifyColumnTimeCase{ // timestamp to datetime {"timestamp", `"2006-01-02 15:04:05"`, "datetime", "2006-01-02 15:04:05", 0}, {"timestamp", `"06-01-02 15:04:05"`, "datetime", "2006-01-02 15:04:05", 0}, @@ -4978,7 +5045,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"timestamp", `20060102150405`, "datetime", "2006-01-02 15:04:05", 0}, {"timestamp", `060102150405`, "datetime", "2006-01-02 15:04:05", 0}, {"timestamp", `"2006-01-02 23:59:59.506"`, "datetime", "2006-01-03 00:00:00", 0}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_DatetimeToYear(c *C) { + tests := []testModifyColumnTimeCase{ // datetime to year {"datetime", `"2006-01-02 15:04:05"`, "year", "2006", 0}, {"datetime", `"06-01-02 15:04:05"`, "year", "2006", 0}, @@ -4990,7 +5062,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { // MySQL will get "Data truncation: Out of range value for column 'a' at row 1. {"datetime", `"1000-01-02 23:59:59"`, "year", "", errno.ErrInvalidYear}, {"datetime", `"9999-01-02 23:59:59"`, "year", "", errno.ErrInvalidYear}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_DatetimeToTime(c *C) { + tests := []testModifyColumnTimeCase{ // datetime to time {"datetime", `"2006-01-02 15:04:05"`, "time", "15:04:05", 0}, {"datetime", `"06-01-02 15:04:05"`, "time", "15:04:05", 0}, @@ -5001,7 +5078,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"datetime", `"2006-01-02 23:59:59.506"`, "time", "00:00:00", 0}, {"datetime", `"1000-01-02 23:59:59"`, "time", "23:59:59", 0}, {"datetime", `"9999-01-02 23:59:59"`, "time", "23:59:59", 0}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_DatetimeToDate(c *C) { + tests := []testModifyColumnTimeCase{ // datetime to date {"datetime", `"2006-01-02 15:04:05"`, "date", "2006-01-02", 0}, {"datetime", `"06-01-02 15:04:05"`, "date", "2006-01-02", 0}, @@ -5012,7 +5094,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"datetime", `"2006-01-02 23:59:59.506"`, "date", "2006-01-03", 0}, {"datetime", `"1000-01-02 23:59:59"`, "date", "1000-01-02", 0}, {"datetime", `"9999-01-02 23:59:59"`, "date", "9999-01-02", 0}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_DatetimeToTimestamp(c *C) { + tests := []testModifyColumnTimeCase{ // datetime to timestamp {"datetime", `"2006-01-02 15:04:05"`, "timestamp", "2006-01-02 15:04:05", 0}, {"datetime", `"06-01-02 15:04:05"`, "timestamp", "2006-01-02 15:04:05", 0}, @@ -5023,7 +5110,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"datetime", `"2006-01-02 23:59:59.506"`, "timestamp", "2006-01-03 00:00:00", 0}, {"datetime", `"1000-01-02 23:59:59"`, "timestamp", "1000-01-02 23:59:59", 0}, {"datetime", `"9999-01-02 23:59:59"`, "timestamp", "9999-01-02 23:59:59", 0}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_YearToTime(c *C) { + tests := []testModifyColumnTimeCase{ // year to time // failed cases are not handled by TiDB {"year", `"2019"`, "time", "00:20:19", 0}, @@ -5036,7 +5128,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"year", `69`, "time", "", errno.ErrTruncatedWrongValue}, {"year", `70`, "time", "", errno.ErrTruncatedWrongValue}, {"year", `99`, "time", "", errno.ErrTruncatedWrongValue}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_YearToDate(c *C) { + tests := []testModifyColumnTimeCase{ // year to date {"year", `"2019"`, "date", "", errno.ErrTruncatedWrongValue}, {"year", `2019`, "date", "", errno.ErrTruncatedWrongValue}, @@ -5049,7 +5146,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"year", `69`, "date", "", errno.ErrTruncatedWrongValue}, {"year", `70`, "date", "", errno.ErrTruncatedWrongValue}, {"year", `99`, "date", "", errno.ErrTruncatedWrongValue}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_YearToDatetime(c *C) { + tests := []testModifyColumnTimeCase{ // year to datetime {"year", `"2019"`, "datetime", "", errno.ErrTruncatedWrongValue}, {"year", `2019`, "datetime", "", errno.ErrTruncatedWrongValue}, @@ -5062,7 +5164,12 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"year", `69`, "datetime", "", errno.ErrTruncatedWrongValue}, {"year", `70`, "datetime", "", errno.ErrTruncatedWrongValue}, {"year", `99`, "datetime", "", errno.ErrTruncatedWrongValue}, + } + testModifyColumnTime(c, s.store, tests) +} +func (s *testDBSuite1) TestModifyColumnTime_YearToTimestamp(c *C) { + tests := []testModifyColumnTimeCase{ // year to timestamp {"year", `"2019"`, "timestamp", "", errno.ErrTruncatedWrongValue}, {"year", `2019`, "timestamp", "", errno.ErrTruncatedWrongValue}, @@ -5076,6 +5183,31 @@ func (s *testDBSuite1) TestModifyColumnTime(c *C) { {"year", `70`, "timestamp", "", errno.ErrTruncatedWrongValue}, {"year", `99`, "timestamp", "", errno.ErrTruncatedWrongValue}, } + testModifyColumnTime(c, s.store, tests) +} + +type testModifyColumnTimeCase struct { + from string + value string + to string + expect string + err uint16 +} + +func testModifyColumnTime(c *C, store kv.Storage, tests []testModifyColumnTimeCase) { + limit := variable.GetDDLErrorCountLimit() + variable.SetDDLErrorCountLimit(3) + + tk := testkit.NewTestKit(c, store) + tk.MustExec("use test_db") + + // Set time zone to UTC. + originalTz := tk.Se.GetSessionVars().TimeZone + tk.Se.GetSessionVars().TimeZone = time.UTC + defer func() { + variable.SetDDLErrorCountLimit(limit) + tk.Se.GetSessionVars().TimeZone = originalTz + }() for _, t := range tests { tk.MustExec("drop table if exists t_mc") @@ -5199,6 +5331,52 @@ func (s *testSerialDBSuite) TestSetTableFlashReplica(c *C) { c.Assert(err.Error(), Equals, "the tiflash replica count: 2 should be less than the total tiflash server count: 0") } +func (s *testSerialDBSuite) TestSetTableFlashReplicaForSystemTable(c *C) { + tk := testkit.NewTestKit(c, s.store) + sysTables := make([]string, 0, 24) + memOrSysDB := []string{"MySQL", "INFORMATION_SCHEMA", "PERFORMANCE_SCHEMA", "METRICS_SCHEMA"} + for _, db := range memOrSysDB { + tk.MustExec("use " + db) + rows := tk.MustQuery("show tables").Rows() + for i := 0; i < len(rows); i++ { + sysTables = append(sysTables, rows[i][0].(string)) + } + for _, one := range sysTables { + _, err := tk.Exec(fmt.Sprintf("alter table `%s` set tiflash replica 1", one)) + c.Assert(err.Error(), Equals, "[ddl:8200]ALTER table replica for tables in system database is currently unsupported") + } + sysTables = sysTables[:0] + } +} + +func (s *testSerialDBSuite) TestSetTiFlashReplicaForTemporaryTable(c *C) { + // test for tiflash replica + c.Assert(failpoint.Enable("github.com/pingcap/tidb/infoschema/mockTiFlashStoreCount", `return(true)`), IsNil) + defer func() { + err := failpoint.Disable("github.com/pingcap/tidb/infoschema/mockTiFlashStoreCount") + c.Assert(err, IsNil) + }() + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("drop table if exists temp") + tk.MustExec("create global temporary table temp(id int) on commit delete rows") + tk.MustGetErrCode("alter table temp set tiflash replica 1", errno.ErrOptOnTemporaryTable) + tk.MustExec("drop table temp") + + tk.MustExec("drop table if exists normal") + tk.MustExec("create table normal(id int)") + defer tk.MustExec("drop table normal") + tk.MustExec("alter table normal set tiflash replica 1") + tk.MustQuery("select REPLICA_COUNT from information_schema.tiflash_replica where table_schema='test' and table_name='normal'").Check(testkit.Rows("1")) + tk.MustExec("create global temporary table temp like normal on commit delete rows") + tk.MustQuery("select REPLICA_COUNT from information_schema.tiflash_replica where table_schema='test' and table_name='temp'").Check(testkit.Rows()) + tk.MustExec("drop table temp") + tk.MustExec("set tidb_enable_noop_functions = 1") + tk.MustExec("create temporary table temp like normal") + tk.MustQuery("select REPLICA_COUNT from information_schema.tiflash_replica where table_schema='test' and table_name='temp'").Check(testkit.Rows()) +} + func (s *testSerialDBSuite) TestAlterShardRowIDBits(c *C) { c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil) defer func() { @@ -5240,6 +5418,19 @@ func (s *testSerialDBSuite) TestAlterShardRowIDBits(c *C) { c.Assert(err.Error(), Equals, "[autoid:1467]Failed to read auto-increment value from storage engine") } +func (s *testSerialDBSuite) TestShardRowIDBitsOnTemporaryTable(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists shard_row_id_temporary") + tk.MustExec("set tidb_enable_global_temporary_table=true") + _, err := tk.Exec("create global temporary table shard_row_id_temporary (a int) shard_row_id_bits = 5 on commit delete rows;") + c.Assert(err.Error(), Equals, core.ErrOptOnTemporaryTable.GenWithStackByArgs("shard_row_id_bits").Error()) + tk.MustExec("create global temporary table shard_row_id_temporary (a int) on commit delete rows;") + defer tk.MustExec("drop table if exists shard_row_id_temporary") + _, err = tk.Exec("alter table shard_row_id_temporary shard_row_id_bits = 4;") + c.Assert(err.Error(), Equals, ddl.ErrOptOnTemporaryTable.GenWithStackByArgs("shard_row_id_bits").Error()) +} + // port from mysql // https://github.com/mysql/mysql-server/blob/124c7ab1d6f914637521fd4463a993aa73403513/mysql-test/t/lock.test func (s *testDBSuite2) TestLock(c *C) { @@ -6307,13 +6498,6 @@ func (s *testSerialDBSuite) TestColumnTypeChangeGenUniqueChangingName(c *C) { tk.MustExec("use test") tk.MustExec("drop table if exists t") - enableChangeColumnType := tk.Se.GetSessionVars().EnableChangeColumnType - tk.Se.GetSessionVars().EnableChangeColumnType = true - defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = enableChangeColumnType - config.RestoreFunc()() - }() - hook := &ddl.TestDDLCallback{} var checkErr error assertChangingColName := "_col$_c2_0" @@ -6431,8 +6615,6 @@ func (s *testSerialDBSuite) TestColumnTypeChangeGenUniqueChangingName(c *C) { func (s *testSerialDBSuite) TestModifyColumnTypeWithWarnings(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - // Enable column change variable. - tk.Se.GetSessionVars().EnableChangeColumnType = true // Test normal warnings. tk.MustExec("drop table if exists t") @@ -6468,8 +6650,6 @@ func (s *testSerialDBSuite) TestModifyColumnTypeWithWarnings(c *C) { func (s *testSerialDBSuite) TestModifyColumnTypeWhenInterception(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - // Enable column change variable. - tk.Se.GetSessionVars().EnableChangeColumnType = true // Test normal warnings. tk.MustExec("drop table if exists t") @@ -6477,7 +6657,7 @@ func (s *testSerialDBSuite) TestModifyColumnTypeWhenInterception(c *C) { count := defaultBatchSize * 4 // Add some rows. - dml := fmt.Sprintf("insert into t values") + dml := "insert into t values" for i := 1; i <= count; i++ { dml += fmt.Sprintf("(%d, %f)", i, 11.22) if i != count { diff --git a/ddl/ddl.go b/ddl/ddl.go index 6f20fe25ccc07..9eb05b86741ed 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -202,7 +202,7 @@ type ddlCtx struct { ddlEventCh chan<- *util.Event lease time.Duration // lease is schema lease. binlogCli *pumpcli.PumpsClient // binlogCli is used for Binlog. - infoHandle *infoschema.Handle + infoCache *infoschema.InfoCache statsHandle *handle.Handle tableLockCkr util.DeadTableLockChecker etcdCli *clientv3.Client @@ -282,6 +282,15 @@ func newDDL(ctx context.Context, options ...Option) *ddl { deadLockCkr = util.NewDeadTableLockChecker(etcdCli) } + // TODO: make store and infoCache explicit arguments + // these two should be ensured to exist + if opt.Store == nil { + panic("store should not be nil") + } + if opt.InfoCache == nil { + panic("infoCache should not be nil") + } + ddlCtx := &ddlCtx{ uuid: id, store: opt.Store, @@ -290,7 +299,7 @@ func newDDL(ctx context.Context, options ...Option) *ddl { ownerManager: manager, schemaSyncer: syncer, binlogCli: binloginfo.GetPumpsClient(), - infoHandle: opt.InfoHandle, + infoCache: opt.InfoCache, tableLockCkr: deadLockCkr, etcdCli: opt.EtcdCli, } @@ -411,7 +420,7 @@ func (d *ddl) GetLease() time.Duration { // Please don't use this function, it is used by TestParallelDDLBeforeRunDDLJob to intercept the calling of d.infoHandle.Get(), use d.infoHandle.Get() instead. // Otherwise, the TestParallelDDLBeforeRunDDLJob will hang up forever. func (d *ddl) GetInfoSchemaWithInterceptor(ctx sessionctx.Context) infoschema.InfoSchema { - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() d.mu.RLock() defer d.mu.RUnlock() @@ -649,10 +658,7 @@ func (d *ddl) startCleanDeadTableLock() { if !d.ownerManager.IsOwner() { continue } - if d.infoHandle == nil || !d.infoHandle.IsValid() { - continue - } - deadLockTables, err := d.tableLockCkr.GetDeadLockedTables(d.ctx, d.infoHandle.Get().AllSchemas()) + deadLockTables, err := d.tableLockCkr.GetDeadLockedTables(d.ctx, d.infoCache.GetLatest().AllSchemas()) if err != nil { logutil.BgLogger().Info("[ddl] get dead table lock failed.", zap.Error(err)) continue diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index b03b4ca66f536..9b670911c4b71 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -1617,7 +1617,10 @@ func checkPartitionDefinitionConstraints(ctx sessionctx.Context, tbInfo *model.T return errors.Trace(err) } if err = checkAddPartitionTooManyPartitions(uint64(len(tbInfo.Partition.Definitions))); err != nil { - return errors.Trace(err) + return err + } + if err = checkAddPartitionOnTemporaryMode(tbInfo); err != nil { + return err } switch tbInfo.Partition.Type { @@ -1640,7 +1643,7 @@ func checkTableInfoValid(tblInfo *model.TableInfo) error { return checkInvisibleIndexOnPK(tblInfo) } -func buildTableInfoWithLike(ident ast.Ident, referTblInfo *model.TableInfo) (*model.TableInfo, error) { +func buildTableInfoWithLike(ident ast.Ident, referTblInfo *model.TableInfo, s *ast.CreateTableStmt) (*model.TableInfo, error) { // Check the referred table is a real table object. if referTblInfo.IsSequence() || referTblInfo.IsView() { return nil, ErrWrongObject.GenWithStackByArgs(ident.Schema, referTblInfo.Name, "BASE TABLE") @@ -1659,7 +1662,10 @@ func buildTableInfoWithLike(ident ast.Ident, referTblInfo *model.TableInfo) (*mo tblInfo.Name = ident.Name tblInfo.AutoIncID = 0 tblInfo.ForeignKeys = nil - if tblInfo.TiFlashReplica != nil { + // Ignore TiFlash replicas for temporary tables. + if s.TemporaryKeyword != ast.TemporaryNone { + tblInfo.TiFlashReplica = nil + } else if tblInfo.TiFlashReplica != nil { replica := *tblInfo.TiFlashReplica // Keep the tiflash replica setting, remove the replica available status. replica.AvailablePartitionIDs = nil @@ -1733,8 +1739,17 @@ func buildTableInfoWithStmt(ctx sessionctx.Context, s *ast.CreateTableStmt, dbCh switch s.TemporaryKeyword { case ast.TemporaryGlobal: tbInfo.TempTableType = model.TempTableGlobal + if !ctx.GetSessionVars().EnableGlobalTemporaryTable { + return nil, errors.New("global temporary table is experimental and it is switched off by tidb_enable_global_temporary_table") + } + // "create global temporary table ... on commit preserve rows" + if !s.OnCommitDelete { + return nil, errors.Trace(errUnsupportedOnCommitPreserve) + } case ast.TemporaryLocal: - tbInfo.TempTableType = model.TempTableLocal + // TODO: set "tbInfo.TempTableType = model.TempTableLocal" after local temporary table is supported. + tbInfo.TempTableType = model.TempTableNone + ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("local TEMPORARY TABLE is not supported yet, TEMPORARY will be parsed but ignored")) case ast.TemporaryNone: tbInfo.TempTableType = model.TempTableNone } @@ -1798,7 +1813,7 @@ func (d *ddl) CreateTable(ctx sessionctx.Context, s *ast.CreateTableStmt) (err e // build tableInfo var tbInfo *model.TableInfo if s.ReferTable != nil { - tbInfo, err = buildTableInfoWithLike(ident, referTbl.Meta()) + tbInfo, err = buildTableInfoWithLike(ident, referTbl.Meta(), s) } else { tbInfo, err = buildTableInfoWithStmt(ctx, s, schema.Charset, schema.Collate) } @@ -1922,6 +1937,9 @@ func (d *ddl) CreateTableWithInfo( // preSplitAndScatter performs pre-split and scatter of the table's regions. // If `pi` is not nil, will only split region for `pi`, this is used when add partition. func (d *ddl) preSplitAndScatter(ctx sessionctx.Context, tbInfo *model.TableInfo, pi *model.PartitionInfo) { + if tbInfo.TempTableType != model.TempTableNone { + return + } sp, ok := d.store.(kv.SplittableStore) if !ok || atomic.LoadUint32(&EnableSplitTableRegion) == 0 { return @@ -2211,9 +2229,18 @@ func handleTableOptions(options []*ast.TableOption, tbInfo *model.TableInfo) err } tbInfo.MaxShardRowIDBits = tbInfo.ShardRowIDBits case ast.TableOptionPreSplitRegion: + if tbInfo.TempTableType != model.TempTableNone { + return errors.Trace(ErrOptOnTemporaryTable.GenWithStackByArgs("pre split regions")) + } tbInfo.PreSplitRegions = op.UintValue case ast.TableOptionCharset, ast.TableOptionCollate: // We don't handle charset and collate here since they're handled in `getCharsetAndCollateInTableOption`. + case ast.TableOptionEngine: + if tbInfo.TempTableType != model.TempTableNone { + if op.StrValue != "" && !strings.EqualFold(op.StrValue, "memory") { + return errors.Trace(errUnsupportedEngineTemporary) + } + } } } shardingBits := shardingBits(tbInfo) @@ -2364,7 +2391,7 @@ func (d *ddl) AlterTable(ctx sessionctx.Context, ident ast.Ident, specs []*ast.A return errors.Trace(err) } - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() if is.TableIsView(ident.Schema, ident.Name) || is.TableIsSequence(ident.Schema, ident.Name) { return ErrWrongObject.GenWithStackByArgs(ident.Schema, ident.Name, "BASE TABLE") } @@ -2626,6 +2653,9 @@ func (d *ddl) ShardRowID(ctx sessionctx.Context, tableIdent ast.Ident, uVal uint if err != nil { return errors.Trace(err) } + if t.Meta().TempTableType != model.TempTableNone { + return ErrOptOnTemporaryTable.GenWithStackByArgs("shard_row_id_bits") + } if uVal == t.Meta().ShardRowIDBits { // Nothing need to do. return nil @@ -2895,7 +2925,7 @@ func (d *ddl) AddColumns(ctx sessionctx.Context, ti ast.Ident, specs []*ast.Alte // AddTablePartitions will add a new partition to the table. func (d *ddl) AddTablePartitions(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error { - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() schema, ok := is.SchemaByName(ident.Schema) if !ok { return errors.Trace(infoschema.ErrDatabaseNotExists.GenWithStackByArgs(schema)) @@ -2956,7 +2986,7 @@ func (d *ddl) AddTablePartitions(ctx sessionctx.Context, ident ast.Ident, spec * // CoalescePartitions coalesce partitions can be used with a table that is partitioned by hash or key to reduce the number of partitions by number. func (d *ddl) CoalescePartitions(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error { - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() schema, ok := is.SchemaByName(ident.Schema) if !ok { return errors.Trace(infoschema.ErrDatabaseNotExists.GenWithStackByArgs(schema)) @@ -2988,7 +3018,7 @@ func (d *ddl) CoalescePartitions(ctx sessionctx.Context, ident ast.Ident, spec * } func (d *ddl) TruncateTablePartition(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error { - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() schema, ok := is.SchemaByName(ident.Schema) if !ok { return errors.Trace(infoschema.ErrDatabaseNotExists.GenWithStackByArgs(schema)) @@ -3036,7 +3066,7 @@ func (d *ddl) TruncateTablePartition(ctx sessionctx.Context, ident ast.Ident, sp } func (d *ddl) DropTablePartition(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error { - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() schema, ok := is.SchemaByName(ident.Schema) if !ok { return errors.Trace(infoschema.ErrDatabaseNotExists.GenWithStackByArgs(schema)) @@ -3457,7 +3487,7 @@ func checkModifyCharsetAndCollation(toCharset, toCollate, origCharset, origColla // There are two cases when types incompatible: // 1. returned canReorg == true: types can be changed by reorg // 2. returned canReorg == false: type change not supported yet -func CheckModifyTypeCompatible(origin *types.FieldType, to *types.FieldType) (canReorg bool, errMsg string, err error) { +func CheckModifyTypeCompatible(origin *types.FieldType, to *types.FieldType) (canReorg bool, err error) { // Deal with the same type. if origin.Tp == to.Tp { if origin.Tp == mysql.TypeEnum || origin.Tp == mysql.TypeSet { @@ -3467,13 +3497,13 @@ func CheckModifyTypeCompatible(origin *types.FieldType, to *types.FieldType) (ca } if len(to.Elems) < len(origin.Elems) { msg := fmt.Sprintf("the number of %s column's elements is less than the original: %d", typeVar, len(origin.Elems)) - return true, msg, errUnsupportedModifyColumn.GenWithStackByArgs(msg) + return true, errUnsupportedModifyColumn.GenWithStackByArgs(msg) } for index, originElem := range origin.Elems { toElem := to.Elems[index] if originElem != toElem { msg := fmt.Sprintf("cannot modify %s column value %s to %s", typeVar, originElem, toElem) - return true, msg, errUnsupportedModifyColumn.GenWithStackByArgs(msg) + return true, errUnsupportedModifyColumn.GenWithStackByArgs(msg) } } } @@ -3484,21 +3514,21 @@ func CheckModifyTypeCompatible(origin *types.FieldType, to *types.FieldType) (ca // remains the same. if to.Flen != origin.Flen || to.Decimal != origin.Decimal || mysql.HasUnsignedFlag(to.Flag) != mysql.HasUnsignedFlag(origin.Flag) { msg := fmt.Sprintf("decimal change from decimal(%d, %d) to decimal(%d, %d)", origin.Flen, origin.Decimal, to.Flen, to.Decimal) - return true, msg, errUnsupportedModifyColumn.GenWithStackByArgs(msg) + return true, errUnsupportedModifyColumn.GenWithStackByArgs(msg) } } needReorg, reason := needReorgToChange(origin, to) if !needReorg { - return false, "", nil + return false, nil } - return true, reason, errUnsupportedModifyColumn.GenWithStackByArgs(reason) + return true, errUnsupportedModifyColumn.GenWithStackByArgs(reason) } // Deal with the different type. if !checkTypeChangeSupported(origin, to) { - unsupportedMsg := fmt.Sprintf("change from original type %v to %v is currently unsupported yet", to.CompactStr(), origin.CompactStr()) - return false, unsupportedMsg, errUnsupportedModifyColumn.GenWithStackByArgs(unsupportedMsg) + unsupportedMsg := fmt.Sprintf("change from original type %v to %v is currently unsupported yet", origin.CompactStr(), to.CompactStr()) + return false, errUnsupportedModifyColumn.GenWithStackByArgs(unsupportedMsg) } // Check if different type can directly convert and no need to reorg. @@ -3507,16 +3537,16 @@ func CheckModifyTypeCompatible(origin *types.FieldType, to *types.FieldType) (ca if stringToString || integerToInteger { needReorg, reason := needReorgToChange(origin, to) if !needReorg { - return false, "", nil + return false, nil } - return true, reason, errUnsupportedModifyColumn.GenWithStackByArgs(reason) + return true, errUnsupportedModifyColumn.GenWithStackByArgs(reason) } notCompatibleMsg := fmt.Sprintf("type %v not match origin %v", to.CompactStr(), origin.CompactStr()) - return true, notCompatibleMsg, errUnsupportedModifyColumn.GenWithStackByArgs(notCompatibleMsg) + return true, errUnsupportedModifyColumn.GenWithStackByArgs(notCompatibleMsg) } -func needReorgToChange(origin *types.FieldType, to *types.FieldType) (needOreg bool, reasonMsg string) { +func needReorgToChange(origin *types.FieldType, to *types.FieldType) (needReorg bool, reasonMsg string) { toFlen := to.Flen originFlen := origin.Flen if mysql.IsIntegerType(to.Tp) && mysql.IsIntegerType(origin.Tp) { @@ -3526,6 +3556,10 @@ func needReorgToChange(origin *types.FieldType, to *types.FieldType) (needOreg b toFlen, _ = mysql.GetDefaultFieldLengthAndDecimal(to.Tp) } + if convertBetweenCharAndVarchar(origin.Tp, to.Tp) { + return true, "conversion between char and varchar string needs reorganization" + } + if toFlen > 0 && toFlen < originFlen { return true, fmt.Sprintf("length %d is less than origin %d", toFlen, originFlen) } @@ -3539,42 +3573,30 @@ func needReorgToChange(origin *types.FieldType, to *types.FieldType) (needOreg b } func checkTypeChangeSupported(origin *types.FieldType, to *types.FieldType) bool { - if types.IsString(origin.Tp) && to.Tp == mysql.TypeBit { - // TODO: Currently string data type cast to bit are not compatible with mysql, should fix here after compatible. - return false - } - - if (origin.Tp == mysql.TypeEnum || origin.Tp == mysql.TypeSet) && - (types.IsTypeTime(to.Tp) || to.Tp == mysql.TypeDuration) { - // TODO: Currently enum/set cast to date/datetime/timestamp/time/bit are not support yet, should fix here after supported. + if (types.IsTypeTime(origin.Tp) || origin.Tp == mysql.TypeDuration || origin.Tp == mysql.TypeYear || + types.IsString(origin.Tp) || origin.Tp == mysql.TypeJSON) && + to.Tp == mysql.TypeBit { + // TODO: Currently date/datetime/timestamp/time/year/string/json data type cast to bit are not compatible with mysql, should fix here after compatible. return false } - if (types.IsTypeTime(origin.Tp) || origin.Tp == mysql.TypeDuration || origin.Tp == mysql.TypeYear) && - (to.Tp == mysql.TypeEnum || to.Tp == mysql.TypeSet || to.Tp == mysql.TypeBit) { - // TODO: Currently date and time cast to enum/set/bit are not support yet, should fix here after supported. + if (types.IsTypeTime(origin.Tp) || origin.Tp == mysql.TypeDuration || origin.Tp == mysql.TypeYear || + origin.Tp == mysql.TypeNewDecimal || origin.Tp == mysql.TypeFloat || origin.Tp == mysql.TypeDouble || origin.Tp == mysql.TypeJSON || origin.Tp == mysql.TypeBit) && + (to.Tp == mysql.TypeEnum || to.Tp == mysql.TypeSet) { + // TODO: Currently date/datetime/timestamp/time/year/decimal/float/double/json/bit cast to enum/set are not support yet, should fix here after supported. return false } - if (origin.Tp == mysql.TypeFloat || origin.Tp == mysql.TypeDouble) && - (types.IsTypeTime(to.Tp) || to.Tp == mysql.TypeEnum || to.Tp == mysql.TypeSet) { - // TODO: Currently float/double cast to date/datetime/timestamp/enum/set type are not support yet, should fix here after supported. + if (origin.Tp == mysql.TypeEnum || origin.Tp == mysql.TypeSet || origin.Tp == mysql.TypeBit || + origin.Tp == mysql.TypeNewDecimal || origin.Tp == mysql.TypeFloat || origin.Tp == mysql.TypeDouble) && + (types.IsTypeTime(to.Tp)) { + // TODO: Currently enum/set/bit/decimal/float/double cast to date/datetime/timestamp type are not support yet, should fix here after supported. return false } - if origin.Tp == mysql.TypeBit && - (types.IsTypeTime(to.Tp) || to.Tp == mysql.TypeDuration || to.Tp == mysql.TypeEnum || to.Tp == mysql.TypeSet) { - // TODO: Currently bit cast to date/datetime/timestamp/time/enum/set are not support yet, should fix here after supported. - return false - } - - if origin.Tp == mysql.TypeNewDecimal && (to.Tp == mysql.TypeEnum || to.Tp == mysql.TypeSet) { - // TODO: Currently decimal cast to enum/set are not support yet, should fix here after supported. - return false - } - - if origin.Tp == mysql.TypeJSON && (to.Tp == mysql.TypeEnum || to.Tp == mysql.TypeSet || to.Tp == mysql.TypeBit) { - // TODO: Currently json cast to enum/set/bit are not support yet, should fix here after supported. + if (origin.Tp == mysql.TypeEnum || origin.Tp == mysql.TypeSet || origin.Tp == mysql.TypeBit) && + to.Tp == mysql.TypeDuration { + // TODO: Currently enum/set/bit cast to time are not support yet, should fix here after supported. return false } @@ -3583,27 +3605,18 @@ func checkTypeChangeSupported(origin *types.FieldType, to *types.FieldType) bool // checkModifyTypes checks if the 'origin' type can be modified to 'to' type no matter directly change // or change by reorg. It returns error if the two types are incompatible and correlated change are not -// supported. However, even the two types can be change, if the flag "tidb_enable_change_column_type" not -// set, or the "origin" type contains primary key, error will be returned. +// supported. However, even the two types can be change, if the "origin" type contains primary key, error will be returned. func checkModifyTypes(ctx sessionctx.Context, origin *types.FieldType, to *types.FieldType, needRewriteCollationData bool) error { - canReorg, changeColumnErrMsg, err := CheckModifyTypeCompatible(origin, to) + canReorg, err := CheckModifyTypeCompatible(origin, to) if err != nil { if !canReorg { return errors.Trace(err) } - enableChangeColumnType := ctx.GetSessionVars().EnableChangeColumnType - if !enableChangeColumnType { - msg := fmt.Sprintf("%s, and tidb_enable_change_column_type is false", changeColumnErrMsg) - return errUnsupportedModifyColumn.GenWithStackByArgs(msg) - } else if mysql.HasPriKeyFlag(origin.Flag) { - msg := "tidb_enable_change_column_type is true and this column has primary key flag" + if mysql.HasPriKeyFlag(origin.Flag) { + msg := "this column has primary key flag" return errUnsupportedModifyColumn.GenWithStackByArgs(msg) } } - if types.IsTypeVarchar(origin.Tp) != types.IsTypeVarchar(to.Tp) { - unsupportedMsg := "column type conversion between 'varchar' and 'non-varchar' is currently unsupported yet" - return errUnsupportedModifyColumn.GenWithStackByArgs(unsupportedMsg) - } err = checkModifyCharsetAndCollation(to.Charset, to.Collate, origin.Charset, origin.Collate, needRewriteCollationData) // column type change can handle the charset change between these two types in the process of the reorg. @@ -3633,13 +3646,29 @@ func setDefaultValue(ctx sessionctx.Context, col *table.Column, option *ast.Colu if err != nil { return hasDefaultValue, errors.Trace(err) } - err = col.SetDefaultValue(value) + err = setDefaultValueWithBinaryPadding(col, value) if err != nil { return hasDefaultValue, errors.Trace(err) } return hasDefaultValue, nil } +func setDefaultValueWithBinaryPadding(col *table.Column, value interface{}) error { + err := col.SetDefaultValue(value) + if err != nil { + return err + } + // https://dev.mysql.com/doc/refman/8.0/en/binary-varbinary.html + // Set the default value for binary type should append the paddings. + if value != nil { + if col.Tp == mysql.TypeString && types.IsBinaryStr(&col.FieldType) && len(value.(string)) < col.Flen { + padding := make([]byte, col.Flen-len(value.(string))) + col.DefaultValue = string(append([]byte(col.DefaultValue.(string)), padding...)) + } + } + return nil +} + func setColumnComment(ctx sessionctx.Context, col *table.Column, option *ast.ColumnOption) error { value, err := expression.EvalAstExpr(ctx, option.Expr) if err != nil { @@ -3749,7 +3778,7 @@ func processAndCheckDefaultValueAndColumn(ctx sessionctx.Context, col *table.Col func (d *ddl) getModifiableColumnJob(ctx sessionctx.Context, ident ast.Ident, originalColName model.CIStr, spec *ast.AlterTableSpec) (*model.Job, error) { specNewColumn := spec.NewColumns[0] - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() schema, ok := is.SchemaByName(ident.Schema) if !ok { return nil, errors.Trace(infoschema.ErrDatabaseNotExists) @@ -3862,12 +3891,12 @@ func (d *ddl) getModifiableColumnJob(ctx sessionctx.Context, ident ast.Ident, or } return nil, errors.Trace(err) } - if ctx.GetSessionVars().EnableChangeColumnType && needChangeColumnData(col.ColumnInfo, newCol.ColumnInfo) { + if needChangeColumnData(col.ColumnInfo, newCol.ColumnInfo) { if err = isGeneratedRelatedColumn(t.Meta(), newCol.ColumnInfo, col.ColumnInfo); err != nil { return nil, errors.Trace(err) } if t.Meta().Partition != nil { - return nil, errUnsupportedModifyColumn.GenWithStackByArgs("tidb_enable_change_column_type is true, table is partition table") + return nil, errUnsupportedModifyColumn.GenWithStackByArgs("table is partition table") } } @@ -4005,7 +4034,13 @@ func checkIndexInModifiableColumns(columns []*model.ColumnInfo, idxColumns []*mo return errKeyColumnDoesNotExits.GenWithStack("column does not exist: %s", ic.Name) } - if err := checkIndexColumn(col, ic.Length); err != nil { + prefixLength := types.UnspecifiedLength + if types.IsTypePrefixable(col.FieldType.Tp) && col.FieldType.Flen > ic.Length { + // When the index column is changed, prefix length is only valid + // if the type is still prefixable and larger than old prefix length. + prefixLength = ic.Length + } + if err := checkIndexColumn(col, prefixLength); err != nil { return err } } @@ -4037,7 +4072,7 @@ func checkAutoRandom(tableInfo *model.TableInfo, originCol *table.Column, specNe autoid.MaxAutoRandomBits, newRandBits, specNewColumn.Name.Name.O) return 0, ErrInvalidAutoRandom.GenWithStackByArgs(errMsg) } - break // Increasing auto_random shard bits is allowed. + // increasing auto_random shard bits is allowed. case oldRandBits > newRandBits: if newRandBits == 0 { return 0, ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomAlterErrMsg) @@ -4200,7 +4235,7 @@ func (d *ddl) ModifyColumn(ctx sessionctx.Context, ident ast.Ident, spec *ast.Al func (d *ddl) AlterColumn(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error { specNewColumn := spec.NewColumns[0] - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() schema, ok := is.SchemaByName(ident.Schema) if !ok { return infoschema.ErrTableNotExists.GenWithStackByArgs(ident.Schema, ident.Name) @@ -4254,7 +4289,7 @@ func (d *ddl) AlterColumn(ctx sessionctx.Context, ident ast.Ident, spec *ast.Alt // AlterTableComment updates the table comment information. func (d *ddl) AlterTableComment(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error { - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() schema, ok := is.SchemaByName(ident.Schema) if !ok { return infoschema.ErrDatabaseNotExists.GenWithStackByArgs(ident.Schema) @@ -4307,7 +4342,7 @@ func (d *ddl) AlterTableCharsetAndCollate(ctx sessionctx.Context, ident ast.Iden return ErrUnknownCharacterSet.GenWithStackByArgs(toCharset) } - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() schema, ok := is.SchemaByName(ident.Schema) if !ok { return infoschema.ErrDatabaseNotExists.GenWithStackByArgs(ident.Schema) @@ -4357,13 +4392,19 @@ func (d *ddl) AlterTableSetTiFlashReplica(ctx sessionctx.Context, ident ast.Iden if err != nil { return errors.Trace(err) } + // Ban setting replica count for tables in system database. + if util.IsMemOrSysDB(schema.Name.L) { + return errors.Trace(errUnsupportedAlterReplicaForSysTable) + } else if tb.Meta().TempTableType != model.TempTableNone { + return ErrOptOnTemporaryTable.GenWithStackByArgs("set tiflash replica") + } tbReplicaInfo := tb.Meta().TiFlashReplica if tbReplicaInfo != nil && tbReplicaInfo.Count == replicaInfo.Count && len(tbReplicaInfo.LocationLabels) == len(replicaInfo.Labels) { changed := false - for i, lable := range tbReplicaInfo.LocationLabels { - if replicaInfo.Labels[i] != lable { + for i, label := range tbReplicaInfo.LocationLabels { + if replicaInfo.Labels[i] != label { changed = true break } @@ -4468,7 +4509,7 @@ func (d *ddl) AlterTableDropStatistics(ctx sessionctx.Context, ident ast.Ident, // UpdateTableReplicaInfo updates the table flash replica infos. func (d *ddl) UpdateTableReplicaInfo(ctx sessionctx.Context, physicalID int64, available bool) error { - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() tb, ok := is.TableByID(physicalID) if !ok { tb, _, _ = is.FindTableByPartitionID(physicalID) @@ -4571,7 +4612,7 @@ func checkAlterTableCharset(tblInfo *model.TableInfo, dbInfo *model.DBInfo, toCh // In TiDB, indexes are case-insensitive (so index 'a' and 'A" are considered the same index), // but index names are case-sensitive (we can rename index 'a' to 'A') func (d *ddl) RenameIndex(ctx sessionctx.Context, ident ast.Ident, spec *ast.AlterTableSpec) error { - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() schema, ok := is.SchemaByName(ident.Schema) if !ok { return infoschema.ErrDatabaseNotExists.GenWithStackByArgs(ident.Schema) @@ -5229,7 +5270,7 @@ func buildFKInfo(fkName model.CIStr, keys []*ast.IndexPartSpecification, refer * } func (d *ddl) CreateForeignKey(ctx sessionctx.Context, ti ast.Ident, fkName model.CIStr, keys []*ast.IndexPartSpecification, refer *ast.ReferenceDef) error { - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() schema, ok := is.SchemaByName(ti.Schema) if !ok { return infoschema.ErrDatabaseNotExists.GenWithStackByArgs(ti.Schema) @@ -5239,6 +5280,9 @@ func (d *ddl) CreateForeignKey(ctx sessionctx.Context, ti ast.Ident, fkName mode if err != nil { return errors.Trace(infoschema.ErrTableNotExists.GenWithStackByArgs(ti.Schema, ti.Name)) } + if t.Meta().TempTableType != model.TempTableNone { + return infoschema.ErrCannotAddForeign + } fkInfo, err := buildFKInfo(fkName, keys, refer, t.Cols(), t.Meta()) if err != nil { @@ -5261,7 +5305,7 @@ func (d *ddl) CreateForeignKey(ctx sessionctx.Context, ti ast.Ident, fkName mode } func (d *ddl) DropForeignKey(ctx sessionctx.Context, ti ast.Ident, fkName model.CIStr) error { - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() schema, ok := is.SchemaByName(ti.Schema) if !ok { return infoschema.ErrDatabaseNotExists.GenWithStackByArgs(ti.Schema) @@ -5287,7 +5331,7 @@ func (d *ddl) DropForeignKey(ctx sessionctx.Context, ti ast.Ident, fkName model. } func (d *ddl) DropIndex(ctx sessionctx.Context, ti ast.Ident, indexName model.CIStr, ifExists bool) error { - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() schema, ok := is.SchemaByName(ti.Schema) if !ok { return errors.Trace(infoschema.ErrDatabaseNotExists) @@ -5892,8 +5936,8 @@ func buildPlacementSpecReplicasAndConstraint(replicas uint64, cnstr string) ([]* } rules = append(rules, &placement.Rule{ - Count: int(replicas), - LabelConstraints: labelConstraints, + Count: int(replicas), + Constraints: labelConstraints, }) return rules, nil @@ -5922,8 +5966,8 @@ func buildPlacementSpecReplicasAndConstraint(replicas uint64, cnstr string) ([]* } rules = append(rules, &placement.Rule{ - Count: cnt, - LabelConstraints: labelConstraints, + Count: cnt, + Constraints: labelConstraints, }) } @@ -6033,7 +6077,7 @@ func (d *ddl) AlterTableAlterPartition(ctx sessionctx.Context, ident ast.Ident, return errors.Trace(err) } - oldBundle := infoschema.GetBundle(d.infoHandle.Get(), []int64{partitionID, meta.ID, schema.ID}) + oldBundle := infoschema.GetBundle(d.infoCache.GetLatest(), []int64{partitionID, meta.ID, schema.ID}) oldBundle.ID = placement.GroupID(partitionID) @@ -6048,14 +6092,14 @@ func (d *ddl) AlterTableAlterPartition(ctx sessionctx.Context, ident ast.Ident, newRules := bundle.Rules[:0] for i, rule := range bundle.Rules { // merge all empty constraints - if len(rule.LabelConstraints) == 0 { + if len(rule.Constraints) == 0 { extraCnt[rule.Role] += rule.Count continue } // refer to tidb#22065. // add -engine=tiflash to every rule to avoid schedules to tiflash instances. // placement rules in SQL is not compatible with `set tiflash replica` yet - if err := rule.LabelConstraints.Add(placement.Constraint{ + if err := rule.Constraints.Add(placement.Constraint{ Op: placement.NotIn, Key: placement.EngineLabelKey, Values: []string{placement.EngineLabelTiFlash}, @@ -6080,7 +6124,7 @@ func (d *ddl) AlterTableAlterPartition(ctx sessionctx.Context, ident ast.Ident, Count: cnt, StartKeyHex: startKey, EndKeyHex: endKey, - LabelConstraints: []placement.Constraint{{ + Constraints: []placement.Constraint{{ Op: placement.NotIn, Key: placement.EngineLabelKey, Values: []string{placement.EngineLabelTiFlash}, diff --git a/ddl/ddl_test.go b/ddl/ddl_test.go index b77c3300c2700..79635bfc0933b 100644 --- a/ddl/ddl_test.go +++ b/ddl/ddl_test.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/domain/infosync" + "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/meta/autoid" @@ -86,6 +87,10 @@ func TestT(t *testing.T) { } func testNewDDLAndStart(ctx context.Context, c *C, options ...Option) *ddl { + // init infoCache and a stub infoSchema + ic := infoschema.NewCache(2) + ic.Insert(infoschema.MockInfoSchemaWithSchemaVer(nil, 0)) + options = append(options, WithInfoCache(ic)) d := newDDL(ctx, options...) err := d.Start(nil) c.Assert(err, IsNil) diff --git a/ddl/ddl_worker_test.go b/ddl/ddl_worker_test.go index 6e745820b04b9..72fef7c96c19e 100644 --- a/ddl/ddl_worker_test.go +++ b/ddl/ddl_worker_test.go @@ -247,7 +247,7 @@ func (s *testDDLSuite) TestTableError(c *C) { // Schema ID is wrong, so dropping table is failed. doDDLJobErr(c, -1, 1, model.ActionDropTable, nil, ctx, d) // Table ID is wrong, so dropping table is failed. - dbInfo := testSchemaInfo(c, d, "test") + dbInfo := testSchemaInfo(c, d, "test_ddl") testCreateSchema(c, testNewContext(d), d, dbInfo) job := doDDLJobErr(c, dbInfo.ID, -1, model.ActionDropTable, nil, ctx, d) @@ -295,7 +295,7 @@ func (s *testDDLSuite) TestViewError(c *C) { c.Assert(err, IsNil) }() ctx := testNewContext(d) - dbInfo := testSchemaInfo(c, d, "test") + dbInfo := testSchemaInfo(c, d, "test_ddl") testCreateSchema(c, testNewContext(d), d, dbInfo) // Table ID or schema ID is wrong, so getting table is failed. @@ -363,7 +363,7 @@ func (s *testDDLSuite) TestForeignKeyError(c *C) { doDDLJobErr(c, -1, 1, model.ActionAddForeignKey, nil, ctx, d) doDDLJobErr(c, -1, 1, model.ActionDropForeignKey, nil, ctx, d) - dbInfo := testSchemaInfo(c, d, "test") + dbInfo := testSchemaInfo(c, d, "test_ddl") tblInfo := testTableInfo(c, d, "t", 3) testCreateSchema(c, ctx, d, dbInfo) testCreateTable(c, ctx, d, dbInfo, tblInfo) @@ -393,7 +393,7 @@ func (s *testDDLSuite) TestIndexError(c *C) { doDDLJobErr(c, -1, 1, model.ActionAddIndex, nil, ctx, d) doDDLJobErr(c, -1, 1, model.ActionDropIndex, nil, ctx, d) - dbInfo := testSchemaInfo(c, d, "test") + dbInfo := testSchemaInfo(c, d, "test_ddl") tblInfo := testTableInfo(c, d, "t", 3) testCreateSchema(c, ctx, d, dbInfo) testCreateTable(c, ctx, d, dbInfo, tblInfo) @@ -435,7 +435,7 @@ func (s *testDDLSuite) TestColumnError(c *C) { }() ctx := testNewContext(d) - dbInfo := testSchemaInfo(c, d, "test") + dbInfo := testSchemaInfo(c, d, "test_ddl") tblInfo := testTableInfo(c, d, "t", 3) testCreateSchema(c, ctx, d, dbInfo) testCreateTable(c, ctx, d, dbInfo, tblInfo) diff --git a/ddl/delete_range.go b/ddl/delete_range.go index e64c122d41e4d..1aeb5ab0354da 100644 --- a/ddl/delete_range.go +++ b/ddl/delete_range.go @@ -27,7 +27,6 @@ import ( "github.com/pingcap/tidb/ddl/util" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx" - "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/sqlexec" @@ -451,7 +450,7 @@ func doBatchInsert(s sqlexec.SQLExecutor, jobID int64, tableIDs []int64, ts uint // getNowTS gets the current timestamp, in TSO. func getNowTSO(ctx sessionctx.Context) (uint64, error) { - currVer, err := ctx.GetStore().CurrentVersion(oracle.GlobalTxnScope) + currVer, err := ctx.GetStore().CurrentVersion(kv.GlobalTxnScope) if err != nil { return 0, errors.Trace(err) } diff --git a/ddl/error.go b/ddl/error.go index 463c9c405a19e..d587e9cb39143 100644 --- a/ddl/error.go +++ b/ddl/error.go @@ -48,6 +48,7 @@ var ( errUnsupportedAlterTableWithValidation = dbterror.ClassDDL.NewStdErr(mysql.ErrUnsupportedDDLOperation, parser_mysql.Message("ALTER TABLE WITH VALIDATION is currently unsupported", nil)) errUnsupportedAlterTableWithoutValidation = dbterror.ClassDDL.NewStdErr(mysql.ErrUnsupportedDDLOperation, parser_mysql.Message("ALTER TABLE WITHOUT VALIDATION is currently unsupported", nil)) errUnsupportedAlterTableOption = dbterror.ClassDDL.NewStdErr(mysql.ErrUnsupportedDDLOperation, parser_mysql.Message("This type of ALTER TABLE is currently unsupported", nil)) + errUnsupportedAlterReplicaForSysTable = dbterror.ClassDDL.NewStdErr(mysql.ErrUnsupportedDDLOperation, parser_mysql.Message("ALTER table replica for tables in system database is currently unsupported", nil)) errBlobKeyWithoutLength = dbterror.ClassDDL.NewStd(mysql.ErrBlobKeyWithoutLength) errKeyPart0 = dbterror.ClassDDL.NewStd(mysql.ErrKeyPart0) errIncorrectPrefixKey = dbterror.ClassDDL.NewStd(mysql.ErrWrongSubKey) @@ -277,4 +278,10 @@ var ( // ErrPartitionNoTemporary returns when partition at temporary mode ErrPartitionNoTemporary = dbterror.ClassDDL.NewStd(mysql.ErrPartitionNoTemporary) + + // ErrOptOnTemporaryTable returns when exec unsupported opt at temporary mode + ErrOptOnTemporaryTable = dbterror.ClassDDL.NewStd(mysql.ErrOptOnTemporaryTable) + + errUnsupportedOnCommitPreserve = dbterror.ClassDDL.NewStdErr(mysql.ErrUnsupportedDDLOperation, parser_mysql.Message("TiDB doesn't support ON COMMIT PRESERVE ROWS for now", nil)) + errUnsupportedEngineTemporary = dbterror.ClassDDL.NewStdErr(mysql.ErrUnsupportedDDLOperation, parser_mysql.Message("TiDB doesn't support this kind of engine for temporary table", nil)) ) diff --git a/ddl/failtest/fail_db_test.go b/ddl/failtest/fail_db_test.go index 8ccb7a5adb594..d70d32699fd84 100644 --- a/ddl/failtest/fail_db_test.go +++ b/ddl/failtest/fail_db_test.go @@ -460,16 +460,10 @@ func (s *testFailDBSuite) TestModifyColumn(c *C) { tk.MustExec("use test") tk.MustExec("drop table if exists t;") - enableChangeColumnType := tk.Se.GetSessionVars().EnableChangeColumnType - tk.Se.GetSessionVars().EnableChangeColumnType = true - defer func() { - tk.Se.GetSessionVars().EnableChangeColumnType = enableChangeColumnType - }() - tk.MustExec("create table t (a int not null default 1, b int default 2, c int not null default 0, primary key(c), index idx(b), index idx1(a), index idx2(b, c))") tk.MustExec("insert into t values(1, 2, 3), (11, 22, 33)") _, err := tk.Exec("alter table t change column c cc mediumint") - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: tidb_enable_change_column_type is true and this column has primary key flag") + c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: this column has primary key flag") tk.MustExec("alter table t change column b bb mediumint first") dom := domain.GetDomain(tk.Se) is := dom.InfoSchema() @@ -516,16 +510,16 @@ func (s *testFailDBSuite) TestModifyColumn(c *C) { // Test unsupport statements. tk.MustExec("create table t1(a int) partition by hash (a) partitions 2") _, err = tk.Exec("alter table t1 modify column a mediumint") - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: tidb_enable_change_column_type is true, table is partition table") + c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: table is partition table") tk.MustExec("create table t2(id int, a int, b int generated always as (abs(a)) virtual, c int generated always as (a+1) stored)") _, err = tk.Exec("alter table t2 modify column b mediumint") - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: tidb_enable_change_column_type is true, newCol IsGenerated false, oldCol IsGenerated true") + c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: newCol IsGenerated false, oldCol IsGenerated true") _, err = tk.Exec("alter table t2 modify column c mediumint") - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: tidb_enable_change_column_type is true, newCol IsGenerated false, oldCol IsGenerated true") + c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: newCol IsGenerated false, oldCol IsGenerated true") _, err = tk.Exec("alter table t2 modify column a mediumint generated always as(id+1) stored") - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: tidb_enable_change_column_type is true, newCol IsGenerated true, oldCol IsGenerated false") + c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: newCol IsGenerated true, oldCol IsGenerated false") _, err = tk.Exec("alter table t2 modify column a mediumint") - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: tidb_enable_change_column_type is true, oldCol is a dependent column 'a' for generated column") + c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: oldCol is a dependent column 'a' for generated column") // Test multiple rows of data. tk.MustExec("create table t3(a int not null default 1, b int default 2, c int not null default 0, primary key(c), index idx(b), index idx1(a), index idx2(b, c))") diff --git a/ddl/generated_column.go b/ddl/generated_column.go index 657a27ec3db4f..21bd3478e8e0f 100644 --- a/ddl/generated_column.go +++ b/ddl/generated_column.go @@ -154,11 +154,11 @@ func hasDependentByGeneratedColumn(tblInfo *model.TableInfo, colName model.CIStr func isGeneratedRelatedColumn(tblInfo *model.TableInfo, newCol, col *model.ColumnInfo) error { if newCol.IsGenerated() || col.IsGenerated() { // TODO: Make it compatible with MySQL error. - msg := fmt.Sprintf("tidb_enable_change_column_type is true, newCol IsGenerated %v, oldCol IsGenerated %v", newCol.IsGenerated(), col.IsGenerated()) + msg := fmt.Sprintf("newCol IsGenerated %v, oldCol IsGenerated %v", newCol.IsGenerated(), col.IsGenerated()) return errUnsupportedModifyColumn.GenWithStackByArgs(msg) } if ok, dep := hasDependentByGeneratedColumn(tblInfo, col.Name); ok { - msg := fmt.Sprintf("tidb_enable_change_column_type is true, oldCol is a dependent column '%s' for generated column", dep) + msg := fmt.Sprintf("oldCol is a dependent column '%s' for generated column", dep) return errUnsupportedModifyColumn.GenWithStackByArgs(msg) } return nil diff --git a/ddl/index.go b/ddl/index.go index f11a595aa8fb3..622e93213929c 100644 --- a/ddl/index.go +++ b/ddl/index.go @@ -33,7 +33,6 @@ import ( "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/store/tikv" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" @@ -1117,7 +1116,7 @@ func (w *addIndexWorker) BackfillDataInTxn(handleRange reorgBackfillTask) (taskC errInTxn = kv.RunInNewTxn(context.Background(), w.sessCtx.GetStore(), true, func(ctx context.Context, txn kv.Transaction) error { taskCtx.addedCount = 0 taskCtx.scanCount = 0 - txn.SetOption(tikvstore.Priority, w.priority) + txn.SetOption(kv.Priority, w.priority) idxRecords, nextKey, taskDone, err := w.fetchRowColVals(txn, handleRange) if err != nil { @@ -1218,10 +1217,9 @@ func (w *worker) updateReorgInfo(t table.PartitionedTable, reorg *reorgInfo) (bo failpoint.Inject("mockUpdateCachedSafePoint", func(val failpoint.Value) { if val.(bool) { - // 18 is for the logical time. - ts := oracle.GetPhysical(time.Now()) << 18 + ts := oracle.GoTimeToTS(time.Now()) s := reorg.d.store.(tikv.Storage) - s.UpdateSPCache(uint64(ts), time.Now()) + s.UpdateSPCache(ts, time.Now()) time.Sleep(time.Millisecond * 3) } }) @@ -1329,7 +1327,7 @@ func (w *cleanUpIndexWorker) BackfillDataInTxn(handleRange reorgBackfillTask) (t errInTxn = kv.RunInNewTxn(context.Background(), w.sessCtx.GetStore(), true, func(ctx context.Context, txn kv.Transaction) error { taskCtx.addedCount = 0 taskCtx.scanCount = 0 - txn.SetOption(tikvstore.Priority, w.priority) + txn.SetOption(kv.Priority, w.priority) idxRecords, nextKey, taskDone, err := w.fetchRowColVals(txn, handleRange) if err != nil { @@ -1344,7 +1342,7 @@ func (w *cleanUpIndexWorker) BackfillDataInTxn(handleRange reorgBackfillTask) (t // we fetch records row by row, so records will belong to // index[0], index[1] ... index[n-1], index[0], index[1] ... // respectively. So indexes[i%n] is the index of idxRecords[i]. - err := w.indexes[i%n].Delete(w.sessCtx.GetSessionVars().StmtCtx, txn.GetUnionStore(), idxRecord.vals, idxRecord.handle) + err := w.indexes[i%n].Delete(w.sessCtx.GetSessionVars().StmtCtx, txn, idxRecord.vals, idxRecord.handle) if err != nil { return errors.Trace(err) } diff --git a/ddl/index_change_test.go b/ddl/index_change_test.go index 0a54b6b25e694..6a34599137c10 100644 --- a/ddl/index_change_test.go +++ b/ddl/index_change_test.go @@ -22,7 +22,6 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" @@ -38,15 +37,18 @@ type testIndexChangeSuite struct { func (s *testIndexChangeSuite) SetUpSuite(c *C) { s.store = testCreateStore(c, "test_index_change") - s.dbInfo = &model.DBInfo{ - Name: model.NewCIStr("test_index_change"), - ID: 1, - } - err := kv.RunInNewTxn(context.Background(), s.store, true, func(ctx context.Context, txn kv.Transaction) error { - t := meta.NewMeta(txn) - return errors.Trace(t.CreateDatabase(s.dbInfo)) - }) - c.Check(err, IsNil, Commentf("err %v", errors.ErrorStack(err))) + d := testNewDDLAndStart( + context.Background(), + c, + WithStore(s.store), + WithLease(testLease), + ) + defer func() { + err := d.Stop() + c.Assert(err, IsNil) + }() + s.dbInfo = testSchemaInfo(c, d, "test_index_change") + testCreateSchema(c, testNewContext(d), d, s.dbInfo) } func (s *testIndexChangeSuite) TearDownSuite(c *C) { @@ -198,7 +200,7 @@ func checkIndexExists(ctx sessionctx.Context, tbl table.Table, indexValue interf if err != nil { return errors.Trace(err) } - doesExist, _, err := idx.Exist(ctx.GetSessionVars().StmtCtx, txn.GetUnionStore(), types.MakeDatums(indexValue), kv.IntHandle(handle)) + doesExist, _, err := idx.Exist(ctx.GetSessionVars().StmtCtx, txn, types.MakeDatums(indexValue), kv.IntHandle(handle)) if err != nil { return errors.Trace(err) } diff --git a/ddl/options.go b/ddl/options.go index 8613a8e9affa9..9238a7c8542ff 100644 --- a/ddl/options.go +++ b/ddl/options.go @@ -26,11 +26,11 @@ type Option func(*Options) // Options represents all the options of the DDL module needs type Options struct { - EtcdCli *clientv3.Client - Store kv.Storage - InfoHandle *infoschema.Handle - Hook Callback - Lease time.Duration + EtcdCli *clientv3.Client + Store kv.Storage + InfoCache *infoschema.InfoCache + Hook Callback + Lease time.Duration } // WithEtcdClient specifies the `clientv3.Client` of DDL used to request the etcd service @@ -47,10 +47,10 @@ func WithStore(store kv.Storage) Option { } } -// WithInfoHandle specifies the `infoschema.Handle` -func WithInfoHandle(ih *infoschema.Handle) Option { +// WithInfoCache specifies the `infoschema.InfoCache` +func WithInfoCache(ic *infoschema.InfoCache) Option { return func(options *Options) { - options.InfoHandle = ih + options.InfoCache = ic } } diff --git a/ddl/options_test.go b/ddl/options_test.go index 294d68731e4c3..22a451d622c71 100644 --- a/ddl/options_test.go +++ b/ddl/options_test.go @@ -33,14 +33,14 @@ func (s *ddlOptionsSuite) TestOptions(c *C) { callback := &ddl.BaseCallback{} lease := time.Second * 3 store := &mock.Store{} - infoHandle := infoschema.NewHandle(store) + infoHandle := infoschema.NewCache(16) options := []ddl.Option{ ddl.WithEtcdClient(client), ddl.WithHook(callback), ddl.WithLease(lease), ddl.WithStore(store), - ddl.WithInfoHandle(infoHandle), + ddl.WithInfoCache(infoHandle), } opt := &ddl.Options{} @@ -52,5 +52,5 @@ func (s *ddlOptionsSuite) TestOptions(c *C) { c.Assert(opt.Hook, Equals, callback) c.Assert(opt.Lease, Equals, lease) c.Assert(opt.Store, Equals, store) - c.Assert(opt.InfoHandle, Equals, infoHandle) + c.Assert(opt.InfoCache, Equals, infoHandle) } diff --git a/ddl/partition.go b/ddl/partition.go index 4cc71eb1c8d74..4e55ec1779e21 100644 --- a/ddl/partition.go +++ b/ddl/partition.go @@ -911,18 +911,15 @@ func getTableInfoWithDroppingPartitions(t *model.TableInfo) *model.TableInfo { } func dropRuleBundles(d *ddlCtx, physicalTableIDs []int64) error { - if d.infoHandle != nil && d.infoHandle.IsValid() { - bundles := make([]*placement.Bundle, 0, len(physicalTableIDs)) - for _, ID := range physicalTableIDs { - oldBundle, ok := d.infoHandle.Get().BundleByName(placement.GroupID(ID)) - if ok && !oldBundle.IsEmpty() { - bundles = append(bundles, placement.BuildPlacementDropBundle(ID)) - } + bundles := make([]*placement.Bundle, 0, len(physicalTableIDs)) + for _, ID := range physicalTableIDs { + oldBundle, ok := d.infoCache.GetLatest().BundleByName(placement.GroupID(ID)) + if ok && !oldBundle.IsEmpty() { + bundles = append(bundles, placement.BuildPlacementDropBundle(ID)) } - err := infosync.PutRuleBundles(context.TODO(), bundles) - return err } - return nil + err := infosync.PutRuleBundles(context.TODO(), bundles) + return err } // onDropTablePartition deletes old partition meta. @@ -1095,22 +1092,20 @@ func onTruncateTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (int64, e } } - if d.infoHandle != nil && d.infoHandle.IsValid() { - bundles := make([]*placement.Bundle, 0, len(oldIDs)) + bundles := make([]*placement.Bundle, 0, len(oldIDs)) - for i, oldID := range oldIDs { - oldBundle, ok := d.infoHandle.Get().BundleByName(placement.GroupID(oldID)) - if ok && !oldBundle.IsEmpty() { - bundles = append(bundles, placement.BuildPlacementDropBundle(oldID)) - bundles = append(bundles, placement.BuildPlacementCopyBundle(oldBundle, newPartitions[i].ID)) - } + for i, oldID := range oldIDs { + oldBundle, ok := d.infoCache.GetLatest().BundleByName(placement.GroupID(oldID)) + if ok && !oldBundle.IsEmpty() { + bundles = append(bundles, placement.BuildPlacementDropBundle(oldID)) + bundles = append(bundles, placement.BuildPlacementCopyBundle(oldBundle, newPartitions[i].ID)) } + } - err = infosync.PutRuleBundles(context.TODO(), bundles) - if err != nil { - job.State = model.JobStateCancelled - return ver, errors.Wrapf(err, "failed to notify PD the placement rules") - } + err = infosync.PutRuleBundles(context.TODO(), bundles) + if err != nil { + job.State = model.JobStateCancelled + return ver, errors.Wrapf(err, "failed to notify PD the placement rules") } newIDs := make([]int64, len(oldIDs)) @@ -1299,27 +1294,25 @@ func (w *worker) onExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo // the follow code is a swap function for rules of two partitions // though partitions has exchanged their ID, swap still take effect - if d.infoHandle != nil && d.infoHandle.IsValid() { - bundles := make([]*placement.Bundle, 0, 2) - ptBundle, ptOK := d.infoHandle.Get().BundleByName(placement.GroupID(partDef.ID)) - ptOK = ptOK && !ptBundle.IsEmpty() - ntBundle, ntOK := d.infoHandle.Get().BundleByName(placement.GroupID(nt.ID)) - ntOK = ntOK && !ntBundle.IsEmpty() - if ptOK && ntOK { - bundles = append(bundles, placement.BuildPlacementCopyBundle(ptBundle, nt.ID)) - bundles = append(bundles, placement.BuildPlacementCopyBundle(ntBundle, partDef.ID)) - } else if ptOK { - bundles = append(bundles, placement.BuildPlacementDropBundle(partDef.ID)) - bundles = append(bundles, placement.BuildPlacementCopyBundle(ptBundle, nt.ID)) - } else if ntOK { - bundles = append(bundles, placement.BuildPlacementDropBundle(nt.ID)) - bundles = append(bundles, placement.BuildPlacementCopyBundle(ntBundle, partDef.ID)) - } - err = infosync.PutRuleBundles(context.TODO(), bundles) - if err != nil { - job.State = model.JobStateCancelled - return ver, errors.Wrapf(err, "failed to notify PD the placement rules") - } + bundles := make([]*placement.Bundle, 0, 2) + ptBundle, ptOK := d.infoCache.GetLatest().BundleByName(placement.GroupID(partDef.ID)) + ptOK = ptOK && !ptBundle.IsEmpty() + ntBundle, ntOK := d.infoCache.GetLatest().BundleByName(placement.GroupID(nt.ID)) + ntOK = ntOK && !ntBundle.IsEmpty() + if ptOK && ntOK { + bundles = append(bundles, placement.BuildPlacementCopyBundle(ptBundle, nt.ID)) + bundles = append(bundles, placement.BuildPlacementCopyBundle(ntBundle, partDef.ID)) + } else if ptOK { + bundles = append(bundles, placement.BuildPlacementDropBundle(partDef.ID)) + bundles = append(bundles, placement.BuildPlacementCopyBundle(ptBundle, nt.ID)) + } else if ntOK { + bundles = append(bundles, placement.BuildPlacementDropBundle(nt.ID)) + bundles = append(bundles, placement.BuildPlacementCopyBundle(ntBundle, partDef.ID)) + } + err = infosync.PutRuleBundles(context.TODO(), bundles) + if err != nil { + job.State = model.JobStateCancelled + return ver, errors.Wrapf(err, "failed to notify PD the placement rules") } ver, err = updateSchemaVersion(t, job) @@ -1476,6 +1469,13 @@ func checkAddPartitionTooManyPartitions(piDefs uint64) error { return nil } +func checkAddPartitionOnTemporaryMode(tbInfo *model.TableInfo) error { + if tbInfo.Partition != nil && tbInfo.TempTableType != model.TempTableNone { + return ErrPartitionNoTemporary + } + return nil +} + func checkNoHashPartitions(ctx sessionctx.Context, partitionNum uint64) error { if partitionNum == 0 { return ast.ErrNoParts.GenWithStackByArgs("partitions") diff --git a/ddl/placement/errors.go b/ddl/placement/errors.go index 19797022a609c..95fce4591c961 100644 --- a/ddl/placement/errors.go +++ b/ddl/placement/errors.go @@ -24,4 +24,10 @@ var ( ErrUnsupportedConstraint = errors.New("unsupported label constraint") // ErrConflictingConstraints is from constraints.go. ErrConflictingConstraints = errors.New("conflicting label constraints") + // ErrInvalidConstraintsMapcnt is from rule.go. + ErrInvalidConstraintsMapcnt = errors.New("label constraints in map syntax have invalid replicas") + // ErrInvalidConstraintsFormat is from rule.go. + ErrInvalidConstraintsFormat = errors.New("invalid label constraints format") + // ErrInvalidConstraintsRelicas is from rule.go. + ErrInvalidConstraintsRelicas = errors.New("label constraints with invalid REPLICAS") ) diff --git a/ddl/placement/rule.go b/ddl/placement/rule.go new file mode 100644 index 0000000000000..134bdd5a610f9 --- /dev/null +++ b/ddl/placement/rule.go @@ -0,0 +1,132 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package placement + +import ( + "fmt" + "strings" + + "github.com/go-yaml/yaml" +) + +// PeerRoleType is the expected peer type of the placement rule. +type PeerRoleType string + +const ( + // Voter can either match a leader peer or follower peer. + Voter PeerRoleType = "voter" + // Leader matches a leader. + Leader PeerRoleType = "leader" + // Follower matches a follower. + Follower PeerRoleType = "follower" + // Learner matches a learner. + Learner PeerRoleType = "learner" +) + +// Rule is the core placement rule struct. Check https://github.com/tikv/pd/blob/master/server/schedule/placement/rule.go. +type Rule struct { + GroupID string `json:"group_id"` + ID string `json:"id"` + Index int `json:"index,omitempty"` + Override bool `json:"override,omitempty"` + StartKeyHex string `json:"start_key"` + EndKeyHex string `json:"end_key"` + Role PeerRoleType `json:"role"` + Count int `json:"count"` + Constraints Constraints `json:"label_constraints,omitempty"` + LocationLabels []string `json:"location_labels,omitempty"` + IsolationLevel string `json:"isolation_level,omitempty"` +} + +// NewRules constructs []*Rule from a yaml-compatible representation of +// array or map of constraints. It converts 'CONSTRAINTS' field in RFC +// https://github.com/pingcap/tidb/blob/master/docs/design/2020-06-24-placement-rules-in-sql.md to structs. +func NewRules(replicas uint64, cnstr string) ([]*Rule, error) { + rules := []*Rule{} + + cnstbytes := []byte(cnstr) + + constraints1 := []string{} + err1 := yaml.UnmarshalStrict(cnstbytes, &constraints1) + if err1 == nil { + // can not emit REPLICAS with an array or empty label + if replicas == 0 { + return rules, fmt.Errorf("%w: should be positive", ErrInvalidConstraintsRelicas) + } + + labelConstraints, err := NewConstraints(constraints1) + if err != nil { + return rules, err + } + + rules = append(rules, &Rule{ + Count: int(replicas), + Constraints: labelConstraints, + }) + + return rules, nil + } + + constraints2 := map[string]int{} + err2 := yaml.UnmarshalStrict(cnstbytes, &constraints2) + if err2 == nil { + ruleCnt := 0 + for labels, cnt := range constraints2 { + if cnt <= 0 { + return rules, fmt.Errorf("%w: count of labels '%s' should be positive, but got %d", ErrInvalidConstraintsMapcnt, labels, cnt) + } + ruleCnt += cnt + } + + if replicas == 0 { + replicas = uint64(ruleCnt) + } + + if int(replicas) < ruleCnt { + return rules, fmt.Errorf("%w: should be larger or equal to the number of total replicas, but REPLICAS=%d < total=%d", ErrInvalidConstraintsRelicas, replicas, ruleCnt) + } + + for labels, cnt := range constraints2 { + labelConstraints, err := NewConstraints(strings.Split(labels, ",")) + if err != nil { + return rules, err + } + + rules = append(rules, &Rule{ + Count: cnt, + Constraints: labelConstraints, + }) + } + + remain := int(replicas) - ruleCnt + if remain > 0 { + rules = append(rules, &Rule{ + Count: remain, + }) + } + + return rules, nil + } + + return nil, fmt.Errorf("%w: should be [constraint1, ...] (error %s), {constraint1: cnt1, ...} (error %s), or any yaml compatible representation", ErrInvalidConstraintsFormat, err1, err2) +} + +// Clone is used to duplicate a RuleOp for safe modification. +// Note that it is a shallow copy: LocationLabels and Constraints +// is not cloned. +func (r *Rule) Clone() *Rule { + n := &Rule{} + *n = *r + return n +} diff --git a/ddl/placement/rule_test.go b/ddl/placement/rule_test.go new file mode 100644 index 0000000000000..85dd492f348e7 --- /dev/null +++ b/ddl/placement/rule_test.go @@ -0,0 +1,206 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package placement + +import ( + "encoding/json" + "errors" + + . "github.com/pingcap/check" +) + +var _ = Suite(&testRuleSuite{}) + +type testRuleSuite struct{} + +func (t *testRuleSuite) TestClone(c *C) { + rule := &Rule{ID: "434"} + newRule := rule.Clone() + newRule.ID = "121" + + c.Assert(rule, DeepEquals, &Rule{ID: "434"}) + c.Assert(newRule, DeepEquals, &Rule{ID: "121"}) +} + +func matchRule(r1 *Rule, t2 []*Rule) bool { + for _, r2 := range t2 { + if ok, _ := DeepEquals.Check([]interface{}{r1, r2}, nil); ok { + return true + } + } + return false +} + +func matchRules(t1, t2 []*Rule, prefix string, c *C) { + expected, err := json.Marshal(t1) + c.Assert(err, IsNil) + got, err := json.Marshal(t2) + c.Assert(err, IsNil) + comment := Commentf("%s, expected %s\nbut got %s", prefix, expected, got) + c.Assert(len(t1), Equals, len(t2), comment) + for _, r1 := range t1 { + c.Assert(matchRule(r1, t2), IsTrue, comment) + } +} + +func (t *testRuleSuite) TestNewRules(c *C) { + type TestCase struct { + name string + input string + replicas uint64 + output []*Rule + err error + } + tests := []TestCase{} + + tests = append(tests, TestCase{ + name: "empty constraints", + input: "", + replicas: 3, + output: []*Rule{ + { + Count: 3, + Constraints: Constraints{}, + }, + }, + }) + + tests = append(tests, TestCase{ + name: "zero replicas", + input: "", + replicas: 0, + err: ErrInvalidConstraintsRelicas, + }) + + labels, err := NewConstraints([]string{"+zone=sh", "+zone=sh"}) + c.Assert(err, IsNil) + tests = append(tests, TestCase{ + name: "normal array constraints", + input: `["+zone=sh", "+zone=sh"]`, + replicas: 3, + output: []*Rule{ + { + Count: 3, + Constraints: labels, + }, + }, + }) + + labels1, err := NewConstraints([]string{"+zone=sh", "-zone=bj"}) + c.Assert(err, IsNil) + labels2, err := NewConstraints([]string{"+zone=sh"}) + c.Assert(err, IsNil) + tests = append(tests, TestCase{ + name: "normal object constraints", + input: `{"+zone=sh,-zone=bj":2, "+zone=sh": 1}`, + replicas: 3, + output: []*Rule{ + { + Count: 2, + Constraints: labels1, + }, + { + Count: 1, + Constraints: labels2, + }, + }, + }) + + tests = append(tests, TestCase{ + name: "normal object constraints, with extra count", + input: "{'+zone=sh,-zone=bj':2, '+zone=sh': 1}", + replicas: 4, + output: []*Rule{ + { + Count: 2, + Constraints: labels1, + }, + { + Count: 1, + Constraints: labels2, + }, + { + Count: 1, + }, + }, + }) + + tests = append(tests, TestCase{ + name: "normal object constraints, without count", + input: "{'+zone=sh,-zone=bj':2, '+zone=sh': 1}", + output: []*Rule{ + { + Count: 2, + Constraints: labels1, + }, + { + Count: 1, + Constraints: labels2, + }, + }, + }) + + tests = append(tests, TestCase{ + name: "zero count in object constraints", + input: `{"+zone=sh,-zone=bj":0, "+zone=sh": 1}`, + replicas: 3, + err: ErrInvalidConstraintsMapcnt, + }) + + tests = append(tests, TestCase{ + name: "overlarge total count in object constraints", + input: `{"+ne=sh,-zone=bj":1, "+zone=sh": 4}`, + replicas: 3, + err: ErrInvalidConstraintsRelicas, + }) + + tests = append(tests, TestCase{ + name: "invalid array", + input: `["+ne=sh", "+zone=sh"`, + replicas: 3, + err: ErrInvalidConstraintsFormat, + }) + + tests = append(tests, TestCase{ + name: "invalid array constraints", + input: `["ne=sh", "+zone=sh"]`, + replicas: 3, + err: ErrInvalidConstraintFormat, + }) + + tests = append(tests, TestCase{ + name: "invalid map", + input: `{+ne=sh,-zone=bj:1, "+zone=sh": 4`, + replicas: 5, + err: ErrInvalidConstraintsFormat, + }) + + tests = append(tests, TestCase{ + name: "invalid map constraints", + input: `{"nesh,-zone=bj":1, "+zone=sh": 4}`, + replicas: 6, + err: ErrInvalidConstraintFormat, + }) + + for _, t := range tests { + comment := Commentf("%s", t.name) + output, err := NewRules(t.replicas, t.input) + if t.err == nil { + c.Assert(err, IsNil, comment) + matchRules(t.output, output, comment.CheckCommentString(), c) + } else { + c.Assert(errors.Is(err, t.err), IsTrue, comment) + } + } +} diff --git a/ddl/placement/types.go b/ddl/placement/types.go index 3bb9da96e3890..72093a2c19c78 100644 --- a/ddl/placement/types.go +++ b/ddl/placement/types.go @@ -22,42 +22,6 @@ import ( // After all, placement rules are communicated using an HTTP API. Loose // coupling is a good feature. -// PeerRoleType is the expected peer type of the placement rule. -type PeerRoleType string - -const ( - // Voter can either match a leader peer or follower peer. - Voter PeerRoleType = "voter" - // Leader matches a leader. - Leader PeerRoleType = "leader" - // Follower matches a follower. - Follower PeerRoleType = "follower" - // Learner matches a learner. - Learner PeerRoleType = "learner" -) - -// Rule is the placement rule. Check https://github.com/tikv/pd/blob/master/server/schedule/placement/rule.go. -type Rule struct { - GroupID string `json:"group_id"` - ID string `json:"id"` - Index int `json:"index,omitempty"` - Override bool `json:"override,omitempty"` - StartKeyHex string `json:"start_key"` - EndKeyHex string `json:"end_key"` - Role PeerRoleType `json:"role"` - Count int `json:"count"` - LabelConstraints Constraints `json:"label_constraints,omitempty"` - LocationLabels []string `json:"location_labels,omitempty"` - IsolationLevel string `json:"isolation_level,omitempty"` -} - -// Clone is used to duplicate a RuleOp for safe modification. -func (r *Rule) Clone() *Rule { - n := &Rule{} - *n = *r - return n -} - // Bundle is a group of all rules and configurations. It is used to support rule cache. type Bundle struct { ID string `json:"group_id"` diff --git a/ddl/placement/types_test.go b/ddl/placement/types_test.go index 77153cb29b692..93ed1a5a80f43 100644 --- a/ddl/placement/types_test.go +++ b/ddl/placement/types_test.go @@ -18,7 +18,6 @@ import ( ) var _ = Suite(&testBundleSuite{}) -var _ = Suite(&testRuleSuite{}) type testBundleSuite struct{} @@ -49,14 +48,3 @@ func (t *testBundleSuite) TestClone(c *C) { c.Assert(bundle, DeepEquals, &Bundle{ID: GroupID(1), Rules: []*Rule{{ID: "434"}}}) c.Assert(newBundle, DeepEquals, &Bundle{ID: GroupID(2), Rules: []*Rule{{ID: "121"}}}) } - -type testRuleSuite struct{} - -func (t *testRuleSuite) TestClone(c *C) { - rule := &Rule{ID: "434"} - newRule := rule.Clone() - newRule.ID = "121" - - c.Assert(rule, DeepEquals, &Rule{ID: "434"}) - c.Assert(newRule, DeepEquals, &Rule{ID: "121"}) -} diff --git a/ddl/placement/utils.go b/ddl/placement/utils.go index 16c0a424dde53..5b12f10e2d243 100644 --- a/ddl/placement/utils.go +++ b/ddl/placement/utils.go @@ -61,7 +61,7 @@ func BuildPlacementCopyBundle(oldBundle *Bundle, newID int64) *Bundle { func GetLeaderDCByBundle(bundle *Bundle, dcLabelKey string) (string, bool) { for _, rule := range bundle.Rules { if isValidLeaderRule(rule, dcLabelKey) { - return rule.LabelConstraints[0].Values[0], true + return rule.Constraints[0].Values[0], true } } return "", false @@ -69,7 +69,7 @@ func GetLeaderDCByBundle(bundle *Bundle, dcLabelKey string) (string, bool) { func isValidLeaderRule(rule *Rule, dcLabelKey string) bool { if rule.Role == Leader && rule.Count == 1 { - for _, con := range rule.LabelConstraints { + for _, con := range rule.Constraints { if con.Op == In && con.Key == dcLabelKey && len(con.Values) == 1 { return true } diff --git a/ddl/placement/utils_test.go b/ddl/placement/utils_test.go index 964382846485e..10941e0663455 100644 --- a/ddl/placement/utils_test.go +++ b/ddl/placement/utils_test.go @@ -58,7 +58,7 @@ func (t *testUtilsSuite) TestGetLeaderDCByBundle(c *C) { { ID: "12", Role: Leader, - LabelConstraints: []Constraint{ + Constraints: []Constraint{ { Key: "zone", Op: In, @@ -84,7 +84,7 @@ func (t *testUtilsSuite) TestGetLeaderDCByBundle(c *C) { { ID: "12", Role: Voter, - LabelConstraints: []Constraint{ + Constraints: []Constraint{ { Key: "zone", Op: In, @@ -110,7 +110,7 @@ func (t *testUtilsSuite) TestGetLeaderDCByBundle(c *C) { { ID: "11", Role: Leader, - LabelConstraints: []Constraint{ + Constraints: []Constraint{ { Key: "zone", Op: In, @@ -127,7 +127,7 @@ func (t *testUtilsSuite) TestGetLeaderDCByBundle(c *C) { { ID: "12", Role: Voter, - LabelConstraints: []Constraint{ + Constraints: []Constraint{ { Key: "zone", Op: In, @@ -153,7 +153,7 @@ func (t *testUtilsSuite) TestGetLeaderDCByBundle(c *C) { { ID: "11", Role: Leader, - LabelConstraints: []Constraint{ + Constraints: []Constraint{ { Key: "fake", Op: In, @@ -179,7 +179,7 @@ func (t *testUtilsSuite) TestGetLeaderDCByBundle(c *C) { { ID: "11", Role: Leader, - LabelConstraints: []Constraint{ + Constraints: []Constraint{ { Key: "zone", Op: NotIn, @@ -205,7 +205,7 @@ func (t *testUtilsSuite) TestGetLeaderDCByBundle(c *C) { { ID: "11", Role: Leader, - LabelConstraints: []Constraint{ + Constraints: []Constraint{ { Key: "zone", Op: In, diff --git a/ddl/placement_rule_test.go b/ddl/placement_rule_test.go index b051092a776e9..a9a916cb5a199 100644 --- a/ddl/placement_rule_test.go +++ b/ddl/placement_rule_test.go @@ -52,7 +52,7 @@ func (s *testPlacementSuite) TestPlacementBuild(c *C) { { Role: placement.Voter, Count: 3, - LabelConstraints: []placement.Constraint{ + Constraints: []placement.Constraint{ {Key: "zone", Op: "in", Values: []string{"sh"}}, }, }, @@ -67,9 +67,9 @@ func (s *testPlacementSuite) TestPlacementBuild(c *C) { Constraints: "", }}, output: []*placement.Rule{{ - Role: placement.Voter, - Count: 3, - LabelConstraints: []placement.Constraint{}, + Role: placement.Voter, + Count: 3, + Constraints: []placement.Constraint{}, }}, }, @@ -83,14 +83,14 @@ func (s *testPlacementSuite) TestPlacementBuild(c *C) { { Role: placement.Voter, Count: 1, - LabelConstraints: []placement.Constraint{ + Constraints: []placement.Constraint{ {Key: "zone", Op: "in", Values: []string{"sh"}}, }, }, { Role: placement.Voter, Count: 2, - LabelConstraints: []placement.Constraint{ + Constraints: []placement.Constraint{ {Key: "zone", Op: "in", Values: []string{"sh"}}, }, }, @@ -108,7 +108,7 @@ func (s *testPlacementSuite) TestPlacementBuild(c *C) { { Role: placement.Voter, Count: 3, - LabelConstraints: []placement.Constraint{ + Constraints: []placement.Constraint{ {Key: "zone", Op: "notIn", Values: []string{"sh"}}, {Key: "zone", Op: "notIn", Values: []string{"bj"}}, }, @@ -127,7 +127,7 @@ func (s *testPlacementSuite) TestPlacementBuild(c *C) { { Role: placement.Voter, Count: 3, - LabelConstraints: []placement.Constraint{ + Constraints: []placement.Constraint{ {Key: "zone", Op: "in", Values: []string{"sh"}}, {Key: "zone", Op: "notIn", Values: []string{"bj"}}, }, @@ -154,7 +154,7 @@ func (s *testPlacementSuite) TestPlacementBuild(c *C) { { Role: placement.Voter, Count: 3, - LabelConstraints: []placement.Constraint{ + Constraints: []placement.Constraint{ {Key: "zone", Op: "in", Values: []string{"sh"}}, {Key: "zone", Op: "notIn", Values: []string{"bj"}}, }, @@ -162,7 +162,7 @@ func (s *testPlacementSuite) TestPlacementBuild(c *C) { { Role: placement.Follower, Count: 2, - LabelConstraints: []placement.Constraint{ + Constraints: []placement.Constraint{ {Key: "zone", Op: "notIn", Values: []string{"sh"}}, {Key: "zone", Op: "in", Values: []string{"bj"}}, }, @@ -189,7 +189,7 @@ func (s *testPlacementSuite) TestPlacementBuild(c *C) { { Role: placement.Voter, Count: 2, - LabelConstraints: []placement.Constraint{ + Constraints: []placement.Constraint{ {Key: "zone", Op: "notIn", Values: []string{"sh"}}, {Key: "zone", Op: "in", Values: []string{"bj"}}, }, @@ -214,14 +214,14 @@ func (s *testPlacementSuite) TestPlacementBuild(c *C) { }, output: []*placement.Rule{ { - Role: placement.Voter, - Count: 1, - LabelConstraints: []placement.Constraint{{Key: "zone", Op: "notIn", Values: []string{"sh"}}}, + Role: placement.Voter, + Count: 1, + Constraints: []placement.Constraint{{Key: "zone", Op: "notIn", Values: []string{"sh"}}}, }, { - Role: placement.Voter, - Count: 1, - LabelConstraints: []placement.Constraint{{Key: "zone", Op: "in", Values: []string{"bj"}}}, + Role: placement.Voter, + Count: 1, + Constraints: []placement.Constraint{{Key: "zone", Op: "in", Values: []string{"bj"}}}, }, { Role: placement.Voter, @@ -306,7 +306,7 @@ func (s *testPlacementSuite) TestPlacementBuild(c *C) { { Role: placement.Voter, Count: 3, - LabelConstraints: []placement.Constraint{ + Constraints: []placement.Constraint{ {Key: "zone", Op: "in", Values: []string{"sh"}}, {Key: "zone", Op: "notIn", Values: []string{"bj"}}, }, diff --git a/ddl/placement_sql_test.go b/ddl/placement_sql_test.go index e77b0ba99d5cf..36812454fb662 100644 --- a/ddl/placement_sql_test.go +++ b/ddl/placement_sql_test.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/tidb/util/testkit" ) -func (s *testDBSuite1) TestAlterTableAlterPartition(c *C) { +func (s *testDBSuite6) TestAlterTableAlterPartition(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t1") @@ -404,7 +404,7 @@ PARTITION BY RANGE (c) ( GroupID: groupID, Role: placement.Leader, Count: 1, - LabelConstraints: []placement.Constraint{ + Constraints: []placement.Constraint{ { Key: placement.DCLabelKey, Op: placement.In, @@ -423,7 +423,7 @@ PARTITION BY RANGE (c) ( GroupID: groupID, Role: placement.Follower, Count: 3, - LabelConstraints: []placement.Constraint{ + Constraints: []placement.Constraint{ { Key: placement.DCLabelKey, Op: placement.In, @@ -619,7 +619,7 @@ PARTITION BY RANGE (c) ( GroupID: groupID, Role: placement.Leader, Count: 1, - LabelConstraints: []placement.Constraint{ + Constraints: []placement.Constraint{ { Key: placement.DCLabelKey, Op: placement.In, diff --git a/ddl/reorg.go b/ddl/reorg.go index fbe42573dbbf7..eb20f31ce5fe7 100644 --- a/ddl/reorg.go +++ b/ddl/reorg.go @@ -33,7 +33,6 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/statistics" - "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/tablecodec" @@ -156,12 +155,41 @@ func (rc *reorgCtx) clean() { rc.doneCh = nil } +// runReorgJob is used as a portal to do the reorganization work. +// eg: +// 1: add index +// 2: alter column type +// 3: clean global index +// +// ddl goroutine >---------+ +// ^ | +// | | +// | | +// | | <---(doneCh)--- f() +// HandleDDLQueue(...) | <---(regular timeout) +// | | <---(ctx done) +// | | +// | | +// A more ddl round <-----+ +// +// How can we cancel reorg job? +// +// The background reorg is continuously running except for several factors, for instances, ddl owner change, +// logic error (kv duplicate when insert index / cast error when alter column), ctx done, and cancel signal. +// +// When `admin cancel ddl jobs xxx` takes effect, we will give this kind of reorg ddl one more round. +// because we should pull the result from doneCh out, otherwise, the reorg worker will hang on `f()` logic, +// which is a kind of goroutine leak. +// +// That's why we couldn't set the job to rollingback state directly in `convertJob2RollbackJob`, which is a +// cancelling portal for admin cancel action. +// +// In other words, the cancelling signal is informed from the bottom up, we set the atomic cancel variable +// in the cancelling portal to notify the lower worker goroutine, and fetch the cancel error from them in +// the additional ddl round. +// +// After that, we can make sure that the worker goroutine is correctly shut down. func (w *worker) runReorgJob(t *meta.Meta, reorgInfo *reorgInfo, tblInfo *model.TableInfo, lease time.Duration, f func() error) error { - // lease = 0 means it's in an integration test. In this case we don't delay so the test won't run too slowly. - if lease > 0 { - delayForAsyncCommit() - } - job := reorgInfo.Job // This is for tests compatible, because most of the early tests try to build the reorg job manually // without reorg meta info, which will cause nil pointer in here. @@ -173,6 +201,12 @@ func (w *worker) runReorgJob(t *meta.Meta, reorgInfo *reorgInfo, tblInfo *model. } } if w.reorgCtx.doneCh == nil { + // Since reorg job will be interrupted for polling the cancel action outside. we don't need to wait for 2.5s + // for the later entrances. + // lease = 0 means it's in an integration test. In this case we don't delay so the test won't run too slowly. + if lease > 0 { + delayForAsyncCommit() + } // start a reorganization job w.wg.Add(1) w.reorgCtx.doneCh = make(chan error, 1) @@ -534,7 +568,7 @@ func getTableRange(d *ddlCtx, tbl table.PhysicalTable, snapshotVer uint64, prior } func getValidCurrentVersion(store kv.Storage) (ver kv.Version, err error) { - ver, err = store.CurrentVersion(oracle.GlobalTxnScope) + ver, err = store.CurrentVersion(kv.GlobalTxnScope) if err != nil { return ver, errors.Trace(err) } else if ver.Ver <= 0 { diff --git a/ddl/reorg_test.go b/ddl/reorg_test.go index 18dd9a975fceb..4c28540e7ad3b 100644 --- a/ddl/reorg_test.go +++ b/ddl/reorg_test.go @@ -217,7 +217,7 @@ func (s *testDDLSuite) TestReorgOwner(c *C) { c.Assert(err, IsNil) }() - dbInfo := testSchemaInfo(c, d1, "test") + dbInfo := testSchemaInfo(c, d1, "test_reorg") testCreateSchema(c, ctx, d1, dbInfo) tblInfo := testTableInfo(c, d1, "t", 3) diff --git a/ddl/restart_test.go b/ddl/restart_test.go index b587d54b80cc8..b7791ef7679bd 100644 --- a/ddl/restart_test.go +++ b/ddl/restart_test.go @@ -120,7 +120,7 @@ func (s *testSchemaSuite) TestSchemaResume(c *C) { testCheckOwner(c, d1, true) - dbInfo := testSchemaInfo(c, d1, "test") + dbInfo := testSchemaInfo(c, d1, "test_restart") job := &model.Job{ SchemaID: dbInfo.ID, Type: model.ActionCreateSchema, @@ -157,7 +157,7 @@ func (s *testStatSuite) TestStat(c *C) { c.Assert(err, IsNil) }() - dbInfo := testSchemaInfo(c, d, "test") + dbInfo := testSchemaInfo(c, d, "test_restart") testCreateSchema(c, testNewContext(d), d, dbInfo) // TODO: Get this information from etcd. diff --git a/ddl/rollingback.go b/ddl/rollingback.go index 845ef175f9204..bd9daab771c82 100644 --- a/ddl/rollingback.go +++ b/ddl/rollingback.go @@ -110,7 +110,15 @@ func convertNotStartAddIdxJob2RollbackJob(t *meta.Meta, job *model.Job, occuredE // Since modifying column job has two types: normal-type and reorg-type, we should handle it respectively. // normal-type has only two states: None -> Public // reorg-type has five states: None -> Delete-only -> Write-only -> Write-org -> Public -func rollingbackModifyColumn(t *meta.Meta, job *model.Job) (ver int64, err error) { +func rollingbackModifyColumn(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) { + // If the value of SnapshotVer isn't zero, it means the reorg workers have been started. + if job.SchemaState == model.StateWriteReorganization && job.SnapshotVer != 0 { + // column type change workers are started. we have to ask them to exit. + logutil.Logger(w.logCtx).Info("[ddl] run the cancelling DDL job", zap.String("job", job.String())) + w.reorgCtx.notifyReorgCancel() + // Give the this kind of ddl one more round to run, the errCancelledDDLJob should be fetched from the bottom up. + return w.onModifyColumn(d, t, job) + } _, tblInfo, oldCol, jp, err := getModifyColumnInfo(t, job) if err != nil { return ver, err @@ -138,7 +146,7 @@ func rollingbackModifyColumn(t *meta.Meta, job *model.Job) (ver int64, err error job.State = model.JobStateCancelled return ver, errCancelledDDLJob } - // The job has been in it's middle state and we roll it back. + // The job has been in its middle state (but the reorg worker hasn't started) and we roll it back here. job.State = model.JobStateRollingback return ver, errCancelledDDLJob } @@ -424,7 +432,7 @@ func convertJob2RollbackJob(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job) case model.ActionTruncateTable: ver, err = rollingbackTruncateTable(t, job) case model.ActionModifyColumn: - ver, err = rollingbackModifyColumn(t, job) + ver, err = rollingbackModifyColumn(w, d, t, job) case model.ActionRebaseAutoID, model.ActionShardRowID, model.ActionAddForeignKey, model.ActionDropForeignKey, model.ActionRenameTable, model.ActionRenameTables, model.ActionModifyTableCharsetAndCollate, model.ActionTruncateTablePartition, diff --git a/ddl/schema.go b/ddl/schema.go index 823e12a551900..a4b14a49bdbc3 100644 --- a/ddl/schema.go +++ b/ddl/schema.go @@ -68,16 +68,12 @@ func onCreateSchema(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error } func checkSchemaNotExists(d *ddlCtx, t *meta.Meta, schemaID int64, dbInfo *model.DBInfo) error { - // d.infoHandle maybe nil in some test. - if d.infoHandle == nil { - return checkSchemaNotExistsFromStore(t, schemaID, dbInfo) - } // Try to use memory schema info to check first. currVer, err := t.GetSchemaVersion() if err != nil { return err } - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() if is.SchemaMetaVersion() == currVer { return checkSchemaNotExistsFromInfoSchema(is, schemaID, dbInfo) } @@ -169,7 +165,7 @@ func onDropSchema(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) oldIDs := getIDs(tables) bundles := make([]*placement.Bundle, 0, len(oldIDs)+1) for _, ID := range append(oldIDs, dbInfo.ID) { - oldBundle, ok := d.infoHandle.Get().BundleByName(placement.GroupID(ID)) + oldBundle, ok := d.infoCache.GetLatest().BundleByName(placement.GroupID(ID)) if ok && !oldBundle.IsEmpty() { bundles = append(bundles, placement.BuildPlacementDropBundle(ID)) } diff --git a/ddl/schema_test.go b/ddl/schema_test.go index c70a0b793bb35..b4c8efee7b089 100644 --- a/ddl/schema_test.go +++ b/ddl/schema_test.go @@ -139,7 +139,7 @@ func (s *testSchemaSuite) TestSchema(c *C) { c.Assert(err, IsNil) }() ctx := testNewContext(d) - dbInfo := testSchemaInfo(c, d, "test") + dbInfo := testSchemaInfo(c, d, "test_schema") // create a database. job := testCreateSchema(c, ctx, d, dbInfo) @@ -228,7 +228,7 @@ func (s *testSchemaSuite) TestSchemaWaitJob(c *C) { // d2 must not be owner. d2.ownerManager.RetireOwner() - dbInfo := testSchemaInfo(c, d2, "test") + dbInfo := testSchemaInfo(c, d2, "test_schema") testCreateSchema(c, ctx, d2, dbInfo) testCheckSchemaState(c, d2, dbInfo, model.StatePublic) diff --git a/ddl/serial_test.go b/ddl/serial_test.go index e532bfc2352af..e6e3164381566 100644 --- a/ddl/serial_test.go +++ b/ddl/serial_test.go @@ -37,6 +37,7 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/meta/autoid" + "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" @@ -55,7 +56,7 @@ import ( var _ = SerialSuites(&testSerialSuite{}) // TODO(tangenta): Move all the parallel tests out of this file. -var _ = Suite(&testIntegrationSuite9{&testIntegrationSuite{}}) +var _ = Suite(&testIntegrationSuite7{&testIntegrationSuite{}}) type testSerialSuite struct { CommonHandleSuite @@ -113,7 +114,7 @@ func (s *testSerialSuite) TestChangeMaxIndexLength(c *C) { tk.MustExec("drop table t, t1") } -func (s *testIntegrationSuite9) TestPrimaryKey(c *C) { +func (s *testIntegrationSuite7) TestPrimaryKey(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("drop database if exists test_primary_key;") tk.MustExec("create database test_primary_key;") @@ -178,7 +179,7 @@ func (s *testIntegrationSuite9) TestPrimaryKey(c *C) { tk.MustExec("drop table t;") } -func (s *testIntegrationSuite9) TestDropAutoIncrementIndex(c *C) { +func (s *testIntegrationSuite7) TestDropAutoIncrementIndex(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t1") @@ -186,7 +187,7 @@ func (s *testIntegrationSuite9) TestDropAutoIncrementIndex(c *C) { tk.MustExec("alter table t1 drop index a") } -func (s *testIntegrationSuite9) TestMultiRegionGetTableEndHandle(c *C) { +func (s *testIntegrationSuite7) TestMultiRegionGetTableEndHandle(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("drop database if exists test_get_endhandle") tk.MustExec("create database test_get_endhandle") @@ -230,7 +231,7 @@ func (s *testIntegrationSuite9) TestMultiRegionGetTableEndHandle(c *C) { c.Assert(maxHandle, Equals, kv.IntHandle(10000)) } -func (s *testIntegrationSuite9) TestGetTableEndHandle(c *C) { +func (s *testIntegrationSuite7) TestGetTableEndHandle(c *C) { // TestGetTableEndHandle test ddl.GetTableMaxHandle method, which will return the max row id of the table. tk := testkit.NewTestKit(c, s.store) tk.MustExec("drop database if exists test_get_endhandle") @@ -322,7 +323,7 @@ func (s *testIntegrationSuite9) TestGetTableEndHandle(c *C) { c.Assert(emptyTable, IsFalse) } -func (s *testIntegrationSuite9) TestMultiRegionGetTableEndCommonHandle(c *C) { +func (s *testIntegrationSuite7) TestMultiRegionGetTableEndCommonHandle(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("drop database if exists test_get_endhandle") tk.MustExec("create database test_get_endhandle") @@ -367,7 +368,7 @@ func (s *testIntegrationSuite9) TestMultiRegionGetTableEndCommonHandle(c *C) { c.Assert(maxHandle, HandleEquals, MustNewCommonHandle(c, "a", 1, 1)) } -func (s *testIntegrationSuite9) TestGetTableEndCommonHandle(c *C) { +func (s *testIntegrationSuite7) TestGetTableEndCommonHandle(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("drop database if exists test_get_endhandle") tk.MustExec("create database test_get_endhandle") @@ -503,15 +504,15 @@ func (s *testSerialSuite) TestCreateTableWithLike(c *C) { // for failure table cases tk.MustExec("use ctwl_db") - failSQL := fmt.Sprintf("create table t1 like test_not_exist.t") + failSQL := "create table t1 like test_not_exist.t" tk.MustGetErrCode(failSQL, mysql.ErrNoSuchTable) - failSQL = fmt.Sprintf("create table t1 like test.t_not_exist") + failSQL = "create table t1 like test.t_not_exist" tk.MustGetErrCode(failSQL, mysql.ErrNoSuchTable) - failSQL = fmt.Sprintf("create table t1 (like test_not_exist.t)") + failSQL = "create table t1 (like test_not_exist.t)" tk.MustGetErrCode(failSQL, mysql.ErrNoSuchTable) - failSQL = fmt.Sprintf("create table test_not_exis.t1 like ctwl_db.t") + failSQL = "create table test_not_exis.t1 like ctwl_db.t" tk.MustGetErrCode(failSQL, mysql.ErrBadDB) - failSQL = fmt.Sprintf("create table t1 like ctwl_db.t") + failSQL = "create table t1 like ctwl_db.t" tk.MustGetErrCode(failSQL, mysql.ErrTableExists) // test failure for wrong object cases @@ -524,6 +525,16 @@ func (s *testSerialSuite) TestCreateTableWithLike(c *C) { tk.MustExec("drop database ctwl_db") tk.MustExec("drop database ctwl_db1") + + // Test create table like at temporary mode. + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("use test") + tk.MustExec("drop table if exists temporary_table;") + tk.MustExec("create global temporary table temporary_table (a int, b int,index(a)) on commit delete rows") + tk.MustExec("drop table if exists temporary_table_t1;") + _, err = tk.Exec("create table temporary_table_t1 like temporary_table") + c.Assert(err.Error(), Equals, core.ErrOptOnTemporaryTable.GenWithStackByArgs("create table like").Error()) + tk.MustExec("drop table if exists temporary_table;") } // TestCancelAddIndex1 tests canceling ddl job when the add index worker is not started. @@ -925,7 +936,19 @@ func (s *testSerialSuite) TestTableLocksEnable(c *C) { checkTableLock(c, tk.Se, "test", "t1", model.TableLockNone) } -func (s *testSerialSuite) TestAutoRandom(c *C) { +func (s *testSerialDBSuite) TestAutoRandomOnTemporaryTable(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists auto_random_temporary") + tk.MustExec("set tidb_enable_global_temporary_table=true") + _, err := tk.Exec("create global temporary table auto_random_temporary (a bigint primary key auto_random(3), b varchar(255)) on commit delete rows;") + c.Assert(err.Error(), Equals, core.ErrOptOnTemporaryTable.GenWithStackByArgs("auto_random").Error()) + tk.MustExec("set @@tidb_enable_noop_functions = 1") + _, err = tk.Exec("create temporary table t(a bigint key auto_random);") + c.Assert(err.Error(), Equals, core.ErrOptOnTemporaryTable.GenWithStackByArgs("auto_random").Error()) +} + +func (s *testSerialDBSuite) TestAutoRandom(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("create database if not exists auto_random_db") defer tk.MustExec("drop database if exists auto_random_db") @@ -1146,7 +1169,7 @@ func (s *testSerialSuite) TestAutoRandom(c *C) { }) } -func (s *testIntegrationSuite9) TestAutoRandomChangeFromAutoInc(c *C) { +func (s *testIntegrationSuite7) TestAutoRandomChangeFromAutoInc(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test;") tk.MustExec("set @@tidb_allow_remove_auto_inc = 1;") @@ -1202,7 +1225,7 @@ func (s *testIntegrationSuite9) TestAutoRandomChangeFromAutoInc(c *C) { tk.MustExec("alter table t modify column a bigint auto_random(4);") } -func (s *testIntegrationSuite9) TestAutoRandomExchangePartition(c *C) { +func (s *testIntegrationSuite7) TestAutoRandomExchangePartition(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("create database if not exists auto_random_db") defer tk.MustExec("drop database if exists auto_random_db") @@ -1236,7 +1259,7 @@ func (s *testIntegrationSuite9) TestAutoRandomExchangePartition(c *C) { tk.MustQuery("select count(*) from e4").Check(testkit.Rows("4")) } -func (s *testIntegrationSuite9) TestAutoRandomIncBitsIncrementAndOffset(c *C) { +func (s *testIntegrationSuite7) TestAutoRandomIncBitsIncrementAndOffset(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("create database if not exists auto_random_db") defer tk.MustExec("drop database if exists auto_random_db") @@ -1388,7 +1411,7 @@ func (s *testSerialSuite) TestForbidUnsupportedCollations(c *C) { // mustGetUnsupportedCollation("alter table t convert to collate utf8mb4_unicode_ci", "utf8mb4_unicode_ci") } -func (s *testIntegrationSuite9) TestInvisibleIndex(c *C) { +func (s *testIntegrationSuite7) TestInvisibleIndex(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -1464,7 +1487,7 @@ func (s *testIntegrationSuite9) TestInvisibleIndex(c *C) { c.Check(len(res.Rows()), Equals, 1) } -func (s *testIntegrationSuite9) TestCreateClusteredIndex(c *C) { +func (s *testIntegrationSuite7) TestCreateClusteredIndex(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk.Se.GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeOn tk.MustExec("CREATE TABLE t1 (a int primary key, b int)") @@ -1515,7 +1538,7 @@ func (s *testIntegrationSuite9) TestCreateClusteredIndex(c *C) { c.Assert(tbl.Meta().IsCommonHandle, IsFalse) } -func (s *testSerialSuite) TestCreateTableNoBlock(c *C) { +func (s *testSerialDBSuite) TestCreateTableNoBlock(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) c.Assert(failpoint.Enable("github.com/pingcap/tidb/ddl/checkOwnerCheckAllVersionsWaitTime", `return(true)`), IsNil) defer func() { diff --git a/ddl/stat_test.go b/ddl/stat_test.go index fe562a0ae0fb8..1ed3cbfe4c7fc 100644 --- a/ddl/stat_test.go +++ b/ddl/stat_test.go @@ -61,7 +61,7 @@ func (s *testSerialStatSuite) TestDDLStatsInfo(c *C) { c.Assert(err, IsNil) }() - dbInfo := testSchemaInfo(c, d, "test") + dbInfo := testSchemaInfo(c, d, "test_stat") testCreateSchema(c, testNewContext(d), d, dbInfo) tblInfo := testTableInfo(c, d, "t", 2) ctx := testNewContext(d) diff --git a/ddl/table.go b/ddl/table.go index acd209a7bb0da..6c113fc855b23 100644 --- a/ddl/table.go +++ b/ddl/table.go @@ -37,6 +37,7 @@ import ( "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/tablecodec" + tidb_util "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/gcutil" ) @@ -56,11 +57,6 @@ func onCreateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) job.State = model.JobStateCancelled return ver, errors.Trace(err) } - if tbInfo.Partition != nil && (tbInfo.TempTableType == model.TempTableGlobal || tbInfo.TempTableType == model.TempTableLocal) { - // unsupported ddl, cancel this job. - job.State = model.JobStateCancelled - return ver, errors.Trace(ErrPartitionNoTemporary) - } tbInfo.State = model.StateNone err := checkTableNotExists(d, t, schemaID, tbInfo.Name.L) @@ -492,34 +488,32 @@ func onTruncateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ erro } } - if d.infoHandle != nil && d.infoHandle.IsValid() { - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() - bundles := make([]*placement.Bundle, 0, len(oldPartitionIDs)+1) - if oldBundle, ok := is.BundleByName(placement.GroupID(tableID)); ok { - bundles = append(bundles, placement.BuildPlacementCopyBundle(oldBundle, newTableID)) - } + bundles := make([]*placement.Bundle, 0, len(oldPartitionIDs)+1) + if oldBundle, ok := is.BundleByName(placement.GroupID(tableID)); ok { + bundles = append(bundles, placement.BuildPlacementCopyBundle(oldBundle, newTableID)) + } - if pi := tblInfo.GetPartitionInfo(); pi != nil { - oldIDs := make([]int64, 0, len(oldPartitionIDs)) - newIDs := make([]int64, 0, len(oldPartitionIDs)) - newDefs := pi.Definitions - for i := range oldPartitionIDs { - newID := newDefs[i].ID - if oldBundle, ok := is.BundleByName(placement.GroupID(oldPartitionIDs[i])); ok && !oldBundle.IsEmpty() { - oldIDs = append(oldIDs, oldPartitionIDs[i]) - newIDs = append(newIDs, newID) - bundles = append(bundles, placement.BuildPlacementCopyBundle(oldBundle, newID)) - } + if pi := tblInfo.GetPartitionInfo(); pi != nil { + oldIDs := make([]int64, 0, len(oldPartitionIDs)) + newIDs := make([]int64, 0, len(oldPartitionIDs)) + newDefs := pi.Definitions + for i := range oldPartitionIDs { + newID := newDefs[i].ID + if oldBundle, ok := is.BundleByName(placement.GroupID(oldPartitionIDs[i])); ok && !oldBundle.IsEmpty() { + oldIDs = append(oldIDs, oldPartitionIDs[i]) + newIDs = append(newIDs, newID) + bundles = append(bundles, placement.BuildPlacementCopyBundle(oldBundle, newID)) } - job.CtxVars = []interface{}{oldIDs, newIDs} } + job.CtxVars = []interface{}{oldIDs, newIDs} + } - err = infosync.PutRuleBundles(context.TODO(), bundles) - if err != nil { - job.State = model.JobStateCancelled - return 0, errors.Wrapf(err, "failed to notify PD the placement rules") - } + err = infosync.PutRuleBundles(context.TODO(), bundles) + if err != nil { + job.State = model.JobStateCancelled + return 0, errors.Wrapf(err, "failed to notify PD the placement rules") } // Clear the tiflash replica available status. @@ -882,6 +876,11 @@ func (w *worker) onSetTableFlashReplica(t *meta.Meta, job *model.Job) (ver int64 return ver, errors.Trace(err) } + // Ban setting replica count for tables in system database. + if tidb_util.IsMemOrSysDB(job.SchemaName) { + return ver, errors.Trace(errUnsupportedAlterReplicaForSysTable) + } + err = w.checkTiFlashReplicaCount(replicaInfo.Count) if err != nil { job.State = model.JobStateCancelled @@ -972,16 +971,12 @@ func onUpdateFlashReplicaStatus(t *meta.Meta, job *model.Job) (ver int64, _ erro } func checkTableNotExists(d *ddlCtx, t *meta.Meta, schemaID int64, tableName string) error { - // d.infoHandle maybe nil in some test. - if d.infoHandle == nil || !d.infoHandle.IsValid() { - return checkTableNotExistsFromStore(t, schemaID, tableName) - } // Try to use memory schema info to check first. currVer, err := t.GetSchemaVersion() if err != nil { return err } - is := d.infoHandle.Get() + is := d.infoCache.GetLatest() if is.SchemaMetaVersion() == currVer { return checkTableNotExistsFromInfoSchema(is, schemaID, tableName) } diff --git a/ddl/table_test.go b/ddl/table_test.go index 5760fc2b152b5..10927908f5289 100644 --- a/ddl/table_test.go +++ b/ddl/table_test.go @@ -355,7 +355,7 @@ func (s *testTableSuite) SetUpSuite(c *C) { WithLease(testLease), ) - s.dbInfo = testSchemaInfo(c, s.d, "test") + s.dbInfo = testSchemaInfo(c, s.d, "test_table") testCreateSchema(c, testNewContext(s.d), s.d, s.dbInfo) } diff --git a/ddl/util/syncer_test.go b/ddl/util/syncer_test.go index b552488ad49de..5a9d41d47e3b8 100644 --- a/ddl/util/syncer_test.go +++ b/ddl/util/syncer_test.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/parser/terror" . "github.com/pingcap/tidb/ddl" . "github.com/pingcap/tidb/ddl/util" + "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/owner" "github.com/pingcap/tidb/store/mockstore" "go.etcd.io/etcd/clientv3" @@ -69,11 +70,14 @@ func TestSyncerSimple(t *testing.T) { defer clus.Terminate(t) cli := clus.RandClient() ctx := goctx.Background() + ic := infoschema.NewCache(2) + ic.Insert(infoschema.MockInfoSchemaWithSchemaVer(nil, 0)) d := NewDDL( ctx, WithEtcdClient(cli), WithStore(store), WithLease(testLease), + WithInfoCache(ic), ) err = d.Start(nil) if err != nil { @@ -110,11 +114,14 @@ func TestSyncerSimple(t *testing.T) { t.Fatalf("client get global version result not match, err %v", err) } + ic2 := infoschema.NewCache(2) + ic2.Insert(infoschema.MockInfoSchemaWithSchemaVer(nil, 0)) d1 := NewDDL( ctx, WithEtcdClient(cli), WithStore(store), WithLease(testLease), + WithInfoCache(ic2), ) err = d1.Start(nil) if err != nil { diff --git a/ddl/util/util.go b/ddl/util/util.go index 0c3ec2608b9eb..0e5eb8fe2051d 100644 --- a/ddl/util/util.go +++ b/ddl/util/util.go @@ -186,7 +186,9 @@ func LoadGlobalVars(ctx sessionctx.Context, varNames []string) error { for _, row := range rows { varName := row.GetString(0) varValue := row.GetString(1) - variable.SetLocalSystemVar(varName, varValue) + if err = ctx.GetSessionVars().SetSystemVar(varName, varValue); err != nil { + return err + } } } return nil diff --git a/distsql/request_builder.go b/distsql/request_builder.go index ce577b993d009..0fd44b044ae3b 100644 --- a/distsql/request_builder.go +++ b/distsql/request_builder.go @@ -27,7 +27,6 @@ import ( "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/statistics" - "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/codec" @@ -230,14 +229,9 @@ func (builder *RequestBuilder) SetFromSessionVars(sv *variable.SessionVars) *Req builder.Request.TaskID = sv.StmtCtx.TaskID builder.Request.Priority = builder.getKVPriority(sv) builder.Request.ReplicaRead = sv.GetReplicaRead() - if sv.SnapshotInfoschema != nil { - builder.Request.SchemaVar = infoschema.GetInfoSchemaBySessionVars(sv).SchemaMetaVersion() - } else { - builder.Request.SchemaVar = sv.TxnCtx.SchemaVersion - } builder.txnScope = sv.TxnCtx.TxnScope builder.IsStaleness = sv.TxnCtx.IsStaleness - if builder.IsStaleness && builder.txnScope != oracle.GlobalTxnScope { + if builder.IsStaleness && builder.txnScope != kv.GlobalTxnScope { builder.MatchStoreLabels = []*metapb.StoreLabel{ { Key: placement.DCLabelKey, @@ -245,6 +239,7 @@ func (builder *RequestBuilder) SetFromSessionVars(sv *variable.SessionVars) *Req }, } } + builder.SetResourceGroupTag(sv.StmtCtx) return builder } @@ -270,19 +265,29 @@ func (builder *RequestBuilder) SetTiDBServerID(serverID uint64) *RequestBuilder // SetFromInfoSchema sets the following fields from infoSchema: // "bundles" -func (builder *RequestBuilder) SetFromInfoSchema(is infoschema.InfoSchema) *RequestBuilder { - if is == nil { +func (builder *RequestBuilder) SetFromInfoSchema(pis interface{}) *RequestBuilder { + is, ok := pis.(infoschema.InfoSchema) + if !ok { return builder } builder.is = is + builder.Request.SchemaVar = is.SchemaMetaVersion() + return builder +} + +// SetResourceGroupTag sets the request resource group tag. +func (builder *RequestBuilder) SetResourceGroupTag(sc *stmtctx.StatementContext) *RequestBuilder { + if variable.TopSQLEnabled() { + builder.Request.ResourceGroupTag = sc.GetResourceGroupTag() + } return builder } func (builder *RequestBuilder) verifyTxnScope() error { if builder.txnScope == "" { - builder.txnScope = oracle.GlobalTxnScope + builder.txnScope = kv.GlobalTxnScope } - if builder.txnScope == oracle.GlobalTxnScope || builder.is == nil { + if builder.txnScope == kv.GlobalTxnScope || builder.is == nil { return nil } visitPhysicalTableID := make(map[int64]struct{}) @@ -601,7 +606,7 @@ func CommonHandleRangesToKVRanges(sc *stmtctx.StatementContext, tids []int64, ra // VerifyTxnScope verify whether the txnScope and visited physical table break the leader rule's dcLocation. func VerifyTxnScope(txnScope string, physicalTableID int64, is infoschema.InfoSchema) bool { - if txnScope == "" || txnScope == oracle.GlobalTxnScope { + if txnScope == "" || txnScope == kv.GlobalTxnScope { return true } bundle, ok := is.BundleByName(placement.GroupID(physicalTableID)) diff --git a/distsql/request_builder_test.go b/distsql/request_builder_test.go index 18c1ee8fc24e4..b9837f77c1ff1 100644 --- a/distsql/request_builder_test.go +++ b/distsql/request_builder_test.go @@ -19,13 +19,11 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/parser/mysql" - "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/statistics" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" @@ -324,7 +322,7 @@ func (s *testSuite) TestRequestBuilder1(c *C) { NotFillCache: false, SyncLog: false, Streaming: false, - ReplicaRead: tikvstore.ReplicaReadLeader, + ReplicaRead: kv.ReplicaReadLeader, } c.Assert(actual, DeepEquals, expect) } @@ -400,7 +398,7 @@ func (s *testSuite) TestRequestBuilder2(c *C) { NotFillCache: false, SyncLog: false, Streaming: false, - ReplicaRead: tikvstore.ReplicaReadLeader, + ReplicaRead: kv.ReplicaReadLeader, } c.Assert(actual, DeepEquals, expect) } @@ -447,7 +445,7 @@ func (s *testSuite) TestRequestBuilder3(c *C) { NotFillCache: false, SyncLog: false, Streaming: false, - ReplicaRead: tikvstore.ReplicaReadLeader, + ReplicaRead: kv.ReplicaReadLeader, } c.Assert(actual, DeepEquals, expect) } @@ -494,7 +492,7 @@ func (s *testSuite) TestRequestBuilder4(c *C) { Streaming: true, NotFillCache: false, SyncLog: false, - ReplicaRead: tikvstore.ReplicaReadLeader, + ReplicaRead: kv.ReplicaReadLeader, } c.Assert(actual, DeepEquals, expect) } @@ -577,10 +575,10 @@ func (s *testSuite) TestRequestBuilder6(c *C) { } func (s *testSuite) TestRequestBuilder7(c *C) { - for _, replicaRead := range []tikvstore.ReplicaReadType{ - tikvstore.ReplicaReadLeader, - tikvstore.ReplicaReadFollower, - tikvstore.ReplicaReadMixed, + for _, replicaRead := range []kv.ReplicaReadType{ + kv.ReplicaReadLeader, + kv.ReplicaReadFollower, + kv.ReplicaReadMixed, } { vars := variable.NewSessionVars() vars.SetReplicaRead(replicaRead) @@ -613,7 +611,6 @@ func (s *testSuite) TestRequestBuilder7(c *C) { func (s *testSuite) TestRequestBuilder8(c *C) { sv := variable.NewSessionVars() - sv.SnapshotInfoschema = infoschema.MockInfoSchemaWithSchemaVer(nil, 10000) actual, err := (&RequestBuilder{}). SetFromSessionVars(sv). Build() @@ -626,8 +623,7 @@ func (s *testSuite) TestRequestBuilder8(c *C) { IsolationLevel: 0, Priority: 0, MemTracker: (*memory.Tracker)(nil), - ReplicaRead: 0x1, - SchemaVar: 10000, + SchemaVar: 0, } c.Assert(actual, DeepEquals, expect) } diff --git a/distsql/select_result.go b/distsql/select_result.go index d5a2d65de2622..ef1a99e193e57 100644 --- a/distsql/select_result.go +++ b/distsql/select_result.go @@ -41,6 +41,7 @@ import ( "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/memory" + "github.com/pingcap/tidb/util/sli" "github.com/pingcap/tipb/go-tipb" "go.uber.org/zap" ) @@ -57,6 +58,7 @@ var ( var ( _ SelectResult = (*selectResult)(nil) _ SelectResult = (*streamResult)(nil) + _ SelectResult = (*serialSelectResults)(nil) ) // SelectResult is an iterator of coprocessor partial results. @@ -69,6 +71,56 @@ type SelectResult interface { Close() error } +// NewSerialSelectResults create a SelectResult which will read each SelectResult serially. +func NewSerialSelectResults(selectResults []SelectResult) SelectResult { + return &serialSelectResults{ + selectResults: selectResults, + cur: 0, + } +} + +// serialSelectResults reads each SelectResult serially +type serialSelectResults struct { + selectResults []SelectResult + cur int +} + +func (ssr *serialSelectResults) NextRaw(ctx context.Context) ([]byte, error) { + for ssr.cur < len(ssr.selectResults) { + resultSubset, err := ssr.selectResults[ssr.cur].NextRaw(ctx) + if err != nil { + return nil, err + } + if len(resultSubset) > 0 { + return resultSubset, nil + } + ssr.cur++ // move to the next SelectResult + } + return nil, nil +} + +func (ssr *serialSelectResults) Next(ctx context.Context, chk *chunk.Chunk) error { + for ssr.cur < len(ssr.selectResults) { + if err := ssr.selectResults[ssr.cur].Next(ctx, chk); err != nil { + return err + } + if chk.NumRows() > 0 { + return nil + } + ssr.cur++ // move to the next SelectResult + } + return nil +} + +func (ssr *serialSelectResults) Close() (err error) { + for _, r := range ssr.selectResults { + if rerr := r.Close(); rerr != nil { + err = rerr + } + } + return +} + type selectResult struct { label string resp kv.Response @@ -290,13 +342,13 @@ func (r *selectResult) updateCopRuntimeStats(ctx context.Context, copStats *copr if r.rootPlanID <= 0 || r.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl == nil || callee == "" { return } - if len(r.selectResp.GetExecutionSummaries()) != len(r.copPlanIDs) { - logutil.Logger(ctx).Error("invalid cop task execution summaries length", - zap.Int("expected", len(r.copPlanIDs)), - zap.Int("received", len(r.selectResp.GetExecutionSummaries()))) - return + if copStats.ScanDetail != nil { + readKeys := copStats.ScanDetail.ProcessedKeys + readTime := copStats.TimeDetail.KvReadWallTimeMs.Seconds() + sli.ObserveReadSLI(uint64(readKeys), readTime) } + if r.stats == nil { id := r.rootPlanID r.stats = &selectResultRuntimeStats{ @@ -311,12 +363,49 @@ func (r *selectResult) updateCopRuntimeStats(ctx context.Context, copStats *copr r.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RecordScanDetail(r.copPlanIDs[len(r.copPlanIDs)-1], r.storeType.Name(), copStats.ScanDetail) } - for i, detail := range r.selectResp.GetExecutionSummaries() { + // If hasExecutor is true, it means the summary is returned from TiFlash. + hasExecutor := false + for _, detail := range r.selectResp.GetExecutionSummaries() { if detail != nil && detail.TimeProcessedNs != nil && detail.NumProducedRows != nil && detail.NumIterations != nil { - planID := r.copPlanIDs[i] - r.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl. - RecordOneCopTask(planID, r.storeType.Name(), callee, detail) + if detail.ExecutorId != nil { + hasExecutor = true + } + break + } + } + if hasExecutor { + var recorededPlanIDs = make(map[int]int) + for i, detail := range r.selectResp.GetExecutionSummaries() { + if detail != nil && detail.TimeProcessedNs != nil && + detail.NumProducedRows != nil && detail.NumIterations != nil { + planID := r.copPlanIDs[i] + recorededPlanIDs[r.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl. + RecordOneCopTask(planID, r.storeType.Name(), callee, detail)] = 0 + } + } + num := uint64(0) + dummySummary := &tipb.ExecutorExecutionSummary{TimeProcessedNs: &num, NumProducedRows: &num, NumIterations: &num, ExecutorId: nil} + for _, planID := range r.copPlanIDs { + if _, ok := recorededPlanIDs[planID]; !ok { + r.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RecordOneCopTask(planID, r.storeType.Name(), callee, dummySummary) + } + } + } else { + // For cop task cases, we still need this protection. + if len(r.selectResp.GetExecutionSummaries()) != len(r.copPlanIDs) { + logutil.Logger(ctx).Error("invalid cop task execution summaries length", + zap.Int("expected", len(r.copPlanIDs)), + zap.Int("received", len(r.selectResp.GetExecutionSummaries()))) + return + } + for i, detail := range r.selectResp.GetExecutionSummaries() { + if detail != nil && detail.TimeProcessedNs != nil && + detail.NumProducedRows != nil && detail.NumIterations != nil { + planID := r.copPlanIDs[i] + r.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl. + RecordOneCopTask(planID, r.storeType.Name(), callee, detail) + } } } } diff --git a/docs/design/2020-06-24-placement-rules-in-sql.md b/docs/design/2020-06-24-placement-rules-in-sql.md index a07c7d5b5e171..d5eb56f690e1e 100644 --- a/docs/design/2020-06-24-placement-rules-in-sql.md +++ b/docs/design/2020-06-24-placement-rules-in-sql.md @@ -14,7 +14,7 @@ The scenarios of defining placement rules in SQL include: - Place data across regions to improve access locality - Add a TiFlash replica for a table -- Limit data within its national border to gaurantee data sovereignty +- Limit data within its national border to guarantee data sovereignty - Place latest data to SSD and history data to HDD - Place the leader of hot data to a high-performance TiKV instance - Increase the replica count of more important data @@ -233,7 +233,7 @@ For example, `CONSTRAINTS="{+zone=sh:1,-zone=bj:2}"` indicates to place 1 replic In the list format, `count` is not specified. The number of replicas for each constraint is not limited, but the total number of replicas should still conform to the `REPLICAS` option. -For example, `CONSTRAINTS="[+zone=sh,+zone=bj]" REPLICAS=3` indicates to place 3 repicas on either `sh` or `bj`. There may be 2 replicas on `sh` and 1 in `bj`, or 2 in `bj` and 1 in `sh`. It's up to PD. +For example, `CONSTRAINTS="[+zone=sh,+zone=bj]" REPLICAS=3` indicates to place 3 replicas on either `sh` or `bj`. There may be 2 replicas on `sh` and 1 in `bj`, or 2 in `bj` and 1 in `sh`. It's up to PD. Label constraints can be implemented by defining `label_constraints` field in PD placement rule configuration. `+` and `-` correspond to property `op`. Specifically, `+` is equivalent to `in` and `-` is equivalent to `notIn`. @@ -553,7 +553,7 @@ However, TiDB also uses placement rules in some cases, as discussed in section " Before choosing the solution, transactional requirements need to be noticed: -- Defining placement rules may fail, and users will probably retry it. As retrying `ADD PLACEMENT POLICY` will add more replicas than expected, the atomacity of the opertion needs to be gauranteed. +- Defining placement rules may fail, and users will probably retry it. As retrying `ADD PLACEMENT POLICY` will add more replicas than expected, the atomicity of the opertion needs to be guaranteed. - `ADD PLACEMENT POLICY` needs to read the original placement rules, combine the 2 rules and then store them to PD, so linearizability should be gauranteed. If the placement rules are stored on both TiKV and PD, the approaches to keep atomicity are as follows: @@ -583,7 +583,7 @@ The comparison shows that both solutions are possible, but storing placement rul The scenarios where TiDB queries placement rules are as follows: 1. The optimizer uses placement rules to decide to route cop request to TiKV or TiFlash. It's already implemented and the TiFlash information is written into table information, which is stored on TiKV. -2. It will be probably used in locality-aware features in the furture, such as follower-read. Follower-read is always used when TiDB wants to read the nearest replica to reduce multi-region latency. In some distributed databases, it’s implemented by labelling data nodes and selecting the nearest replica according to the labels. +2. It will be probably used in locality-aware features in the future, such as follower-read. Follower-read is always used when TiDB wants to read the nearest replica to reduce multi-region latency. In some distributed databases, it’s implemented by labelling data nodes and selecting the nearest replica according to the labels. 3. Local transactions need to know the binding relationship between Raft leader and region, which is also defined by placement rules. 4. Once a rule is defined on a table, all the subsequent partitions added to the table should also inherit the rule. So the `ADD PARTITION` operation should query the rules on the table. The same is true for creating tables and indices. 5. `SHOW PLACEMENT POLICY` statement should output the placement rules correctly. @@ -615,7 +615,7 @@ The fact that the DDL procedure in TiDB is mature helps to achieve some features - Placement rules are defined in serial as there's only one DDL owner at the same time - DDL is capable of disaster recovery as the middle states are persistent in TiKV - DDL is rollbackable as the middle states can transform from one to another -- Updating schema version guarantees all active transactions are based on the same version of placement ruels +- Updating schema version guarantees all active transactions are based on the same version of placement rules ### Rule priorities @@ -712,7 +712,7 @@ ALTER TABLE t ALTER PLACEMENT POLICY CONSTRAINTS="{+zone=bj:2,+zone=sh:1}" ROLE=voter; ``` -It needs 2 placement rules for `voter` in the PD placment rule configuration, because each rule can only specify one `count`. To make `id` unique, a unique identifier must be appended to `id`. DDL job ID plus an index in the job is a good choice. +It needs 2 placement rules for `voter` in the PD placement rule configuration, because each rule can only specify one `count`. To make `id` unique, a unique identifier must be appended to `id`. DDL job ID plus an index in the job is a good choice. Take the case above for example, assuming the table ID of `t` is 100, the ID of the DDL job executing this statement is 200, then `id` of the placement rules are `100-200-1` and `100-200-2`. diff --git a/docs/design/2021-03-09-dynamic-privileges.md b/docs/design/2021-03-09-dynamic-privileges.md index 7ad0d59d2c54e..c85c0dc0c5305 100644 --- a/docs/design/2021-03-09-dynamic-privileges.md +++ b/docs/design/2021-03-09-dynamic-privileges.md @@ -1,7 +1,7 @@ # Proposal: - Author(s): [morgo](https://github.com/morgo) -- Last updated: April 25, 2021 +- Last updated: May 04, 2021 - Discussion at: N/A ## Table of Contents @@ -238,7 +238,7 @@ No change | Privilege Name | Description | Notes | | --------------- | --------------- | --------------- | -| `RESTRICTED_SYSTEM_VARIABLES_ADMIN` | Allows changing a restricted `GLOBAL` system variable. | Currently in SEM all high risk variables are unloaded. TBD, it might be required in future that they are only visible/settable to those with this privilege and not SUPER. | +| `RESTRICTED_VARIABLES_ADMIN` | Allows changing a restricted `GLOBAL` system variable. | Currently in SEM all high risk variables are unloaded. TBD, it might be required in future that they are only visible/settable to those with this privilege and not SUPER. | | `RESTRICTED_STATUS_ADMIN` | Allows observing restricted status variables. | i.e. `SHOW GLOBAL STATUS` by default hides some statistics when `SEM` is enabled. | | `RESTRICTED_CONNECTION_ADMIN` | A special privilege to say that their connections, etc. can’t be killed by SUPER users AND they can kill connections by all other users. Affects `KILL`, `KILL TIDB` commands. | It is intended for the CloudAdmin user in DBaaS. | | `RESTRICTED_USER_ADMIN` | A special privilege to say that their access can’t be changed by `SUPER` users. Statements `DROP USER`, `SET PASSWORD`, `ALTER USER`, `REVOKE` are all limited. | It is intended for the CloudAdmin user in DbaaS. | diff --git a/docs/design/2021-03-09-security-enhanced-mode.md b/docs/design/2021-03-09-security-enhanced-mode.md index e939fec67c154..efc5b79f499e4 100644 --- a/docs/design/2021-03-09-security-enhanced-mode.md +++ b/docs/design/2021-03-09-security-enhanced-mode.md @@ -1,7 +1,7 @@ # Proposal: - Author(s): [morgo](https://github.com/morgo) -- Last updated: April 25, 2021 +- Last updated: May 04, 2021 - Discussion at: N/A ## Table of Contents @@ -49,7 +49,7 @@ A boolean option called `EnableEnhancedSecurity` (default `FALSE`) will be added ### System Variables -The following system variables will be hidden unless the user has the `RESTRICTED_SYSTEM_VARIABLES_ADMIN` privilege: +The following system variables will be hidden unless the user has the `RESTRICTED_VARIABLES_ADMIN` privilege: * variable.TiDBDDLSlowOprThreshold, * variable.TiDBAllowRemoveAutoInc, diff --git a/docs/design/2021-04-20-temporary-table.md b/docs/design/2021-04-20-temporary-table.md new file mode 100644 index 0000000000000..dae11f7fcce97 --- /dev/null +++ b/docs/design/2021-04-20-temporary-table.md @@ -0,0 +1,403 @@ +# Temporary Table Design + +- Authors: [Kangli Mao](http://github.com/tiancaiamao), [Ming Zhang](http://github.com/djshow832) +- Discussion PR: https://github.com/pingcap/tidb/pull/24168 +- Tracking Issue: https://github.com/pingcap/tidb/issues/24169 + +# Introduction + +A temporary table is a special table whose rows data are only temporarily available. + +TiDB will implement both local temporary tables and global temporary tables. The local temporary table is basically compatible with MySQL temporary tables. The global temporary table is a subset of the SQL standard, which supports only one table commit action. + +For a local temporary table, it is visible only within the current session, and the table is dropped automatically when the session is closed. + +For a global temporary table, the table schema is visible to all the sessions, but the changes into the table are only available within the current transaction, when the transaction commits, all changes to the global temporary table are discarded. + +## Syntax + +The syntax of creating and dropping a global temporary table: + +```sql +CREATE GLOBAL TEMPORARY TABLE tbl_name (create_definition) +[ENGINE=engine_name] +ON COMMIT DELETE ROWS; + +DROP TABLE tbl_name; +``` + +The syntax of creating and dropping a session(local) temporary table: + +```sql +CREATE TEMPORARY TABLE tbl_name (create_definition) +[ENGINE=engine_name]; + +DROP [TEMPORARY] TABLE tbl_name; +``` + +In the following sectionsn session temporary tables and local temporary tables are used interchangeably. + +## Visibility of table definition + +There are 2 kinds of table definition visibility. + +- Global: global temporary tables are visible to all sessions. These tables only need to be created once and the metadata is persistent. +- Local: local temporary tables are visible to the session that creates them. These tables must be created in the session before being used and the metadata is only kept in memory. + +## Visibility of data + +There are 2 kinds of data visibility: + +- Session: when the table is a session temporary table, the data will be kept after a transaction commits. Subsequent transactions will see the data committed by the previous transactions. +- Transaction: when the table is defined as `ON COMMIT DELETE ROWS`, this data will be cleared automatically after a transaction commits. Subsequent transactions won’t see the data committed by the previous transactions. + +## Storage engines + +TiDB uses TiKV and TiFlash to store the data of normal tables. This is also true for temporary tables by default because MySQL does so. However, temporary tables sometimes are used to boost performance, so it's also reasonable to support in-memory temporary tables. There are 2 kinds of storage engines available: + +- Memory or TempTable: TiDB keeps data in memory, and if the memory consumed by a temporary table exceeds a threshold, the data will be spilled to the local disk on TiDB. This is the default storage engine, which is different from MySQL. +- InnoDB: TiDB stores data on TiKV and TiFlash with more than one replica, just like normal tables. According to MySQL, this is the default storage engine, even for temporary tables. + +# Motivation or Background + +Temporary tables are useful in applications where a result set is to be buffered (temporarily persisted), perhaps because it is constructed by running multiple DML operations. + +The purposes of temporary tables include: + +- Usability. Temporary tables are typically for a temporary use. Applications don't need to frequently truncate the table. +- Performance. In-memory temporary tables are stored in memory, which boosts performance. +- Materialize middle data for queries. Some queries apply internal temporary tables for materializing middle data, such as CTE. + +# Detailed Design + +## Metadata + +The table ID of global temporary tables must be globally unique, while the local temporary tables don't. However, logical or physical plans involve table IDs, which means the temporary table IDs must be different from the normal table IDs. To achieve this goal, it's straightforward to also allocate local temporary tables globally. + +For a global temporary table, its table name should not be duplicated with a normal table. For a local temporary table, when its name conflicts with an existing table, it will take a higher priority. We can keep the temporary tables in a local schema, and overload the original one. The databases where temporary tables belong depend on the identifier in `CREATE TABLE` statements, just like normal tables. + +Since the metadata of global temporary tables are persistent on TiKV, it's straightforward to execute DDL in the same procedure as normal tables. However, the metadata of local temporary tables are only kept in the memory of the current TiDB instance, so we can bypass the complex online DDL procedure. We need only to generate the metadata locally and then merge it into the information schema. Thus, users cannot see the DDL jobs of local temporary tables through `ADMIN SHOW DDL JOBS`. + +Local temporary tables don’t support altering table operations because few users will do that. TiDB should report errors when users try to do that. + +As all DDL statements do, any DDL on a global temporary table will cause an implicit commit. However, creating and dropping a local temporary table doesn’t cause an implicit commit, according to [the MySQL documentation](https://dev.mysql.com/doc/refman/8.0/en/implicit-commit.html). + +Each temporary table belongs to its own database. Local temporary tables have a very loose relationship with databases. Dropping a database does not automatically drop any local temporary tables created within that database. The local temporary tables still stay in a virtual database with the same name. + +Truncating global temporary tables also conforms to the online DDL procedure, which affects other sessions. However, it's different for local temporary tables because the metadata is kept in memory. Truncating local temporary tables just drops the current metadata and creates a new one in memory. + +DDL operations, including those using `INSTANT` and `COPY` algorithms, are simple to apply on global temporary tables. For example, adding an index on a global temporary table is easy, because the table must be empty before adding the index. This benefits from implicit commit before adding the index. Local temporary tables, on the other hand, do support adding indexes, as other altering table operations. + +Some options in `CREATE TABLE` statements are not suitable for temporary tables. These options include: `AUTO_RANDOM`, `SHARD_ROW_ID_BITS`, `PRE_SPLIT_REGIONS`, `PARTITION BY`, `FOREIGN KEY`. Similarly, some related DDL operations are not supported, such as `SPLIT TABLE`, `SHOW TABLE REGIONS`, `ALTER PLACEMENT POLICY`. Table partition option is useless to a temporary table in the real use cases, so it's also not supported. Errors should be reported when users declare such options in the statements. + +Since some options are not suitable for temporary tables, when a user creates a temporary table from the `CREATE TABLE LIKE` statement and the source table has these options, an error should be reported. + +Other options will be kept. For example, clustered indexes and secondary indexes are kept because they can improve performance. + +- Altering table types is not allowed, since few users will do that: +- Altering a temporary table to a normal table or conversely. +- Altering a global temporary table to a local temporary table or conversely. +- Altering the storage engine of temporary tables. + +The result of `SHOW TABLES` contains global temporary tables, but not local temporary tables. The motivation is that local temporary tables are only for temporary usage, so the user should know what he is doing. However, `SHOW CREATE TABLE` works for all temporary tables. + +Similarly, system tables `TABLES` and `COLUMNS` in `information_schema` do not contain local temporary tables, but they contain global temporary tables. For global temporary tables, the value of field `TABLE_TYPE` in `information_schema.TABLES` is `GLOBAL TEMPORARY TABLE`. + +`ADMIN CHECK TABLE` and `ADMIN CHECKSUM TABLE` are used to check data integrity of the table. Data of temporary tables might also be corrupted due to unexpected problems, but it's impossible to check them because they are invisible to other sessions. So TiDB doesn’t support these commands. + +Creating views on a global temporary table makes sense, and the view will also be persistent on TiKV. However, it's unreasonable to persist a view that is based on a local temporary table, because the table will be discarded when the session ends. + +As the metadata of global temporary tables are persistent on TiKV, the binlog of DDL should also be exported. However, this is unnecessary for local temporary tables. + +## Optimizer + +Statistics of temporary tables are very different from those of normal tables. The data of each temporary table is invisible to other sessions, so it's unnecessary to persist statistics on TiKV. On the other hand, even if the sizes of temporary tables are relatively small, it's also necessary to consider statistics since it may improve the query performance significantly. + +Updating statistics is a little different from normal tables. We can’t rely on `AUTO ANALYZE` anymore, because the lifecycle of one session is relatively short, it's unlikely to wait for `AUTO ANALYZE`. What’s more, `AUTO ANALYZE` runs in background sessions, which means they can’t visit the temporary data. + +it's also unreasonable to force users to run `ANALYZE TABLE` periodically in applications. Intuitively, there are 2 ways to maintain statistics: +Update statistics once updating the table. When users run `ANALYZE TABLE`, TiDB updates statistics in the current session, instead of in background sessions. +Instead of maintaining statistics, collecting needed statistics before each query is another option. The collection can be boosted by sampling. `ANALYZE TABLE` needs to do nothing in this way. + +Both ways are also easy to implement, and it's not concrete that we really need statistics for temporary tables for now. So we only maintain row count, and just skip maintaining NDV, selectivity and others until there’s a need. + +Statistics of the same global temporary table are different in different sessions using, so every session keeps a copy of statistics for each global temporary table. However, the statistics of normal tables are stored in a global cache, which can be visited concurrently. So there needs to be a way to store the statistics of global temporary tables separately. + +Obviously, the cost of reading temporary tables is much lower than reading normal tables, since TiDB doesn’t need to request TiKV or consume any network resources. So the factors of such operations should be lower or 0. + +SQL binding is used to bind statements with specific execution plans. Global SQL binding affects all sessions and session SQL binding affects the current session. Since local temporary tables are invisible to other sessions, global SQL binding is meaningless. TiDB should report errors when users try to use global SQL binding on local temporary tables. Creating global or session SQL binding on global temporary tables, and creating session SQL binding on local temporary tables are both allowed. + +Baseline is used to capture the plans of statements that appear more than once. The appearance of statements is counted by SQL digests. Even if all sessions share the same global temporary table definitions, the data and statistics is different from one session to another. Thus baseline and SPM is useless for temporary tables. TiDB will ignore this feature for temporary tables. + +Prepared plan cache is used to cache plans for prepared statements to avoid duplicate optimization. Each session has a cache and the scope of each cache is the current session. Even if the cached plan stays in the cache after the temporary table is dropped, the plan won’t take effect and will be removed by the LRU list finally. So we just leave it as it was. + +## Storage + +Before going further to the executor, we need to determine the storage form of temporary tables. + +Basically, there are a few alternatives to store the data of in-memory temporary tables, and the most promising ones are: + +- Unistore. Unistore is a module that simulates TiKV on a standalone TiDB instance. +- UnionScan. UnionScan is a module that unit's membuffer and TiKV data. Membuffer buffers the dirty data a transaction writ's. Query operators read UnionScan and UnionScan will read both buffered data and persistent data. Thus, if the persistent part is always empty, then the UnionScan itself is a temporary table. + +| | Unistore | UnionScan | +| ------------- | :-------------: | -----: | +| execution | Y | Y | +| indexes | Y | Y | +| spilling to disk | Y | N | +| MVCC | Y | N | + + +TiDB uses UnionScan to store the data of temporary tables for the following reasons: + +- The performance should be better. It can cut down the marshal/unmarshal cost and a lot of the coprocessor code path, which is inevitable in the Unistore as a standalone storage engine. +- The implementation is easier. As long as we don’t take the spilling to disk feature into consideration for now, the global temporary table is almost handly. And we do not bother by the tons of background goroutines of the Unistore engine when dealing with resource releasing. How to keep the atomicity is another headache if we choose the Unistore, imaging that a transaction would write to both temporary tables and normal tables at the same time. +- A lower risk of introducing bugs. Although we implement the coprocessor in the Unistore, we just use it for the testing purpose, and we also have some experimental features first implemented in the Unistore, so its behavior may slightly differ from the real TiKV, and that difference would introduce bugs. + +Nothing needs to be changed for the KV encoding, the temporary table uses the same strategy with the normal table. + +When the memory consumed by a temporary table exceeds a threshold, the data should be spilled to the local disk to avoid OOM. The threshold is defined by the system variable `temptable_max_ram`, which is 1G by default. Membuffer does not support disk storage for now, so we need to implement it. + +A possible implementation is to use the AOF (append-only file) persistent storage, the membuffer is implemented as a red-black tree and its node is allocated from a customized allocator. That allocator manages the memory in an arena manner. A continuous block of memory becomes an arena block and an arena consists of several arena blocks. We can dump those blocks into the disk and maintain some LRU records for them. A precondition of the AOF implementation is that once a block is dumped, it is never modified. Currently, the value of a node is append-only, but the key is modified in-place, so some changes are necessary. We can keep all the red-black tree nodes in memory, while keeping part of the key-value data in the disk. + +Another option is changing the red-black tree to a B+ tree, this option is more disk friendly but the change takes more effort. + +When a temporary table needs to be cleared, its disk data should also be cleared. That can be done asynchronously by a background goroutine. The goroutine needs to know whether a file is in use. So TiDB needs to maintain a map which contains in-use files, each session updates the map periodically if it'still needs the file. If some files are not touched for a long time, we can treat the session as crashed and collect the disk files. + +For on-disk temporary tables, it's straightforward to store them on TiKV like normal tables. Since each local temporary table has a unique table ID, the data from different sessions are separate. However, multiple sessions using the same global temporary table will share the same table ID, which means the data from multiple sessions is continuous and stored together. Sessions will affect each other in this case. + +Clearing the data of on-disk temporary tables is a little different from in-memory temporary tables. When a TiDB instance is down, the storage space needs to be collected by other TiDB instances. Thus the maintenance of in-use temporary tables should be on the TiKV side. + +## Executor + +For normal tables, auto-increment IDs and row IDs are allocated in batches from TiKV, which significantly improves performance. However, as temporary tables are usually inserted, it may cause write hotspot on TiKV. So the best way is to allocate IDs locally on TiDB. The ID of a global temporary table is allocated separately among sessions and rebases to 0 every time a transaction ends. That means, each session needs a copy of allocators, rather than sharing the same allocators. + +Besides, `last_insert_id` is also affected by inserting into temporary tables. + +Since the data of in-memory temporary tables are not needed to be cached anymore, coprocessor cache and point-get cache are ignored. But they still work for on-disk temporary tables. + +Follower read indicates TiDB to read the follower replicas to release the load of leaders. For in-memory temporary tables, this hint is ignored. But it'still works for on-disk temporary tables. + +Users can also choose the storage engine to read by setting `tidb_isolation_read_engines`. For in-memory temporary tables, this setting will also be ignored. But it'still works for on-disk temporary tables. + +Since in-memory temporary tables are not persistent on TiKV or TiFlash, writing binlog for DML is also unnecessary. This also stays true for on-disk temporary tables, because data is invisible to other sessions. + +it's straightforward to support MPP on on-disk temporary tables, because the data is also synchronized to TiFlash. Most operators on in-memory temporary tables can still be processed in TiDB, such as Aggregation and TopN. These operators will not cost much memory because the sizes of in-memory temporary tables are relatively small. + +However, joining normal tables with in-memory temporary tables might be a problem, because the sizes of normal tables might be huge and thus merge sort join will cause OOM, while hash join and index lookup join will be too slow. Supporting broadcast join and shuffled hash join on in-memory temporary tables is very difficult. Fortunately, MPP typically happens in OLAP applications, where writing and scanning duration is relatively short compared to computing duration. So users can choose to define on-disk temporary tables in this case. + +## Transaction + +Because it's rare to read historical data of a temporary table, temporary tables don’t support features that rely on MVCC, like flashback table, recover table, stale read, and historical read. Errors will be reported when users execute such statements on temporary tables. + +If a transaction only writ's to temporary tables without normal tables, it does not really need a TSO for committing, because MVCC is unsupported and the data is invisible to other sessions. But for the sake of simplicity, we still fetch commit's just like normal tables. + +When a transaction commits, the data in in-memory temporary tables should not be committed on TiKV. When a transaction rollbacks, operations on normal tables and temporary tables should be rollbacked together. The data in on-disk temporary tables will be committed. However, this can be omitted by global on-disk temporary tables, because the data will be cleared anyway. TiDB can iterate KV ranges to filter out the data. + +Since there won’t be concurrent modifications on a same temporary table, there won’t be lock conflicts. So `FOR UPDATE` and `LOCK IN SHARE MODE` clauses will be ignored. + +Transactions might retry write operations when commit fails. DML on normal tables might rely on the data on temporary tables, so DML on temporary tables should also be retried. For example: + +``` +INSERT INTO normal_table SELECT * FROM temp_table +UPDATE normal_table, temp_table SET … WHERE normal_table.id=temp_table.id +``` + +If DML on temporary tables is not retried, such statements won’t write any data. + +Specially, as mentioned above, creating and dropping local temporary tables might also be in a transaction, but they needn’t be retried. + +TiDB comes with an optimization when the variable `tidb_constraint_check_in_place` is disabled: checking for duplicate values in UNIQUE indexes is deferred until the transaction commits. For those cases where temporary tables skip 2PC, this optimization should be disabled. + +Local transactions are special transactions that fetch TSO from the local PD. They can not access the data that is bound to the current available zone. Although temporary tables are not bound to any zones, they are invisible to other sessions, which means local transactions can still guarantee linearizability even when they access temporary tables. + +Schema change on a global temporary table may happen during a transaction which writ's to the temporary table. Unlike normal tables, the transaction won’t overwrite other transactions, so it's fine to commit. Schema change on a local temporary table will never happen during a transaction which writ's to the temporary table. + +## Privileges + +Creating a local temporary table checks the `CREATE TEMPORARY TABLES` privilege. No access rights are checked when dropping a local temporary table, according to [the MySQL documentation](https://dev.mysql.com/doc/refman/8.0/en/drop-table.html). + +All DDL on global temporary tables check the corresponding privileges like normal tables do. + +Writing to a global temporary table checks the privileges like the normal table. But there is no privilege check for a local temporary table. + +Privileges can not be granted to local temporary tables, because the tables are invisible to other users. Granting privileges to global temporary tables is possible. + +Ecosystem Tools +As mentioned above, DDL binlog of global temporary tables needs to be recorded, but not for local temporary tables. DML binlog is always skipped for temporary tables. DDL of global temporary tables should be supported by all data migration tools whose upstream is TiDB, such as Dumpling, TiDB-binlog, and TiCDC. + +Since `information_schema.tables` lists global temporary tables, these tables will be processed by tools like Dumpling. Fortunately, querying global temporary tables in a new session just returns empty results, so nothing needs to be handled. + +When backup tools read TiKV data, the data of temporary tables should never be read. However, on-disk temporary tables are stored on TiKV and TiFlash, so they need to be ignored by those tools, such as BR and TiCDC. Since these tools can see the metadata, they should also be capable of skipping tables that are not normal tables. + +Telemetry is used to report the usage information of various features in TiDB. Global and local temporary tables will be reported by telemetry separately, because the scenarios of them are different. + +Internal temporary tables +In MySQL, temporary tables will be used internally as infrastructures of other features, such as CTE, derived tables, and UNION statements. These temporary tables are called [internal temporary tables](https://dev.mysql.com/doc/refman/8.0/en/internal-temporary-tables.html). + +In the future, temporary tables will probably be used internally for some new features, such as CTE. So TiDB should be capable of creating a local temporary table when it's given a list of columns. Internal temporary tables might be in-memory or on-disk, depending on the optimizer. Physical plans that don’t apply MPP should use in-memory temporary tables, otherwise they will use on-disk temporary tables + +Transactions that apply internal temporary tables might be read-only, but there are some steps which write to TiKV: + +- Assigning table ID for temporary tables +- Writing to on-disk temporary tables +- Writing to on-disk temporary tables is inevitable, so the best way is NOT to report errors in this case. + +When executing a CTE in MPP mode, TiFlash has to write to the temporary table by itself, because the middle data is generated by TiFlash. + +## Modification to the write path + +The data of the temporary table is all in the membuffer of UnionScan. Writing to the temporary table is writing to the transaction cache. + +The data writing path in TiDB is typically through the `table.Table` interface, rows data are converted into key-values, and then written to TiKV via 2PC. For normal transactions, the data is cached in the membuffer until the transaction commits. The temporary table should never be written to the real TiKV. + +A transaction can write to both the temporary table and the normal table. Should the temporary table share the membuffer with the normal table, or use a separate one? +it's better to share the membuffer so the commit/rollback operation can be atomic. The risk is that in case of a bug, the temporary table data may be written to the real table. + +For global temporary tables, since the transaction commits automatically clears the temporary table, we can filter out and discard the transaction cache data from the temporary table when committing, and the implementation is relatively simple. + +For local temporary tables, the temporary table data is cleared at the end of the session, so the data should survive the transaction’s commit. We can copy the key-value data belonging to the temporary table to another place and use it in the later read operation. + +## Modification to the read path + +In TiDB, reading is implemented by the coprocessor. Read operations should see their own writ's. TiDB uses a UnionScan operator on top of the coprocessor's executor. This operator takes the data read by the coprocessor as a snapshot, then merges it with the membuffer, and passes the result to its parent operator. + +For transactional temporary tables, there is no need to do any additional changes, the current code works well. + +For the local temporary table, the key-value data of the temporary table in the current transaction should be taken out. We keep a temporary table membuffer in the session variable and if it is not null, the UnionScan operator needs to combine it with the original data. So now the data hierarchy looks like this: + + TiKV snapshot => Old membuffer => current transaction’s membuffer + +## Upgrade and Downgrade Compatibility + +When users downgrade TiDB to an older version after they have created a global temporary table, the table should not be seen by the older version. Otherwise, they might write to the table and then upgrade TiDB, which will be a problem. + +# Test Design + +A brief description of how the implementation will be tested. Both the integration test and the unit test should be considered. + +## Functional Tests + +it's used to ensure the basic feature function works as expected. Both the integration test and the unit test should be considered. + +- DDL + - Create table / create table like + - Drop table + - Truncate table + - Other DDL (for global) +- DML + - Insert / replace / update / delete / load data + - All kinds of query operators + - Show create table / show tables +- Transaction + - Atomicity (for local) + - Isolation (RC / SI, linearizability) + - Data visibility between transactions + - Optimistic / Pessimistic transactions +- Information_schema + - Tables + - Columns +- Privileges + +## Scenario Tests + +it's used to ensure this feature works as expected in some common scenarios. + +Before we implement the spilling to disk feature, we have to know the memory usage for some scenarios. For example, 100M for each temporary table, and 500-600 concurrent connections, how much memory does TiDB use. + +## Compatibility Tests + +| Feature | Compatibility | Temporary Table type | Reason | +| ------------- | :-------------: | :-----: | :---: | +| placement rules | Report error: not support | | Meaningless | +| partition | Report error: not support | | Meaningless | +| show table regions / split table / pre_split_regions | Report error: not support | | Meaningless | +| stale read / historical read | Report error: not support | | Meaningless | +| auto_random / `shard_row_id_bits` | Report error: not support | | No need to release writing hotspots | +| flashback / recover table | Report error: not support | | Meaningless | +| global SQL binding | Report error: not support | local | Bindings are meaningless after session ends & Tables are different amo ng sessions | +| view | Report error: not support | local | Views are meaningless after session ends & Tables are different among sessions | +| copr cache | Ignore this setting | in-memory | No need to cache | +| point get cache | Ignore this setting | in-memory | No need to cache | +| follower read | Ignore this setting | in-memory | Data is neither on TiKV nor on TiFlash | +| read engine | Ignore this setting | in-memory | Data is neither on TiKV nor on TiFlash | +| GC | Ignore this setting | | No need to GC data | +| select for update / lock in share mode | Ignore this setting | | No lock is needed | +| broadcast join / shuffle hash join | Ignore this setting | | Data is not on TiFlash | +| baseline / SPM | Ignore this setting | | | +| `tidb_constraint_check_in_place` | Ignore this setting | in-memory & local | Data is not committed on TiKV | +| auto analyze | Ignore this setting | | Background sessions can’t access private data | +| global session-specific | Need to backfill in memory | | | +| global txn / local txn | Need to deal with | | No limitation for read / write | +| analyze | Need to deal with | on-disk | Update in-memory stats in the current session instead of in system sessions | +| telemetry | Need to deal with | | Report the usage of temporary tables | +| view | Need to test | global | Drop views on dropping temporary tables | +| `auto_increment` / `last_insert_id` | Need to deal with | | AutoID is allocated locally | +| add index | Need to deal with | | The table is always empty before adding index | +| all hints | Need to test | | | +| plan cache | Need to test | | plan cache is session-scope | +| show fields / index / keys | Need to test | | | +| SQL binding | Need to test | | | +| clustered index | Need to test | | | +| async commit / 1PC | Need to test | | | +| checksum / check table | Report error: not support | | Meaningless | +| collation / charset | Need to test | | | +| batch insert | Need to test | | | +| feedback | Need to test | | | +| `statements_summary` / `slow_query` | Need to test | | SQL normalization | +| big transaction | Need to test | | | +| memory tracker | Need to test | | | +| explain / explain analyze | Need to test | | | + +Compatibility with other external components, like TiDB, PD, TiKV, TiFlash, BR, TiCDC, Dumpling, TiUP, K8s, etc. + +Upgrade compatibility + +Downgrade compatibility + +## Benchmark Tests + +The following two parts need to be measured: + +- measure the performance of this feature under different parameters +- measure the performance influence on the online workload + +sysbench for the temporary table, comparing its performance with the normal table. It means comparing the performance of an in-memory system with a distributed system. + +# Investigation & Alternatives + +How do other systems solve this issue? What other designs have been considered and what is the rationale for not choosing them? + +[MySQL documentation](https://dev.mysql.com/doc/refman/8.0/en/internal-temporary-tables.html) about the temporary table + +[CockroachDB](https://github.com/cockroachdb/cockroach/issues/5807) just uses the normal table as the temporary table. All the temporary tables are stored in a special schema, and it is scanned and cleaned periodically. If a session is finished, the temporary tables of that session are also cleaned. + +[Oracle](https://docs.oracle.com/cd/B28359_01/server.111/b28310/tables003.htm#ADMIN11633) uses global temporary tables in the old versions, and later on they also have the private temporary tables. The private temporary table in Oracle looks like MySQL, those tables are visible to the session rather than global. For global temporary tables, Oracle does not need to handle the schema change, because `alter table` is not allowed when some transactions are using the table. + +# Development Plan + +Stage 1(Support transactional, in-memory temporary table) + +- The architecture design for global/local, session/transaction, in-memory/disk spilling (L) +- DDL(2XL) + - Basic DDL e.g. create/drop table/add index etc + - Show tables/show create table for global temporary table +- DML (2XL) + + +- Backup and recover the temporary table meta information (L) +- Sync the temporary table DDL through CDC or Binlog to the downstream +- Privilege (L) +- Compatibility with other features (2XL) +- Performance test (L) + +Stage 2 (Support for session-specific temporary table) + +- create/drop session temporary table (3XL) +- Duplicated table name handling +- Compatibility with MySQL 8.0 temporary table (in-memory) +- Compatibility with other features (2XL) + +Stage 3 + +- Support spilling to disk for all kind of temporary tables (4XL) diff --git a/docs/design/2021-04-21-unify-log-library.md b/docs/design/2021-04-21-unify-log-library.md new file mode 100644 index 0000000000000..deb467a7b05fc --- /dev/null +++ b/docs/design/2021-04-21-unify-log-library.md @@ -0,0 +1,165 @@ +# Proposal: + +- Author(s): [Yifan Xu](https://github.com/SabaPing) +- Last updated: May 11, 2021 +- Discussion at: https://github.com/pingcap/tidb/pull/24181 +- Tracking issue: https://github.com/pingcap/tidb/issues/24190 + +## Abstract + +There are heterogeneous logging libraries in PingCAP's golang projects. These different logging libraries affect the development efficiency and even the user configuration experience. + +It is necessary to unify those heterogeneous logging libraries. + +## Background + +Except for slow query logs, all other logs must satisfy the [unified-log-format RFC standard](https://github.com/tikv/rfcs/blob/master/text/2018-12-19-unified-log-format.md). + +However, in practice, it was found that the format of logs is confusing: + +- There are few logging configuration instructions in the document. We need to enrich document especially the type and the format of logs each component would emit. +- The configured logging parameters do not match the runtime logging, e.g. `tidb_stderr` is configured with text format, but the log is in json format. +- The logs of some components do not meet the [unified-log-format RFC standard](https://github.com/tikv/rfcs/blob/23d4f9aed68a295b678e8bd909ee8479e3ba0bd1/text/2018-12-19-unified-log-format.md), e.g. `tiflash_cluster_manager`. +- Duplicate logs, e.g. `pd_stderr` will emit both text and json logs with duplicate content (but with a few subtle differences in timestamps). + +## Proposal + +There must be something wrong with the engineering of these codes above, and they must be changed. The cost to change them is not small. + +Rationale - for long-term consideration, we should maintain code quality. The speed of output can be sacrificed in time if necessary. + +Implementation plan: + +1. Unify log library for `pingcap/tidb` first. For dependent parts we write dummy code to satisfy. +2. Unify log library for `pingcap/br`, remove the dependency on `pingcap/tidb/util/logutil`, and clear dummy code of `pingcap/tidb`. +3. Unify log library for `tikv/pd`. + +After the implementation, we have `pingcap/tidb`, `pingcap/br`, `tikv/pd` all depend directly on `pingcap/log` and do not depend on any other log libraries (including `std/log`) or each other. + +## Rationale + +The following rationales are organized by GitHub repositories. + +### [pingcap/log](https://github.com/pingcap/log) + +As a common logging library in PingCAP, it does the following things: + +- Provides the standard config schema. +- Provides a factory method for creating log handlers. +- Hard code the log format according to [unified-log-format RFC standard](https://github.com/tikv/rfcs/blob/23d4f9aed68a295b678e8bd909ee8479e3ba0bd1/text/2018-12-19-unified-log-format.md). +- Encapsulates the logic of the rolling file. +- Provides global log handler and related methods for package dimension. + +### TiDB + +Log library dependencies: + +![tidb-log-dependency](./imgs/tidb-log-dependency.png) + +For historical reasons, TiDB has two third-party logging libraries, `logrus` and `pingcap/log`. `pingcap/log` is a wrapper of `zap`. + +Logs of TiDB can be divided into two types, slow query logs and the other logs. +As mentioned above, these two types of logs are emitted through two different logging libraries, which results in separate configurations for the two types of logs and requires writing additional configuration conversion code. + +TiDB-specific logging logic is written inside `util/logutil/log.go`, e.g., logger initialization, logger configuration, and so on. + +Note this file, which is one of the main culprits of circular dependencies. The following briefly describes the key logic in `util/logutil/log.go`, two init methods and four log handlers. + +#### logrus + +The init method of `logrus` may initialize two `logrus` handlers. + +First, it is necessary to initialize the standard log handler (package level handler). `InitLogger` first initializes the standard logger according to the configuration. + +Then, determine whether the configuration has enabled slow query log, and if so, create a log handler specific to slow query. + +[Here is the code](https://github.com/pingcap/tidb/blob/e79fa8c6b654e5b94e9ed0a1c0f997d6564e95be/util/logutil/log.go#L261). + +Regarding where these two handlers are used. + +- Some historical legacy code, such as `cmd/importer/parser.go`, which uses the standard logger by `logrus`. +- Slow query log all uses the slow query log handler created by `logrus`, code in `executor/adapter.go`. + +#### [pingcap/log](https://github.com/pingcap/log) + +`pingcap/log` is a wrapper around zap, and as mentioned below the two terms are equivalently interchangeable. + +Similar to `logrus`, the init method of zap `func InitZapLogger(cfg *LogConfig) error` may initialize two zap handlers. + +- The global zap handler, the default log handler for the entire repo, through which the vast majority of logs are emitted. +- Slow query zap handler, which is only initialized and not used. + +`InitZapLogger`'s logic is very similar to `logrus`' above. + +#### gRPC Logger + +In `main.go` there is a bunch of grpc logger initialization code, which is not in `util/logutil/log.go`. + +[Here is the code](https://github.com/pingcap/tidb/blob/e79fa8c6b654e5b94e9ed0a1c0f997d6564e95be/tidb-server/main.go#L591). + +The `NewLoggerV2` method creates a go native logger handler and is only used in grpc. + +### PD + +PD is similar to TiDB in that it also relies on `logrus` and `pingcap/log`, but with an additional layer of `capnslog` as a proxy. + +Log library dependencies: + +![pd-log-dependency](./imgs/pd-log-dependency.png) + +#### Logrus + +The standard logger is then passed down to the etcd (via the `capnslog` proxy), grpc and draft components as the log handler for these packages. + +There is only one `logrus` handler inside the entire PD codebase. + +Only the etcd, grpc, and draft components use the `logrus` handler. + +The initialization of `logrus` locates at `pkg/logutil/log.go`. [Here is the code](https://github.com/tikv/pd/blob/b07be86fb91aef07e8a68258ff6149256ab511f8/pkg/logutil/log.go#L260). + +#### [pingcap/log](https://github.com/pingcap/log) + +There is only one zap log handler inside the entire PD codebase, and its initialization is inline `cmd/pd-server/main.go`. + +[Here is the code](https://github.com/tikv/pd/blob/b07be86fb91aef07e8a68258ff6149256ab511f8/cmd/pd-server/main.go#L66). + +The logic is simple, create a new handler based on the configuration and replace the global handler at the `pingcap/log` package level. + +Most of the logging logic in PD will use the global zap handler. + +### [pingcap/br](https://github.com/pingcap/br) + +TiDB depends on BR, which in turn depends on tidb's `util/logutil/log.go`, constituting a circular dependency. + +Not only is it a circular dependency, it also happens to depend on the log component. This creates a considerable obstacle for the refactor. + +The following code is from `pkg/lightning/log/log.go`, which calls TiDB's `InitLogger` and then `pingcap/log`'s InitLogger. + +[Here is the code](https://github.com/pingcap/br/blob/b09611d526a754cee82e6d3b12edf67e4cc885ae/pkg/lightning/log/log.go#L77). + +BR also relies on TiDB's slow log, which he initializes in the main function as `SlowQueryLogger`. + +BR also calls TiDB's `InitLogger` twice in two places. + +BR also created two different zap handlers in two places, one of which is not used. + +These problem codes are not listed here. + +To refactor TiDB's logging functionality, you must first change BR to remove the dependency on TiDB log, then let TiDB depend on the new version of BR, and finally refactor TiDB's logging. + +## Compatibility and Migration Plan + +Must ensure that refactoring is compatible with historical logging logic. + +Guaranteed by unit testing. + +## Implementation + +See meta issue: https://github.com/pingcap/tidb/issues/24190. + +## Testing Plan + +Mainly unit testing. + +## Open issues (if applicable) +## Logging code for each component diff --git a/docs/design/2021-04-26-lock-view.md b/docs/design/2021-04-26-lock-view.md index 56d16e8e86194..79afbce94abef 100644 --- a/docs/design/2021-04-26-lock-view.md +++ b/docs/design/2021-04-26-lock-view.md @@ -1,7 +1,7 @@ # TiDB Design Documents - Author(s): [longfangsong](https://github.com/longfangsong), [MyonKeminta](http://github.com/MyonKeminta) -- Last updated: May 6, 2021 +- Last updated: May 18, 2021 - Discussion PR: N/A - Tracking Issue: https://github.com/pingcap/tidb/issues/24199 @@ -35,21 +35,22 @@ Several tables will be provided in `information_schema`. Some tables has both lo | Field | Type | Comment | |------------|------------|---------| -|`TRX_ID` | `unsigned bigint` | The transaction ID (aka. start ts) | -|`TRX_STARTED`|`time`| Human readable start time of the transaction | -|`DIGEST`|`text`| The digest of the current executing SQL statement | -|`SQLS` | `text` | A list of all executed SQL statements' digests | -|`STATE`| `enum('Running', 'Lock waiting', 'Committing', 'RollingBack')`| The state of the transaction | +| `TRX_ID` | `unsigned bigint` | The transaction ID (aka. start ts) | +| `TRX_STARTED`|`time`| Human readable start time of the transaction | +| `CURRENT_SQL_DIGEST`|`text`| The digest of the current executing SQL statement | +| `ALL_SQL_DIGESTS` | `text` | A list of all executed SQL statements' digests | +| `STATE`| `enum('Running', 'Lock waiting', 'Committing', 'RollingBack')`| The state of the transaction | | `WAITING_START_TIME` | `time` | The elapsed time since the start of the current lock waiting (if any) | | `SCOPE` | `enum('Global', 'Local')` | The scope of the transaction | -| `ISOLATION_LEVEL` | `enum('RR', 'RC')` | | +| `ISOLATION_LEVEL` | `enum('REPEATABLE-READ', 'READ-COMMITTED')` | | | `AUTOCOMMIT` | `bool` | | | `SESSION_ID` | `unsigned bigint` | | | `USER` | `varchar` | | | `DB` | `varchar` | | | `SET_COUNT` | `int` | Modified keys of the current transaction | | `LOCKED_COUNT` | `int` | Locked keys of the current transaction | -| `MEM_BUFFER_SIZE` | `int` | Size occupied by the transaction's membuffer | +| `MEM_BUFFER_KEYS` | `int` | Entries in transaction's membuffer | +| `MEM_BUFFER_BYTES` | `int` | Size occupied by the transaction's membuffer | * Life span of rows: * Create on first writing or locking operation in a transaction @@ -64,7 +65,6 @@ Several tables will be provided in `information_schema`. Some tables has both lo | Field | Type | Comment | |------------|------------|---------| -| `HASH` | `bigint` | The hash of the lock in TiKV's LockManager | | `KEY` | `varchar` | The key that's being waiting on | | `TRX_ID` | `unsigned bigint` | The current transaction that's waiting for the lock | | `SQL_DIGEST` | `text` | The digest of the SQL that's trying to acquire the lock | @@ -79,24 +79,28 @@ Several tables will be provided in `information_schema`. Some tables has both lo * Permission: * `PROCESS` privilege is needed to access this table. -### Table `(CLUSTER_)DEAD_LOCK` +### Table `(CLUSTER_)DEADLOCKS` | Field | Type | Comment | |------------|------------|---------| | `DEADLOCK_ID` | `int` | There needs multiple rows to represent information of a single deadlock event. This field is used to distinguish different events. | | `OCCUR_TIME` | `time` | The physical time when the deadlock occurs | +| `RETRYABLE` | `bool` | Is the deadlock retryable. TiDB tries to determine if the current statement is (indirectly) waiting for a lock locked by the current statement. | | `TRY_LOCK_TRX_ID` | `unsigned bigint` | The transaction ID (start ts) of the transaction that's trying to acquire the lock | | `CURRENT_SQL_DIGEST` | `text` | The SQL that's being blocked | | `KEY` | `varchar` | The key that's being locked, but locked by another transaction in the deadlock event | -| `SQLS` | `text` | A list of the digest of SQL statements that the transaction has executed | +| `ALL_SQL_DIGESTS` | `text` | A list of the digest of SQL statements that the transaction has executed | | `TRX_HOLDING_LOCK` | `unsigned bigint` | The transaction that's currently holding the lock. There will be another record in the table with the same `DEADLOCK_ID` for that transaction. | * Life span of rows: * Create after TiDB receive a deadlock error * FIFO,clean the oldest after buffer is full * Collecting, storing and querying: - * All of these information can be collected on TiDB side. It just need to add the information to the table when receives deadlock error from TiKV. The information of other transactions involved in the deadlock circle needed to be fetched from elsewhere (the `TIDB_TRX` table) when handling the deadlock error. - * Currently there are no much information in the deadlock error (it doesn't has the SQLs and keys' information), which needs to be improved. + * All of these information can be collected on TiDB side. It just need to add the information to the table when receives deadlock error from TiKV. The information of other transactions involved in the deadlock circle needed to be fetched from elsewhere (the `CLUSTER_TIDB_TRX` table) when handling the deadlock error. + * TiKV needs to report more rich information in the deadlock error for collecting. + * There are two types of deadlock errors internally: retryable or non-retryable. The transaction will internally retry on retryable deadlocks and won't report error to the client. Therefore, the user are typically more interested in the non-retryable deadlocks. + * Retryable deadlock errors are by default not collected, and can be enabled with configuration. + * Collecting `CLUSTER_TIDB_TRX` for more rich information for retryable deadlock is possible to make the performance worse. Whether it will be collected for retryable deadlock will be decided after some tests. * Permission: * `PROCESS` privilege is needed to access this table. @@ -151,9 +155,25 @@ The locking key and `resource_group_tag` that comes from the `Context` of the pe The wait chain will be added to the `Deadlock` error which is returned by the `PessimisticLock` request, so that when deadlock happens, the full wait chain information can be passed to TiDB. +### Configurations + +#### TiDB Config File `pessimistic-txn.tidb_deadlock_history_capacity` + +Specifies how many recent deadlock events each TiDB node should keep. +Dynamically changeable via HTTP API. +Value: 0 to 10000 +Default: 10 + +#### TiDB Config File `pessimistic-txn.tidb_deadlock_history_collect_retryable` + +Specifies whether to collect retryable deadlock errors to the `(CLUSTER_)DEADLOCKS` table. +Dynamically changeable via HTTP API. +Value: 0 (do not collect) or 1 (collect) +Default: 0 + ## Compatibility -This feature is not expected to be incompatible with other features. During upgrading, when there are different versions of TiDB nodes exists at the same time, it's possible that the `CLUSTER_` prefixed tables may encounter errors. But since this feature is typically used by user manually, this shouldn't be a severe problem. So we don't need to care much about that. +This feature is not expected to be incompatible with other features. During upgrading, when there are different versions of TiDB nodes exists at the same time, it's possible that the `CLUSTER_` prefixed tables may encounter errors. However, since this feature is typically used by user manually, this shouldn't be a severe problem. So we don't need to care much about that. ## Test Design @@ -190,7 +210,7 @@ This feature is not expected to be incompatible with other features. During upgr * Since lock waiting on TiKV may timeout and retry, it's possible that in a single query to `DATA_LOCK_WAIT` table doesn't shows all (logical) lock waiting. * Information about internal transactions may not be collected in our first version of implementation. -* Since TiDB need to query transaction information after it receives the deadlock error, the transactions' status may be changed during that time. As a result the information in `(CLUSTER_)DEAD_LOCK` table can't be promised to be accurate and complete. +* Since TiDB need to query transaction information after it receives the deadlock error, the transactions' status may be changed during that time. As a result the information in `(CLUSTER_)DEADLOCKS` table can't be promised to be accurate and complete. * Statistics about transaction conflicts is still not enough. * Historical information of `TIDB_TRX` and `DATA_LOCK_WAITS` is not kept, which possibly makes it still difficult to investigate some kind of problems. * The SQL digest that's holding lock and blocking the current transaction is hard to retrieve and is not included in the current design. diff --git a/docs/design/imgs/pd-log-dependency.png b/docs/design/imgs/pd-log-dependency.png new file mode 100644 index 0000000000000..27241f7ee6936 Binary files /dev/null and b/docs/design/imgs/pd-log-dependency.png differ diff --git a/docs/design/imgs/tidb-log-dependency.png b/docs/design/imgs/tidb-log-dependency.png new file mode 100644 index 0000000000000..04227db9c8683 Binary files /dev/null and b/docs/design/imgs/tidb-log-dependency.png differ diff --git a/domain/domain.go b/domain/domain.go index f4b0ac8900f24..a22d647066ea1 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -48,7 +48,6 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/statistics/handle" "github.com/pingcap/tidb/store/tikv" - "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/telemetry" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/dbterror" @@ -67,7 +66,7 @@ import ( // Multiple domains can be used in parallel without synchronization. type Domain struct { store kv.Storage - infoHandle *infoschema.Handle + infoCache *infoschema.InfoCache privHandle *privileges.Handle bindHandle *bindinfo.BindHandle statsHandle unsafe.Pointer @@ -79,7 +78,7 @@ type Domain struct { sysSessionPool *sessionPool exit chan struct{} etcdClient *clientv3.Client - gvc GlobalVariableCache + sysVarCache SysVarCache // replaces GlobalVariableCache slowQuery *topNSlowQueries expensiveQueryHandle *expensivequery.Handle wg sync.WaitGroup @@ -92,78 +91,75 @@ type Domain struct { isLostConnectionToPD sync2.AtomicInt32 // !0: true, 0: false. } -// loadInfoSchema loads infoschema at startTS into handle, usedSchemaVersion is the currently used -// infoschema version, if it is the same as the schema version at startTS, we don't need to reload again. -// It returns the latest schema version, the changed table IDs, whether it's a full load and an error. -func (do *Domain) loadInfoSchema(handle *infoschema.Handle, usedSchemaVersion int64, - startTS uint64) (neededSchemaVersion int64, change *tikv.RelatedSchemaChange, fullLoad bool, err error) { +// loadInfoSchema loads infoschema at startTS. +// It returns: +// 1. the needed infoschema +// 2. cache hit indicator +// 3. currentSchemaVersion(before loading) +// 4. the changed table IDs if it is not full load +// 5. an error if any +func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, int64, *tikv.RelatedSchemaChange, error) { snapshot := do.store.GetSnapshot(kv.NewVersion(startTS)) m := meta.NewSnapshotMeta(snapshot) - neededSchemaVersion, err = m.GetSchemaVersion() + neededSchemaVersion, err := m.GetSchemaVersion() if err != nil { - return 0, nil, fullLoad, err - } - if usedSchemaVersion != 0 && usedSchemaVersion == neededSchemaVersion { - return neededSchemaVersion, nil, fullLoad, nil + return nil, false, 0, nil, err } - // Update self schema version to etcd. - defer func() { - // There are two possibilities for not updating the self schema version to etcd. - // 1. Failed to loading schema information. - // 2. When users use history read feature, the neededSchemaVersion isn't the latest schema version. - if err != nil || neededSchemaVersion < do.InfoSchema().SchemaMetaVersion() { - logutil.BgLogger().Info("do not update self schema version to etcd", - zap.Int64("usedSchemaVersion", usedSchemaVersion), - zap.Int64("neededSchemaVersion", neededSchemaVersion), zap.Error(err)) - return - } + if is := do.infoCache.GetByVersion(neededSchemaVersion); is != nil { + return is, true, 0, nil, nil + } - err = do.ddl.SchemaSyncer().UpdateSelfVersion(context.Background(), neededSchemaVersion) - if err != nil { - logutil.BgLogger().Info("update self version failed", - zap.Int64("usedSchemaVersion", usedSchemaVersion), - zap.Int64("neededSchemaVersion", neededSchemaVersion), zap.Error(err)) - } - }() + currentSchemaVersion := int64(0) + if oldInfoSchema := do.infoCache.GetLatest(); oldInfoSchema != nil { + currentSchemaVersion = oldInfoSchema.SchemaMetaVersion() + } + // TODO: tryLoadSchemaDiffs has potential risks of failure. And it becomes worse in history reading cases. + // It is only kept because there is no alternative diff/partial loading solution. + // And it is only used to diff upgrading the current latest infoschema, if: + // 1. Not first time bootstrap loading, which needs a full load. + // 2. It is newer than the current one, so it will be "the current one" after this function call. + // 3. There are less 100 diffs. startTime := time.Now() - ok, relatedChanges, err := do.tryLoadSchemaDiffs(m, usedSchemaVersion, neededSchemaVersion) - if err != nil { + if currentSchemaVersion != 0 && neededSchemaVersion > currentSchemaVersion && neededSchemaVersion-currentSchemaVersion < 100 { + is, relatedChanges, err := do.tryLoadSchemaDiffs(m, currentSchemaVersion, neededSchemaVersion) + if err == nil { + do.infoCache.Insert(is) + logutil.BgLogger().Info("diff load InfoSchema success", + zap.Int64("currentSchemaVersion", currentSchemaVersion), + zap.Int64("neededSchemaVersion", neededSchemaVersion), + zap.Duration("start time", time.Since(startTime)), + zap.Int64s("phyTblIDs", relatedChanges.PhyTblIDS), + zap.Uint64s("actionTypes", relatedChanges.ActionTypes)) + return is, false, currentSchemaVersion, relatedChanges, nil + } // We can fall back to full load, don't need to return the error. logutil.BgLogger().Error("failed to load schema diff", zap.Error(err)) } - if ok { - logutil.BgLogger().Info("diff load InfoSchema success", - zap.Int64("usedSchemaVersion", usedSchemaVersion), - zap.Int64("neededSchemaVersion", neededSchemaVersion), - zap.Duration("start time", time.Since(startTime)), - zap.Int64s("phyTblIDs", relatedChanges.PhyTblIDS), - zap.Uint64s("actionTypes", relatedChanges.ActionTypes)) - return neededSchemaVersion, relatedChanges, fullLoad, nil - } - fullLoad = true schemas, err := do.fetchAllSchemasWithTables(m) if err != nil { - return 0, nil, fullLoad, err + return nil, false, currentSchemaVersion, nil, err } bundles, err := infosync.GetAllRuleBundles(context.TODO()) if err != nil { - return 0, nil, fullLoad, err + return nil, false, currentSchemaVersion, nil, err } - newISBuilder, err := infoschema.NewBuilder(handle).InitWithDBInfos(schemas, bundles, neededSchemaVersion) + newISBuilder, err := infoschema.NewBuilder(do.Store()).InitWithDBInfos(schemas, bundles, neededSchemaVersion) if err != nil { - return 0, nil, fullLoad, err + return nil, false, currentSchemaVersion, nil, err } logutil.BgLogger().Info("full load InfoSchema success", - zap.Int64("usedSchemaVersion", usedSchemaVersion), + zap.Int64("currentSchemaVersion", currentSchemaVersion), zap.Int64("neededSchemaVersion", neededSchemaVersion), zap.Duration("start time", time.Since(startTime))) - newISBuilder.Build() - return neededSchemaVersion, nil, fullLoad, nil + + is := newISBuilder.Build() + do.infoCache.Insert(is) + return is, false, currentSchemaVersion, nil, nil } func (do *Domain) fetchAllSchemasWithTables(m *meta.Meta) ([]*model.DBInfo, error) { @@ -238,48 +234,31 @@ func (do *Domain) fetchSchemasWithTables(schemas []*model.DBInfo, m *meta.Meta, done <- nil } -const ( - initialVersion = 0 - maxNumberOfDiffsToLoad = 100 -) - -func isTooOldSchema(usedVersion, newVersion int64) bool { - if usedVersion == initialVersion || newVersion-usedVersion > maxNumberOfDiffsToLoad { - return true - } - return false -} - // tryLoadSchemaDiffs tries to only load latest schema changes. // Return true if the schema is loaded successfully. // Return false if the schema can not be loaded by schema diff, then we need to do full load. // The second returned value is the delta updated table and partition IDs. -func (do *Domain) tryLoadSchemaDiffs(m *meta.Meta, usedVersion, newVersion int64) (bool, *tikv.RelatedSchemaChange, error) { - // If there isn't any used version, or used version is too old, we do full load. - // And when users use history read feature, we will set usedVersion to initialVersion, then full load is needed. - if isTooOldSchema(usedVersion, newVersion) { - return false, nil, nil - } +func (do *Domain) tryLoadSchemaDiffs(m *meta.Meta, usedVersion, newVersion int64) (infoschema.InfoSchema, *tikv.RelatedSchemaChange, error) { var diffs []*model.SchemaDiff for usedVersion < newVersion { usedVersion++ diff, err := m.GetSchemaDiff(usedVersion) if err != nil { - return false, nil, err + return nil, nil, err } if diff == nil { // If diff is missing for any version between used and new version, we fall back to full reload. - return false, nil, nil + return nil, nil, fmt.Errorf("failed to get schemadiff") } diffs = append(diffs, diff) } - builder := infoschema.NewBuilder(do.infoHandle).InitWithOldInfoSchema() + builder := infoschema.NewBuilder(do.Store()).InitWithOldInfoSchema(do.infoCache.GetLatest()) phyTblIDs := make([]int64, 0, len(diffs)) actions := make([]uint64, 0, len(diffs)) for _, diff := range diffs { IDs, err := builder.ApplyDiff(m, diff) if err != nil { - return false, nil, err + return nil, nil, err } if canSkipSchemaCheckerDDL(diff.Type) { continue @@ -289,11 +268,11 @@ func (do *Domain) tryLoadSchemaDiffs(m *meta.Meta, usedVersion, newVersion int64 actions = append(actions, uint64(1< 10 { + time.Sleep(time.Duration(count) * time.Second) + } + continue + } + count = 0 + logutil.BgLogger().Debug("Rebuilding sysvar cache from etcd watch event.") + err := do.sysVarCache.RebuildSysVarCache(ctx) + metrics.LoadSysVarCacheCounter.WithLabelValues(metrics.RetLabel(err)).Inc() + if err != nil { + logutil.BgLogger().Error("LoadSysVarCacheLoop failed", zap.Error(err)) + } + } + }() + return nil +} + // PrivilegeHandle returns the MySQLPrivilege. func (do *Domain) PrivilegeHandle() *privileges.Handle { return do.privHandle @@ -1300,7 +1326,10 @@ func (do *Domain) ExpensiveQueryHandle() *expensivequery.Handle { return do.expensiveQueryHandle } -const privilegeKey = "/tidb/privilege" +const ( + privilegeKey = "/tidb/privilege" + sysVarCacheKey = "/tidb/sysvars" +) // NotifyUpdatePrivilege updates privilege key in etcd, TiDB client that watches // the key will get notification. @@ -1322,6 +1351,23 @@ func (do *Domain) NotifyUpdatePrivilege(ctx sessionctx.Context) { } } +// NotifyUpdateSysVarCache updates the sysvar cache key in etcd, which other TiDB +// clients are subscribed to for updates. For the caller, the cache is also built +// synchronously so that the effect is immediate. +func (do *Domain) NotifyUpdateSysVarCache(ctx sessionctx.Context) { + if do.etcdClient != nil { + row := do.etcdClient.KV + _, err := row.Put(context.Background(), sysVarCacheKey, "") + if err != nil { + logutil.BgLogger().Warn("notify update sysvar cache failed", zap.Error(err)) + } + } + // update locally + if err := do.sysVarCache.RebuildSysVarCache(ctx); err != nil { + logutil.BgLogger().Error("rebuilding sysvar cache failed", zap.Error(err)) + } +} + // ServerID gets serverID. func (do *Domain) ServerID() uint64 { return atomic.LoadUint64(&do.serverID) diff --git a/domain/domain_test.go b/domain/domain_test.go index 7c9d9ff633bc5..099e3234c8279 100644 --- a/domain/domain_test.go +++ b/domain/domain_test.go @@ -35,6 +35,7 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/metrics" + "github.com/pingcap/tidb/session/txninfo" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/store/tikv" @@ -127,7 +128,7 @@ func TestInfo(t *testing.T) { goCtx, ddl.WithEtcdClient(dom.GetEtcdClient()), ddl.WithStore(s), - ddl.WithInfoHandle(dom.infoHandle), + ddl.WithInfoCache(dom.infoCache), ddl.WithLease(ddlLease), ) err = dom.ddl.Start(nil) @@ -241,6 +242,10 @@ type mockSessionManager struct { PS []*util.ProcessInfo } +func (msm *mockSessionManager) ShowTxnList() []*txninfo.TxnInfo { + panic("unimplemented!") +} + func (msm *mockSessionManager) ShowProcessList() map[uint64]*util.ProcessInfo { ret := make(map[uint64]*util.ProcessInfo) for _, item := range msm.PS { @@ -282,7 +287,7 @@ func (*testSuite) TestT(c *C) { c.Assert(dd, NotNil) c.Assert(dd.GetLease(), Equals, 80*time.Millisecond) - snapTS := oracle.EncodeTSO(oracle.GetPhysical(time.Now())) + snapTS := oracle.GoTimeToTS(time.Now()) cs := &ast.CharsetOpt{ Chs: "utf8", Col: "utf8_bin", @@ -312,7 +317,7 @@ func (*testSuite) TestT(c *C) { c.Assert(err, IsNil) // for GetSnapshotInfoSchema - currSnapTS := oracle.EncodeTSO(oracle.GetPhysical(time.Now())) + currSnapTS := oracle.GoTimeToTS(time.Now()) currSnapIs, err := dom.GetSnapshotInfoSchema(currSnapTS) c.Assert(err, IsNil) c.Assert(currSnapIs, NotNil) @@ -342,7 +347,7 @@ func (*testSuite) TestT(c *C) { // for schemaValidator schemaVer := dom.SchemaValidator.(*schemaValidator).LatestSchemaVersion() - ver, err := store.CurrentVersion(oracle.GlobalTxnScope) + ver, err := store.CurrentVersion(kv.GlobalTxnScope) c.Assert(err, IsNil) ts := ver.Ver @@ -355,7 +360,7 @@ func (*testSuite) TestT(c *C) { c.Assert(succ, Equals, ResultSucc) time.Sleep(ddlLease) - ver, err = store.CurrentVersion(oracle.GlobalTxnScope) + ver, err = store.CurrentVersion(kv.GlobalTxnScope) c.Assert(err, IsNil) ts = ver.Ver _, succ = dom.SchemaValidator.Check(ts, schemaVer, nil) diff --git a/domain/global_vars_cache.go b/domain/global_vars_cache.go deleted file mode 100644 index 52aa12a5ac955..0000000000000 --- a/domain/global_vars_cache.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package domain - -import ( - "fmt" - "sync" - "time" - - "github.com/pingcap/parser/ast" - "github.com/pingcap/tidb/sessionctx/variable" - "github.com/pingcap/tidb/util/chunk" - "github.com/pingcap/tidb/util/logutil" - "github.com/pingcap/tidb/util/stmtsummary" - "go.uber.org/zap" - "golang.org/x/sync/singleflight" -) - -// GlobalVariableCache caches global variables. -type GlobalVariableCache struct { - sync.RWMutex - lastModify time.Time - rows []chunk.Row - fields []*ast.ResultField - - // Unit test may like to disable it. - disable bool - SingleFight singleflight.Group -} - -// GlobalVariableCacheExpiry is the global variable cache TTL. -const GlobalVariableCacheExpiry = 2 * time.Second - -// Update updates the global variable cache. -func (gvc *GlobalVariableCache) Update(rows []chunk.Row, fields []*ast.ResultField) { - gvc.Lock() - gvc.lastModify = time.Now() - gvc.rows = rows - gvc.fields = fields - gvc.Unlock() - - checkEnableServerGlobalVar(rows) -} - -// Get gets the global variables from cache. -func (gvc *GlobalVariableCache) Get() (succ bool, rows []chunk.Row, fields []*ast.ResultField) { - gvc.RLock() - defer gvc.RUnlock() - if time.Since(gvc.lastModify) < GlobalVariableCacheExpiry { - succ, rows, fields = !gvc.disable, gvc.rows, gvc.fields - return - } - succ = false - return -} - -type loadResult struct { - rows []chunk.Row - fields []*ast.ResultField -} - -// LoadGlobalVariables will load from global cache first, loadFn will be executed if cache is not valid -func (gvc *GlobalVariableCache) LoadGlobalVariables(loadFn func() ([]chunk.Row, []*ast.ResultField, error)) ([]chunk.Row, []*ast.ResultField, error) { - succ, rows, fields := gvc.Get() - if succ { - return rows, fields, nil - } - fn := func() (interface{}, error) { - resRows, resFields, loadErr := loadFn() - if loadErr != nil { - return nil, loadErr - } - gvc.Update(resRows, resFields) - return &loadResult{resRows, resFields}, nil - } - res, err, _ := gvc.SingleFight.Do("loadGlobalVariable", fn) - if err != nil { - return nil, nil, err - } - loadRes := res.(*loadResult) - return loadRes.rows, loadRes.fields, nil -} - -// Disable disables the global variable cache, used in test only. -func (gvc *GlobalVariableCache) Disable() { - gvc.Lock() - defer gvc.Unlock() - gvc.disable = true -} - -// checkEnableServerGlobalVar processes variables that acts in server and global level. -func checkEnableServerGlobalVar(rows []chunk.Row) { - for _, row := range rows { - sVal := "" - if !row.IsNull(1) { - sVal = row.GetString(1) - } - var err error - switch row.GetString(0) { - case variable.TiDBEnableStmtSummary: - err = stmtsummary.StmtSummaryByDigestMap.SetEnabled(sVal, false) - case variable.TiDBStmtSummaryInternalQuery: - err = stmtsummary.StmtSummaryByDigestMap.SetEnabledInternalQuery(sVal, false) - case variable.TiDBStmtSummaryRefreshInterval: - err = stmtsummary.StmtSummaryByDigestMap.SetRefreshInterval(sVal, false) - case variable.TiDBStmtSummaryHistorySize: - err = stmtsummary.StmtSummaryByDigestMap.SetHistorySize(sVal, false) - case variable.TiDBStmtSummaryMaxStmtCount: - err = stmtsummary.StmtSummaryByDigestMap.SetMaxStmtCount(sVal, false) - case variable.TiDBStmtSummaryMaxSQLLength: - err = stmtsummary.StmtSummaryByDigestMap.SetMaxSQLLength(sVal, false) - case variable.TiDBCapturePlanBaseline: - variable.CapturePlanBaseline.Set(sVal, false) - } - if err != nil { - logutil.BgLogger().Error(fmt.Sprintf("load global variable %s error", row.GetString(0)), zap.Error(err)) - } - } -} - -// GetGlobalVarsCache gets the global variable cache. -func (do *Domain) GetGlobalVarsCache() *GlobalVariableCache { - return &do.gvc -} diff --git a/domain/global_vars_cache_test.go b/domain/global_vars_cache_test.go deleted file mode 100644 index 7358d709986af..0000000000000 --- a/domain/global_vars_cache_test.go +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package domain - -import ( - "sync" - "sync/atomic" - "time" - - . "github.com/pingcap/check" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/charset" - "github.com/pingcap/parser/model" - "github.com/pingcap/parser/mysql" - "github.com/pingcap/tidb/sessionctx/variable" - "github.com/pingcap/tidb/store/mockstore" - "github.com/pingcap/tidb/types" - "github.com/pingcap/tidb/util/chunk" - "github.com/pingcap/tidb/util/stmtsummary" - "github.com/pingcap/tidb/util/testleak" -) - -var _ = SerialSuites(&testGVCSuite{}) - -type testGVCSuite struct{} - -func (gvcSuite *testGVCSuite) TestSimple(c *C) { - defer testleak.AfterTest(c)() - testleak.BeforeTest() - - store, err := mockstore.NewMockStore() - c.Assert(err, IsNil) - defer func() { - err := store.Close() - c.Assert(err, IsNil) - }() - ddlLease := 50 * time.Millisecond - dom := NewDomain(store, ddlLease, 0, 0, mockFactory) - err = dom.Init(ddlLease, sysMockFactory) - c.Assert(err, IsNil) - defer dom.Close() - - // Get empty global vars cache. - gvc := dom.GetGlobalVarsCache() - succ, rows, fields := gvc.Get() - c.Assert(succ, IsFalse) - c.Assert(rows, IsNil) - c.Assert(fields, IsNil) - // Get a variable from global vars cache. - rf := getResultField("c", 1, 0) - rf1 := getResultField("c1", 2, 1) - ft := &types.FieldType{ - Tp: mysql.TypeString, - Charset: charset.CharsetBin, - Collate: charset.CollationBin, - } - ft1 := &types.FieldType{ - Tp: mysql.TypeString, - Charset: charset.CharsetBin, - Collate: charset.CollationBin, - } - ck := chunk.NewChunkWithCapacity([]*types.FieldType{ft, ft1}, 1024) - ck.AppendString(0, "variable1") - ck.AppendString(1, "value1") - row := ck.GetRow(0) - gvc.Update([]chunk.Row{row}, []*ast.ResultField{rf, rf1}) - succ, rows, fields = gvc.Get() - c.Assert(succ, IsTrue) - c.Assert(rows[0], Equals, row) - c.Assert(fields, DeepEquals, []*ast.ResultField{rf, rf1}) - // Disable the cache. - gvc.Disable() - succ, rows, fields = gvc.Get() - c.Assert(succ, IsFalse) - c.Assert(rows[0], Equals, row) - c.Assert(fields, DeepEquals, []*ast.ResultField{rf, rf1}) -} - -func getResultField(colName string, id, offset int) *ast.ResultField { - return &ast.ResultField{ - Column: &model.ColumnInfo{ - Name: model.NewCIStr(colName), - ID: int64(id), - Offset: offset, - FieldType: types.FieldType{ - Tp: mysql.TypeString, - Charset: charset.CharsetUTF8, - Collate: charset.CollationUTF8, - }, - }, - TableAsName: model.NewCIStr("tbl"), - DBName: model.NewCIStr("test"), - } -} - -func (gvcSuite *testGVCSuite) TestConcurrentOneFlight(c *C) { - defer testleak.AfterTest(c)() - testleak.BeforeTest() - gvc := &GlobalVariableCache{} - succ, rows, fields := gvc.Get() - c.Assert(succ, IsFalse) - c.Assert(rows, IsNil) - c.Assert(fields, IsNil) - - // Get a variable from global vars cache. - rf := getResultField("c", 1, 0) - rf1 := getResultField("c1", 2, 1) - ft := &types.FieldType{ - Tp: mysql.TypeString, - Charset: charset.CharsetBin, - Collate: charset.CollationBin, - } - ft1 := &types.FieldType{ - Tp: mysql.TypeString, - Charset: charset.CharsetBin, - Collate: charset.CollationBin, - } - ckLow := chunk.NewChunkWithCapacity([]*types.FieldType{ft, ft1}, 1024) - val := "fromStorage" - val1 := "fromStorage1" - ckLow.AppendString(0, val) - ckLow.AppendString(1, val1) - - // Let cache become invalid, and try concurrent load - counter := int32(0) - waitToStart := new(sync.WaitGroup) - waitToStart.Add(1) - gvc.lastModify = time.Now().Add(time.Duration(-10) * time.Second) - loadFunc := func() ([]chunk.Row, []*ast.ResultField, error) { - time.Sleep(100 * time.Millisecond) - atomic.AddInt32(&counter, 1) - return []chunk.Row{ckLow.GetRow(0)}, []*ast.ResultField{rf, rf1}, nil - } - wg := new(sync.WaitGroup) - worker := 100 - resArray := make([]loadResult, worker) - for i := 0; i < worker; i++ { - wg.Add(1) - go func(idx int) { - defer wg.Done() - waitToStart.Wait() - resRow, resField, _ := gvc.LoadGlobalVariables(loadFunc) - resArray[idx].rows = resRow - resArray[idx].fields = resField - }(i) - } - waitToStart.Done() - wg.Wait() - succ, rows, fields = gvc.Get() - c.Assert(counter, Equals, int32(1)) - c.Assert(resArray[0].rows[0].GetString(0), Equals, val) - c.Assert(resArray[0].rows[0].GetString(1), Equals, val1) - for i := 0; i < worker; i++ { - c.Assert(resArray[0].rows[0], Equals, resArray[i].rows[0]) - c.Assert(resArray[i].rows[0].GetString(0), Equals, val) - c.Assert(resArray[i].rows[0].GetString(1), Equals, val1) - } - // Validate cache - c.Assert(succ, IsTrue) - c.Assert(rows[0], Equals, resArray[0].rows[0]) - c.Assert(fields, DeepEquals, []*ast.ResultField{rf, rf1}) -} - -func (gvcSuite *testGVCSuite) TestCheckEnableStmtSummary(c *C) { - defer testleak.AfterTest(c)() - testleak.BeforeTest() - - store, err := mockstore.NewMockStore() - c.Assert(err, IsNil) - defer func() { - err := store.Close() - c.Assert(err, IsNil) - }() - ddlLease := 50 * time.Millisecond - dom := NewDomain(store, ddlLease, 0, 0, mockFactory) - err = dom.Init(ddlLease, sysMockFactory) - c.Assert(err, IsNil) - defer dom.Close() - - gvc := dom.GetGlobalVarsCache() - - rf := getResultField("c", 1, 0) - rf1 := getResultField("c1", 2, 1) - ft := &types.FieldType{ - Tp: mysql.TypeString, - Charset: charset.CharsetBin, - Collate: charset.CollationBin, - } - ft1 := &types.FieldType{ - Tp: mysql.TypeString, - Charset: charset.CharsetBin, - Collate: charset.CollationBin, - } - - err = stmtsummary.StmtSummaryByDigestMap.SetEnabled("0", false) - c.Assert(err, IsNil) - ck := chunk.NewChunkWithCapacity([]*types.FieldType{ft, ft1}, 1024) - ck.AppendString(0, variable.TiDBEnableStmtSummary) - ck.AppendString(1, "1") - row := ck.GetRow(0) - gvc.Update([]chunk.Row{row}, []*ast.ResultField{rf, rf1}) - c.Assert(stmtsummary.StmtSummaryByDigestMap.Enabled(), Equals, true) - - ck = chunk.NewChunkWithCapacity([]*types.FieldType{ft, ft1}, 1024) - ck.AppendString(0, variable.TiDBEnableStmtSummary) - ck.AppendString(1, "0") - row = ck.GetRow(0) - gvc.Update([]chunk.Row{row}, []*ast.ResultField{rf, rf1}) - c.Assert(stmtsummary.StmtSummaryByDigestMap.Enabled(), Equals, false) -} diff --git a/domain/infosync/info.go b/domain/infosync/info.go index cbd76d4e6f266..8596dcc6ce477 100644 --- a/domain/infosync/info.go +++ b/domain/infosync/info.go @@ -19,7 +19,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "os" "path" @@ -334,7 +333,7 @@ func doRequest(ctx context.Context, addrs []string, route, method string, body i } }) if err == nil { - bodyBytes, err := ioutil.ReadAll(res.Body) + bodyBytes, err := io.ReadAll(res.Body) if err != nil { return nil, err } @@ -554,12 +553,12 @@ func (is *InfoSyncer) ReportMinStartTS(store kv.Storage) { pl := is.manager.ShowProcessList() // Calculate the lower limit of the start timestamp to avoid extremely old transaction delaying GC. - currentVer, err := store.CurrentVersion(oracle.GlobalTxnScope) + currentVer, err := store.CurrentVersion(kv.GlobalTxnScope) if err != nil { logutil.BgLogger().Error("update minStartTS failed", zap.Error(err)) return } - now := time.Unix(0, oracle.ExtractPhysical(currentVer.Ver)*1e6) + now := oracle.GetTimeFromTS(currentVer.Ver) startTSLowerLimit := oracle.GoTimeToLowerLimitStartTS(now, tikv.MaxTxnTimeUse) minStartTS := oracle.GoTimeToTS(now) diff --git a/domain/schema_validator.go b/domain/schema_validator.go index b983eff1d6203..a8baa49db93b9 100644 --- a/domain/schema_validator.go +++ b/domain/schema_validator.go @@ -234,8 +234,9 @@ func (s *schemaValidator) Check(txnTS uint64, schemaVer int64, relatedPhysicalTa // Schema changed, result decided by whether related tables change. if schemaVer < s.latestSchemaVer { - // The DDL relatedPhysicalTableIDs is empty. - if len(relatedPhysicalTableIDs) == 0 { + // When a transaction executes a DDL, relatedPhysicalTableIDs is nil. + // When a transaction only contains DML on temporary tables, relatedPhysicalTableIDs is []. + if relatedPhysicalTableIDs == nil { logutil.BgLogger().Info("the related physical table ID is empty", zap.Int64("schemaVer", schemaVer), zap.Int64("latestSchemaVer", s.latestSchemaVer)) return nil, ResultFail diff --git a/domain/sysvar_cache.go b/domain/sysvar_cache.go new file mode 100644 index 0000000000000..0cb8b8977f329 --- /dev/null +++ b/domain/sysvar_cache.go @@ -0,0 +1,173 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package domain + +import ( + "context" + "fmt" + "sync" + + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/sessionctx/variable" + "github.com/pingcap/tidb/util/logutil" + "github.com/pingcap/tidb/util/sqlexec" + "github.com/pingcap/tidb/util/stmtsummary" + "go.uber.org/zap" +) + +// The sysvar cache replaces the GlobalVariableCache. +// It is an improvement because it operates similar to privilege cache, +// where it caches for 5 minutes instead of 2 seconds, plus it listens on etcd +// for updates from other servers. + +// SysVarCache represents the cache of system variables broken up into session and global scope. +type SysVarCache struct { + sync.RWMutex + global map[string]string + session map[string]string +} + +// GetSysVarCache gets the global variable cache. +func (do *Domain) GetSysVarCache() *SysVarCache { + return &do.sysVarCache +} + +func (svc *SysVarCache) rebuildCacheIfNeeded(ctx sessionctx.Context) (err error) { + svc.RLock() + cacheNeedsRebuild := len(svc.session) == 0 || len(svc.global) == 0 + svc.RUnlock() + if cacheNeedsRebuild { + logutil.BgLogger().Warn("sysvar cache is empty, triggering rebuild") + if err = svc.RebuildSysVarCache(ctx); err != nil { + logutil.BgLogger().Error("rebuilding sysvar cache failed", zap.Error(err)) + } + } + return err +} + +// GetSessionCache gets a copy of the session sysvar cache. +// The intention is to copy it directly to the systems[] map +// on creating a new session. +func (svc *SysVarCache) GetSessionCache(ctx sessionctx.Context) (map[string]string, error) { + if err := svc.rebuildCacheIfNeeded(ctx); err != nil { + return nil, err + } + svc.RLock() + defer svc.RUnlock() + // Perform a deep copy since this will be assigned directly to the session + newMap := make(map[string]string, len(svc.session)) + for k, v := range svc.session { + newMap[k] = v + } + return newMap, nil +} + +// GetGlobalVar gets an individual global var from the sysvar cache. +func (svc *SysVarCache) GetGlobalVar(ctx sessionctx.Context, name string) (string, error) { + if err := svc.rebuildCacheIfNeeded(ctx); err != nil { + return "", err + } + svc.RLock() + defer svc.RUnlock() + + if val, ok := svc.global[name]; ok { + return val, nil + } + logutil.BgLogger().Warn("could not find key in global cache", zap.String("name", name)) + return "", variable.ErrUnknownSystemVar.GenWithStackByArgs(name) +} + +func (svc *SysVarCache) fetchTableValues(ctx sessionctx.Context) (map[string]string, error) { + tableContents := make(map[string]string) + // Copy all variables from the table to tableContents + exec := ctx.(sqlexec.RestrictedSQLExecutor) + stmt, err := exec.ParseWithParams(context.Background(), `SELECT variable_name, variable_value FROM mysql.global_variables`) + if err != nil { + return tableContents, err + } + rows, _, err := exec.ExecRestrictedStmt(context.TODO(), stmt) + if err != nil { + return nil, err + } + for _, row := range rows { + name := row.GetString(0) + val := row.GetString(1) + tableContents[name] = val + } + return tableContents, nil +} + +// RebuildSysVarCache rebuilds the sysvar cache both globally and for session vars. +// It needs to be called when sysvars are added or removed. +func (svc *SysVarCache) RebuildSysVarCache(ctx sessionctx.Context) error { + newSessionCache := make(map[string]string) + newGlobalCache := make(map[string]string) + tableContents, err := svc.fetchTableValues(ctx) + if err != nil { + return err + } + + for _, sv := range variable.GetSysVars() { + sVal := sv.Value + if _, ok := tableContents[sv.Name]; ok { + sVal = tableContents[sv.Name] + } + // session cache stores non-skippable variables, which essentially means session scope. + // for historical purposes there are some globals, but these should eventually be removed. + if !sv.SkipInit() { + newSessionCache[sv.Name] = sVal + } + if sv.HasGlobalScope() { + newGlobalCache[sv.Name] = sVal + } + // Propagate any changes to the server scoped variables + checkEnableServerGlobalVar(sv.Name, sVal) + } + + logutil.BgLogger().Debug("rebuilding sysvar cache") + + svc.Lock() + defer svc.Unlock() + svc.session = newSessionCache + svc.global = newGlobalCache + return nil +} + +// checkEnableServerGlobalVar processes variables that acts in server and global level. +// This is required because the SetGlobal function on the sysvar struct only executes on +// the initiating tidb-server. There is no current method to say "run this function on all +// tidb servers when the value of this variable changes". If you do not require changes to +// be applied on all servers, use a getter/setter instead! You don't need to add to this list. +func checkEnableServerGlobalVar(name, sVal string) { + var err error + switch name { + case variable.TiDBEnableStmtSummary: + err = stmtsummary.StmtSummaryByDigestMap.SetEnabled(sVal, false) + case variable.TiDBStmtSummaryInternalQuery: + err = stmtsummary.StmtSummaryByDigestMap.SetEnabledInternalQuery(sVal, false) + case variable.TiDBStmtSummaryRefreshInterval: + err = stmtsummary.StmtSummaryByDigestMap.SetRefreshInterval(sVal, false) + case variable.TiDBStmtSummaryHistorySize: + err = stmtsummary.StmtSummaryByDigestMap.SetHistorySize(sVal, false) + case variable.TiDBStmtSummaryMaxStmtCount: + err = stmtsummary.StmtSummaryByDigestMap.SetMaxStmtCount(sVal, false) + case variable.TiDBStmtSummaryMaxSQLLength: + err = stmtsummary.StmtSummaryByDigestMap.SetMaxSQLLength(sVal, false) + case variable.TiDBCapturePlanBaseline: + variable.CapturePlanBaseline.Set(sVal, false) + } + if err != nil { + logutil.BgLogger().Error(fmt.Sprintf("load global variable %s error", name), zap.Error(err)) + } +} diff --git a/errno/errcode.go b/errno/errcode.go index 2ed488242dd10..2a3fe2733b5bf 100644 --- a/errno/errcode.go +++ b/errno/errcode.go @@ -855,6 +855,11 @@ const ( ErrGrantRole = 3523 ErrRoleNotGranted = 3530 ErrLockAcquireFailAndNoWaitSet = 3572 + ErrCTERecursiveRequiresUnion = 3573 + ErrCTERecursiveRequiresNonRecursiveFirst = 3574 + ErrCTERecursiveForbidsAggregation = 3575 + ErrCTERecursiveForbiddenJoinOrder = 3576 + ErrInvalidRequiresSingleReference = 3577 ErrWindowNoSuchWindow = 3579 ErrWindowCircularityInWindowGraph = 3580 ErrWindowNoChildPartitioning = 3581 @@ -877,6 +882,7 @@ const ( ErrWindowExplainJSON = 3598 ErrWindowFunctionIgnoresFrame = 3599 ErrIllegalPrivilegeLevel = 3619 + ErrCTEMaxRecursionDepth = 3636 ErrNotHintUpdatable = 3637 ErrDataTruncatedFunctionalIndex = 3751 ErrDataOutOfRangeFunctionalIndex = 3752 @@ -916,6 +922,7 @@ const ( ErrAdminCheckTable = 8003 ErrTxnTooLarge = 8004 ErrWriteConflictInTiDB = 8005 + ErrOptOnTemporaryTable = 8006 ErrUnsupportedReloadPlugin = 8018 ErrUnsupportedReloadPluginVar = 8019 ErrTableLocked = 8020 @@ -996,6 +1003,8 @@ const ( ErrMultiStatementDisabled = 8130 ErrPartitionStatsMissing = 8131 ErrNotSupportedWithSem = 8132 + ErrDataInConsistentExtraIndex = 8133 + ErrDataInConsistentMisMatchIndex = 8134 // Error codes used by TiDB ddl package ErrUnsupportedDDLOperation = 8200 diff --git a/errno/errname.go b/errno/errname.go index 5afdbbb91c4c0..05a8f476f835b 100644 --- a/errno/errname.go +++ b/errno/errname.go @@ -355,7 +355,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{ ErrViewSelectClause: mysql.Message("View's SELECT contains a '%s' clause", nil), ErrViewSelectVariable: mysql.Message("View's SELECT contains a variable or parameter", nil), ErrViewSelectTmptable: mysql.Message("View's SELECT refers to a temporary table '%-.192s'", nil), - ErrViewWrongList: mysql.Message("View's SELECT and view's field list have different column counts", nil), + ErrViewWrongList: mysql.Message("In definition of view, derived table or common table expression, SELECT list and column names list have different column counts", nil), ErrWarnViewMerge: mysql.Message("View merge algorithm can't be used here for now (assumed undefined algorithm)", nil), ErrWarnViewWithoutKey: mysql.Message("View being updated does not have complete key of underlying table in it", nil), ErrViewInvalid: mysql.Message("View '%-.192s.%-.192s' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them", nil), @@ -902,6 +902,12 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{ ErrUnsupportedConstraintCheck: mysql.Message("%s is not supported", nil), ErrDynamicPrivilegeNotRegistered: mysql.Message("Dynamic privilege '%s' is not registered with the server.", nil), ErrIllegalPrivilegeLevel: mysql.Message("Illegal privilege level specified for %s", nil), + ErrCTERecursiveRequiresUnion: mysql.Message("Recursive Common Table Expression '%s' should contain a UNION", nil), + ErrCTERecursiveRequiresNonRecursiveFirst: mysql.Message("Recursive Common Table Expression '%s' should have one or more non-recursive query blocks followed by one or more recursive ones", nil), + ErrCTERecursiveForbidsAggregation: mysql.Message("Recursive Common Table Expression '%s' can contain neither aggregation nor window functions in recursive query block", nil), + ErrCTERecursiveForbiddenJoinOrder: mysql.Message("In recursive query block of Recursive Common Table Expression '%s', the recursive table must neither be in the right argument of a LEFT JOIN, nor be forced to be non-first with join order hints", nil), + ErrInvalidRequiresSingleReference: mysql.Message("In recursive query block of Recursive Common Table Expression '%s', the recursive table must be referenced only once, and not in any subquery", nil), + ErrCTEMaxRecursionDepth: mysql.Message("Recursive query aborted after %d iterations. Try increasing @@cte_max_recursion_depth to a larger value", nil), // MariaDB errors. ErrOnlyOneDefaultPartionAllowed: mysql.Message("Only one DEFAULT partition allowed", nil), ErrWrongPartitionTypeExpectedSystemTime: mysql.Message("Wrong partitioning type, expected type: `SYSTEM_TIME`", nil), @@ -915,88 +921,91 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{ ErrSequenceInvalidTableStructure: mysql.Message("Sequence '%-.64s.%-.64s' table structure is invalid (%s)", nil), // TiDB errors. - ErrMemExceedThreshold: mysql.Message("%s holds %dB memory, exceeds threshold %dB.%s", nil), - ErrForUpdateCantRetry: mysql.Message("[%d] can not retry select for update statement", nil), - ErrAdminCheckTable: mysql.Message("TiDB admin check table failed.", nil), - ErrTxnTooLarge: mysql.Message("Transaction is too large, size: %d", nil), - ErrWriteConflictInTiDB: mysql.Message("Write conflict, txnStartTS %d is stale", nil), - ErrInvalidPluginID: mysql.Message("Wrong plugin id: %s, valid plugin id is [name]-[version], both name and version should not contain '-'", nil), - ErrInvalidPluginManifest: mysql.Message("Cannot read plugin %s's manifest", nil), - ErrInvalidPluginName: mysql.Message("Plugin load with %s but got wrong name %s", nil), - ErrInvalidPluginVersion: mysql.Message("Plugin load with %s but got %s", nil), - ErrDuplicatePlugin: mysql.Message("Plugin [%s] is redeclared", nil), - ErrInvalidPluginSysVarName: mysql.Message("Plugin %s's sysVar %s must start with its plugin name %s", nil), - ErrRequireVersionCheckFail: mysql.Message("Plugin %s require %s be %v but got %v", nil), - ErrUnsupportedReloadPlugin: mysql.Message("Plugin %s isn't loaded so cannot be reloaded", nil), - ErrUnsupportedReloadPluginVar: mysql.Message("Reload plugin with different sysVar is unsupported %v", nil), - ErrTableLocked: mysql.Message("Table '%s' was locked in %s by %v", nil), - ErrNotExist: mysql.Message("Error: key not exist", nil), - ErrTxnRetryable: mysql.Message("Error: KV error safe to retry %s ", []int{0}), - ErrCannotSetNilValue: mysql.Message("can not set nil value", nil), - ErrInvalidTxn: mysql.Message("invalid transaction", nil), - ErrEntryTooLarge: mysql.Message("entry too large, the max entry size is %d, the size of data is %d", nil), - ErrNotImplemented: mysql.Message("not implemented", nil), - ErrInfoSchemaExpired: mysql.Message("Information schema is out of date: schema failed to update in 1 lease, please make sure TiDB can connect to TiKV", nil), - ErrInfoSchemaChanged: mysql.Message("Information schema is changed during the execution of the statement(for example, table definition may be updated by other DDL ran in parallel). If you see this error often, try increasing `tidb_max_delta_schema_count`", nil), - ErrBadNumber: mysql.Message("Bad Number", nil), - ErrCastAsSignedOverflow: mysql.Message("Cast to signed converted positive out-of-range integer to it's negative complement", nil), - ErrCastNegIntAsUnsigned: mysql.Message("Cast to unsigned converted negative integer to it's positive complement", nil), - ErrInvalidYearFormat: mysql.Message("invalid year format", nil), - ErrInvalidYear: mysql.Message("invalid year", nil), - ErrIncorrectDatetimeValue: mysql.Message("Incorrect datetime value: '%s'", []int{0}), - ErrInvalidTimeFormat: mysql.Message("invalid time format: '%v'", []int{0}), - ErrInvalidWeekModeFormat: mysql.Message("invalid week mode format: '%v'", nil), - ErrFieldGetDefaultFailed: mysql.Message("Field '%s' get default value fail", nil), - ErrIndexOutBound: mysql.Message("Index column %s offset out of bound, offset: %d, row: %v", []int{2}), - ErrUnsupportedOp: mysql.Message("operation not supported", nil), - ErrRowNotFound: mysql.Message("can not find the row: %s", []int{0}), - ErrTableStateCantNone: mysql.Message("table %s can't be in none state", nil), - ErrColumnStateCantNone: mysql.Message("column %s can't be in none state", nil), - ErrColumnStateNonPublic: mysql.Message("can not use non-public column", nil), - ErrIndexStateCantNone: mysql.Message("index %s can't be in none state", nil), - ErrInvalidRecordKey: mysql.Message("invalid record key", nil), - ErrUnsupportedValueForVar: mysql.Message("variable '%s' does not yet support value: %s", nil), - ErrUnsupportedIsolationLevel: mysql.Message("The isolation level '%s' is not supported. Set tidb_skip_isolation_level_check=1 to skip this error", nil), - ErrInvalidDDLWorker: mysql.Message("Invalid DDL worker", nil), - ErrUnsupportedDDLOperation: mysql.Message("Unsupported %s", nil), - ErrNotOwner: mysql.Message("TiDB server is not a DDL owner", nil), - ErrCantDecodeRecord: mysql.Message("Cannot decode %s value, because %v", nil), - ErrInvalidDDLJob: mysql.Message("Invalid DDL job", nil), - ErrInvalidDDLJobFlag: mysql.Message("Invalid DDL job flag", nil), - ErrWaitReorgTimeout: mysql.Message("Timeout waiting for data reorganization", nil), - ErrInvalidStoreVersion: mysql.Message("Invalid storage current version: %d", nil), - ErrUnknownTypeLength: mysql.Message("Unknown length for type %d", nil), - ErrUnknownFractionLength: mysql.Message("Unknown length for type %d and fraction %d", nil), - ErrInvalidDDLJobVersion: mysql.Message("Version %d of DDL job is greater than current one: %d", nil), - ErrInvalidSplitRegionRanges: mysql.Message("Failed to split region ranges: %s", nil), - ErrReorgPanic: mysql.Message("Reorg worker panic", nil), - ErrInvalidDDLState: mysql.Message("Invalid %s state: %v", nil), - ErrCancelledDDLJob: mysql.Message("Cancelled DDL job", nil), - ErrRepairTable: mysql.Message("Failed to repair table: %s", nil), - ErrLoadPrivilege: mysql.Message("Load privilege table fail: %s", nil), - ErrInvalidPrivilegeType: mysql.Message("unknown privilege type %s", nil), - ErrUnknownFieldType: mysql.Message("unknown field type", nil), - ErrInvalidSequence: mysql.Message("invalid sequence", nil), - ErrInvalidType: mysql.Message("invalid type", nil), - ErrCantGetValidID: mysql.Message("Cannot get a valid auto-ID when retrying the statement", nil), - ErrCantSetToNull: mysql.Message("cannot set variable to null", nil), - ErrSnapshotTooOld: mysql.Message("snapshot is older than GC safe point %s", nil), - ErrInvalidTableID: mysql.Message("invalid TableID", nil), - ErrInvalidAutoRandom: mysql.Message("Invalid auto random: %s", nil), - ErrInvalidHashKeyFlag: mysql.Message("invalid encoded hash key flag", nil), - ErrInvalidListIndex: mysql.Message("invalid list index", nil), - ErrInvalidListMetaData: mysql.Message("invalid list meta data", nil), - ErrWriteOnSnapshot: mysql.Message("write on snapshot", nil), - ErrInvalidKey: mysql.Message("invalid key", nil), - ErrInvalidIndexKey: mysql.Message("invalid index key", nil), - ErrDataInConsistent: mysql.Message("data isn't equal", nil), - ErrDDLReorgElementNotExist: mysql.Message("DDL reorg element does not exist", nil), - ErrDDLJobNotFound: mysql.Message("DDL Job:%v not found", nil), - ErrCancelFinishedDDLJob: mysql.Message("This job:%v is finished, so can't be cancelled", nil), - ErrCannotCancelDDLJob: mysql.Message("This job:%v is almost finished, can't be cancelled now", nil), - ErrUnknownAllocatorType: mysql.Message("Invalid allocator type", nil), - ErrAutoRandReadFailed: mysql.Message("Failed to read auto-random value from storage engine", nil), - ErrInvalidIncrementAndOffset: mysql.Message("Invalid auto_increment settings: auto_increment_increment: %d, auto_increment_offset: %d, both of them must be in range [1..65535]", nil), + ErrMemExceedThreshold: mysql.Message("%s holds %dB memory, exceeds threshold %dB.%s", nil), + ErrForUpdateCantRetry: mysql.Message("[%d] can not retry select for update statement", nil), + ErrAdminCheckTable: mysql.Message("TiDB admin check table failed.", nil), + ErrOptOnTemporaryTable: mysql.Message("`%s` is unsupported on temporary tables.", nil), + ErrTxnTooLarge: mysql.Message("Transaction is too large, size: %d", nil), + ErrWriteConflictInTiDB: mysql.Message("Write conflict, txnStartTS %d is stale", nil), + ErrInvalidPluginID: mysql.Message("Wrong plugin id: %s, valid plugin id is [name]-[version], both name and version should not contain '-'", nil), + ErrInvalidPluginManifest: mysql.Message("Cannot read plugin %s's manifest", nil), + ErrInvalidPluginName: mysql.Message("Plugin load with %s but got wrong name %s", nil), + ErrInvalidPluginVersion: mysql.Message("Plugin load with %s but got %s", nil), + ErrDuplicatePlugin: mysql.Message("Plugin [%s] is redeclared", nil), + ErrInvalidPluginSysVarName: mysql.Message("Plugin %s's sysVar %s must start with its plugin name %s", nil), + ErrRequireVersionCheckFail: mysql.Message("Plugin %s require %s be %v but got %v", nil), + ErrUnsupportedReloadPlugin: mysql.Message("Plugin %s isn't loaded so cannot be reloaded", nil), + ErrUnsupportedReloadPluginVar: mysql.Message("Reload plugin with different sysVar is unsupported %v", nil), + ErrTableLocked: mysql.Message("Table '%s' was locked in %s by %v", nil), + ErrNotExist: mysql.Message("Error: key not exist", nil), + ErrTxnRetryable: mysql.Message("Error: KV error safe to retry %s ", []int{0}), + ErrCannotSetNilValue: mysql.Message("can not set nil value", nil), + ErrInvalidTxn: mysql.Message("invalid transaction", nil), + ErrEntryTooLarge: mysql.Message("entry too large, the max entry size is %d, the size of data is %d", nil), + ErrNotImplemented: mysql.Message("not implemented", nil), + ErrInfoSchemaExpired: mysql.Message("Information schema is out of date: schema failed to update in 1 lease, please make sure TiDB can connect to TiKV", nil), + ErrInfoSchemaChanged: mysql.Message("Information schema is changed during the execution of the statement(for example, table definition may be updated by other DDL ran in parallel). If you see this error often, try increasing `tidb_max_delta_schema_count`", nil), + ErrBadNumber: mysql.Message("Bad Number", nil), + ErrCastAsSignedOverflow: mysql.Message("Cast to signed converted positive out-of-range integer to it's negative complement", nil), + ErrCastNegIntAsUnsigned: mysql.Message("Cast to unsigned converted negative integer to it's positive complement", nil), + ErrInvalidYearFormat: mysql.Message("invalid year format", nil), + ErrInvalidYear: mysql.Message("invalid year", nil), + ErrIncorrectDatetimeValue: mysql.Message("Incorrect datetime value: '%s'", []int{0}), + ErrInvalidTimeFormat: mysql.Message("invalid time format: '%v'", []int{0}), + ErrInvalidWeekModeFormat: mysql.Message("invalid week mode format: '%v'", nil), + ErrFieldGetDefaultFailed: mysql.Message("Field '%s' get default value fail", nil), + ErrIndexOutBound: mysql.Message("Index column %s offset out of bound, offset: %d, row: %v", []int{2}), + ErrUnsupportedOp: mysql.Message("operation not supported", nil), + ErrRowNotFound: mysql.Message("can not find the row: %s", []int{0}), + ErrTableStateCantNone: mysql.Message("table %s can't be in none state", nil), + ErrColumnStateCantNone: mysql.Message("column %s can't be in none state", nil), + ErrColumnStateNonPublic: mysql.Message("can not use non-public column", nil), + ErrIndexStateCantNone: mysql.Message("index %s can't be in none state", nil), + ErrInvalidRecordKey: mysql.Message("invalid record key", nil), + ErrUnsupportedValueForVar: mysql.Message("variable '%s' does not yet support value: %s", nil), + ErrUnsupportedIsolationLevel: mysql.Message("The isolation level '%s' is not supported. Set tidb_skip_isolation_level_check=1 to skip this error", nil), + ErrInvalidDDLWorker: mysql.Message("Invalid DDL worker", nil), + ErrUnsupportedDDLOperation: mysql.Message("Unsupported %s", nil), + ErrNotOwner: mysql.Message("TiDB server is not a DDL owner", nil), + ErrCantDecodeRecord: mysql.Message("Cannot decode %s value, because %v", nil), + ErrInvalidDDLJob: mysql.Message("Invalid DDL job", nil), + ErrInvalidDDLJobFlag: mysql.Message("Invalid DDL job flag", nil), + ErrWaitReorgTimeout: mysql.Message("Timeout waiting for data reorganization", nil), + ErrInvalidStoreVersion: mysql.Message("Invalid storage current version: %d", nil), + ErrUnknownTypeLength: mysql.Message("Unknown length for type %d", nil), + ErrUnknownFractionLength: mysql.Message("Unknown length for type %d and fraction %d", nil), + ErrInvalidDDLJobVersion: mysql.Message("Version %d of DDL job is greater than current one: %d", nil), + ErrInvalidSplitRegionRanges: mysql.Message("Failed to split region ranges: %s", nil), + ErrReorgPanic: mysql.Message("Reorg worker panic", nil), + ErrInvalidDDLState: mysql.Message("Invalid %s state: %v", nil), + ErrCancelledDDLJob: mysql.Message("Cancelled DDL job", nil), + ErrRepairTable: mysql.Message("Failed to repair table: %s", nil), + ErrLoadPrivilege: mysql.Message("Load privilege table fail: %s", nil), + ErrInvalidPrivilegeType: mysql.Message("unknown privilege type %s", nil), + ErrUnknownFieldType: mysql.Message("unknown field type", nil), + ErrInvalidSequence: mysql.Message("invalid sequence", nil), + ErrInvalidType: mysql.Message("invalid type", nil), + ErrCantGetValidID: mysql.Message("Cannot get a valid auto-ID when retrying the statement", nil), + ErrCantSetToNull: mysql.Message("cannot set variable to null", nil), + ErrSnapshotTooOld: mysql.Message("snapshot is older than GC safe point %s", nil), + ErrInvalidTableID: mysql.Message("invalid TableID", nil), + ErrInvalidAutoRandom: mysql.Message("Invalid auto random: %s", nil), + ErrInvalidHashKeyFlag: mysql.Message("invalid encoded hash key flag", nil), + ErrInvalidListIndex: mysql.Message("invalid list index", nil), + ErrInvalidListMetaData: mysql.Message("invalid list meta data", nil), + ErrWriteOnSnapshot: mysql.Message("write on snapshot", nil), + ErrInvalidKey: mysql.Message("invalid key", nil), + ErrInvalidIndexKey: mysql.Message("invalid index key", nil), + ErrDataInConsistent: mysql.Message("index:%#v != record:%#v", []int{0, 1}), + ErrDDLReorgElementNotExist: mysql.Message("DDL reorg element does not exist", nil), + ErrDDLJobNotFound: mysql.Message("DDL Job:%v not found", nil), + ErrCancelFinishedDDLJob: mysql.Message("This job:%v is finished, so can't be cancelled", nil), + ErrCannotCancelDDLJob: mysql.Message("This job:%v is almost finished, can't be cancelled now", nil), + ErrUnknownAllocatorType: mysql.Message("Invalid allocator type", nil), + ErrAutoRandReadFailed: mysql.Message("Failed to read auto-random value from storage engine", nil), + ErrInvalidIncrementAndOffset: mysql.Message("Invalid auto_increment settings: auto_increment_increment: %d, auto_increment_offset: %d, both of them must be in range [1..65535]", nil), + ErrDataInConsistentExtraIndex: mysql.Message("handle %#v, index:%#v != record:%#v", []int{0, 1, 2}), + ErrDataInConsistentMisMatchIndex: mysql.Message("col %s, handle %#v, index:%#v != record:%#v, compare err:%#v", []int{1, 2, 3, 4}), ErrWarnOptimizerHintInvalidInteger: mysql.Message("integer value is out of range in '%s'", nil), ErrWarnOptimizerHintUnsupportedHint: mysql.Message("Optimizer hint %s is not supported by TiDB and is ignored", nil), @@ -1020,7 +1029,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{ ErrBuildExecutor: mysql.Message("Failed to build executor", nil), ErrBatchInsertFail: mysql.Message("Batch insert failed, please clean the table and try again.", nil), ErrGetStartTS: mysql.Message("Can not get start ts", nil), - ErrPrivilegeCheckFail: mysql.Message("privilege check fail", nil), // this error message should begin lowercased to be compatible with the test + ErrPrivilegeCheckFail: mysql.Message("privilege check for '%s' fail", nil), // this error message should begin lowercased to be compatible with the test ErrInvalidWildCard: mysql.Message("Wildcard fields without any table name appears in wrong place", nil), ErrMixOfGroupFuncAndFieldsIncompatible: mysql.Message("In aggregated query without GROUP BY, expression #%d of SELECT list contains nonaggregated column '%s'; this is incompatible with sql_mode=only_full_group_by", nil), ErrUnsupportedSecondArgumentType: mysql.Message("JSON_OBJECTAGG: unsupported second argument type %v", nil), diff --git a/errors.toml b/errors.toml index 458af951629d8..1c6903d1905e5 100644 --- a/errors.toml +++ b/errors.toml @@ -8,7 +8,7 @@ TiDB admin check table failed. ["admin:8223"] error = ''' -data isn't equal +index:%#v != record:%#v ''' ["admin:8224"] @@ -193,7 +193,7 @@ View's SELECT contains a '%s' clause ["ddl:1353"] error = ''' -View's SELECT and view's field list have different column counts +In definition of view, derived table or common table expression, SELECT list and column names list have different column counts ''' ["ddl:1481"] @@ -401,6 +401,11 @@ error = ''' Unknown SEQUENCE: '%-.300s' ''' +["ddl:8006"] +error = ''' +`%s` is unsupported on temporary tables. +''' + ["ddl:8200"] error = ''' Unsupported partition by range columns @@ -501,6 +506,11 @@ error = ''' Deadlock found when trying to get lock; try restarting transaction ''' +["executor:1221"] +error = ''' +Incorrect usage of %s and %s +''' + ["executor:1242"] error = ''' Subquery returns more than 1 row @@ -556,6 +566,11 @@ error = ''' Illegal privilege level specified for %s ''' +["executor:3636"] +error = ''' +Recursive query aborted after %d iterations. Try increasing @@cte_max_recursion_depth to a larger value +''' + ["executor:3929"] error = ''' Dynamic privilege '%s' is not registered with the server. @@ -621,6 +636,16 @@ error = ''' Export failed: %s ''' +["executor:8133"] +error = ''' +handle %#v, index:%#v != record:%#v +''' + +["executor:8134"] +error = ''' +col %s, handle %#v, index:%#v != record:%#v, compare err:%#v +''' + ["executor:8212"] error = ''' Failed to split region ranges: %s @@ -1011,6 +1036,31 @@ error = ''' Unresolved name '%s' for %s hint ''' +["planner:3573"] +error = ''' +Recursive Common Table Expression '%s' should contain a UNION +''' + +["planner:3574"] +error = ''' +Recursive Common Table Expression '%s' should have one or more non-recursive query blocks followed by one or more recursive ones +''' + +["planner:3575"] +error = ''' +Recursive Common Table Expression '%s' can contain neither aggregation nor window functions in recursive query block +''' + +["planner:3576"] +error = ''' +In recursive query block of Recursive Common Table Expression '%s', the recursive table must neither be in the right argument of a LEFT JOIN, nor be forced to be non-first with join order hints +''' + +["planner:3577"] +error = ''' +In recursive query block of Recursive Common Table Expression '%s', the recursive table must be referenced only once, and not in any subquery +''' + ["planner:3579"] error = ''' Window name '%s' is not defined. @@ -1101,6 +1151,11 @@ error = ''' Variable '%s' cannot be set using SET_VAR hint. ''' +["planner:8006"] +error = ''' +`%s` is unsupported on temporary tables. +''' + ["planner:8108"] error = ''' Unsupported type %T @@ -1133,7 +1188,7 @@ Schema has changed ["planner:8121"] error = ''' -privilege check fail +privilege check for '%s' fail ''' ["planner:8122"] @@ -1281,6 +1336,11 @@ error = ''' Unknown SEQUENCE: '%-.300s' ''' +["schema:8003"] +error = ''' +TiDB admin check table failed. +''' + ["schema:8020"] error = ''' Table '%s' was locked in %s by %v @@ -1321,6 +1381,11 @@ error = ''' Unknown column '%-.192s' in '%-.192s' ''' +["table:1114"] +error = ''' +The table '%-.192s' is full +''' + ["table:1192"] error = ''' Can't execute the given command because you have active locked tables or an active transaction diff --git a/executor/adapter.go b/executor/adapter.go index 5e5b7990f61d9..b324fdb331a69 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -26,7 +26,9 @@ import ( "github.com/cznic/mathutil" "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/log" + "github.com/pingcap/parser" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" @@ -43,7 +45,7 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" tikverr "github.com/pingcap/tidb/store/tikv/error" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" + "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/store/tikv/util" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" @@ -55,6 +57,7 @@ import ( "github.com/pingcap/tidb/util/sqlexec" "github.com/pingcap/tidb/util/stmtsummary" "github.com/pingcap/tidb/util/stringutil" + "github.com/pingcap/tidb/util/topsql" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -173,10 +176,21 @@ func (a *recordSet) OnFetchReturned() { a.stmt.LogSlowQuery(a.txnStartTS, a.lastErr == nil, true) } +// TelemetryInfo records some telemetry information during execution. +type TelemetryInfo struct { + UseNonRecursive bool + UseRecursive bool +} + // ExecStmt implements the sqlexec.Statement interface, it builds a planner.Plan to an sqlexec.Statement. type ExecStmt struct { // GoCtx stores parent go context.Context for a stmt. GoCtx context.Context + // SnapshotTS stores the timestamp for stale read. + // It is not equivalent to session variables's snapshot ts, it only use to build the executor. + SnapshotTS uint64 + // ExplicitStaleness means whether the 'SELECT' clause are using 'AS OF TIMESTAMP' to perform stale read explicitly. + ExplicitStaleness bool // InfoSchema stores a reference to the schema information. InfoSchema infoschema.InfoSchema // Plan stores a reference to the final physical plan. @@ -198,6 +212,7 @@ type ExecStmt struct { // OutputNames will be set if using cached plan OutputNames []*types.FieldName PsStmt *plannercore.CachedPrepareStmt + Ti *TelemetryInfo } // PointGet short path for point exec directly from plan, keep only necessary steps @@ -208,6 +223,7 @@ func (a *ExecStmt) PointGet(ctx context.Context, is infoschema.InfoSchema) (*rec defer span1.Finish() ctx = opentracing.ContextWithSpan(ctx, span1) } + ctx = a.setPlanLabelForTopSQL(ctx) startTs := uint64(math.MaxUint64) err := a.Ctx.InitTxnWithStartTS(startTs) if err != nil { @@ -229,7 +245,7 @@ func (a *ExecStmt) PointGet(ctx context.Context, is infoschema.InfoSchema) (*rec } } if a.PsStmt.Executor == nil { - b := newExecutorBuilder(a.Ctx, is) + b := newExecutorBuilder(a.Ctx, is, a.Ti) newExecutor := b.build(a.Plan) if b.err != nil { return nil, b.err @@ -268,18 +284,29 @@ func (a *ExecStmt) IsReadOnly(vars *variable.SessionVars) bool { // RebuildPlan rebuilds current execute statement plan. // It returns the current information schema version that 'a' is using. func (a *ExecStmt) RebuildPlan(ctx context.Context) (int64, error) { - is := infoschema.GetInfoSchema(a.Ctx) - a.InfoSchema = is - if err := plannercore.Preprocess(a.Ctx, a.StmtNode, is, plannercore.InTxnRetry); err != nil { + ret := &plannercore.PreprocessorReturn{} + if err := plannercore.Preprocess(a.Ctx, a.StmtNode, plannercore.InTxnRetry, plannercore.WithPreprocessorReturn(ret)); err != nil { return 0, err } - p, names, err := planner.Optimize(ctx, a.Ctx, a.StmtNode, is) + a.InfoSchema = ret.InfoSchema + a.SnapshotTS = ret.SnapshotTS + a.ExplicitStaleness = ret.ExplicitStaleness + p, names, err := planner.Optimize(ctx, a.Ctx, a.StmtNode, a.InfoSchema) if err != nil { return 0, err } a.OutputNames = names a.Plan = p - return is.SchemaMetaVersion(), nil + return a.InfoSchema.SchemaMetaVersion(), nil +} + +func (a *ExecStmt) setPlanLabelForTopSQL(ctx context.Context) context.Context { + if a.Plan == nil || !variable.TopSQLEnabled() { + return ctx + } + normalizedSQL, sqlDigest := a.Ctx.GetSessionVars().StmtCtx.SQLDigest() + normalizedPlan, planDigest := getPlanDigest(a.Ctx, a.Plan) + return topsql.AttachSQLInfo(ctx, normalizedSQL, sqlDigest, normalizedPlan, planDigest) } // Exec builds an Executor from a plan. If the Executor doesn't return result, @@ -305,6 +332,25 @@ func (a *ExecStmt) Exec(ctx context.Context) (_ sqlexec.RecordSet, err error) { logutil.Logger(ctx).Error("execute sql panic", zap.String("sql", a.GetTextToLog()), zap.Stack("stack")) }() + failpoint.Inject("assertStaleTSO", func(val failpoint.Value) { + if n, ok := val.(int); ok { + startTS := oracle.ExtractPhysical(a.SnapshotTS) / 1000 + if n != int(startTS) { + panic("different tso") + } + failpoint.Return() + } + }) + failpoint.Inject("assertStaleTSOWithTolerance", func(val failpoint.Value) { + if n, ok := val.(int); ok { + // Convert to seconds + startTS := oracle.ExtractPhysical(a.SnapshotTS) / 1000 + if int(startTS) <= n-1 || n+1 <= int(startTS) { + panic("tso violate tolerance") + } + failpoint.Return() + } + }) sctx := a.Ctx ctx = util.SetSessionID(ctx, sctx.GetSessionVars().ConnectionID) if _, ok := a.Plan.(*plannercore.Analyze); ok && sctx.GetSessionVars().InRestrictedSQL { @@ -332,6 +378,8 @@ func (a *ExecStmt) Exec(ctx context.Context) (_ sqlexec.RecordSet, err error) { if err != nil { return nil, err } + // ExecuteExec will rewrite `a.Plan`, so set plan label should be executed after `a.buildExecutor`. + ctx = a.setPlanLabelForTopSQL(ctx) if err = e.Open(ctx); err != nil { terror.Call(e.Close) @@ -377,6 +425,7 @@ func (a *ExecStmt) Exec(ctx context.Context) (_ sqlexec.RecordSet, err error) { if txn.Valid() { txnStartTS = txn.StartTS() } + return &recordSet{ executor: e, stmt: a, @@ -576,6 +625,7 @@ func (a *ExecStmt) handlePessimisticDML(ctx context.Context, e Executor) error { if len(keys) == 0 { return nil } + keys = filterTemporaryTableKeys(sctx.GetSessionVars(), keys) seVars := sctx.GetSessionVars() lockCtx := newLockCtx(seVars, seVars.LockWaitTimeout) var lockKeyStats *util.LockKeysDetails @@ -590,6 +640,7 @@ func (a *ExecStmt) handlePessimisticDML(ctx context.Context, e Executor) error { } e, err = a.handlePessimisticLockError(ctx, err) if err != nil { + // todo: Report deadlock if ErrDeadlock.Equal(err) { metrics.StatementDeadlockDetectDuration.Observe(time.Since(startLocking).Seconds()) } @@ -626,7 +677,7 @@ func UpdateForUpdateTS(seCtx sessionctx.Context, newForUpdateTS uint64) error { newForUpdateTS = version.Ver } seCtx.GetSessionVars().TxnCtx.SetForUpdateTS(newForUpdateTS) - txn.SetOption(tikvstore.SnapshotTS, seCtx.GetSessionVars().TxnCtx.GetForUpdateTS()) + txn.SetOption(kv.SnapshotTS, seCtx.GetSessionVars().TxnCtx.GetForUpdateTS()) return nil } @@ -741,7 +792,9 @@ func (a *ExecStmt) buildExecutor() (Executor, error) { ctx.GetSessionVars().StmtCtx.Priority = kv.PriorityLow } - b := newExecutorBuilder(ctx, a.InfoSchema) + b := newExecutorBuilder(ctx, a.InfoSchema, a.Ti) + b.snapshotTS = a.SnapshotTS + b.explicitStaleness = a.ExplicitStaleness e := b.build(a.Plan) if b.err != nil { return nil, errors.Trace(b.err) @@ -812,6 +865,7 @@ var ( // 2. record summary statement. // 3. record execute duration metric. // 4. update the `PrevStmt` in session variable. +// 5. reset `DurationParse` in session variable. func (a *ExecStmt) FinishExecuteStmt(txnTS uint64, succ bool, hasMoreResults bool) { sessVars := a.Ctx.GetSessionVars() execDetail := sessVars.StmtCtx.GetExecDetails() @@ -849,6 +903,8 @@ func (a *ExecStmt) FinishExecuteStmt(txnTS uint64, succ bool, hasMoreResults boo } else { sessionExecuteRunDurationGeneral.Observe(executeDuration.Seconds()) } + // Reset DurationParse due to the next statement may not need to be parsed (not a text protocol query). + sessVars.DurationParse = 0 } // CloseRecordSet will finish the execution of current statement and do some record work @@ -921,7 +977,7 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) { slowItems := &variable.SlowQueryLogItems{ TxnTS: txnTS, SQL: sql.String(), - Digest: digest, + Digest: digest.String(), TimeTotal: costTime, TimeParse: sessVars.DurationParse, TimeCompile: sessVars.DurationCompile, @@ -935,7 +991,7 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) { DiskMax: diskMax, Succ: succ, Plan: getPlanTree(a.Ctx, a.Plan), - PlanDigest: planDigest, + PlanDigest: planDigest.String(), Prepared: a.isPreparedStmt, HasMoreResults: hasMoreResults, PlanFromCache: sessVars.FoundInPlanCache, @@ -979,7 +1035,7 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) { } domain.GetDomain(a.Ctx).LogSlowQuery(&domain.SlowQueryInfo{ SQL: sql.String(), - Digest: digest, + Digest: digest.String(), Start: sessVars.StartTime, Duration: costTime, Detail: sessVars.StmtCtx.GetExecDetails(), @@ -1009,14 +1065,15 @@ func getPlanTree(sctx sessionctx.Context, p plannercore.Plan) string { } // getPlanDigest will try to get the select plan tree if the plan is select or the select plan of delete/update/insert statement. -func getPlanDigest(sctx sessionctx.Context, p plannercore.Plan) (normalized, planDigest string) { - normalized, planDigest = sctx.GetSessionVars().StmtCtx.GetPlanDigest() - if len(normalized) > 0 { - return +func getPlanDigest(sctx sessionctx.Context, p plannercore.Plan) (string, *parser.Digest) { + sc := sctx.GetSessionVars().StmtCtx + normalized, planDigest := sc.GetPlanDigest() + if len(normalized) > 0 && planDigest != nil { + return normalized, planDigest } normalized, planDigest = plannercore.NormalizePlan(p) - sctx.GetSessionVars().StmtCtx.SetPlanDigest(normalized, planDigest) - return + sc.SetPlanDigest(normalized, planDigest) + return normalized, planDigest } // getEncodedPlan gets the encoded plan, and generates the hint string if indicated. @@ -1077,7 +1134,7 @@ func (a *ExecStmt) SummaryStmt(succ bool) { } prevSQL = sessVars.PrevStmt.String() } - sessVars.SetPrevStmtDigest(digest) + sessVars.SetPrevStmtDigest(digest.String()) // No need to encode every time, so encode lazily. planGenerator := func() (string, string) { @@ -1091,10 +1148,11 @@ func (a *ExecStmt) SummaryStmt(succ bool) { if a.Plan.TP() == plancodec.TypePointGet { planDigestGen = func() string { _, planDigest := getPlanDigest(a.Ctx, a.Plan) - return planDigest + return planDigest.String() } } else { - _, planDigest = getPlanDigest(a.Ctx, a.Plan) + _, tmp := getPlanDigest(a.Ctx, a.Plan) + planDigest = tmp.String() } execDetail := stmtCtx.GetExecDetails() @@ -1118,7 +1176,7 @@ func (a *ExecStmt) SummaryStmt(succ bool) { Charset: charset, Collation: collation, NormalizedSQL: normalizedSQL, - Digest: digest, + Digest: digest.String(), PrevSQL: prevSQL, PrevSQLDigest: prevSQLDigest, PlanGenerator: planGenerator, diff --git a/executor/admin.go b/executor/admin.go index 7e15a24e667ca..df29c6a62aa8b 100644 --- a/executor/admin.go +++ b/executor/admin.go @@ -126,6 +126,7 @@ func (e *CheckIndexRangeExec) Open(ctx context.Context) error { SetStartTS(txn.StartTS()). SetKeepOrder(true). SetFromSessionVars(e.ctx.GetSessionVars()). + SetFromInfoSchema(e.ctx.GetInfoSchema()). Build() if err != nil { return err @@ -272,6 +273,7 @@ func (e *RecoverIndexExec) buildTableScan(ctx context.Context, txn kv.Transactio SetStartTS(txn.StartTS()). SetKeepOrder(true). SetFromSessionVars(e.ctx.GetSessionVars()). + SetFromInfoSchema(e.ctx.GetInfoSchema()). Build() if err != nil { return nil, err @@ -575,7 +577,7 @@ func (e *CleanupIndexExec) deleteDanglingIdx(txn kv.Transaction, values map[stri return errors.Trace(errors.Errorf("batch keys are inconsistent with handles")) } for _, handleIdxVals := range handleIdxValsGroup.([][]types.Datum) { - if err := e.index.Delete(e.ctx.GetSessionVars().StmtCtx, txn.GetUnionStore(), handleIdxVals, handle); err != nil { + if err := e.index.Delete(e.ctx.GetSessionVars().StmtCtx, txn, handleIdxVals, handle); err != nil { return err } e.removeCnt++ @@ -735,6 +737,7 @@ func (e *CleanupIndexExec) buildIndexScan(ctx context.Context, txn kv.Transactio SetStartTS(txn.StartTS()). SetKeepOrder(true). SetFromSessionVars(e.ctx.GetSessionVars()). + SetFromInfoSchema(e.ctx.GetInfoSchema()). Build() if err != nil { return nil, err diff --git a/executor/admin_test.go b/executor/admin_test.go index c9cda897a4745..da555ef999578 100644 --- a/executor/admin_test.go +++ b/executor/admin_test.go @@ -24,6 +24,7 @@ import ( mysql "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" @@ -76,6 +77,34 @@ func (s *testSuite5) TestAdminCheckIndex(c *C) { check() } +func (s *testSuite5) TestAdminCheckIndexInTemporaryMode(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists temporary_admin_test;") + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("create global temporary table temporary_admin_test (c1 int, c2 int, c3 int default 1, primary key (c1), index (c1), unique key(c2)) ON COMMIT DELETE ROWS;") + tk.MustExec("insert temporary_admin_test (c1, c2) values (1, 1), (2, 2), (3, 3);") + _, err := tk.Exec("admin check table temporary_admin_test;") + c.Assert(err.Error(), Equals, core.ErrOptOnTemporaryTable.GenWithStackByArgs("admin check table").Error()) + tk.MustExec("drop table if exists temporary_admin_test;") + + tk.MustExec("drop table if exists non_temporary_admin_test;") + tk.MustExec("create table non_temporary_admin_test (c1 int, c2 int, c3 int default 1, primary key (c1), index (c1), unique key(c2));") + tk.MustExec("insert non_temporary_admin_test (c1, c2) values (1, 1), (2, 2), (3, 3);") + tk.MustExec("admin check table non_temporary_admin_test;") + tk.MustExec("drop table if exists non_temporary_admin_test;") + + tk.MustExec("drop table if exists temporary_admin_checksum_table_with_index_test;") + tk.MustExec("drop table if exists temporary_admin_checksum_table_without_index_test;") + tk.MustExec("create global temporary table temporary_admin_checksum_table_with_index_test (id int, count int, PRIMARY KEY(id), KEY(count)) ON COMMIT DELETE ROWS;") + tk.MustExec("create global temporary table temporary_admin_checksum_table_without_index_test (id int, count int, PRIMARY KEY(id)) ON COMMIT DELETE ROWS;") + _, err = tk.Exec("admin checksum table temporary_admin_checksum_table_with_index_test;") + c.Assert(err.Error(), Equals, core.ErrOptOnTemporaryTable.GenWithStackByArgs("admin checksum table").Error()) + _, err = tk.Exec("admin checksum table temporary_admin_checksum_table_without_index_test;") + c.Assert(err.Error(), Equals, core.ErrOptOnTemporaryTable.GenWithStackByArgs("admin checksum table").Error()) + tk.MustExec("drop table if exists temporary_admin_checksum_table_with_index_test,temporary_admin_checksum_table_without_index_test;") +} + func (s *testSuite5) TestAdminRecoverIndex(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -119,7 +148,7 @@ func (s *testSuite5) TestAdminRecoverIndex(c *C) { sc := s.ctx.GetSessionVars().StmtCtx txn, err := s.store.Begin() c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(1), kv.IntHandle(1)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(1), kv.IntHandle(1)) c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) @@ -142,7 +171,7 @@ func (s *testSuite5) TestAdminRecoverIndex(c *C) { txn, err = s.store.Begin() c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(10), kv.IntHandle(10)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(10), kv.IntHandle(10)) c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) @@ -156,15 +185,15 @@ func (s *testSuite5) TestAdminRecoverIndex(c *C) { txn, err = s.store.Begin() c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(1), kv.IntHandle(1)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(1), kv.IntHandle(1)) c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(2), kv.IntHandle(2)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(2), kv.IntHandle(2)) c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(3), kv.IntHandle(3)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(3), kv.IntHandle(3)) c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(10), kv.IntHandle(10)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(10), kv.IntHandle(10)) c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(20), kv.IntHandle(20)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(20), kv.IntHandle(20)) c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) @@ -220,7 +249,7 @@ func (s *testSuite5) TestClusteredIndexAdminRecoverIndex(c *C) { txn, err := s.store.Begin() c.Assert(err, IsNil) cHandle := testutil.MustNewCommonHandle(c, "1", "3") - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(2), cHandle) + err = indexOpr.Delete(sc, txn, types.MakeDatums(2), cHandle) c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) @@ -253,7 +282,7 @@ func (s *testSuite5) TestAdminRecoverPartitionTableIndex(c *C) { sc := s.ctx.GetSessionVars().StmtCtx txn, err := s.store.Begin() c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(idxValue), kv.IntHandle(idxValue)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(idxValue), kv.IntHandle(idxValue)) c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) @@ -329,13 +358,13 @@ func (s *testSuite5) TestAdminRecoverIndex1(c *C) { txn, err := s.store.Begin() c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums("1"), kv.IntHandle(1)) + err = indexOpr.Delete(sc, txn, types.MakeDatums("1"), kv.IntHandle(1)) c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums("2"), kv.IntHandle(2)) + err = indexOpr.Delete(sc, txn, types.MakeDatums("2"), kv.IntHandle(2)) c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums("3"), kv.IntHandle(3)) + err = indexOpr.Delete(sc, txn, types.MakeDatums("3"), kv.IntHandle(3)) c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums("10"), kv.IntHandle(4)) + err = indexOpr.Delete(sc, txn, types.MakeDatums("10"), kv.IntHandle(4)) c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) @@ -729,7 +758,7 @@ func (s *testSuite3) TestAdminCheckPartitionTableFailed(c *C) { indexOpr := tables.NewIndex(tblInfo.GetPartitionInfo().Definitions[partitionIdx].ID, tblInfo, idxInfo) txn, err := s.store.Begin() c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(i), kv.IntHandle(i)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(i), kv.IntHandle(i)) c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) @@ -764,11 +793,11 @@ func (s *testSuite3) TestAdminCheckPartitionTableFailed(c *C) { c.Assert(err, IsNil) err = tk.ExecToErr("admin check table admin_test_p") c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, fmt.Sprintf("handle %d, index:types.Datum{k:0x1, decimal:0x0, length:0x0, i:%d, collation:\"\", b:[]uint8(nil), x:interface {}(nil)} != record:", i+8, i+8)) + c.Assert(err.Error(), Equals, fmt.Sprintf("[executor:8133]handle %d, index:types.Datum{k:0x1, decimal:0x0, length:0x0, i:%d, collation:\"\", b:[]uint8(nil), x:interface {}(nil)} != record:", i+8, i+8)) // TODO: fix admin recover for partition table. txn, err = s.store.Begin() c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(i+8), kv.IntHandle(i+8)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(i+8), kv.IntHandle(i+8)) c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) @@ -787,11 +816,11 @@ func (s *testSuite3) TestAdminCheckPartitionTableFailed(c *C) { c.Assert(err, IsNil) err = tk.ExecToErr("admin check table admin_test_p") c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, fmt.Sprintf("col c2, handle %d, index:types.Datum{k:0x1, decimal:0x0, length:0x0, i:%d, collation:\"\", b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, decimal:0x0, length:0x0, i:%d, collation:\"\", b:[]uint8(nil), x:interface {}(nil)}", i, i+8, i)) + c.Assert(err.Error(), Equals, fmt.Sprintf("[executor:8134]col c2, handle %d, index:types.Datum{k:0x1, decimal:0x0, length:0x0, i:%d, collation:\"\", b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, decimal:0x0, length:0x0, i:%d, collation:\"\", b:[]uint8(nil), x:interface {}(nil)}, compare err:", i, i+8, i)) // TODO: fix admin recover for partition table. txn, err = s.store.Begin() c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(i+8), kv.IntHandle(i)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(i+8), kv.IntHandle(i)) c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) @@ -799,7 +828,7 @@ func (s *testSuite3) TestAdminCheckPartitionTableFailed(c *C) { } } -func (s *testSuite5) TestAdminCheckTableFailed(c *C) { +func (s *testSuiteJoinSerial) TestAdminCheckTableFailed(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists admin_test") @@ -826,7 +855,7 @@ func (s *testSuite5) TestAdminCheckTableFailed(c *C) { // Index c2 is missing 11. txn, err := s.store.Begin() c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(-10), kv.IntHandle(-1)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(-10), kv.IntHandle(-1)) c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) @@ -835,6 +864,11 @@ func (s *testSuite5) TestAdminCheckTableFailed(c *C) { c.Assert(err.Error(), Equals, "[executor:8003]admin_test err:[admin:8223]index: != record:&admin.RecordData{Handle:-1, Values:[]types.Datum{types.Datum{k:0x1, decimal:0x0, length:0x0, i:-10, collation:\"\", b:[]uint8(nil), x:interface {}(nil)}}}") c.Assert(executor.ErrAdminCheckTable.Equal(err), IsTrue) + tk.MustExec("set @@tidb_redact_log=1;") + err = tk.ExecToErr("admin check table admin_test") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[executor:8003]admin_test err:[admin:8223]index:\"?\" != record:\"?\"") + tk.MustExec("set @@tidb_redact_log=0;") r := tk.MustQuery("admin recover index admin_test c2") r.Check(testkit.Rows("1 7")) tk.MustExec("admin check table admin_test") @@ -850,14 +884,19 @@ func (s *testSuite5) TestAdminCheckTableFailed(c *C) { c.Assert(err, IsNil) err = tk.ExecToErr("admin check table admin_test") c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "handle 0, index:types.Datum{k:0x1, decimal:0x0, length:0x0, i:0, collation:\"\", b:[]uint8(nil), x:interface {}(nil)} != record:") + c.Assert(err.Error(), Equals, "[executor:8133]handle 0, index:types.Datum{k:0x1, decimal:0x0, length:0x0, i:0, collation:\"\", b:[]uint8(nil), x:interface {}(nil)} != record:") + tk.MustExec("set @@tidb_redact_log=1;") + err = tk.ExecToErr("admin check table admin_test") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[executor:8133]handle \"?\", index:\"?\" != record:\"?\"") + tk.MustExec("set @@tidb_redact_log=0;") // Add one row of index. // Table count < index count. // Index c2 has two more values than table data: 10, 13, and these handles have correlative record. txn, err = s.store.Begin() c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(0), kv.IntHandle(0)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(0), kv.IntHandle(0)) c.Assert(err, IsNil) // Make sure the index value "19" is smaller "21". Then we scan to "19" before "21". _, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(19), kv.IntHandle(10), nil) @@ -868,21 +907,31 @@ func (s *testSuite5) TestAdminCheckTableFailed(c *C) { c.Assert(err, IsNil) err = tk.ExecToErr("admin check table admin_test") c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "col c2, handle 2, index:types.Datum{k:0x1, decimal:0x0, length:0x0, i:13, collation:\"\", b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, decimal:0x0, length:0x0, i:12, collation:\"\", b:[]uint8(nil), x:interface {}(nil)}") + c.Assert(err.Error(), Equals, "[executor:8134]col c2, handle 2, index:types.Datum{k:0x1, decimal:0x0, length:0x0, i:13, collation:\"\", b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, decimal:0x0, length:0x0, i:12, collation:\"\", b:[]uint8(nil), x:interface {}(nil)}, compare err:") + tk.MustExec("set @@tidb_redact_log=1;") + err = tk.ExecToErr("admin check table admin_test") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[executor:8134]col c2, handle \"?\", index:\"?\" != record:\"?\", compare err:\"?\"") + tk.MustExec("set @@tidb_redact_log=0;") // Table count = index count. // Two indices have the same handle. txn, err = s.store.Begin() c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(13), kv.IntHandle(2)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(13), kv.IntHandle(2)) c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(12), kv.IntHandle(2)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(12), kv.IntHandle(2)) c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) err = tk.ExecToErr("admin check table admin_test") c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "col c2, handle 10, index:types.Datum{k:0x1, decimal:0x0, length:0x0, i:19, collation:\"\", b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, decimal:0x0, length:0x0, i:20, collation:\"\", b:[]uint8(nil), x:interface {}(nil)}") + c.Assert(err.Error(), Equals, "[executor:8134]col c2, handle 10, index:types.Datum{k:0x1, decimal:0x0, length:0x0, i:19, collation:\"\", b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, decimal:0x0, length:0x0, i:20, collation:\"\", b:[]uint8(nil), x:interface {}(nil)}, compare err:") + tk.MustExec("set @@tidb_redact_log=1;") + err = tk.ExecToErr("admin check table admin_test") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[executor:8134]col c2, handle \"?\", index:\"?\" != record:\"?\", compare err:\"?\"") + tk.MustExec("set @@tidb_redact_log=0;") // Table count = index count. // Index c2 has one line of data is 19, the corresponding table data is 20. @@ -890,18 +939,23 @@ func (s *testSuite5) TestAdminCheckTableFailed(c *C) { c.Assert(err, IsNil) _, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(12), kv.IntHandle(2), nil) c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(20), kv.IntHandle(10)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(20), kv.IntHandle(10)) c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) err = tk.ExecToErr("admin check table admin_test") c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "col c2, handle 10, index:types.Datum{k:0x1, decimal:0x0, length:0x0, i:19, collation:\"\", b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, decimal:0x0, length:0x0, i:20, collation:\"\", b:[]uint8(nil), x:interface {}(nil)}") + c.Assert(err.Error(), Equals, "[executor:8134]col c2, handle 10, index:types.Datum{k:0x1, decimal:0x0, length:0x0, i:19, collation:\"\", b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, decimal:0x0, length:0x0, i:20, collation:\"\", b:[]uint8(nil), x:interface {}(nil)}, compare err:") + tk.MustExec("set @@tidb_redact_log=1;") + err = tk.ExecToErr("admin check table admin_test") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[executor:8134]col c2, handle \"?\", index:\"?\" != record:\"?\", compare err:\"?\"") + tk.MustExec("set @@tidb_redact_log=0;") // Recover records. txn, err = s.store.Begin() c.Assert(err, IsNil) - err = indexOpr.Delete(sc, txn.GetUnionStore(), types.MakeDatums(19), kv.IntHandle(10)) + err = indexOpr.Delete(sc, txn, types.MakeDatums(19), kv.IntHandle(10)) c.Assert(err, IsNil) _, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(20), kv.IntHandle(10), nil) c.Assert(err, IsNil) @@ -1022,8 +1076,7 @@ func (s *testSuite8) TestAdminCheckTable(c *C) { tk.MustExec(`create table t1 (a decimal(2,1), index(a))`) tk.MustExec(`insert into t1 set a='1.9'`) err = tk.ExecToErr(`alter table t1 modify column a decimal(3,2);`) - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "[ddl:8200]Unsupported modify column: decimal change from decimal(2, 1) to decimal(3, 2), and tidb_enable_change_column_type is false") + c.Assert(err, IsNil) tk.MustExec(`delete from t1;`) tk.MustExec(`admin check table t1;`) } diff --git a/executor/aggfuncs/aggfuncs.go b/executor/aggfuncs/aggfuncs.go index cacb418203cc2..6dd807c2ea5e1 100644 --- a/executor/aggfuncs/aggfuncs.go +++ b/executor/aggfuncs/aggfuncs.go @@ -201,7 +201,7 @@ type SlidingWindowAggFunc interface { // PartialResult stores the intermediate result which will be used in the next // sliding window, ensure call ResetPartialResult after a frame are evaluated // completely. - Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error + Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error } // MaxMinSlidingWindowAggFunc is the interface to evaluate the max/min agg function using sliding window diff --git a/executor/aggfuncs/builder.go b/executor/aggfuncs/builder.go index 666743e4c137c..c914ea4838f2d 100644 --- a/executor/aggfuncs/builder.go +++ b/executor/aggfuncs/builder.go @@ -479,7 +479,7 @@ func buildGroupConcat(ctx sessionctx.Context, aggFuncDesc *aggregation.AggFuncDe panic(fmt.Sprintf("Error happened when buildGroupConcat: %s", err.Error())) } var s string - s, err = variable.GetSessionSystemVar(ctx.GetSessionVars(), variable.GroupConcatMaxLen) + s, err = variable.GetSessionOrGlobalSystemVar(ctx.GetSessionVars(), variable.GroupConcatMaxLen) if err != nil { panic(fmt.Sprintf("Error happened when buildGroupConcat: no system variable named '%s'", variable.GroupConcatMaxLen)) } diff --git a/executor/aggfuncs/func_avg.go b/executor/aggfuncs/func_avg.go index 216778d643c53..a62565ab43fa6 100644 --- a/executor/aggfuncs/func_avg.go +++ b/executor/aggfuncs/func_avg.go @@ -105,10 +105,12 @@ func (e *avgOriginal4Decimal) UpdatePartialResult(sctx sessionctx.Context, rowsI return 0, nil } -func (e *avgOriginal4Decimal) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &avgOriginal4Decimal{} + +func (e *avgOriginal4Decimal) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4AvgDecimal)(pr) for i := uint64(0); i < shiftEnd; i++ { - input, isNull, err := e.args[0].EvalDecimal(sctx, rows[lastEnd+i]) + input, isNull, err := e.args[0].EvalDecimal(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -124,7 +126,7 @@ func (e *avgOriginal4Decimal) Slide(sctx sessionctx.Context, rows []chunk.Row, l p.count++ } for i := uint64(0); i < shiftStart; i++ { - input, isNull, err := e.args[0].EvalDecimal(sctx, rows[lastStart+i]) + input, isNull, err := e.args[0].EvalDecimal(sctx, getRow(lastStart+i)) if err != nil { return err } @@ -325,10 +327,12 @@ type avgOriginal4Float64 struct { avgOriginal4Float64HighPrecision } -func (e *avgOriginal4Float64) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &avgOriginal4Float64{} + +func (e *avgOriginal4Float64) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4AvgFloat64)(pr) for i := uint64(0); i < shiftEnd; i++ { - input, isNull, err := e.args[0].EvalReal(sctx, rows[lastEnd+i]) + input, isNull, err := e.args[0].EvalReal(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -339,7 +343,7 @@ func (e *avgOriginal4Float64) Slide(sctx sessionctx.Context, rows []chunk.Row, l p.count++ } for i := uint64(0); i < shiftStart; i++ { - input, isNull, err := e.args[0].EvalReal(sctx, rows[lastStart+i]) + input, isNull, err := e.args[0].EvalReal(sctx, getRow(lastStart+i)) if err != nil { return err } diff --git a/executor/aggfuncs/func_bitfuncs.go b/executor/aggfuncs/func_bitfuncs.go index 214a1a9da0f1d..00aee7af2e364 100644 --- a/executor/aggfuncs/func_bitfuncs.go +++ b/executor/aggfuncs/func_bitfuncs.go @@ -91,10 +91,12 @@ func (e *bitXorUint64) UpdatePartialResult(sctx sessionctx.Context, rowsInGroup return memDelta, nil } -func (e *bitXorUint64) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &bitXorUint64{} + +func (e *bitXorUint64) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4BitFunc)(pr) for i := uint64(0); i < shiftStart; i++ { - inputValue, isNull, err := e.args[0].EvalInt(sctx, rows[lastStart+i]) + inputValue, isNull, err := e.args[0].EvalInt(sctx, getRow(lastStart+i)) if err != nil { return err } @@ -104,7 +106,7 @@ func (e *bitXorUint64) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStar *p ^= uint64(inputValue) } for i := uint64(0); i < shiftEnd; i++ { - inputValue, isNull, err := e.args[0].EvalInt(sctx, rows[lastEnd+i]) + inputValue, isNull, err := e.args[0].EvalInt(sctx, getRow(lastEnd+i)) if err != nil { return err } diff --git a/executor/aggfuncs/func_count.go b/executor/aggfuncs/func_count.go index abc2bc17cde17..8b0c4f558744b 100644 --- a/executor/aggfuncs/func_count.go +++ b/executor/aggfuncs/func_count.go @@ -68,10 +68,12 @@ func (e *countOriginal4Int) UpdatePartialResult(sctx sessionctx.Context, rowsInG return 0, nil } -func (e *countOriginal4Int) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &countOriginal4Int{} + +func (e *countOriginal4Int) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4Count)(pr) for i := uint64(0); i < shiftStart; i++ { - _, isNull, err := e.args[0].EvalInt(sctx, rows[lastStart+i]) + _, isNull, err := e.args[0].EvalInt(sctx, getRow(lastStart+i)) if err != nil { return err } @@ -81,7 +83,7 @@ func (e *countOriginal4Int) Slide(sctx sessionctx.Context, rows []chunk.Row, las *p-- } for i := uint64(0); i < shiftEnd; i++ { - _, isNull, err := e.args[0].EvalInt(sctx, rows[lastEnd+i]) + _, isNull, err := e.args[0].EvalInt(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -115,10 +117,12 @@ func (e *countOriginal4Real) UpdatePartialResult(sctx sessionctx.Context, rowsIn return 0, nil } -func (e *countOriginal4Real) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &countOriginal4Real{} + +func (e *countOriginal4Real) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4Count)(pr) for i := uint64(0); i < shiftStart; i++ { - _, isNull, err := e.args[0].EvalReal(sctx, rows[lastStart+i]) + _, isNull, err := e.args[0].EvalReal(sctx, getRow(lastStart+i)) if err != nil { return err } @@ -128,7 +132,7 @@ func (e *countOriginal4Real) Slide(sctx sessionctx.Context, rows []chunk.Row, la *p-- } for i := uint64(0); i < shiftEnd; i++ { - _, isNull, err := e.args[0].EvalReal(sctx, rows[lastEnd+i]) + _, isNull, err := e.args[0].EvalReal(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -162,10 +166,12 @@ func (e *countOriginal4Decimal) UpdatePartialResult(sctx sessionctx.Context, row return 0, nil } -func (e *countOriginal4Decimal) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &countOriginal4Decimal{} + +func (e *countOriginal4Decimal) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4Count)(pr) for i := uint64(0); i < shiftStart; i++ { - _, isNull, err := e.args[0].EvalDecimal(sctx, rows[lastStart+i]) + _, isNull, err := e.args[0].EvalDecimal(sctx, getRow(lastStart+i)) if err != nil { return err } @@ -175,7 +181,7 @@ func (e *countOriginal4Decimal) Slide(sctx sessionctx.Context, rows []chunk.Row, *p-- } for i := uint64(0); i < shiftEnd; i++ { - _, isNull, err := e.args[0].EvalDecimal(sctx, rows[lastEnd+i]) + _, isNull, err := e.args[0].EvalDecimal(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -209,10 +215,12 @@ func (e *countOriginal4Time) UpdatePartialResult(sctx sessionctx.Context, rowsIn return 0, nil } -func (e *countOriginal4Time) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &countOriginal4Time{} + +func (e *countOriginal4Time) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4Count)(pr) for i := uint64(0); i < shiftStart; i++ { - _, isNull, err := e.args[0].EvalTime(sctx, rows[lastStart+i]) + _, isNull, err := e.args[0].EvalTime(sctx, getRow(lastStart+i)) if err != nil { return err } @@ -222,7 +230,7 @@ func (e *countOriginal4Time) Slide(sctx sessionctx.Context, rows []chunk.Row, la *p-- } for i := uint64(0); i < shiftEnd; i++ { - _, isNull, err := e.args[0].EvalTime(sctx, rows[lastEnd+i]) + _, isNull, err := e.args[0].EvalTime(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -256,10 +264,12 @@ func (e *countOriginal4Duration) UpdatePartialResult(sctx sessionctx.Context, ro return 0, nil } -func (e *countOriginal4Duration) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &countOriginal4Duration{} + +func (e *countOriginal4Duration) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4Count)(pr) for i := uint64(0); i < shiftStart; i++ { - _, isNull, err := e.args[0].EvalDuration(sctx, rows[lastStart+i]) + _, isNull, err := e.args[0].EvalDuration(sctx, getRow(lastStart+i)) if err != nil { return err } @@ -269,7 +279,7 @@ func (e *countOriginal4Duration) Slide(sctx sessionctx.Context, rows []chunk.Row *p-- } for i := uint64(0); i < shiftEnd; i++ { - _, isNull, err := e.args[0].EvalDuration(sctx, rows[lastEnd+i]) + _, isNull, err := e.args[0].EvalDuration(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -303,10 +313,12 @@ func (e *countOriginal4JSON) UpdatePartialResult(sctx sessionctx.Context, rowsIn return 0, nil } -func (e *countOriginal4JSON) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &countOriginal4JSON{} + +func (e *countOriginal4JSON) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4Count)(pr) for i := uint64(0); i < shiftStart; i++ { - _, isNull, err := e.args[0].EvalJSON(sctx, rows[lastStart+i]) + _, isNull, err := e.args[0].EvalJSON(sctx, getRow(lastStart+i)) if err != nil { return err } @@ -316,7 +328,7 @@ func (e *countOriginal4JSON) Slide(sctx sessionctx.Context, rows []chunk.Row, la *p-- } for i := uint64(0); i < shiftEnd; i++ { - _, isNull, err := e.args[0].EvalJSON(sctx, rows[lastEnd+i]) + _, isNull, err := e.args[0].EvalJSON(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -350,10 +362,12 @@ func (e *countOriginal4String) UpdatePartialResult(sctx sessionctx.Context, rows return 0, nil } -func (e *countOriginal4String) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &countOriginal4String{} + +func (e *countOriginal4String) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4Count)(pr) for i := uint64(0); i < shiftStart; i++ { - _, isNull, err := e.args[0].EvalString(sctx, rows[lastStart+i]) + _, isNull, err := e.args[0].EvalString(sctx, getRow(lastStart+i)) if err != nil { return err } @@ -363,7 +377,7 @@ func (e *countOriginal4String) Slide(sctx sessionctx.Context, rows []chunk.Row, *p-- } for i := uint64(0); i < shiftEnd; i++ { - _, isNull, err := e.args[0].EvalString(sctx, rows[lastEnd+i]) + _, isNull, err := e.args[0].EvalString(sctx, getRow(lastEnd+i)) if err != nil { return err } diff --git a/executor/aggfuncs/func_max_min.go b/executor/aggfuncs/func_max_min.go index 63a00ddd5bede..2c10b9251a463 100644 --- a/executor/aggfuncs/func_max_min.go +++ b/executor/aggfuncs/func_max_min.go @@ -343,10 +343,12 @@ func (e *maxMin4IntSliding) UpdatePartialResult(sctx sessionctx.Context, rowsInG return 0, nil } -func (e *maxMin4IntSliding) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &maxMin4IntSliding{} + +func (e *maxMin4IntSliding) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4MaxMinInt)(pr) for i := uint64(0); i < shiftEnd; i++ { - input, isNull, err := e.args[0].EvalInt(sctx, rows[lastEnd+i]) + input, isNull, err := e.args[0].EvalInt(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -480,10 +482,12 @@ func (e *maxMin4UintSliding) UpdatePartialResult(sctx sessionctx.Context, rowsIn return 0, nil } -func (e *maxMin4UintSliding) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &maxMin4UintSliding{} + +func (e *maxMin4UintSliding) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4MaxMinUint)(pr) for i := uint64(0); i < shiftEnd; i++ { - input, isNull, err := e.args[0].EvalInt(sctx, rows[lastEnd+i]) + input, isNull, err := e.args[0].EvalInt(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -617,10 +621,12 @@ func (e *maxMin4Float32Sliding) UpdatePartialResult(sctx sessionctx.Context, row return 0, nil } -func (e *maxMin4Float32Sliding) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &maxMin4Float32Sliding{} + +func (e *maxMin4Float32Sliding) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4MaxMinFloat32)(pr) for i := uint64(0); i < shiftEnd; i++ { - input, isNull, err := e.args[0].EvalReal(sctx, rows[lastEnd+i]) + input, isNull, err := e.args[0].EvalReal(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -752,10 +758,12 @@ func (e *maxMin4Float64Sliding) UpdatePartialResult(sctx sessionctx.Context, row return 0, nil } -func (e *maxMin4Float64Sliding) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &maxMin4Float64Sliding{} + +func (e *maxMin4Float64Sliding) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4MaxMinFloat64)(pr) for i := uint64(0); i < shiftEnd; i++ { - input, isNull, err := e.args[0].EvalReal(sctx, rows[lastEnd+i]) + input, isNull, err := e.args[0].EvalReal(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -904,10 +912,12 @@ func (e *maxMin4DecimalSliding) UpdatePartialResult(sctx sessionctx.Context, row return 0, nil } -func (e *maxMin4DecimalSliding) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &maxMin4DecimalSliding{} + +func (e *maxMin4DecimalSliding) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4MaxMinDecimal)(pr) for i := uint64(0); i < shiftEnd; i++ { - input, isNull, err := e.args[0].EvalDecimal(sctx, rows[lastEnd+i]) + input, isNull, err := e.args[0].EvalDecimal(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -1052,10 +1062,12 @@ func (e *maxMin4StringSliding) UpdatePartialResult(sctx sessionctx.Context, rows return 0, nil } -func (e *maxMin4StringSliding) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &maxMin4StringSliding{} + +func (e *maxMin4StringSliding) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4MaxMinString)(pr) for i := uint64(0); i < shiftEnd; i++ { - input, isNull, err := e.args[0].EvalString(sctx, rows[lastEnd+i]) + input, isNull, err := e.args[0].EvalString(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -1190,10 +1202,12 @@ func (e *maxMin4TimeSliding) UpdatePartialResult(sctx sessionctx.Context, rowsIn return 0, nil } -func (e *maxMin4TimeSliding) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &maxMin4DurationSliding{} + +func (e *maxMin4TimeSliding) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4MaxMinTime)(pr) for i := uint64(0); i < shiftEnd; i++ { - input, isNull, err := e.args[0].EvalTime(sctx, rows[lastEnd+i]) + input, isNull, err := e.args[0].EvalTime(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -1328,10 +1342,12 @@ func (e *maxMin4DurationSliding) UpdatePartialResult(sctx sessionctx.Context, ro return 0, nil } -func (e *maxMin4DurationSliding) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &maxMin4DurationSliding{} + +func (e *maxMin4DurationSliding) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4MaxMinDuration)(pr) for i := uint64(0); i < shiftEnd; i++ { - input, isNull, err := e.args[0].EvalDuration(sctx, rows[lastEnd+i]) + input, isNull, err := e.args[0].EvalDuration(sctx, getRow(lastEnd+i)) if err != nil { return err } diff --git a/executor/aggfuncs/func_percentile.go b/executor/aggfuncs/func_percentile.go index 31855f791fb0f..ff13392276c61 100644 --- a/executor/aggfuncs/func_percentile.go +++ b/executor/aggfuncs/func_percentile.go @@ -53,9 +53,7 @@ func (e *basePercentile) AllocPartialResult() (pr PartialResult, memDelta int64) return } -func (e *basePercentile) ResetPartialResult(pr PartialResult) { - return -} +func (e *basePercentile) ResetPartialResult(pr PartialResult) {} func (e *basePercentile) UpdatePartialResult(sctx sessionctx.Context, rowsInGroup []chunk.Row, pr PartialResult) (memDelta int64, err error) { return diff --git a/executor/aggfuncs/func_sum.go b/executor/aggfuncs/func_sum.go index 127e743b000a9..77cc6745ddf65 100644 --- a/executor/aggfuncs/func_sum.go +++ b/executor/aggfuncs/func_sum.go @@ -115,10 +115,12 @@ type sum4Float64 struct { baseSum4Float64 } -func (e *sum4Float64) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &sum4Float64{} + +func (e *sum4Float64) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4SumFloat64)(pr) for i := uint64(0); i < shiftEnd; i++ { - input, isNull, err := e.args[0].EvalReal(sctx, rows[lastEnd+i]) + input, isNull, err := e.args[0].EvalReal(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -129,7 +131,7 @@ func (e *sum4Float64) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart p.notNullRowCount++ } for i := uint64(0); i < shiftStart; i++ { - input, isNull, err := e.args[0].EvalReal(sctx, rows[lastStart+i]) + input, isNull, err := e.args[0].EvalReal(sctx, getRow(lastStart+i)) if err != nil { return err } @@ -201,10 +203,12 @@ func (e *sum4Decimal) UpdatePartialResult(sctx sessionctx.Context, rowsInGroup [ return 0, nil } -func (e *sum4Decimal) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { +var _ SlidingWindowAggFunc = &sum4Decimal{} + +func (e *sum4Decimal) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { p := (*partialResult4SumDecimal)(pr) for i := uint64(0); i < shiftEnd; i++ { - input, isNull, err := e.args[0].EvalDecimal(sctx, rows[lastEnd+i]) + input, isNull, err := e.args[0].EvalDecimal(sctx, getRow(lastEnd+i)) if err != nil { return err } @@ -225,7 +229,7 @@ func (e *sum4Decimal) Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart p.notNullRowCount++ } for i := uint64(0); i < shiftStart; i++ { - input, isNull, err := e.args[0].EvalDecimal(sctx, rows[lastStart+i]) + input, isNull, err := e.args[0].EvalDecimal(sctx, getRow(lastStart+i)) if err != nil { return err } diff --git a/executor/aggfuncs/row_number.go b/executor/aggfuncs/row_number.go index f1257d15f0411..b1f95a008ec9b 100644 --- a/executor/aggfuncs/row_number.go +++ b/executor/aggfuncs/row_number.go @@ -52,3 +52,9 @@ func (rn *rowNumber) AppendFinalResult2Chunk(sctx sessionctx.Context, pr Partial chk.AppendInt64(rn.ordinal, p.curIdx) return nil } + +var _ SlidingWindowAggFunc = &rowNumber{} + +func (rn *rowNumber) Slide(sctx sessionctx.Context, getRow func(uint64) chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error { + return nil +} diff --git a/executor/aggregate_test.go b/executor/aggregate_test.go index 94820a028123d..32041a29df8fd 100644 --- a/executor/aggregate_test.go +++ b/executor/aggregate_test.go @@ -824,6 +824,21 @@ func (s *testSuiteAgg) TestIssue16279(c *C) { tk.MustQuery("select count(a) , date_format(a, '%Y-%m-%d') as xx from s group by xx") } +func (s *testSuiteAgg) TestIssue24676(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("set sql_mode = 'ONLY_FULL_GROUP_BY'") + tk.MustExec("drop table if exists t1") + tk.MustExec(`create table t1( + id int(11) NOT NULL PRIMARY KEY, + c1 int(11) NOT NULL DEFAULT '0' + )`) + tk.MustQuery("SELECT c1 FROM t1 GROUP BY c1 ORDER BY c1 ASC;") + tk.MustQuery("SELECT ((floor(((`c1` - 0.0) / 50000)) * 50000) + 0.0) AS `c1` FROM `t1` GROUP BY ((floor(((`c1` - 0.0) / 50000)) * 50000) + 0.0) ORDER BY ((floor(((`c1` - 0.0) / 50000)) * 50000) + 0.0) ASC;") + err := tk.ExecToErr("SELECT ((floor(((`c1` - 10) / 300)) * 50000) + 0.0) AS `c1` FROM `t1` GROUP BY ((floor(((`c1` - 0.0) / 50000)) * 50000) + 0.0) ORDER BY ((floor(((`c1` - 0.0) / 50000)) * 50000) + 0.0) ASC;") + c.Assert(terror.ErrorEqual(err, plannercore.ErrFieldNotInGroupBy), IsTrue, Commentf("err %v", err)) +} + func (s *testSuiteAgg) TestAggPushDownPartitionTable(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -1293,50 +1308,59 @@ func (s *testSuiteAgg) TestIssue20658(c *C) { tk.MustExec("CREATE TABLE t(a bigint, b bigint);") tk.MustExec("set tidb_init_chunk_size=1;") tk.MustExec("set tidb_max_chunk_size=32;") - var insertSQL string + randSeed := time.Now().UnixNano() + r := rand.New(rand.NewSource(randSeed)) + var insertSQL strings.Builder for i := 0; i < 1000; i++ { - if i == 0 { - insertSQL += fmt.Sprintf("(%d, %d)", rand.Intn(100), rand.Intn(100)) - } else { - insertSQL += fmt.Sprintf(",(%d, %d)", rand.Intn(100), rand.Intn(100)) + insertSQL.WriteString("(") + insertSQL.WriteString(strconv.Itoa(r.Intn(10))) + insertSQL.WriteString(",") + insertSQL.WriteString(strconv.Itoa(r.Intn(10))) + insertSQL.WriteString(")") + if i < 1000-1 { + insertSQL.WriteString(",") } } - tk.MustExec(fmt.Sprintf("insert into t values %s;", insertSQL)) - - concurrencies := []int{1, 2, 4, 8} + tk.MustExec(fmt.Sprintf("insert into t values %s;", insertSQL.String())) + + mustParseAndSort := func(rows [][]interface{}, cmt CommentInterface) []float64 { + ret := make([]float64, len(rows)) + for i := 0; i < len(rows); i++ { + rowStr := rows[i][0].(string) + if rowStr == "" { + ret[i] = 0 + continue + } + v, err := strconv.ParseFloat(rowStr, 64) + c.Assert(err, IsNil, cmt) + ret[i] = v + } + sort.Float64s(ret) + return ret + } for _, sql := range sqls { - var expected [][]interface{} - for _, con := range concurrencies { - comment := Commentf("sql: %s; concurrency: %d", sql, con) + tk.MustExec("set @@tidb_streamagg_concurrency = 1;") + exp := tk.MustQuery(sql).Rows() + expected := mustParseAndSort(exp, Commentf("sql: %s; seed: %d", sql, randSeed)) + for _, con := range []int{2, 4, 8} { + comment := Commentf("sql: %s; concurrency: %d, seed: %d", sql, con, randSeed) tk.MustExec(fmt.Sprintf("set @@tidb_streamagg_concurrency=%d;", con)) - if con == 1 { - expected = tk.MustQuery(sql).Sort().Rows() - } else { - er := tk.MustQuery("explain format = 'brief' " + sql).Rows() - ok := false - for _, l := range er { - str := fmt.Sprintf("%v", l) - if strings.Contains(str, "Shuffle") { - ok = true - break - } - } - c.Assert(ok, Equals, true, comment) - rows := tk.MustQuery(sql).Sort().Rows() - - c.Assert(len(rows), Equals, len(expected), comment) - for i := range rows { - rowStr, expStr := rows[i][0].(string), expected[i][0].(string) - if rowStr == "" && expStr == "" { - continue - } - v1, err := strconv.ParseFloat(rowStr, 64) - c.Assert(err, IsNil, comment) - v2, err := strconv.ParseFloat(expStr, 64) - c.Assert(err, IsNil, comment) - c.Assert(math.Abs(v1-v2), Less, 1e-3, comment) + er := tk.MustQuery("explain format = 'brief' " + sql).Rows() + ok := false + for _, l := range er { + str := fmt.Sprintf("%v", l) + if strings.Contains(str, "Shuffle") { + ok = true + break } } + c.Assert(ok, Equals, true, comment) + rows := mustParseAndSort(tk.MustQuery(sql).Rows(), comment) + + c.Assert(len(rows), Equals, len(expected), comment) + for i := range rows { + c.Assert(math.Abs(rows[i]-expected[i]), Less, 1e-3, comment) + } } } } diff --git a/executor/analyze.go b/executor/analyze.go index 2d3187842845c..f905c2c3f5b84 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -36,6 +36,7 @@ import ( "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/distsql" "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/metrics" @@ -45,7 +46,6 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/store/tikv" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" @@ -54,6 +54,7 @@ import ( "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/ranger" "github.com/pingcap/tidb/util/sqlexec" + "github.com/pingcap/tidb/util/timeutil" "github.com/pingcap/tipb/go-tipb" "go.uber.org/zap" ) @@ -147,6 +148,10 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { } statisticsID := result.TableID.GetStatisticsID() for i, hg := range result.Hist { + // It's normal virtual column, skip. + if hg == nil { + continue + } if result.TableID.IsPartitionTable() && needGlobalStats { // If it does not belong to the statistics of index, we need to set it to -1 to distinguish. idxID := int64(-1) @@ -159,10 +164,10 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { } } var err1 error - if result.StatsVer == statistics.Version3 { - err1 = statsHandle.SaveStatsToStorage(statisticsID, result.Count, result.IsIndex, hg, nil, result.TopNs[i], result.Fms[i], result.StatsVer, 1) + if result.StatsVer == statistics.Version2 { + err1 = statsHandle.SaveStatsToStorage(statisticsID, result.Count, result.IsIndex, hg, nil, result.TopNs[i], result.Fms[i], result.StatsVer, 1, result.TableID.IsPartitionTable() && needGlobalStats) } else { - err1 = statsHandle.SaveStatsToStorage(statisticsID, result.Count, result.IsIndex, hg, result.Cms[i], result.TopNs[i], result.Fms[i], result.StatsVer, 1) + err1 = statsHandle.SaveStatsToStorage(statisticsID, result.Count, result.IsIndex, hg, result.Cms[i], result.TopNs[i], result.Fms[i], result.StatsVer, 1, result.TableID.IsPartitionTable() && needGlobalStats) } if err1 != nil { err = err1 @@ -187,7 +192,7 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { } if needGlobalStats { for globalStatsID, info := range globalStatsMap { - globalStats, err := statsHandle.MergePartitionStats2GlobalStatsByTableID(e.ctx, e.opts, infoschema.GetInfoSchema(e.ctx), globalStatsID.tableID, info.isIndex, info.idxID) + globalStats, err := statsHandle.MergePartitionStats2GlobalStatsByTableID(e.ctx, e.opts, e.ctx.GetInfoSchema().(infoschema.InfoSchema), globalStatsID.tableID, info.isIndex, info.idxID) if err != nil { if types.ErrPartitionStatsMissing.Equal(err) { // When we find some partition-level stats are missing, we need to report warning. @@ -198,19 +203,20 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { } for i := 0; i < globalStats.Num; i++ { hg, cms, topN, fms := globalStats.Hg[i], globalStats.Cms[i], globalStats.TopN[i], globalStats.Fms[i] - err = statsHandle.SaveStatsToStorage(globalStatsID.tableID, globalStats.Count, info.isIndex, hg, cms, topN, fms, info.statsVersion, 1) + // fms for global stats doesn't need to dump to kv. + err = statsHandle.SaveStatsToStorage(globalStatsID.tableID, globalStats.Count, info.isIndex, hg, cms, topN, fms, info.statsVersion, 1, false) if err != nil { logutil.Logger(ctx).Error("save global-level stats to storage failed", zap.Error(err)) } } } } - return statsHandle.Update(infoschema.GetInfoSchema(e.ctx)) + return statsHandle.Update(e.ctx.GetInfoSchema().(infoschema.InfoSchema)) } func getBuildStatsConcurrency(ctx sessionctx.Context) (int, error) { sessionVars := ctx.GetSessionVars() - concurrency, err := variable.GetSessionSystemVar(sessionVars, variable.TiDBBuildStatsConcurrency) + concurrency, err := variable.GetSessionOrGlobalSystemVar(sessionVars, variable.TiDBBuildStatsConcurrency) if err != nil { return 0, err } @@ -330,6 +336,36 @@ func analyzeIndexPushdown(idxExec *AnalyzeIndexExec) analyzeResult { return result } +func analyzeIndexNDVPushDown(idxExec *AnalyzeIndexExec) analyzeResult { + ranges := ranger.FullRange() + // For single-column index, we do not load null rows from TiKV, so the built histogram would not include + // null values, and its `NullCount` would be set by result of another distsql call to get null rows. + // For multi-column index, we cannot define null for the rows, so we still use full range, and the rows + // containing null fields would exist in built histograms. Note that, the `NullCount` of histograms for + // multi-column index is always 0 then. + if len(idxExec.idxInfo.Columns) == 1 { + ranges = ranger.FullNotNullRange() + } + fms, nullHist, err := idxExec.buildSimpleStats(ranges, len(idxExec.idxInfo.Columns) == 1) + if err != nil { + return analyzeResult{Err: err, job: idxExec.job} + } + result := analyzeResult{ + TableID: idxExec.tableID, + Fms: []*statistics.FMSketch{fms}, + // We use histogram to get the Index's ID. + Hist: []*statistics.Histogram{statistics.NewHistogram(idxExec.idxInfo.ID, 0, 0, statistics.Version1, types.NewFieldType(mysql.TypeBlob), 0, 0)}, + IsIndex: 1, + job: idxExec.job, + // TODO: avoid reusing Version1. + StatsVer: statistics.Version1, + } + if nullHist != nil && nullHist.Len() > 0 { + result.Count = nullHist.Buckets[nullHist.Len()-1].Count + } + return result +} + // AnalyzeIndexExec represents analyze index push down executor. type AnalyzeIndexExec struct { ctx sessionctx.Context @@ -355,6 +391,7 @@ func (e *AnalyzeIndexExec) fetchAnalyzeResult(ranges []*ranger.Range, isNullRang } else { kvReqBuilder = builder.SetIndexRangesForTables(e.ctx.GetSessionVars().StmtCtx, []int64{e.tableID.GetStatisticsID()}, e.idxInfo.ID, ranges) } + kvReqBuilder.SetResourceGroupTag(e.ctx.GetSessionVars().StmtCtx) kvReq, err := kvReqBuilder. SetAnalyzeRequest(e.analyzePB). SetStartTS(math.MaxUint64). @@ -511,6 +548,29 @@ func (e *AnalyzeIndexExec) buildStats(ranges []*ranger.Range, considerNull bool) return hist, cms, fms, topN, nil } +func (e *AnalyzeIndexExec) buildSimpleStats(ranges []*ranger.Range, considerNull bool) (fms *statistics.FMSketch, nullHist *statistics.Histogram, err error) { + if err = e.open(ranges, considerNull); err != nil { + return nil, nil, err + } + defer func() { + err1 := closeAll(e.result, e.countNullRes) + if err == nil { + err = err1 + } + }() + _, _, fms, _, err = e.buildStatsFromResult(e.result, false) + if e.countNullRes != nil { + nullHist, _, _, _, err := e.buildStatsFromResult(e.countNullRes, false) + if err != nil { + return nil, nil, err + } + if l := nullHist.Len(); l > 0 { + return fms, nullHist, nil + } + } + return fms, nil, nil +} + func analyzeColumnsPushdown(colExec *AnalyzeColumnsExec) []analyzeResult { var ranges []*ranger.Range if hc := colExec.handleCols; hc != nil { @@ -522,34 +582,64 @@ func analyzeColumnsPushdown(colExec *AnalyzeColumnsExec) []analyzeResult { } else { ranges = ranger.FullIntRange(false) } - if colExec.analyzeVer == statistics.Version3 { - count, hists, topns, fmSketches, err := colExec.buildSamplingStats(ranges) + collExtStats := colExec.ctx.GetSessionVars().EnableExtendedStats + if colExec.StatsVersion == statistics.Version2 { + specialIndexes := make([]*model.IndexInfo, 0, len(colExec.indexes)) + specialIndexesOffsets := make([]int, 0, len(colExec.indexes)) + for i, idx := range colExec.indexes { + isSpecial := false + for _, col := range idx.Columns { + colInfo := colExec.colsInfo[col.Offset] + isVirtualCol := colInfo.IsGenerated() && !colInfo.GeneratedStored + isPrefixCol := col.Length != types.UnspecifiedLength + if isVirtualCol || isPrefixCol { + isSpecial = true + break + } + } + if isSpecial { + specialIndexesOffsets = append(specialIndexesOffsets, i) + specialIndexes = append(specialIndexes, idx) + } + } + idxNDVPushDownCh := make(chan analyzeIndexNDVTotalResult, 1) + go colExec.handleNDVForSpecialIndexes(specialIndexes, idxNDVPushDownCh) + count, hists, topns, fmSketches, extStats, err := colExec.buildSamplingStats(ranges, collExtStats, specialIndexesOffsets, idxNDVPushDownCh) if err != nil { return []analyzeResult{{Err: err, job: colExec.job}} } cLen := len(colExec.analyzePB.ColReq.ColumnsInfo) - colResult := analyzeResult{ - TableID: colExec.tableID, - Hist: hists[:cLen], - TopNs: topns[:cLen], - Fms: fmSketches[:cLen], - job: colExec.job, - StatsVer: colExec.analyzeVer, - Count: count, - } colGroupResult := analyzeResult{ - TableID: colExec.tableID, + TableID: colExec.TableID, Hist: hists[cLen:], TopNs: topns[cLen:], Fms: fmSketches[cLen:], job: colExec.job, - StatsVer: colExec.analyzeVer, + StatsVer: colExec.StatsVersion, Count: count, IsIndex: 1, } + // Discard stats of _tidb_rowid. + // Because the process of analyzing will keep the order of results be the same as the colsInfo in the analyze task, + // and in `buildAnalyzeFullSamplingTask` we always place the _tidb_rowid at the last of colsInfo, so if there are + // stats for _tidb_rowid, it must be at the end of the column stats. + // Virtual column has no histogram yet. So we check nil here. + if hists[cLen-1] != nil && hists[cLen-1].ID == -1 { + cLen -= 1 + } + colResult := analyzeResult{ + TableID: colExec.TableID, + Hist: hists[:cLen], + TopNs: topns[:cLen], + Fms: fmSketches[:cLen], + ExtStats: extStats, + job: colExec.job, + StatsVer: colExec.StatsVersion, + Count: count, + } + return []analyzeResult{colResult, colGroupResult} } - collExtStats := colExec.ctx.GetSessionVars().EnableExtendedStats hists, cms, topNs, fms, extStats, err := colExec.buildStats(ranges, collExtStats) if err != nil { return []analyzeResult{{Err: err, job: colExec.job}} @@ -557,7 +647,7 @@ func analyzeColumnsPushdown(colExec *AnalyzeColumnsExec) []analyzeResult { if hasPkHist(colExec.handleCols) { PKresult := analyzeResult{ - TableID: colExec.tableID, + TableID: colExec.TableID, Hist: hists[:1], Cms: cms[:1], TopNs: topNs[:1], @@ -568,14 +658,14 @@ func analyzeColumnsPushdown(colExec *AnalyzeColumnsExec) []analyzeResult { } PKresult.Count = int64(PKresult.Hist[0].TotalRowCount()) restResult := analyzeResult{ - TableID: colExec.tableID, + TableID: colExec.TableID, Hist: hists[1:], Cms: cms[1:], TopNs: topNs[1:], Fms: fms[1:], ExtStats: extStats, job: colExec.job, - StatsVer: colExec.analyzeVer, + StatsVer: colExec.StatsVersion, } restResult.Count = PKresult.Count return []analyzeResult{PKresult, restResult} @@ -583,31 +673,31 @@ func analyzeColumnsPushdown(colExec *AnalyzeColumnsExec) []analyzeResult { var result []analyzeResult if colExec.analyzePB.Tp == tipb.AnalyzeType_TypeMixed { result = append(result, analyzeResult{ - TableID: colExec.tableID, + TableID: colExec.TableID, Hist: []*statistics.Histogram{hists[0]}, Cms: []*statistics.CMSketch{cms[0]}, TopNs: []*statistics.TopN{topNs[0]}, Fms: []*statistics.FMSketch{nil}, IsIndex: 1, job: colExec.job, - StatsVer: colExec.analyzeVer, + StatsVer: colExec.StatsVersion, }) hists = hists[1:] cms = cms[1:] topNs = topNs[1:] } colResult := analyzeResult{ - TableID: colExec.tableID, + TableID: colExec.TableID, Hist: hists, Cms: cms, TopNs: topNs, Fms: fms, ExtStats: extStats, job: colExec.job, - StatsVer: colExec.analyzeVer, + StatsVer: colExec.StatsVersion, } colResult.Count = int64(colResult.Hist[0].TotalRowCount()) - if colResult.StatsVer == statistics.Version2 { + if colResult.StatsVer >= statistics.Version2 { colResult.Count += int64(topNs[0].TotalCount()) } return append(result, colResult) @@ -616,7 +706,7 @@ func analyzeColumnsPushdown(colExec *AnalyzeColumnsExec) []analyzeResult { // AnalyzeColumnsExec represents Analyze columns push down executor. type AnalyzeColumnsExec struct { ctx sessionctx.Context - tableID core.AnalyzeTableID + tableInfo *model.TableInfo colsInfo []*model.ColumnInfo handleCols core.HandleCols concurrency int @@ -625,8 +715,14 @@ type AnalyzeColumnsExec struct { resultHandler *tableResultHandler opts map[ast.AnalyzeOptionType]uint64 job *statistics.AnalyzeJob - analyzeVer int indexes []*model.IndexInfo + core.AnalyzeInfo + + subIndexWorkerWg *sync.WaitGroup + samplingBuilderWg *sync.WaitGroup + samplingMergeWg *sync.WaitGroup + + schemaForVirtualColEval *expression.Schema } func (e *AnalyzeColumnsExec) open(ranges []*ranger.Range) error { @@ -652,7 +748,8 @@ func (e *AnalyzeColumnsExec) open(ranges []*ranger.Range) error { func (e *AnalyzeColumnsExec) buildResp(ranges []*ranger.Range) (distsql.SelectResult, error) { var builder distsql.RequestBuilder - reqBuilder := builder.SetHandleRangesForTables(e.ctx.GetSessionVars().StmtCtx, []int64{e.tableID.GetStatisticsID()}, e.handleCols != nil && !e.handleCols.IsInt(), ranges, nil) + reqBuilder := builder.SetHandleRangesForTables(e.ctx.GetSessionVars().StmtCtx, []int64{e.TableID.GetStatisticsID()}, e.handleCols != nil && !e.handleCols.IsInt(), ranges, nil) + builder.SetResourceGroupTag(e.ctx.GetSessionVars().StmtCtx) // Always set KeepOrder of the request to be true, in order to compute // correct `correlation` of columns. kvReq, err := reqBuilder. @@ -672,21 +769,65 @@ func (e *AnalyzeColumnsExec) buildResp(ranges []*ranger.Range) (distsql.SelectRe return result, nil } -func (e *AnalyzeColumnsExec) buildSamplingStats(ranges []*ranger.Range) ( +// decodeSampleDataWithVirtualColumn constructs the virtual column by evaluating from the deocded normal columns. +// If it failed, it would return false to trigger normal decoding way without the virtual column. +func (e AnalyzeColumnsExec) decodeSampleDataWithVirtualColumn( + collector *statistics.RowSampleCollector, + fieldTps []*types.FieldType, + virtualColIdx []int, + schema *expression.Schema, +) error { + totFts := make([]*types.FieldType, 0, e.schemaForVirtualColEval.Len()) + for _, col := range e.schemaForVirtualColEval.Columns { + totFts = append(totFts, col.RetType) + } + chk := chunk.NewChunkWithCapacity(totFts, len(collector.Samples)) + decoder := codec.NewDecoder(chk, e.ctx.GetSessionVars().TimeZone) + for _, sample := range collector.Samples { + for i := range sample.Columns { + if schema.Columns[i].VirtualExpr != nil { + continue + } + _, err := decoder.DecodeOne(sample.Columns[i].GetBytes(), i, e.schemaForVirtualColEval.Columns[i].RetType) + if err != nil { + return err + } + } + } + err := FillVirtualColumnValue(fieldTps, virtualColIdx, schema, e.colsInfo, e.ctx, chk) + if err != nil { + return err + } + iter := chunk.NewIterator4Chunk(chk) + for row, i := iter.Begin(), 0; row != iter.End(); row, i = iter.Next(), i+1 { + datums := row.GetDatumRow(totFts) + collector.Samples[i].Columns = datums + } + return nil +} + +func (e *AnalyzeColumnsExec) buildSamplingStats( + ranges []*ranger.Range, + needExtStats bool, + indexesWithVirtualColOffsets []int, + idxNDVPushDownCh chan analyzeIndexNDVTotalResult, +) ( count int64, hists []*statistics.Histogram, topns []*statistics.TopN, fmSketches []*statistics.FMSketch, + extStats *statistics.ExtendedStatsColl, err error, ) { if err = e.open(ranges); err != nil { - return 0, nil, nil, nil, err + return 0, nil, nil, nil, nil, err } defer func() { if err1 := e.resultHandler.Close(); err1 != nil { err = err1 } }() + l := len(e.analyzePB.ColReq.ColumnsInfo) + len(e.analyzePB.ColReq.ColumnGroups) rootRowCollector := &statistics.RowSampleCollector{ NullCount: make([]int64, l), @@ -699,100 +840,498 @@ func (e *AnalyzeColumnsExec) buildSamplingStats(ranges []*ranger.Range) ( rootRowCollector.FMSketches = append(rootRowCollector.FMSketches, statistics.NewFMSketch(maxSketchSize)) } sc := e.ctx.GetSessionVars().StmtCtx + statsConcurrency, err := getBuildStatsConcurrency(e.ctx) + if err != nil { + return 0, nil, nil, nil, nil, err + } + mergeResultCh := make(chan *samplingMergeResult, statsConcurrency) + mergeTaskCh := make(chan []byte, statsConcurrency) + e.samplingMergeWg = &sync.WaitGroup{} + e.samplingMergeWg.Add(statsConcurrency) + for i := 0; i < statsConcurrency; i++ { + go e.subMergeWorker(mergeResultCh, mergeTaskCh, l, i == 0) + } for { data, err1 := e.resultHandler.nextRaw(context.TODO()) if err1 != nil { - return 0, nil, nil, nil, err1 + return 0, nil, nil, nil, nil, err1 } if data == nil { break } - colResp := &tipb.AnalyzeColumnsResp{} - err = colResp.Unmarshal(data) + mergeTaskCh <- data + } + close(mergeTaskCh) + mergeWorkerPanicCnt := 0 + for mergeWorkerPanicCnt < statsConcurrency { + mergeResult, ok := <-mergeResultCh + if !ok { + break + } + if mergeResult.err != nil { + err = mergeResult.err + if mergeResult.err == errAnalyzeWorkerPanic { + mergeWorkerPanicCnt++ + } + continue + } + rootRowCollector.MergeCollector(mergeResult.collector) + } + if err != nil { + return 0, nil, nil, nil, nil, err + } + + // handling virtual columns + virtualColIdx := buildVirtualColumnIndex(e.schemaForVirtualColEval, e.colsInfo) + if len(virtualColIdx) > 0 { + fieldTps := make([]*types.FieldType, 0, len(virtualColIdx)) + for _, colOffset := range virtualColIdx { + fieldTps = append(fieldTps, e.schemaForVirtualColEval.Columns[colOffset].RetType) + } + err = e.decodeSampleDataWithVirtualColumn(rootRowCollector, fieldTps, virtualColIdx, e.schemaForVirtualColEval) if err != nil { - return 0, nil, nil, nil, err + return 0, nil, nil, nil, nil, err } - subCollector := &statistics.RowSampleCollector{ - MaxSampleSize: int(e.analyzePB.ColReq.SampleSize), + } else { + // If there's no virtual column or we meet error during eval virtual column, we fallback to normal decode otherwise. + for _, sample := range rootRowCollector.Samples { + for i := range sample.Columns { + sample.Columns[i], err = tablecodec.DecodeColumnValue(sample.Columns[i].GetBytes(), &e.colsInfo[i].FieldType, sc.TimeZone) + if err != nil { + return 0, nil, nil, nil, nil, err + } + } } - subCollector.FromProto(colResp.RowCollector) - e.job.Update(subCollector.Count) - rootRowCollector.MergeCollector(subCollector) } + for _, sample := range rootRowCollector.Samples { for i := range sample.Columns { - sample.Columns[i], err = tablecodec.DecodeColumnValue(sample.Columns[i].GetBytes(), &e.colsInfo[i].FieldType, sc.TimeZone) - if err != nil { - return 0, nil, nil, nil, err - } if sample.Columns[i].Kind() == types.KindBytes { sample.Columns[i].SetBytes(sample.Columns[i].GetBytes()) } } + // Calculate handle from the row data for each row. It will be used to sort the samples. + sample.Handle, err = e.handleCols.BuildHandleByDatums(sample.Columns) + if err != nil { + return 0, nil, nil, nil, nil, err + } } - hists = make([]*statistics.Histogram, 0, len(e.colsInfo)) - topns = make([]*statistics.TopN, 0, len(e.colsInfo)) - fmSketches = make([]*statistics.FMSketch, 0, len(e.colsInfo)) + + colLen := len(e.colsInfo) + + // The order of the samples are broken when merging samples from sub-collectors. + // So now we need to sort the samples according to the handle in order to calculate correlation. + sort.Slice(rootRowCollector.Samples, func(i, j int) bool { + return rootRowCollector.Samples[i].Handle.Compare(rootRowCollector.Samples[j].Handle) < 0 + }) + + totalLen := len(e.colsInfo) + len(e.indexes) + hists = make([]*statistics.Histogram, totalLen) + topns = make([]*statistics.TopN, totalLen) + fmSketches = make([]*statistics.FMSketch, 0, totalLen) + buildResultChan := make(chan error, totalLen) + buildTaskChan := make(chan *samplingBuildTask, totalLen) + e.samplingBuilderWg = &sync.WaitGroup{} + e.samplingBuilderWg.Add(statsConcurrency) + sampleCollectors := make([]*statistics.SampleCollector, len(e.colsInfo)) + for i := 0; i < statsConcurrency; i++ { + go e.subBuildWorker(buildResultChan, buildTaskChan, hists, topns, sampleCollectors, i == 0) + } + for i, col := range e.colsInfo { - sampleItems := make([]*statistics.SampleItem, 0, rootRowCollector.MaxSampleSize) - for _, row := range rootRowCollector.Samples { - if row.Columns[i].IsNull() { - continue + buildTaskChan <- &samplingBuildTask{ + id: col.ID, + rootRowCollector: rootRowCollector, + tp: &col.FieldType, + isColumn: true, + slicePos: i, + } + fmSketches = append(fmSketches, rootRowCollector.FMSketches[i]) + } + + indexPushedDownResult := <-idxNDVPushDownCh + if indexPushedDownResult.err != nil { + return 0, nil, nil, nil, nil, indexPushedDownResult.err + } + for _, offset := range indexesWithVirtualColOffsets { + ret := indexPushedDownResult.results[e.indexes[offset].ID] + rootRowCollector.NullCount[colLen+offset] = ret.Count + rootRowCollector.FMSketches[colLen+offset] = ret.Fms[0] + } + + // build index stats + for i, idx := range e.indexes { + buildTaskChan <- &samplingBuildTask{ + id: idx.ID, + rootRowCollector: rootRowCollector, + tp: types.NewFieldType(mysql.TypeBlob), + isColumn: false, + slicePos: colLen + i, + } + fmSketches = append(fmSketches, rootRowCollector.FMSketches[colLen+i]) + } + close(buildTaskChan) + panicCnt := 0 + for panicCnt < statsConcurrency { + err1, ok := <-buildResultChan + if !ok { + break + } + if err1 != nil { + err = err1 + if err1 == errAnalyzeWorkerPanic { + panicCnt++ } - sampleItems = append(sampleItems, &statistics.SampleItem{ - Value: row.Columns[i], - }) - } - collector := &statistics.SampleCollector{ - Samples: sampleItems, - NullCount: rootRowCollector.NullCount[i], - Count: rootRowCollector.Count - rootRowCollector.NullCount[i], - FMSketch: rootRowCollector.FMSketches[i], - TotalSize: rootRowCollector.TotalSizes[i], - } - hg, topn, err := statistics.BuildHistAndTopNOnRowSample(e.ctx, int(e.opts[ast.AnalyzeOptNumBuckets]), int(e.opts[ast.AnalyzeOptNumTopN]), col.ID, collector, &col.FieldType, true) + continue + } + } + if err != nil { + return 0, nil, nil, nil, nil, err + } + count = rootRowCollector.Count + if needExtStats { + statsHandle := domain.GetDomain(e.ctx).StatsHandle() + extStats, err = statsHandle.BuildExtendedStats(e.TableID.GetStatisticsID(), e.colsInfo, sampleCollectors) if err != nil { - return 0, nil, nil, nil, err + return 0, nil, nil, nil, nil, err } - hists = append(hists, hg) - topns = append(topns, topn) - fmSketches = append(fmSketches, rootRowCollector.FMSketches[i]) } + return +} + +type analyzeIndexNDVTotalResult struct { + results map[int64]analyzeResult + err error +} + +// handleNDVForSpecialIndexes deals with the logic to analyze the index containing the virtual column when the mode is full sampling. +func (e *AnalyzeColumnsExec) handleNDVForSpecialIndexes(indexInfos []*model.IndexInfo, totalResultCh chan analyzeIndexNDVTotalResult) { + defer func() { + if r := recover(); r != nil { + buf := make([]byte, 4096) + stackSize := runtime.Stack(buf, false) + buf = buf[:stackSize] + logutil.BgLogger().Error("analyze ndv for special index panicked", zap.String("stack", string(buf))) + metrics.PanicCounter.WithLabelValues(metrics.LabelAnalyze).Inc() + totalResultCh <- analyzeIndexNDVTotalResult{ + err: errAnalyzeWorkerPanic, + } + } + }() + tasks := e.buildSubIndexJobForSpecialIndex(indexInfos) + statsConcurrncy, err := getBuildStatsConcurrency(e.ctx) + taskCh := make(chan *analyzeTask, len(tasks)) + for _, task := range tasks { + statistics.AddNewAnalyzeJob(task.job) + } + resultCh := make(chan analyzeResult, len(tasks)) + e.subIndexWorkerWg = &sync.WaitGroup{} + e.subIndexWorkerWg.Add(statsConcurrncy) + for i := 0; i < statsConcurrncy; i++ { + go e.subIndexWorkerForNDV(taskCh, resultCh, i == 0) + } + for _, task := range tasks { + taskCh <- task + } + close(taskCh) + panicCnt := 0 + totalResult := analyzeIndexNDVTotalResult{ + results: make(map[int64]analyzeResult, len(indexInfos)), + } + for panicCnt < statsConcurrncy { + result, ok := <-resultCh + if !ok { + break + } + if result.Err != nil { + result.job.Finish(true) + err = result.Err + if err == errAnalyzeWorkerPanic { + panicCnt++ + } + continue + } + result.job.Finish(false) + statistics.MoveToHistory(result.job) + totalResult.results[result.Hist[0].ID] = result + } + if err != nil { + totalResult.err = err + } + totalResultCh <- totalResult +} + +// subIndexWorker receive the task for each index and return the result for them. +func (e *AnalyzeColumnsExec) subIndexWorkerForNDV(taskCh chan *analyzeTask, resultCh chan analyzeResult, isFirstToCloseCh bool) { + var task *analyzeTask + defer func() { + if r := recover(); r != nil { + buf := make([]byte, 4096) + stackSize := runtime.Stack(buf, false) + buf = buf[:stackSize] + logutil.BgLogger().Error("analyze worker panicked", zap.String("stack", string(buf))) + metrics.PanicCounter.WithLabelValues(metrics.LabelAnalyze).Inc() + resultCh <- analyzeResult{ + Err: errAnalyzeWorkerPanic, + job: task.job, + } + } + e.subIndexWorkerWg.Done() + if isFirstToCloseCh { + e.subIndexWorkerWg.Wait() + close(resultCh) + } + }() + for { + var ok bool + task, ok = <-taskCh + if !ok { + break + } + task.job.Start() + if task.taskType != idxTask { + resultCh <- analyzeResult{ + Err: errors.Errorf("incorrect analyze type"), + job: task.job, + } + continue + } + task.idxExec.job = task.job + resultCh <- analyzeIndexNDVPushDown(task.idxExec) + } +} + +// buildSubIndexJobForSpecialIndex builds sub index pushed down task to calculate the NDV information for indexes containing virtual column. +// This is because we cannot push the calculation of the virtual column down to the tikv side. +func (e *AnalyzeColumnsExec) buildSubIndexJobForSpecialIndex(indexInfos []*model.IndexInfo) []*analyzeTask { + _, offset := timeutil.Zone(e.ctx.GetSessionVars().Location()) + tasks := make([]*analyzeTask, 0, len(indexInfos)) + sc := e.ctx.GetSessionVars().StmtCtx + for _, indexInfo := range indexInfos { + idxExec := &AnalyzeIndexExec{ + ctx: e.ctx, + tableID: e.TableID, + isCommonHandle: e.tableInfo.IsCommonHandle, + idxInfo: indexInfo, + concurrency: e.ctx.GetSessionVars().IndexSerialScanConcurrency(), + analyzePB: &tipb.AnalyzeReq{ + Tp: tipb.AnalyzeType_TypeIndex, + Flags: sc.PushDownFlags(), + TimeZoneOffset: offset, + }, + } + idxExec.opts = make(map[ast.AnalyzeOptionType]uint64, len(ast.AnalyzeOptionString)) + idxExec.opts[ast.AnalyzeOptNumTopN] = 0 + idxExec.opts[ast.AnalyzeOptCMSketchDepth] = 0 + idxExec.opts[ast.AnalyzeOptCMSketchWidth] = 0 + idxExec.opts[ast.AnalyzeOptNumSamples] = 0 + idxExec.opts[ast.AnalyzeOptNumBuckets] = 1 + statsVersion := new(int32) + *statsVersion = statistics.Version1 + // No Top-N + topnSize := int32(0) + idxExec.analyzePB.IdxReq = &tipb.AnalyzeIndexReq{ + // One bucket to store the null for null histogram. + BucketSize: 1, + NumColumns: int32(len(indexInfo.Columns)), + TopNSize: &topnSize, + Version: statsVersion, + SketchSize: maxSketchSize, + } + if idxExec.isCommonHandle && indexInfo.Primary { + idxExec.analyzePB.Tp = tipb.AnalyzeType_TypeCommonHandle + } + // No CM-Sketch. + depth := int32(0) + width := int32(0) + idxExec.analyzePB.IdxReq.CmsketchDepth = &depth + idxExec.analyzePB.IdxReq.CmsketchWidth = &width + autoAnalyze := "" + if e.ctx.GetSessionVars().InRestrictedSQL { + autoAnalyze = "auto " + } + job := &statistics.AnalyzeJob{DBName: e.job.DBName, TableName: e.job.TableName, PartitionName: e.job.PartitionName, JobInfo: autoAnalyze + "analyze ndv for index " + indexInfo.Name.O} + tasks = append(tasks, &analyzeTask{ + taskType: idxTask, + idxExec: idxExec, + job: job, + }) + } + return tasks +} + +type samplingMergeResult struct { + collector *statistics.RowSampleCollector + err error +} + +func (e *AnalyzeColumnsExec) subMergeWorker(resultCh chan<- *samplingMergeResult, taskCh <-chan []byte, l int, isClosedChanThread bool) { + defer func() { + if r := recover(); r != nil { + buf := make([]byte, 4096) + stackSize := runtime.Stack(buf, false) + buf = buf[:stackSize] + logutil.BgLogger().Error("analyze worker panicked", zap.String("stack", string(buf))) + metrics.PanicCounter.WithLabelValues(metrics.LabelAnalyze).Inc() + resultCh <- &samplingMergeResult{err: errAnalyzeWorkerPanic} + } + // Consume the remaining things. + for { + _, ok := <-taskCh + if !ok { + break + } + } + e.samplingMergeWg.Done() + if isClosedChanThread { + e.samplingMergeWg.Wait() + close(resultCh) + } + }() + failpoint.Inject("mockAnalyzeSamplingMergeWorkerPanic", func() { + panic("failpoint triggered") + }) + retCollector := &statistics.RowSampleCollector{ + NullCount: make([]int64, l), + FMSketches: make([]*statistics.FMSketch, 0, l), + TotalSizes: make([]int64, l), + Samples: make(statistics.WeightedRowSampleHeap, 0, e.analyzePB.ColReq.SampleSize), + MaxSampleSize: int(e.analyzePB.ColReq.SampleSize), + } + for i := 0; i < l; i++ { + retCollector.FMSketches = append(retCollector.FMSketches, statistics.NewFMSketch(maxSketchSize)) + } + for { + data, ok := <-taskCh + if !ok { + break + } + colResp := &tipb.AnalyzeColumnsResp{} + err := colResp.Unmarshal(data) + if err != nil { + resultCh <- &samplingMergeResult{err: err} + return + } + subCollector := &statistics.RowSampleCollector{ + MaxSampleSize: int(e.analyzePB.ColReq.SampleSize), + } + subCollector.FromProto(colResp.RowCollector) + e.job.Update(subCollector.Count) + retCollector.MergeCollector(subCollector) + } + resultCh <- &samplingMergeResult{collector: retCollector} +} + +type samplingBuildTask struct { + id int64 + rootRowCollector *statistics.RowSampleCollector + tp *types.FieldType + isColumn bool + slicePos int +} + +func (e *AnalyzeColumnsExec) subBuildWorker(resultCh chan error, taskCh chan *samplingBuildTask, hists []*statistics.Histogram, topns []*statistics.TopN, collectors []*statistics.SampleCollector, isClosedChanThread bool) { + defer func() { + if r := recover(); r != nil { + buf := make([]byte, 4096) + stackSize := runtime.Stack(buf, false) + buf = buf[:stackSize] + logutil.BgLogger().Error("analyze worker panicked", zap.String("stack", string(buf))) + metrics.PanicCounter.WithLabelValues(metrics.LabelAnalyze).Inc() + resultCh <- errAnalyzeWorkerPanic + } + e.samplingBuilderWg.Done() + if isClosedChanThread { + e.samplingBuilderWg.Wait() + close(resultCh) + } + }() + failpoint.Inject("mockAnalyzeSamplingBuildWorkerPanic", func() { + panic("failpoint triggered") + }) colLen := len(e.colsInfo) - for i, idx := range e.indexes { - sampleItems := make([]*statistics.SampleItem, 0, rootRowCollector.MaxSampleSize) - for _, row := range rootRowCollector.Samples { - if len(idx.Columns) == 1 && row.Columns[idx.Columns[0].Offset].IsNull() { +workLoop: + for { + task, ok := <-taskCh + if !ok { + break + } + var collector *statistics.SampleCollector + if task.isColumn { + if e.colsInfo[task.slicePos].IsGenerated() && !e.colsInfo[task.slicePos].GeneratedStored { + hists[task.slicePos] = nil + topns[task.slicePos] = nil continue } - b := make([]byte, 0, 8) - for _, col := range idx.Columns { - b, err = codec.EncodeKey(e.ctx.GetSessionVars().StmtCtx, b, row.Columns[col.Offset]) - if err != nil { - return 0, nil, nil, nil, err + sampleItems := make([]*statistics.SampleItem, 0, task.rootRowCollector.MaxSampleSize) + for j, row := range task.rootRowCollector.Samples { + if row.Columns[task.slicePos].IsNull() { + continue + } + sampleItems = append(sampleItems, &statistics.SampleItem{ + Value: row.Columns[task.slicePos], + Ordinal: j, + }) + } + collector = &statistics.SampleCollector{ + Samples: sampleItems, + NullCount: task.rootRowCollector.NullCount[task.slicePos], + Count: task.rootRowCollector.Count - task.rootRowCollector.NullCount[task.slicePos], + FMSketch: task.rootRowCollector.FMSketches[task.slicePos], + TotalSize: task.rootRowCollector.TotalSizes[task.slicePos], + } + } else { + var tmpDatum types.Datum + var err error + idx := e.indexes[task.slicePos-colLen] + sampleItems := make([]*statistics.SampleItem, 0, task.rootRowCollector.MaxSampleSize) + for _, row := range task.rootRowCollector.Samples { + if len(idx.Columns) == 1 && row.Columns[idx.Columns[0].Offset].IsNull() { + continue + } + + b := make([]byte, 0, 8) + for _, col := range idx.Columns { + if col.Length != types.UnspecifiedLength { + row.Columns[col.Offset].Copy(&tmpDatum) + ranger.CutDatumByPrefixLen(&tmpDatum, col.Length, &e.colsInfo[col.Offset].FieldType) + b, err = codec.EncodeKey(e.ctx.GetSessionVars().StmtCtx, b, tmpDatum) + if err != nil { + resultCh <- err + continue workLoop + } + continue + } + b, err = codec.EncodeKey(e.ctx.GetSessionVars().StmtCtx, b, row.Columns[col.Offset]) + if err != nil { + resultCh <- err + continue workLoop + } } + sampleItems = append(sampleItems, &statistics.SampleItem{ + Value: types.NewBytesDatum(b), + }) + } + collector = &statistics.SampleCollector{ + Samples: sampleItems, + NullCount: task.rootRowCollector.NullCount[task.slicePos], + Count: task.rootRowCollector.Count - task.rootRowCollector.NullCount[task.slicePos], + FMSketch: task.rootRowCollector.FMSketches[task.slicePos], + TotalSize: task.rootRowCollector.TotalSizes[task.slicePos], } - sampleItems = append(sampleItems, &statistics.SampleItem{ - Value: types.NewBytesDatum(b), - }) - } - collector := &statistics.SampleCollector{ - Samples: sampleItems, - NullCount: rootRowCollector.NullCount[colLen+i], - Count: rootRowCollector.Count - rootRowCollector.NullCount[colLen+i], - FMSketch: rootRowCollector.FMSketches[colLen+i], - TotalSize: rootRowCollector.TotalSizes[colLen+i], - } - hg, topn, err := statistics.BuildHistAndTopNOnRowSample(e.ctx, int(e.opts[ast.AnalyzeOptNumBuckets]), int(e.opts[ast.AnalyzeOptNumTopN]), idx.ID, collector, types.NewFieldType(mysql.TypeBlob), false) + } + if task.isColumn { + collectors[task.slicePos] = collector + } + hist, topn, err := statistics.BuildHistAndTopN(e.ctx, int(e.opts[ast.AnalyzeOptNumBuckets]), int(e.opts[ast.AnalyzeOptNumTopN]), task.id, collector, task.tp, task.isColumn) if err != nil { - return 0, nil, nil, nil, err + resultCh <- err + continue } - hists = append(hists, hg) - topns = append(topns, topn) - fmSketches = append(fmSketches, rootRowCollector.FMSketches[colLen+i]) + hists[task.slicePos] = hist + topns[task.slicePos] = topn + resultCh <- nil } - count = rootRowCollector.Count - return } func (e *AnalyzeColumnsExec) buildStats(ranges []*ranger.Range, needExtStats bool) (hists []*statistics.Histogram, cms []*statistics.CMSketch, topNs []*statistics.TopN, fms []*statistics.FMSketch, extStats *statistics.ExtendedStatsColl, err error) { @@ -889,7 +1428,7 @@ func (e *AnalyzeColumnsExec) buildStats(ranges []*ranger.Range, needExtStats boo fms = append(fms, nil) } for i, col := range e.colsInfo { - if e.analyzeVer < 2 { + if e.StatsVersion < 2 { // In analyze version 2, we don't collect TopN this way. We will collect TopN from samples in `BuildColumnHistAndTopN()` below. err := collectors[i].ExtractTopN(uint32(e.opts[ast.AnalyzeOptNumTopN]), e.ctx.GetSessionVars().StmtCtx, &col.FieldType, timeZone) if err != nil { @@ -912,10 +1451,10 @@ func (e *AnalyzeColumnsExec) buildStats(ranges []*ranger.Range, needExtStats boo var hg *statistics.Histogram var err error var topn *statistics.TopN - if e.analyzeVer < 2 { + if e.StatsVersion < 2 { hg, err = statistics.BuildColumn(e.ctx, int64(e.opts[ast.AnalyzeOptNumBuckets]), col.ID, collectors[i], &col.FieldType) } else { - hg, topn, err = statistics.BuildColumnHistAndTopN(e.ctx, int(e.opts[ast.AnalyzeOptNumBuckets]), int(e.opts[ast.AnalyzeOptNumTopN]), col.ID, collectors[i], &col.FieldType) + hg, topn, err = statistics.BuildHistAndTopN(e.ctx, int(e.opts[ast.AnalyzeOptNumBuckets]), int(e.opts[ast.AnalyzeOptNumTopN]), col.ID, collectors[i], &col.FieldType, true) topNs = append(topNs, topn) } if err != nil { @@ -928,7 +1467,7 @@ func (e *AnalyzeColumnsExec) buildStats(ranges []*ranger.Range, needExtStats boo } if needExtStats { statsHandle := domain.GetDomain(e.ctx).StatsHandle() - extStats, err = statsHandle.BuildExtendedStats(e.tableID.GetStatisticsID(), e.colsInfo, collectors) + extStats, err = statsHandle.BuildExtendedStats(e.TableID.GetStatisticsID(), e.colsInfo, collectors) if err != nil { return nil, nil, nil, nil, nil, err } @@ -1120,9 +1659,9 @@ func (e *AnalyzeFastExec) activateTxnForRowCount() (rollbackFn func() error, err return nil, errors.Trace(err) } } - txn.SetOption(tikvstore.Priority, kv.PriorityLow) - txn.SetOption(tikvstore.IsolationLevel, kv.RC) - txn.SetOption(tikvstore.NotFillCache, true) + txn.SetOption(kv.Priority, kv.PriorityLow) + txn.SetOption(kv.IsolationLevel, kv.RC) + txn.SetOption(kv.NotFillCache, true) return rollbackFn, nil } @@ -1321,8 +1860,9 @@ func (e *AnalyzeFastExec) handleScanIter(iter kv.Iterator) (scanKeysSize int, er func (e *AnalyzeFastExec) handleScanTasks(bo *tikv.Backoffer) (keysSize int, err error) { snapshot := e.ctx.GetStore().GetSnapshot(kv.MaxVersion) if e.ctx.GetSessionVars().GetReplicaRead().IsFollowerRead() { - snapshot.SetOption(tikvstore.ReplicaRead, tikvstore.ReplicaReadFollower) + snapshot.SetOption(kv.ReplicaRead, kv.ReplicaReadFollower) } + setResourceGroupTagForTxn(e.ctx.GetSessionVars().StmtCtx, snapshot) for _, t := range e.scanTasks { iter, err := snapshot.Iter(kv.Key(t.StartKey), kv.Key(t.EndKey)) if err != nil { @@ -1340,11 +1880,12 @@ func (e *AnalyzeFastExec) handleScanTasks(bo *tikv.Backoffer) (keysSize int, err func (e *AnalyzeFastExec) handleSampTasks(workID int, step uint32, err *error) { defer e.wg.Done() snapshot := e.ctx.GetStore().GetSnapshot(kv.MaxVersion) - snapshot.SetOption(tikvstore.NotFillCache, true) - snapshot.SetOption(tikvstore.IsolationLevel, kv.RC) - snapshot.SetOption(tikvstore.Priority, kv.PriorityLow) + snapshot.SetOption(kv.NotFillCache, true) + snapshot.SetOption(kv.IsolationLevel, kv.RC) + snapshot.SetOption(kv.Priority, kv.PriorityLow) + setResourceGroupTagForTxn(e.ctx.GetSessionVars().StmtCtx, snapshot) if e.ctx.GetSessionVars().GetReplicaRead().IsFollowerRead() { - snapshot.SetOption(tikvstore.ReplicaRead, tikvstore.ReplicaReadFollower) + snapshot.SetOption(kv.ReplicaRead, kv.ReplicaReadFollower) } rander := rand.New(rand.NewSource(e.randSeed)) @@ -1355,7 +1896,7 @@ func (e *AnalyzeFastExec) handleSampTasks(workID int, step uint32, err *error) { lower, upper := step-uint32(2*math.Sqrt(float64(step))), step step = uint32(rander.Intn(int(upper-lower))) + lower } - snapshot.SetOption(tikvstore.SampleStep, step) + snapshot.SetOption(kv.SampleStep, step) kvMap := make(map[string][]byte) var iter kv.Iterator iter, *err = snapshot.Iter(kv.Key(task.StartKey), kv.Key(task.EndKey)) @@ -1598,7 +2139,7 @@ func analyzeIndexIncremental(idxExec *analyzeIndexIncrementalExec) analyzeResult } cms.CalcDefaultValForAnalyze(uint64(hist.NDV)) } - if statsVer == statistics.Version2 { + if statsVer >= statistics.Version2 { poped := statistics.MergeTopNAndUpdateCMSketch(topN, idxExec.oldTopN, cms, uint32(idxExec.opts[ast.AnalyzeOptNumTopN])) hist.AddIdxVals(poped) } @@ -1644,7 +2185,7 @@ func analyzePKIncremental(colExec *analyzePKIncrementalExec) analyzeResult { return analyzeResult{Err: err, job: colExec.job} } result := analyzeResult{ - TableID: colExec.tableID, + TableID: colExec.TableID, Hist: []*statistics.Histogram{hist}, Cms: []*statistics.CMSketch{nil}, TopNs: []*statistics.TopN{nil}, diff --git a/executor/analyze_test.go b/executor/analyze_test.go index f8eff902fcb3d..f9765aa08a096 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx" @@ -38,7 +39,6 @@ import ( "github.com/pingcap/tidb/statistics/handle" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/store/tikv" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/mockstore/cluster" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/table" @@ -46,6 +46,7 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/collate" + "github.com/pingcap/tidb/util/israce" "github.com/pingcap/tidb/util/testkit" ) @@ -56,6 +57,7 @@ func (s *testSuite1) TestAnalyzePartition(c *C) { testkit.WithPruneMode(tk, variable.Static, func() { tk.MustExec("use test") tk.MustExec("drop table if exists t") + tk.MustExec("set @@tidb_analyze_version=2") createTable := `CREATE TABLE t (a int, b int, c varchar(10), primary key(a), index idx(b)) PARTITION BY RANGE ( a ) ( PARTITION p0 VALUES LESS THAN (6), @@ -69,7 +71,7 @@ PARTITION BY RANGE ( a ) ( } tk.MustExec("analyze table t") - is := infoschema.GetInfoSchema(tk.Se.(sessionctx.Context)) + is := tk.Se.(sessionctx.Context).GetInfoSchema().(infoschema.InfoSchema) table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) pi := table.Meta().GetPartitionInfo() @@ -83,10 +85,10 @@ PARTITION BY RANGE ( a ) ( c.Assert(len(statsTbl.Columns), Equals, 3) c.Assert(len(statsTbl.Indices), Equals, 1) for _, col := range statsTbl.Columns { - c.Assert(col.Len(), Greater, 0) + c.Assert(col.Len()+col.Num(), Greater, 0) } for _, idx := range statsTbl.Indices { - c.Assert(idx.Len(), Greater, 0) + c.Assert(idx.Len()+idx.Num(), Greater, 0) } } @@ -96,7 +98,7 @@ PARTITION BY RANGE ( a ) ( tk.MustExec(fmt.Sprintf(`insert into t values (%d, %d, "hello")`, i, i)) } tk.MustExec("alter table t analyze partition p0") - is = infoschema.GetInfoSchema(tk.Se.(sessionctx.Context)) + is = tk.Se.(sessionctx.Context).GetInfoSchema().(infoschema.InfoSchema) table, err = is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) pi = table.Meta().GetPartitionInfo() @@ -121,7 +123,7 @@ func (s *testSuite1) TestAnalyzeReplicaReadFollower(c *C) { tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int)") ctx := tk.Se.(sessionctx.Context) - ctx.GetSessionVars().SetReplicaRead(tikvstore.ReplicaReadFollower) + ctx.GetSessionVars().SetReplicaRead(kv.ReplicaReadFollower) tk.MustExec("analyze table t") } @@ -175,8 +177,9 @@ func (s *testSuite1) TestAnalyzeParameters(c *C) { tk.MustExec("insert into t values (19), (19), (19)") tk.MustExec("set @@tidb_enable_fast_analyze = 1") + tk.MustExec("set @@tidb_analyze_version = 1") tk.MustExec("analyze table t with 30 samples") - is := infoschema.GetInfoSchema(tk.Se.(sessionctx.Context)) + is := tk.Se.(sessionctx.Context).GetInfoSchema().(infoschema.InfoSchema) table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) tableInfo := table.Meta() @@ -227,7 +230,7 @@ func (s *testSuite1) TestAnalyzeTooLongColumns(c *C) { tk.MustExec(fmt.Sprintf("insert into t values ('%s')", value)) tk.MustExec("analyze table t") - is := infoschema.GetInfoSchema(tk.Se.(sessionctx.Context)) + is := tk.Se.(sessionctx.Context).GetInfoSchema().(infoschema.InfoSchema) table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) tableInfo := table.Meta() @@ -237,7 +240,7 @@ func (s *testSuite1) TestAnalyzeTooLongColumns(c *C) { } func (s *testSuite1) TestAnalyzeIndexExtractTopN(c *C) { - c.Skip("unstable") + c.Skip("unstable, skip it and fix it before 20210618") store, err := mockstore.NewMockStore() c.Assert(err, IsNil) defer func() { @@ -252,21 +255,21 @@ func (s *testSuite1) TestAnalyzeIndexExtractTopN(c *C) { defer dom.Close() tk := testkit.NewTestKit(c, store) - tk.MustExec("use test") + tk.MustExec("create database test_index_extract_topn") + tk.MustExec("use test_index_extract_topn") tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int, b int, index idx(a, b))") tk.MustExec("insert into t values(1, 1), (1, 1), (1, 2), (1, 2)") tk.MustExec("set @@session.tidb_analyze_version=2") - tk.MustExec("analyze table t with 10 cmsketch width") + tk.MustExec("analyze table t") - is := infoschema.GetInfoSchema(tk.Se.(sessionctx.Context)) - table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + is := tk.Se.(sessionctx.Context).GetInfoSchema().(infoschema.InfoSchema) + table, err := is.TableByName(model.NewCIStr("test_index_extract_topn"), model.NewCIStr("t")) c.Assert(err, IsNil) tableInfo := table.Meta() tbl := dom.StatsHandle().GetTableStats(tableInfo) // Construct TopN, should be (1, 1) -> 2 and (1, 2) -> 2 - cms := statistics.NewCMSketch(5, 10) topn := statistics.NewTopN(2) { key1, err := codec.EncodeKey(tk.Se.GetSessionVars().StmtCtx, nil, types.NewIntDatum(1), types.NewIntDatum(1)) @@ -275,19 +278,11 @@ func (s *testSuite1) TestAnalyzeIndexExtractTopN(c *C) { key2, err := codec.EncodeKey(tk.Se.GetSessionVars().StmtCtx, nil, types.NewIntDatum(1), types.NewIntDatum(2)) c.Assert(err, IsNil) topn.AppendTopN(key2, 2) - prefixKey, err := codec.EncodeKey(tk.Se.GetSessionVars().StmtCtx, nil, types.NewIntDatum(1)) - c.Assert(err, IsNil) - cms.InsertBytes(prefixKey) - cms.InsertBytes(prefixKey) - cms.InsertBytes(prefixKey) - cms.InsertBytes(prefixKey) - cms.CalcDefaultValForAnalyze(2) } for _, idx := range tbl.Indices { ok, err := checkHistogram(tk.Se.GetSessionVars().StmtCtx, &idx.Histogram) c.Assert(err, IsNil) c.Assert(ok, IsTrue) - c.Assert(idx.CMSketch.Equal(cms), IsTrue) c.Assert(idx.TopN.Equal(topn), IsTrue) } } @@ -420,6 +415,7 @@ func (s *testFastAnalyze) TestFastAnalyze(c *C) { tk.MustExec("create table t(a int primary key, b int, c char(10), index index_b(b))") tk.MustExec("set @@session.tidb_enable_fast_analyze=1") tk.MustExec("set @@session.tidb_build_stats_concurrency=1") + tk.MustExec("set @@tidb_analyze_version = 1") // Should not panic. tk.MustExec("analyze table t") tblInfo, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) @@ -435,7 +431,7 @@ func (s *testFastAnalyze) TestFastAnalyze(c *C) { } tk.MustExec("analyze table t with 5 buckets, 6 samples") - is := infoschema.GetInfoSchema(tk.Se.(sessionctx.Context)) + is := tk.Se.(sessionctx.Context).GetInfoSchema().(infoschema.InfoSchema) table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) tableInfo := table.Meta() @@ -511,18 +507,21 @@ func (s *testFastAnalyze) TestFastAnalyze(c *C) { } func (s *testSerialSuite2) TestFastAnalyze4GlobalStats(c *C) { - c.Skip("unstable") + if israce.RaceEnabled { + c.Skip("unstable, skip race test") + } tk := testkit.NewTestKit(c, s.store) - tk.MustExec("use test") + tk.MustExec(`create database if not exists test_fast_gstats`) + tk.MustExec("use test_fast_gstats") tk.MustExec("set @@session.tidb_enable_fast_analyze=1") tk.MustExec("set @@session.tidb_build_stats_concurrency=1") // test fast analyze in dynamic mode tk.MustExec("set @@session.tidb_analyze_version = 2;") tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic';") - tk.MustExec("drop table if exists t4;") - tk.MustExec("create table t4(a int, b int) PARTITION BY HASH(a) PARTITIONS 2;") - tk.MustExec("insert into t4 values(1,1),(3,3),(4,4),(2,2),(5,5);") - err := tk.ExecToErr("analyze table t4;") + tk.MustExec("drop table if exists test_fast_gstats;") + tk.MustExec("create table test_fast_gstats(a int, b int) PARTITION BY HASH(a) PARTITIONS 2;") + tk.MustExec("insert into test_fast_gstats values(1,1),(3,3),(4,4),(2,2),(5,5);") + err := tk.ExecToErr("analyze table test_fast_gstats;") c.Assert(err.Error(), Equals, "Fast analyze hasn't reached General Availability and only support analyze version 1 currently.") } @@ -532,6 +531,7 @@ func (s *testSuite1) TestIssue15993(c *C) { tk.MustExec("drop table if exists t0") tk.MustExec("CREATE TABLE t0(c0 INT PRIMARY KEY);") tk.MustExec("set @@tidb_enable_fast_analyze=1;") + tk.MustExec("set @@tidb_analyze_version = 1") tk.MustExec("ANALYZE TABLE t0 INDEX PRIMARY;") } @@ -542,6 +542,7 @@ func (s *testSuite1) TestIssue15751(c *C) { tk.MustExec("CREATE TABLE t0(c0 INT, c1 INT, PRIMARY KEY(c0, c1))") tk.MustExec("INSERT INTO t0 VALUES (0, 0)") tk.MustExec("set @@tidb_enable_fast_analyze=1") + tk.MustExec("set @@tidb_analyze_version = 1") tk.MustExec("ANALYZE TABLE t0") } @@ -553,30 +554,39 @@ func (s *testSuite1) TestIssue15752(c *C) { tk.MustExec("INSERT INTO t0 VALUES (0)") tk.MustExec("CREATE INDEX i0 ON t0(c0)") tk.MustExec("set @@tidb_enable_fast_analyze=1") + tk.MustExec("set @@tidb_analyze_version = 1") tk.MustExec("ANALYZE TABLE t0 INDEX i0") } func (s *testSuite1) TestAnalyzeIndex(c *C) { + c.Skip("unstable, skip it and fix it before 20210622") tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t1") tk.MustExec("create table t1 (id int, v int, primary key(id), index k(v))") tk.MustExec("insert into t1(id, v) values(1, 2), (2, 2), (3, 2), (4, 2), (5, 1), (6, 3), (7, 4)") + tk.MustExec("set @@tidb_analyze_version=1") tk.MustExec("analyze table t1 index k") c.Assert(len(tk.MustQuery("show stats_buckets where table_name = 't1' and column_name = 'k' and is_index = 1").Rows()), Greater, 0) + tk.MustExec("set @@tidb_analyze_version=default") + tk.MustExec("analyze table t1") + c.Assert(len(tk.MustQuery("show stats_topn where table_name = 't1' and column_name = 'k' and is_index = 1").Rows()), Greater, 0) func() { defer tk.MustExec("set @@session.tidb_enable_fast_analyze=0") tk.MustExec("drop stats t1") tk.MustExec("set @@session.tidb_enable_fast_analyze=1") + tk.MustExec("set @@tidb_analyze_version=1") tk.MustExec("analyze table t1 index k") c.Assert(len(tk.MustQuery("show stats_buckets where table_name = 't1' and column_name = 'k' and is_index = 1").Rows()), Greater, 1) }() } func (s *testSuite1) TestAnalyzeIncremental(c *C) { + c.Skip("unstable, skip it and fix it before 20210622") tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") + tk.MustExec("set @@tidb_analyze_version = 1") tk.Se.GetSessionVars().EnableStreaming = false s.testAnalyzeIncremental(tk, c) } @@ -723,6 +733,7 @@ func (s *testFastAnalyze) TestFastAnalyzeRetryRowCount(c *C) { c.Assert(dom.StatsHandle().Update(dom.InfoSchema()), IsNil) tk.MustExec("set @@session.tidb_enable_fast_analyze=1") tk.MustExec("set @@session.tidb_build_stats_concurrency=1") + tk.MustExec("set @@tidb_analyze_version = 1") for i := 0; i < 30; i++ { tk.MustExec(fmt.Sprintf("insert into retry_row_count values (%d)", i)) } @@ -740,6 +751,7 @@ func (s *testSuite10) TestFailedAnalyzeRequest(c *C) { tk.MustExec("use test") tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int primary key, b int, index index_b(b))") + tk.MustExec("set @@tidb_analyze_version = 1") c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/buildStatsFromResult", `return(true)`), IsNil) _, err := tk.Exec("analyze table t") c.Assert(err.Error(), Equals, "mock buildStatsFromResult error") @@ -747,21 +759,24 @@ func (s *testSuite10) TestFailedAnalyzeRequest(c *C) { } func (s *testSuite1) TestExtractTopN(c *C) { - c.Skip("unstable") + if israce.RaceEnabled { + c.Skip("unstable, skip race test") + } tk := testkit.NewTestKit(c, s.store) - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec("create table t(a int primary key, b int, index index_b(b))") + tk.MustExec("create database if not exists test_extract_topn") + tk.MustExec("use test_extract_topn") + tk.MustExec("drop table if exists test_extract_topn") + tk.MustExec("create table test_extract_topn(a int primary key, b int, index index_b(b))") tk.MustExec("set @@session.tidb_analyze_version=2") for i := 0; i < 10; i++ { - tk.MustExec(fmt.Sprintf("insert into t values (%d, %d)", i, i)) + tk.MustExec(fmt.Sprintf("insert into test_extract_topn values (%d, %d)", i, i)) } for i := 0; i < 10; i++ { - tk.MustExec(fmt.Sprintf("insert into t values (%d, 0)", i+10)) + tk.MustExec(fmt.Sprintf("insert into test_extract_topn values (%d, 0)", i+10)) } - tk.MustExec("analyze table t") + tk.MustExec("analyze table test_extract_topn") is := s.dom.InfoSchema() - table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + table, err := is.TableByName(model.NewCIStr("test_extract_topn"), model.NewCIStr("test_extract_topn")) c.Assert(err, IsNil) tblInfo := table.Meta() tblStats := s.dom.StatsHandle().GetTableStats(tblInfo) @@ -774,26 +789,26 @@ func (s *testSuite1) TestExtractTopN(c *C) { idxItem := idxStats.TopN.TopN[0] c.Assert(idxItem.Count, Equals, uint64(11)) // The columns are: DBName, table name, column name, is index, value, count. - tk.MustQuery("show stats_topn").Sort().Check(testkit.Rows("test t b 0 0 11", - "test t b 0 1 1", - "test t b 0 2 1", - "test t b 0 3 1", - "test t b 0 4 1", - "test t b 0 5 1", - "test t b 0 6 1", - "test t b 0 7 1", - "test t b 0 8 1", - "test t b 0 9 1", - "test t index_b 1 0 11", - "test t index_b 1 1 1", - "test t index_b 1 2 1", - "test t index_b 1 3 1", - "test t index_b 1 4 1", - "test t index_b 1 5 1", - "test t index_b 1 6 1", - "test t index_b 1 7 1", - "test t index_b 1 8 1", - "test t index_b 1 9 1", + tk.MustQuery("show stats_topn where column_name in ('b', 'index_b')").Sort().Check(testkit.Rows("test_extract_topn test_extract_topn b 0 0 11", + "test_extract_topn test_extract_topn b 0 1 1", + "test_extract_topn test_extract_topn b 0 2 1", + "test_extract_topn test_extract_topn b 0 3 1", + "test_extract_topn test_extract_topn b 0 4 1", + "test_extract_topn test_extract_topn b 0 5 1", + "test_extract_topn test_extract_topn b 0 6 1", + "test_extract_topn test_extract_topn b 0 7 1", + "test_extract_topn test_extract_topn b 0 8 1", + "test_extract_topn test_extract_topn b 0 9 1", + "test_extract_topn test_extract_topn index_b 1 0 11", + "test_extract_topn test_extract_topn index_b 1 1 1", + "test_extract_topn test_extract_topn index_b 1 2 1", + "test_extract_topn test_extract_topn index_b 1 3 1", + "test_extract_topn test_extract_topn index_b 1 4 1", + "test_extract_topn test_extract_topn index_b 1 5 1", + "test_extract_topn test_extract_topn index_b 1 6 1", + "test_extract_topn test_extract_topn index_b 1 7 1", + "test_extract_topn test_extract_topn index_b 1 8 1", + "test_extract_topn test_extract_topn index_b 1 9 1", )) } @@ -810,6 +825,7 @@ func (s *testSuite1) TestHashInTopN(c *C) { for i := 0; i < 3; i++ { tk.MustExec("insert into t select * from t") } + tk.MustExec("set @@tidb_analyze_version = 1") // get stats of normal analyze tk.MustExec("analyze table t") is := s.dom.InfoSchema() @@ -845,6 +861,8 @@ func (s *testSuite1) TestNormalAnalyzeOnCommonHandle(c *C) { tk.MustExec("CREATE TABLE t3 (a int, b int, c int, primary key (a, b), key(c))") tk.MustExec("insert into t3 values(1,1,1), (2,2,2), (3,3,3)") + // Version2 is tested in TestStatsVer2. + tk.MustExec("set@@tidb_analyze_version=1") tk.MustExec("analyze table t1, t2, t3") tk.MustQuery(`show stats_buckets where table_name in ("t1", "t2", "t3")`).Sort().Check(testkit.Rows( @@ -923,6 +941,7 @@ func (s *testSerialSuite2) TestIssue20874(c *C) { tk.MustExec("drop table if exists t") tk.MustExec("create table t (a char(10) collate utf8mb4_unicode_ci not null, b char(20) collate utf8mb4_general_ci not null, key idxa(a), key idxb(b))") tk.MustExec("insert into t values ('#', 'C'), ('$', 'c'), ('a', 'a')") + tk.MustExec("set @@tidb_analyze_version=1") tk.MustExec("analyze table t") tk.MustQuery("show stats_buckets where db_name = 'test' and table_name = 't'").Sort().Check(testkit.Rows( "test t a 0 0 1 1 \x02\xd2 \x02\xd2 0", @@ -947,9 +966,66 @@ func (s *testSuite1) TestAnalyzeClusteredIndexPrimary(c *C) { tk.MustExec("create table t1(a varchar(20), primary key(a))") tk.MustExec("insert into t0 values('1111')") tk.MustExec("insert into t1 values('1111')") + tk.MustExec("set @@session.tidb_analyze_version = 1") tk.MustExec("analyze table t0 index primary") tk.MustExec("analyze table t1 index primary") tk.MustQuery("show stats_buckets").Check(testkit.Rows( "test t0 PRIMARY 1 0 1 1 1111 1111 0", "test t1 PRIMARY 1 0 1 1 1111 1111 0")) + tk.MustExec("set @@session.tidb_analyze_version = 2") + tk.MustExec("analyze table t0") + tk.MustExec("analyze table t1") + tk.MustQuery("show stats_topn").Sort().Check(testkit.Rows(""+ + "test t0 PRIMARY 1 1111 1", + "test t0 a 0 1111 1", + "test t1 PRIMARY 1 1111 1", + "test t1 a 0 1111 1")) +} + +func (s *testSuite1) TestAnalyzeFullSamplingOnIndexWithVirtualColumnOrPrefixColumn(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists sampling_index_virtual_col") + tk.MustExec("create table sampling_index_virtual_col(a int, b int as (a+1), index idx(b))") + tk.MustExec("insert into sampling_index_virtual_col (a) values (1), (2), (null), (3), (4), (null), (5), (5), (5), (5)") + tk.MustExec("set @@session.tidb_analyze_version = 2") + tk.MustExec("analyze table sampling_index_virtual_col with 1 topn") + tk.MustQuery("show stats_buckets where table_name = 'sampling_index_virtual_col' and column_name = 'idx'").Check(testkit.Rows( + "test sampling_index_virtual_col idx 1 0 1 1 2 2 0", + "test sampling_index_virtual_col idx 1 1 2 1 3 3 0", + "test sampling_index_virtual_col idx 1 2 3 1 4 4 0", + "test sampling_index_virtual_col idx 1 3 4 1 5 5 0")) + tk.MustQuery("show stats_topn where table_name = 'sampling_index_virtual_col' and column_name = 'idx'").Check(testkit.Rows("test sampling_index_virtual_col idx 1 6 4")) + row := tk.MustQuery(`show stats_histograms where db_name = "test" and table_name = "sampling_index_virtual_col"`).Rows()[0] + // The NDV. + c.Assert(row[6], Equals, "5") + // The NULLs. + c.Assert(row[7], Equals, "2") + tk.MustExec("drop table if exists sampling_index_prefix_col") + tk.MustExec("create table sampling_index_prefix_col(a varchar(3), index idx(a(1)))") + tk.MustExec("insert into sampling_index_prefix_col (a) values ('aa'), ('ab'), ('ac'), ('bb')") + tk.MustExec("analyze table sampling_index_prefix_col with 1 topn") + tk.MustQuery("show stats_buckets where table_name = 'sampling_index_prefix_col' and column_name = 'idx'").Check(testkit.Rows( + "test sampling_index_prefix_col idx 1 0 1 1 b b 0", + )) + tk.MustQuery("show stats_topn where table_name = 'sampling_index_prefix_col' and column_name = 'idx'").Check(testkit.Rows("test sampling_index_prefix_col idx 1 a 3")) +} + +func (s *testSuite2) TestAnalyzeSamplingWorkPanic(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("set @@session.tidb_analyze_version = 2") + tk.MustExec("create table t(a int)") + tk.MustExec("insert into t values(1), (2), (3), (4), (5), (6), (7), (8), (9), (10), (11), (12)") + tk.MustExec("split table t between (-9223372036854775808) and (9223372036854775807) regions 12") + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/mockAnalyzeSamplingBuildWorkerPanic", "return(1)"), IsNil) + err := tk.ExecToErr("analyze table t") + c.Assert(err, NotNil) + c.Assert(failpoint.Disable("github.com/pingcap/tidb/executor/mockAnalyzeSamplingBuildWorkerPanic"), IsNil) + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/mockAnalyzeSamplingMergeWorkerPanic", "return(1)"), IsNil) + err = tk.ExecToErr("analyze table t") + c.Assert(err, NotNil) + c.Assert(failpoint.Disable("github.com/pingcap/tidb/executor/mockAnalyzeSamplingMergeWorkerPanic"), IsNil) } diff --git a/executor/batch_point_get.go b/executor/batch_point_get.go index f3ec18106fb21..35ccecec65670 100644 --- a/executor/batch_point_get.go +++ b/executor/batch_point_get.go @@ -19,18 +19,19 @@ import ( "sort" "sync/atomic" + "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/ddl/placement" + "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" driver "github.com/pingcap/tidb/store/driver/txn" "github.com/pingcap/tidb/store/tikv" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" - "github.com/pingcap/tidb/store/tikv/oracle" + "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" @@ -48,6 +49,7 @@ type BatchPointGetExec struct { idxInfo *model.IndexInfo handles []kv.Handle physIDs []int64 + partExpr *tables.PartitionExpr partPos int singlePart bool partTblID int64 @@ -91,7 +93,9 @@ func (e *BatchPointGetExec) buildVirtualColumnInfo() { // Open implements the Executor interface. func (e *BatchPointGetExec) Open(context.Context) error { e.snapshotTS = e.startTS - txnCtx := e.ctx.GetSessionVars().TxnCtx + sessVars := e.ctx.GetSessionVars() + txnCtx := sessVars.TxnCtx + stmtCtx := sessVars.StmtCtx if e.lock { e.snapshotTS = txnCtx.GetForUpdateTS() } @@ -113,23 +117,29 @@ func (e *BatchPointGetExec) Open(context.Context) error { e.stats = &runtimeStatsWithSnapshot{ SnapshotRuntimeStats: snapshotStats, } - snapshot.SetOption(tikvstore.CollectRuntimeStats, snapshotStats) - e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats) + snapshot.SetOption(kv.CollectRuntimeStats, snapshotStats) + stmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats) } if e.ctx.GetSessionVars().GetReplicaRead().IsFollowerRead() { - snapshot.SetOption(tikvstore.ReplicaRead, tikvstore.ReplicaReadFollower) + snapshot.SetOption(kv.ReplicaRead, kv.ReplicaReadFollower) } - snapshot.SetOption(tikvstore.TaskID, e.ctx.GetSessionVars().StmtCtx.TaskID) + snapshot.SetOption(kv.TaskID, stmtCtx.TaskID) + snapshot.SetOption(kv.TxnScope, e.ctx.GetSessionVars().TxnCtx.TxnScope) isStaleness := e.ctx.GetSessionVars().TxnCtx.IsStaleness - snapshot.SetOption(tikvstore.IsStalenessReadOnly, isStaleness) - if isStaleness && e.ctx.GetSessionVars().TxnCtx.TxnScope != oracle.GlobalTxnScope { - snapshot.SetOption(tikvstore.MatchStoreLabels, []*metapb.StoreLabel{ + snapshot.SetOption(kv.IsStalenessReadOnly, isStaleness) + if isStaleness && e.ctx.GetSessionVars().TxnCtx.TxnScope != kv.GlobalTxnScope { + snapshot.SetOption(kv.MatchStoreLabels, []*metapb.StoreLabel{ { Key: placement.DCLabelKey, Value: e.ctx.GetSessionVars().TxnCtx.TxnScope, }, }) } + setResourceGroupTagForTxn(stmtCtx, snapshot) + // Avoid network requests for the temporary table. + if e.tblInfo.TempTableType == model.TempTableGlobal { + snapshot = globalTemporaryTableSnapshot{snapshot} + } var batchGetter kv.BatchGetter = snapshot if txn.Valid() { lock := e.tblInfo.Lock @@ -146,10 +156,20 @@ func (e *BatchPointGetExec) Open(context.Context) error { return nil } +// Global temporary table would always be empty, so get the snapshot data of it is meanless. +// globalTemporaryTableSnapshot inherits kv.Snapshot and override the BatchGet methods to return empty. +type globalTemporaryTableSnapshot struct { + kv.Snapshot +} + +func (s globalTemporaryTableSnapshot) BatchGet(ctx context.Context, keys []kv.Key) (map[string][]byte, error) { + return make(map[string][]byte), nil +} + // Close implements the Executor interface. func (e *BatchPointGetExec) Close() error { if e.runtimeStats != nil && e.snapshot != nil { - e.snapshot.DelOption(tikvstore.CollectRuntimeStats) + e.snapshot.SetOption(kv.CollectRuntimeStats, nil) } e.inited = 0 e.index = 0 @@ -212,7 +232,11 @@ func (e *BatchPointGetExec) initialize(ctx context.Context) error { continue } - physID := getPhysID(e.tblInfo, idxVals[e.partPos].GetInt64()) + physID, err := getPhysID(e.tblInfo, e.partExpr, idxVals[e.partPos].GetInt64()) + if err != nil { + continue + } + // If this BatchPointGetExec is built only for the specific table partition, skip those filters not matching this partition. if e.singlePart && e.partTblID != physID { continue @@ -340,13 +364,19 @@ func (e *BatchPointGetExec) initialize(ctx context.Context) error { tID = e.physIDs[i] } else { if handle.IsInt() { - tID = getPhysID(e.tblInfo, handle.IntValue()) + tID, err = getPhysID(e.tblInfo, e.partExpr, handle.IntValue()) + if err != nil { + continue + } } else { _, d, err1 := codec.DecodeOne(handle.EncodedCol(e.partPos)) if err1 != nil { return err1 } - tID = getPhysID(e.tblInfo, d.GetInt64()) + tID, err = getPhysID(e.tblInfo, e.partExpr, d.GetInt64()) + if err != nil { + continue + } } } // If this BatchPointGetExec is built only for the specific table partition, skip those handles not matching this partition. @@ -418,8 +448,7 @@ func LockKeys(ctx context.Context, seCtx sessionctx.Context, lockWaitTime int64, txnCtx := seCtx.GetSessionVars().TxnCtx lctx := newLockCtx(seCtx.GetSessionVars(), lockWaitTime) if txnCtx.IsPessimistic { - lctx.ReturnValues = true - lctx.Values = make(map[string]tikvstore.ReturnedValue, len(keys)) + lctx.InitReturnValues(len(keys)) } err := doLockKeys(ctx, seCtx, lctx, keys...) if err != nil { @@ -429,9 +458,8 @@ func LockKeys(ctx context.Context, seCtx sessionctx.Context, lockWaitTime int64, // When doLockKeys returns without error, no other goroutines access the map, // it's safe to read it without mutex. for _, key := range keys { - rv := lctx.Values[string(key)] - if !rv.AlreadyLocked { - txnCtx.SetPessimisticLockCache(key, rv.Value) + if v, ok := lctx.GetValueNotLocked(key); ok { + txnCtx.SetPessimisticLockCache(key, v) } } } @@ -453,13 +481,44 @@ func (getter *PessimisticLockCacheGetter) Get(_ context.Context, key kv.Key) ([] return nil, kv.ErrNotExist } -func getPhysID(tblInfo *model.TableInfo, intVal int64) int64 { +func getPhysID(tblInfo *model.TableInfo, partitionExpr *tables.PartitionExpr, intVal int64) (int64, error) { pi := tblInfo.GetPartitionInfo() if pi == nil { - return tblInfo.ID + return tblInfo.ID, nil } - partIdx := math.Abs(intVal % int64(pi.Num)) - return pi.Definitions[partIdx].ID + + if partitionExpr == nil { + return tblInfo.ID, nil + } + + switch pi.Type { + case model.PartitionTypeHash: + partIdx := math.Abs(intVal % int64(pi.Num)) + return pi.Definitions[partIdx].ID, nil + case model.PartitionTypeRange: + // we've check the type assertions in func TryFastPlan + col, ok := partitionExpr.Expr.(*expression.Column) + if !ok { + return 0, errors.Errorf("unsupported partition type in BatchGet") + } + unsigned := mysql.HasUnsignedFlag(col.GetType().Flag) + ranges := partitionExpr.ForRangePruning + length := len(ranges.LessThan) + partIdx := sort.Search(length, func(i int) bool { + return ranges.Compare(i, intVal, unsigned) > 0 + }) + if partIdx >= 0 && partIdx < length { + return pi.Definitions[partIdx].ID, nil + } + case model.PartitionTypeList: + isNull := false // we've guaranteed this in the build process of either TryFastPlan or buildBatchPointGet + partIdx := partitionExpr.ForListPruning.LocatePartition(intVal, isNull) + if partIdx >= 0 { + return pi.Definitions[partIdx].ID, nil + } + } + + return 0, errors.Errorf("dual partition") } type cacheBatchGetter struct { diff --git a/executor/batch_point_get_test.go b/executor/batch_point_get_test.go index 926834dc9281b..5db23ed39b0b8 100644 --- a/executor/batch_point_get_test.go +++ b/executor/batch_point_get_test.go @@ -20,6 +20,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/session" @@ -156,6 +157,16 @@ func (s *testBatchPointGetSuite) TestIssue18843(c *C) { tk.MustQuery("select * from t18843 where f is null").Check(testkit.Rows("2 ")) } +func (s *testBatchPointGetSuite) TestIssue24562(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists ttt") + tk.MustExec("create table ttt(a enum(\"a\",\"b\",\"c\",\"d\"), primary key(a));") + tk.MustExec("insert into ttt values(1)") + tk.MustQuery("select * from ttt where ttt.a in (\"1\",\"b\")").Check(testkit.Rows()) + tk.MustQuery("select * from ttt where ttt.a in (1,\"b\")").Check(testkit.Rows("a")) +} + func (s *testBatchPointGetSuite) TestBatchPointGetUnsignedHandleWithSort(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -310,3 +321,28 @@ func (s *testBatchPointGetSuite) TestBatchPointGetLockExistKey(c *C) { c.Assert(err, IsNil) } } + +func (s *testBatchPointGetSuite) TestPointGetForTemporaryTable(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("create global temporary table t1 (id int primary key, val int) on commit delete rows") + tk.MustExec("begin") + tk.MustExec("insert into t1 values (1,1)") + tk.MustQuery("explain format = 'brief' select * from t1 where id in (1, 2, 3)"). + Check(testkit.Rows("Batch_Point_Get 3.00 root table:t1 handle:[1 2 3], keep order:false, desc:false")) + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/unistore/rpcServerBusy", "return(true)"), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/unistore/rpcServerBusy"), IsNil) + }() + + // Batch point get. + tk.MustQuery("select * from t1 where id in (1, 2, 3)").Check(testkit.Rows("1 1")) + tk.MustQuery("select * from t1 where id in (2, 3)").Check(testkit.Rows()) + + // Point get. + tk.MustQuery("select * from t1 where id = 1").Check(testkit.Rows("1 1")) + tk.MustQuery("select * from t1 where id = 2").Check(testkit.Rows()) +} diff --git a/executor/benchmark_test.go b/executor/benchmark_test.go index 48fd34d4bb2a0..8e19d39ddb23d 100644 --- a/executor/benchmark_test.go +++ b/executor/benchmark_test.go @@ -17,7 +17,6 @@ import ( "context" "encoding/base64" "fmt" - "io/ioutil" "math/rand" "os" "sort" @@ -291,7 +290,7 @@ func buildHashAggExecutor(ctx sessionctx.Context, src Executor, schema *expressi plan.SetSchema(schema) plan.Init(ctx, nil, 0) plan.SetChildren(nil) - b := newExecutorBuilder(ctx, nil) + b := newExecutorBuilder(ctx, nil, nil) exec := b.build(plan) hashAgg := exec.(*HashAggExec) hashAgg.children[0] = src @@ -343,7 +342,7 @@ func buildStreamAggExecutor(ctx sessionctx.Context, srcExec Executor, schema *ex plan = sg } - b := newExecutorBuilder(ctx, nil) + b := newExecutorBuilder(ctx, nil, nil) return b.build(plan) } @@ -576,7 +575,7 @@ func buildWindowExecutor(ctx sessionctx.Context, windowFunc string, funcs int, f plan = win } - b := newExecutorBuilder(ctx, nil) + b := newExecutorBuilder(ctx, nil, nil) exec := b.build(plan) return exec } @@ -589,6 +588,7 @@ type windowTestCase struct { ndv int // the number of distinct group-by keys rows int concurrency int + pipelined int dataSourceSorted bool ctx sessionctx.Context rawDataSmall string @@ -596,15 +596,15 @@ type windowTestCase struct { } func (a windowTestCase) String() string { - return fmt.Sprintf("(func:%v, aggColType:%s, numFunc:%v, ndv:%v, rows:%v, sorted:%v, concurrency:%v)", - a.windowFunc, a.columns[0].RetType, a.numFunc, a.ndv, a.rows, a.dataSourceSorted, a.concurrency) + return fmt.Sprintf("(func:%v, aggColType:%s, numFunc:%v, ndv:%v, rows:%v, sorted:%v, concurrency:%v, pipelined:%v)", + a.windowFunc, a.columns[0].RetType, a.numFunc, a.ndv, a.rows, a.dataSourceSorted, a.concurrency, a.pipelined) } func defaultWindowTestCase() *windowTestCase { ctx := mock.NewContext() ctx.GetSessionVars().InitChunkSize = variable.DefInitChunkSize ctx.GetSessionVars().MaxChunkSize = variable.DefMaxChunkSize - return &windowTestCase{ast.WindowFuncRowNumber, 1, nil, 1000, 10000000, 1, true, ctx, strings.Repeat("x", 16), + return &windowTestCase{ast.WindowFuncRowNumber, 1, nil, 1000, 10000000, 1, 0, true, ctx, strings.Repeat("x", 16), []*expression.Column{ {Index: 0, RetType: types.NewFieldType(mysql.TypeDouble)}, {Index: 1, RetType: types.NewFieldType(mysql.TypeLonglong)}, @@ -618,6 +618,9 @@ func benchmarkWindowExecWithCase(b *testing.B, casTest *windowTestCase) { if err := ctx.GetSessionVars().SetSystemVar(variable.TiDBWindowConcurrency, fmt.Sprintf("%v", casTest.concurrency)); err != nil { b.Fatal(err) } + if err := ctx.GetSessionVars().SetSystemVar(variable.TiDBEnablePipelinedWindowFunction, fmt.Sprintf("%v", casTest.pipelined)); err != nil { + b.Fatal(err) + } cols := casTest.columns dataSource := buildMockDataSource(mockDataSourceParameters{ @@ -658,10 +661,10 @@ func benchmarkWindowExecWithCase(b *testing.B, casTest *windowTestCase) { } } -func BenchmarkWindowRows(b *testing.B) { +func baseBenchmarkWindowRows(b *testing.B, pipelined int) { b.ReportAllocs() rows := []int{1000, 100000} - ndvs := []int{10, 1000} + ndvs := []int{1, 10, 1000} concs := []int{1, 2, 4} for _, row := range rows { for _, ndv := range ndvs { @@ -672,6 +675,7 @@ func BenchmarkWindowRows(b *testing.B) { cas.concurrency = con cas.dataSourceSorted = false cas.windowFunc = ast.WindowFuncRowNumber // cheapest + cas.pipelined = pipelined b.Run(fmt.Sprintf("%v", cas), func(b *testing.B) { benchmarkWindowExecWithCase(b, cas) }) @@ -680,20 +684,25 @@ func BenchmarkWindowRows(b *testing.B) { } } -func BenchmarkWindowFunctions(b *testing.B) { +func BenchmarkWindowRows(b *testing.B) { + baseBenchmarkWindowRows(b, 0) + baseBenchmarkWindowRows(b, 1) +} + +func baseBenchmarkWindowFunctions(b *testing.B, pipelined int) { b.ReportAllocs() windowFuncs := []string{ - ast.WindowFuncRowNumber, - ast.WindowFuncRank, - ast.WindowFuncDenseRank, - ast.WindowFuncCumeDist, - ast.WindowFuncPercentRank, - ast.WindowFuncNtile, - ast.WindowFuncLead, + // ast.WindowFuncRowNumber, + // ast.WindowFuncRank, + // ast.WindowFuncDenseRank, + // ast.WindowFuncCumeDist, + // ast.WindowFuncPercentRank, + // ast.WindowFuncNtile, + // ast.WindowFuncLead, ast.WindowFuncLag, - ast.WindowFuncFirstValue, - ast.WindowFuncLastValue, - ast.WindowFuncNthValue, + // ast.WindowFuncFirstValue, + // ast.WindowFuncLastValue, + // ast.WindowFuncNthValue, } concs := []int{1, 4} for _, windowFunc := range windowFuncs { @@ -704,6 +713,7 @@ func BenchmarkWindowFunctions(b *testing.B) { cas.concurrency = con cas.dataSourceSorted = false cas.windowFunc = windowFunc + cas.pipelined = pipelined b.Run(fmt.Sprintf("%v", cas), func(b *testing.B) { benchmarkWindowExecWithCase(b, cas) }) @@ -711,7 +721,12 @@ func BenchmarkWindowFunctions(b *testing.B) { } } -func BenchmarkWindowFunctionsWithFrame(b *testing.B) { +func BenchmarkWindowFunctions(b *testing.B) { + baseBenchmarkWindowFunctions(b, 0) + baseBenchmarkWindowFunctions(b, 1) +} + +func baseBenchmarkWindowFunctionsWithFrame(b *testing.B, pipelined int) { b.ReportAllocs() windowFuncs := []string{ ast.WindowFuncRowNumber, @@ -737,6 +752,7 @@ func BenchmarkWindowFunctionsWithFrame(b *testing.B) { if i < len(frames) { cas.frame = frames[i] } + cas.pipelined = pipelined b.Run(fmt.Sprintf("%v", cas), func(b *testing.B) { benchmarkWindowExecWithCase(b, cas) }) @@ -744,9 +760,15 @@ func BenchmarkWindowFunctionsWithFrame(b *testing.B) { } } } + } -func BenchmarkWindowFunctionsAggWindowProcessorAboutFrame(b *testing.B) { +func BenchmarkWindowFunctionsWithFrame(b *testing.B) { + baseBenchmarkWindowFunctionsWithFrame(b, 0) + baseBenchmarkWindowFunctionsWithFrame(b, 1) +} + +func baseBenchmarkWindowFunctionsAggWindowProcessorAboutFrame(b *testing.B, pipelined int) { b.ReportAllocs() windowFunc := ast.AggFuncMax frame := &core.WindowFrame{Type: ast.Rows, Start: &core.FrameBound{UnBounded: true}, End: &core.FrameBound{UnBounded: true}} @@ -758,12 +780,18 @@ func BenchmarkWindowFunctionsAggWindowProcessorAboutFrame(b *testing.B) { cas.windowFunc = windowFunc cas.numFunc = 1 cas.frame = frame + cas.pipelined = pipelined b.Run(fmt.Sprintf("%v", cas), func(b *testing.B) { benchmarkWindowExecWithCase(b, cas) }) } -func baseBenchmarkWindowFunctionsWithSlidingWindow(b *testing.B, frameType ast.FrameType) { +func BenchmarkWindowFunctionsAggWindowProcessorAboutFrame(b *testing.B) { + baseBenchmarkWindowFunctionsAggWindowProcessorAboutFrame(b, 0) + baseBenchmarkWindowFunctionsAggWindowProcessorAboutFrame(b, 1) +} + +func baseBenchmarkWindowFunctionsWithSlidingWindow(b *testing.B, frameType ast.FrameType, pipelined int) { b.ReportAllocs() windowFuncs := []struct { aggFunc string @@ -795,6 +823,7 @@ func baseBenchmarkWindowFunctionsWithSlidingWindow(b *testing.B, frameType ast.F cas.windowFunc = windowFunc.aggFunc cas.frame = frame cas.columns[0].RetType.Tp = windowFunc.aggColTypes + cas.pipelined = pipelined b.Run(fmt.Sprintf("%v", cas), func(b *testing.B) { benchmarkWindowExecWithCase(b, cas) }) @@ -802,8 +831,10 @@ func baseBenchmarkWindowFunctionsWithSlidingWindow(b *testing.B, frameType ast.F } func BenchmarkWindowFunctionsWithSlidingWindow(b *testing.B) { - baseBenchmarkWindowFunctionsWithSlidingWindow(b, ast.Rows) - baseBenchmarkWindowFunctionsWithSlidingWindow(b, ast.Ranges) + baseBenchmarkWindowFunctionsWithSlidingWindow(b, ast.Rows, 0) + baseBenchmarkWindowFunctionsWithSlidingWindow(b, ast.Ranges, 0) + baseBenchmarkWindowFunctionsWithSlidingWindow(b, ast.Rows, 1) + baseBenchmarkWindowFunctionsWithSlidingWindow(b, ast.Ranges, 1) } type hashJoinTestCase struct { @@ -1291,7 +1322,7 @@ func prepare4IndexInnerHashJoin(tc *indexJoinTestCase, outerDS *mockDataSource, hashCols: tc.outerHashKeyIdx, }, innerCtx: innerCtx{ - readerBuilder: &dataReaderBuilder{Plan: &mockPhysicalIndexReader{e: innerDS}, executorBuilder: newExecutorBuilder(tc.ctx, nil)}, + readerBuilder: &dataReaderBuilder{Plan: &mockPhysicalIndexReader{e: innerDS}, executorBuilder: newExecutorBuilder(tc.ctx, nil, nil)}, rowTypes: rightTypes, colLens: colLens, keyCols: tc.innerJoinKeyIdx, @@ -1357,7 +1388,7 @@ func prepare4IndexMergeJoin(tc *indexJoinTestCase, outerDS *mockDataSource, inne compareFuncs: outerCompareFuncs, }, innerMergeCtx: innerMergeCtx{ - readerBuilder: &dataReaderBuilder{Plan: &mockPhysicalIndexReader{e: innerDS}, executorBuilder: newExecutorBuilder(tc.ctx, nil)}, + readerBuilder: &dataReaderBuilder{Plan: &mockPhysicalIndexReader{e: innerDS}, executorBuilder: newExecutorBuilder(tc.ctx, nil, nil)}, rowTypes: rightTypes, joinKeys: innerJoinKeys, colLens: colLens, @@ -1991,7 +2022,7 @@ func BenchmarkReadLastLinesOfHugeLine(b *testing.B) { hugeLine[i] = 'a' + byte(i%26) } fileName := "tidb.log" - err := ioutil.WriteFile(fileName, hugeLine, 0644) + err := os.WriteFile(fileName, hugeLine, 0644) if err != nil { b.Fatal(err) } @@ -2062,3 +2093,8 @@ func BenchmarkAggPartialResultMapperMemoryUsage(b *testing.B) { }) } } + +func BenchmarkPipelinedRowNumberWindowFunctionExecution(b *testing.B) { + b.ReportAllocs() + +} diff --git a/executor/brie.go b/executor/brie.go index e26da72d3647b..a4438afd49575 100644 --- a/executor/brie.go +++ b/executor/brie.go @@ -403,9 +403,14 @@ func (gs *tidbGlueSession) CreateSession(store kv.Storage) (glue.Session, error) } // Execute implements glue.Session +// These queries execute without privilege checking, since the calling statements +// such as BACKUP and RESTORE have already been privilege checked. func (gs *tidbGlueSession) Execute(ctx context.Context, sql string) error { - // FIXME: br relies on a deprecated API, it may be unsafe - _, err := gs.se.(sqlexec.SQLExecutor).Execute(ctx, sql) + stmt, err := gs.se.(sqlexec.RestrictedSQLExecutor).ParseWithParams(ctx, sql) + if err != nil { + return err + } + _, _, err = gs.se.(sqlexec.RestrictedSQLExecutor).ExecRestrictedStmt(ctx, stmt) return err } diff --git a/executor/builder.go b/executor/builder.go index e82db0d6aaccc..70f76120d5dcf 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -49,6 +49,8 @@ import ( "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/admin" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/cteutil" + "github.com/pingcap/tidb/util/dbterror" "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/ranger" @@ -80,12 +82,24 @@ type executorBuilder struct { snapshotTSCached bool err error // err is set when there is error happened during Executor building process. hasLock bool + Ti *TelemetryInfo + // ExplicitStaleness means whether the 'SELECT' clause are using 'AS OF TIMESTAMP' to perform stale read explicitly. + explicitStaleness bool } -func newExecutorBuilder(ctx sessionctx.Context, is infoschema.InfoSchema) *executorBuilder { +// CTEStorages stores resTbl and iterInTbl for CTEExec. +// There will be a map[CTEStorageID]*CTEStorages in StmtCtx, +// which will store all CTEStorages to make all shared CTEs use same the CTEStorages. +type CTEStorages struct { + ResTbl cteutil.Storage + IterInTbl cteutil.Storage +} + +func newExecutorBuilder(ctx sessionctx.Context, is infoschema.InfoSchema, ti *TelemetryInfo) *executorBuilder { return &executorBuilder{ ctx: ctx, is: is, + Ti: ti, } } @@ -234,6 +248,10 @@ func (b *executorBuilder) build(p plannercore.Plan) Executor { return b.buildAdminShowTelemetry(v) case *plannercore.AdminResetTelemetryID: return b.buildAdminResetTelemetryID(v) + case *plannercore.PhysicalCTE: + return b.buildCTE(v) + case *plannercore.PhysicalCTETable: + return b.buildCTETableReader(v) default: if mp, ok := p.(MockPhysicalPlan); ok { return mp.GetExecutor() @@ -639,7 +657,6 @@ func (b *executorBuilder) buildPrepare(v *plannercore.Prepare) Executor { base.initCap = chunk.ZeroCapacity return &PrepareExec{ baseExecutor: base, - is: b.is, name: v.Name, sqlText: v.SQLText, } @@ -705,10 +722,11 @@ func (b *executorBuilder) buildSimple(v *plannercore.Simple) Executor { base := newBaseExecutor(b.ctx, v.Schema(), v.ID()) base.initCap = chunk.ZeroCapacity e := &SimpleExec{ - baseExecutor: base, - Statement: v.Statement, - IsFromRemote: v.IsFromRemote, - is: b.is, + baseExecutor: base, + Statement: v.Statement, + IsFromRemote: v.IsFromRemote, + is: b.is, + staleTxnStartTS: v.StaleTxnStartTS, } return e } @@ -1526,12 +1544,18 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo strings.ToLower(infoschema.TableTiKVStoreStatus), strings.ToLower(infoschema.TableStatementsSummary), strings.ToLower(infoschema.TableStatementsSummaryHistory), + strings.ToLower(infoschema.TableStatementsSummaryEvicted), strings.ToLower(infoschema.ClusterTableStatementsSummary), strings.ToLower(infoschema.ClusterTableStatementsSummaryHistory), strings.ToLower(infoschema.TablePlacementPolicy), strings.ToLower(infoschema.TableClientErrorsSummaryGlobal), strings.ToLower(infoschema.TableClientErrorsSummaryByUser), - strings.ToLower(infoschema.TableClientErrorsSummaryByHost): + strings.ToLower(infoschema.TableClientErrorsSummaryByHost), + strings.ToLower(infoschema.TableTiDBTrx), + strings.ToLower(infoschema.ClusterTableTiDBTrx), + strings.ToLower(infoschema.TableDeadlocks), + strings.ToLower(infoschema.ClusterTableDeadlocks), + strings.ToLower(infoschema.TableDataLockWaits): return &MemTableReaderExec{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()), table: v.Table, @@ -2016,7 +2040,7 @@ func (b *executorBuilder) buildAnalyzeIndexIncremental(task plannercore.AnalyzeI oldHist = idx.TruncateHistogram(bktID) } var oldTopN *statistics.TopN - if analyzeTask.idxExec.analyzePB.IdxReq.GetVersion() == statistics.Version2 { + if analyzeTask.idxExec.analyzePB.IdxReq.GetVersion() >= statistics.Version2 { oldTopN = idx.TopN.Copy() oldTopN.RemoveVal(oldHist.Bounds.GetRow(len(oldHist.Buckets)*2 - 1).GetBytes(0)) } @@ -2027,11 +2051,7 @@ func (b *executorBuilder) buildAnalyzeIndexIncremental(task plannercore.AnalyzeI return analyzeTask } -func (b *executorBuilder) buildAnalyzeSamplingPushdown( - task plannercore.AnalyzeColumnsTask, - opts map[ast.AnalyzeOptionType]uint64, - autoAnalyze string, -) *analyzeTask { +func (b *executorBuilder) buildAnalyzeSamplingPushdown(task plannercore.AnalyzeColumnsTask, opts map[ast.AnalyzeOptionType]uint64, autoAnalyze string, schemaForVirtualColEval *expression.Schema) *analyzeTask { availableIdx := make([]*model.IndexInfo, 0, len(task.Indexes)) colGroups := make([]*tipb.AnalyzeColumnGroup, 0, len(task.Indexes)) if len(task.Indexes) > 0 { @@ -2051,7 +2071,7 @@ func (b *executorBuilder) buildAnalyzeSamplingPushdown( sc := b.ctx.GetSessionVars().StmtCtx e := &AnalyzeColumnsExec{ ctx: b.ctx, - tableID: task.TableID, + tableInfo: task.TblInfo, colsInfo: task.ColsInfo, handleCols: task.HandleCols, concurrency: b.ctx.GetSessionVars().DistSQLScanConcurrency(), @@ -2060,9 +2080,10 @@ func (b *executorBuilder) buildAnalyzeSamplingPushdown( Flags: sc.PushDownFlags(), TimeZoneOffset: offset, }, - opts: opts, - analyzeVer: task.StatsVersion, - indexes: availableIdx, + opts: opts, + indexes: availableIdx, + AnalyzeInfo: task.AnalyzeInfo, + schemaForVirtualColEval: schemaForVirtualColEval, } e.analyzePB.ColReq = &tipb.AnalyzeColumnsReq{ BucketSize: int64(opts[ast.AnalyzeOptNumBuckets]), @@ -2082,9 +2103,9 @@ func (b *executorBuilder) buildAnalyzeSamplingPushdown( return &analyzeTask{taskType: colTask, colExec: e, job: job} } -func (b *executorBuilder) buildAnalyzeColumnsPushdown(task plannercore.AnalyzeColumnsTask, opts map[ast.AnalyzeOptionType]uint64, autoAnalyze string) *analyzeTask { - if task.StatsVersion == statistics.Version3 { - return b.buildAnalyzeSamplingPushdown(task, opts, autoAnalyze) +func (b *executorBuilder) buildAnalyzeColumnsPushdown(task plannercore.AnalyzeColumnsTask, opts map[ast.AnalyzeOptionType]uint64, autoAnalyze string, schemaForVirtualColEval *expression.Schema) *analyzeTask { + if task.StatsVersion == statistics.Version2 { + return b.buildAnalyzeSamplingPushdown(task, opts, autoAnalyze, schemaForVirtualColEval) } cols := task.ColsInfo if hasPkHist(task.HandleCols) { @@ -2103,7 +2124,6 @@ func (b *executorBuilder) buildAnalyzeColumnsPushdown(task plannercore.AnalyzeCo sc := b.ctx.GetSessionVars().StmtCtx e := &AnalyzeColumnsExec{ ctx: b.ctx, - tableID: task.TableID, colsInfo: task.ColsInfo, handleCols: task.HandleCols, concurrency: b.ctx.GetSessionVars().DistSQLScanConcurrency(), @@ -2112,8 +2132,8 @@ func (b *executorBuilder) buildAnalyzeColumnsPushdown(task plannercore.AnalyzeCo Flags: sc.PushDownFlags(), TimeZoneOffset: offset, }, - opts: opts, - analyzeVer: task.StatsVersion, + opts: opts, + AnalyzeInfo: task.AnalyzeInfo, } depth := int32(opts[ast.AnalyzeOptCMSketchDepth]) width := int32(opts[ast.AnalyzeOptCMSketchWidth]) @@ -2159,7 +2179,7 @@ func (b *executorBuilder) buildAnalyzeColumnsPushdown(task plannercore.AnalyzeCo func (b *executorBuilder) buildAnalyzePKIncremental(task plannercore.AnalyzeColumnsTask, opts map[ast.AnalyzeOptionType]uint64) *analyzeTask { h := domain.GetDomain(b.ctx).StatsHandle() statsTbl := h.GetPartitionStats(&model.TableInfo{}, task.TableID.GetStatisticsID()) - analyzeTask := b.buildAnalyzeColumnsPushdown(task, opts, "") + analyzeTask := b.buildAnalyzeColumnsPushdown(task, opts, "", nil) if statsTbl.Pseudo { return analyzeTask } @@ -2285,7 +2305,13 @@ func (b *executorBuilder) buildAnalyze(v *plannercore.Analyze) Executor { if enableFastAnalyze { b.buildAnalyzeFastColumn(e, task, v.Opts) } else { - e.tasks = append(e.tasks, b.buildAnalyzeColumnsPushdown(task, v.Opts, autoAnalyze)) + columns, _, err := expression.ColumnInfos2ColumnsAndNames(b.ctx, model.NewCIStr(task.AnalyzeInfo.DBName), task.TblInfo.Name, task.ColsInfo, task.TblInfo) + if err != nil { + b.err = err + return nil + } + schema := expression.NewSchema(columns...) + e.tasks = append(e.tasks, b.buildAnalyzeColumnsPushdown(task, v.Opts, autoAnalyze, schema)) } } if b.err != nil { @@ -2474,11 +2500,14 @@ func (b *executorBuilder) buildIndexLookUpJoin(v *plannercore.PhysicalIndexJoin) outerKeyCols[i] = v.OuterJoinKeys[i].Index } innerKeyCols := make([]int, len(v.InnerJoinKeys)) + innerKeyColIDs := make([]int64, len(v.InnerJoinKeys)) for i := 0; i < len(v.InnerJoinKeys); i++ { innerKeyCols[i] = v.InnerJoinKeys[i].Index + innerKeyColIDs[i] = v.InnerJoinKeys[i].ID } e.outerCtx.keyCols = outerKeyCols e.innerCtx.keyCols = innerKeyCols + e.innerCtx.keyColIDs = innerKeyColIDs outerHashCols, innerHashCols := make([]int, len(v.OuterHashKeys)), make([]int, len(v.InnerHashKeys)) for i := 0; i < len(v.OuterHashKeys); i++ { @@ -2615,6 +2644,10 @@ func buildNoRangeTableReader(b *executorBuilder, v *plannercore.PhysicalTableRea return nil, err } ts := v.GetTableScan() + if err = b.validCanReadTemporaryTable(ts.Table); err != nil { + return nil, err + } + tbl, _ := b.is.TableByID(ts.Table.ID) isPartition, physicalTableID := ts.IsPartition() if isPartition { @@ -2709,6 +2742,11 @@ func (b *executorBuilder) buildTableReader(v *plannercore.PhysicalTableReader) E } ts := v.GetTableScan() + if err = b.validCanReadTemporaryTable(ts.Table); err != nil { + b.err = err + return nil + } + ret.ranges = ts.Ranges sctx := b.ctx.GetSessionVars().StmtCtx sctx.TableIDs = append(sctx.TableIDs, ts.Table.ID) @@ -2739,7 +2777,6 @@ func (b *executorBuilder) buildTableReader(v *plannercore.PhysicalTableReader) E ret.kvRangeBuilder = kvRangeBuilderFromRangeAndPartition{ sctx: b.ctx, partitions: partitions, - ranges: ts.Ranges, } return ret @@ -2783,7 +2820,6 @@ func keyColumnsIncludeAllPartitionColumns(keyColumns []int, pe *tables.Partition func prunePartitionForInnerExecutor(ctx sessionctx.Context, tbl table.Table, schema *expression.Schema, partitionInfo *plannercore.PartitionInfo, lookUpContent []*indexJoinLookUpContent) (usedPartition []table.PhysicalTable, canPrune bool, contentPos []int64, err error) { partitionTbl := tbl.(table.PartitionedTable) - locateKey := make([]types.Datum, schema.Len()) // TODO: condition based pruning can be do in advance. condPruneResult, err := partitionPruning(ctx, partitionTbl, partitionInfo.PruningConds, partitionInfo.PartitionNames, partitionInfo.Columns, partitionInfo.ColumnNames) if err != nil { @@ -2798,22 +2834,44 @@ func prunePartitionForInnerExecutor(ctx sessionctx.Context, tbl table.Table, sch if err != nil { return nil, false, nil, err } + + // recalculate key column offsets + if lookUpContent[0].keyColIDs == nil { + return nil, false, nil, + dbterror.ClassOptimizer.NewStd(mysql.ErrInternal).GenWithStack("cannot get column IDs when dynamic pruning") + } + keyColOffsets := make([]int, len(lookUpContent[0].keyColIDs)) + for i, colID := range lookUpContent[0].keyColIDs { + offset := -1 + for j, col := range partitionTbl.Cols() { + if colID == col.ID { + offset = j + break + } + } + if offset == -1 { + return nil, false, nil, + dbterror.ClassOptimizer.NewStd(mysql.ErrInternal).GenWithStack("invalid column offset when dynamic pruning") + } + keyColOffsets[i] = offset + } + offsetMap := make(map[int]bool) - for _, offset := range lookUpContent[0].keyCols { + for _, offset := range keyColOffsets { offsetMap[offset] = true } for _, offset := range pe.ColumnOffset { if _, ok := offsetMap[offset]; !ok { - logutil.BgLogger().Warn("can not runtime prune in index join") return condPruneResult, false, nil, nil } } + locateKey := make([]types.Datum, len(partitionTbl.Cols())) partitions := make(map[int64]table.PhysicalTable) contentPos = make([]int64, len(lookUpContent)) for idx, content := range lookUpContent { for i, date := range content.keys { - locateKey[content.keyCols[i]] = date + locateKey[keyColOffsets[i]] = date } p, err := partitionTbl.GetPartitionByRow(ctx, locateKey) if err != nil { @@ -2897,13 +2955,18 @@ func buildNoRangeIndexReader(b *executorBuilder, v *plannercore.PhysicalIndexRea } func (b *executorBuilder) buildIndexReader(v *plannercore.PhysicalIndexReader) Executor { + is := v.IndexPlans[0].(*plannercore.PhysicalIndexScan) + if err := b.validCanReadTemporaryTable(is.Table); err != nil { + b.err = err + return nil + } + ret, err := buildNoRangeIndexReader(b, v) if err != nil { b.err = err return nil } - is := v.IndexPlans[0].(*plannercore.PhysicalIndexScan) ret.ranges = is.Ranges sctx := b.ctx.GetSessionVars().StmtCtx sctx.IndexNames = append(sctx.IndexNames, is.Table.Name.O+":"+is.Index.Name.O) @@ -3044,13 +3107,18 @@ func buildNoRangeIndexLookUpReader(b *executorBuilder, v *plannercore.PhysicalIn } func (b *executorBuilder) buildIndexLookUpReader(v *plannercore.PhysicalIndexLookUpReader) Executor { + is := v.IndexPlans[0].(*plannercore.PhysicalIndexScan) + if err := b.validCanReadTemporaryTable(is.Table); err != nil { + b.err = err + return nil + } + ret, err := buildNoRangeIndexLookUpReader(b, v) if err != nil { b.err = err return nil } - is := v.IndexPlans[0].(*plannercore.PhysicalIndexScan) ts := v.TablePlans[0].(*plannercore.PhysicalTableScan) ret.ranges = is.Ranges @@ -3150,6 +3218,12 @@ func buildNoRangeIndexMergeReader(b *executorBuilder, v *plannercore.PhysicalInd } func (b *executorBuilder) buildIndexMergeReader(v *plannercore.PhysicalIndexMergeReader) Executor { + ts := v.TablePlans[0].(*plannercore.PhysicalTableScan) + if err := b.validCanReadTemporaryTable(ts.Table); err != nil { + b.err = err + return nil + } + ret, err := buildNoRangeIndexMergeReader(b, v) if err != nil { b.err = err @@ -3169,7 +3243,6 @@ func (b *executorBuilder) buildIndexMergeReader(v *plannercore.PhysicalIndexMerg } } } - ts := v.TablePlans[0].(*plannercore.PhysicalTableScan) sctx.TableIDs = append(sctx.TableIDs, ts.Table.ID) executorCounterIndexMergeReaderExecutor.Inc() @@ -3403,15 +3476,30 @@ func dedupHandles(lookUpContents []*indexJoinLookUpContent) ([]kv.Handle, []*ind type kvRangeBuilderFromRangeAndPartition struct { sctx sessionctx.Context partitions []table.PhysicalTable - ranges []*ranger.Range } -func (h kvRangeBuilderFromRangeAndPartition) buildKeyRange(int64) ([]kv.KeyRange, error) { +func (h kvRangeBuilderFromRangeAndPartition) buildKeyRangeSeparately(ranges []*ranger.Range) ([]int64, [][]kv.KeyRange, error) { + var ret [][]kv.KeyRange + var pids []int64 + for _, p := range h.partitions { + pid := p.GetPhysicalID() + meta := p.Meta() + kvRange, err := distsql.TableHandleRangesToKVRanges(h.sctx.GetSessionVars().StmtCtx, []int64{pid}, meta != nil && meta.IsCommonHandle, ranges, nil) + if err != nil { + return nil, nil, err + } + pids = append(pids, pid) + ret = append(ret, kvRange) + } + return pids, ret, nil +} + +func (h kvRangeBuilderFromRangeAndPartition) buildKeyRange(_ int64, ranges []*ranger.Range) ([]kv.KeyRange, error) { var ret []kv.KeyRange for _, p := range h.partitions { pid := p.GetPhysicalID() meta := p.Meta() - kvRange, err := distsql.TableHandleRangesToKVRanges(h.sctx.GetSessionVars().StmtCtx, []int64{pid}, meta != nil && meta.IsCommonHandle, h.ranges, nil) + kvRange, err := distsql.TableHandleRangesToKVRanges(h.sctx.GetSessionVars().StmtCtx, []int64{pid}, meta != nil && meta.IsCommonHandle, ranges, nil) if err != nil { return nil, err } @@ -3432,7 +3520,7 @@ func (builder *dataReaderBuilder) buildTableReaderBase(ctx context.Context, e *T SetKeepOrder(e.keepOrder). SetStreaming(e.streaming). SetFromSessionVars(e.ctx.GetSessionVars()). - SetFromInfoSchema(infoschema.GetInfoSchema(e.ctx)). + SetFromInfoSchema(e.ctx.GetInfoSchema()). Build() if err != nil { return nil, err @@ -3697,7 +3785,7 @@ func buildKvRangesForIndexJoin(ctx sessionctx.Context, tableID, indexID int64, l return distsql.IndexRangesToKVRanges(ctx.GetSessionVars().StmtCtx, tableID, indexID, tmpDatumRanges, nil) } -func (b *executorBuilder) buildWindow(v *plannercore.PhysicalWindow) *WindowExec { +func (b *executorBuilder) buildWindow(v *plannercore.PhysicalWindow) Executor { childExec := b.build(v.Children()[0]) if b.err != nil { return nil @@ -3726,6 +3814,40 @@ func (b *executorBuilder) buildWindow(v *plannercore.PhysicalWindow) *WindowExec partialResults = append(partialResults, partialResult) resultColIdx++ } + + if b.ctx.GetSessionVars().EnablePipelinedWindowExec { + exec := &PipelinedWindowExec{ + baseExecutor: base, + groupChecker: newVecGroupChecker(b.ctx, groupByItems), + numWindowFuncs: len(v.WindowFuncDescs), + } + + exec.windowFuncs = windowFuncs + exec.partialResults = partialResults + if v.Frame == nil { + exec.start = &plannercore.FrameBound{ + Type: ast.Preceding, + UnBounded: true, + } + exec.end = &plannercore.FrameBound{ + Type: ast.Following, + UnBounded: true, + } + } else { + exec.start = v.Frame.Start + exec.end = v.Frame.End + if v.Frame.Type == ast.Ranges { + cmpResult := int64(-1) + if len(v.OrderBy) > 0 && v.OrderBy[0].Desc { + cmpResult = 1 + } + exec.orderByCols = orderByCols + exec.expectedCmpResult = cmpResult + exec.isRangeFrame = true + } + } + return exec + } var processor windowProcessor if v.Frame == nil { processor = &aggWindowProcessor{ @@ -3892,6 +4014,11 @@ func NewRowDecoder(ctx sessionctx.Context, schema *expression.Schema, tbl *model } func (b *executorBuilder) buildBatchPointGet(plan *plannercore.BatchPointGetPlan) Executor { + if err := b.validCanReadTemporaryTable(plan.TblInfo); err != nil { + b.err = err + return nil + } + startTS, err := b.getSnapshotTS() if err != nil { b.err = err @@ -3908,6 +4035,7 @@ func (b *executorBuilder) buildBatchPointGet(plan *plannercore.BatchPointGetPlan desc: plan.Desc, lock: plan.Lock, waitTime: plan.LockWaitTime, + partExpr: plan.PartitionExpr, partPos: plan.PartitionColPos, singlePart: plan.SinglePart, partTblID: plan.PartTblID, @@ -4023,6 +4151,11 @@ func fullRangePartition(idxArr []int) bool { } func (b *executorBuilder) buildTableSample(v *plannercore.PhysicalTableSample) *TableSampleExecutor { + if v.TableInfo.Meta().TempTableType != model.TempTableNone { + b.err = errors.New("TABLESAMPLE clause can not be applied to temporary tables") + return nil + } + startTS, err := b.getSnapshotTS() if err != nil { b.err = err @@ -4040,3 +4173,116 @@ func (b *executorBuilder) buildTableSample(v *plannercore.PhysicalTableSample) * } return e } + +func (b *executorBuilder) buildCTE(v *plannercore.PhysicalCTE) Executor { + // 1. Build seedPlan. + if b.Ti != nil { + b.Ti.UseNonRecursive = true + } + seedExec := b.build(v.SeedPlan) + if b.err != nil { + return nil + } + + // 2. Build iterInTbl. + chkSize := b.ctx.GetSessionVars().MaxChunkSize + tps := seedExec.base().retFieldTypes + iterOutTbl := cteutil.NewStorageRowContainer(tps, chkSize) + if err := iterOutTbl.OpenAndRef(); err != nil { + b.err = err + return nil + } + + var resTbl cteutil.Storage + var iterInTbl cteutil.Storage + storageMap, ok := b.ctx.GetSessionVars().StmtCtx.CTEStorageMap.(map[int]*CTEStorages) + if !ok { + b.err = errors.New("type assertion for CTEStorageMap failed") + return nil + } + storages, ok := storageMap[v.CTE.IDForStorage] + if ok { + // Storage already setup. + resTbl = storages.ResTbl + iterInTbl = storages.IterInTbl + } else { + resTbl = cteutil.NewStorageRowContainer(tps, chkSize) + if err := resTbl.OpenAndRef(); err != nil { + b.err = err + return nil + } + iterInTbl = cteutil.NewStorageRowContainer(tps, chkSize) + if err := iterInTbl.OpenAndRef(); err != nil { + b.err = err + return nil + } + storageMap[v.CTE.IDForStorage] = &CTEStorages{ResTbl: resTbl, IterInTbl: iterInTbl} + } + + // 3. Build recursive part. + if v.RecurPlan != nil && b.Ti != nil { + b.Ti.UseRecursive = true + } + recursiveExec := b.build(v.RecurPlan) + if b.err != nil { + return nil + } + + var sel []int + if v.CTE.IsDistinct { + sel = make([]int, chkSize) + for i := 0; i < chkSize; i++ { + sel[i] = i + } + } + + return &CTEExec{ + baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()), + seedExec: seedExec, + recursiveExec: recursiveExec, + resTbl: resTbl, + iterInTbl: iterInTbl, + iterOutTbl: iterOutTbl, + chkIdx: 0, + isDistinct: v.CTE.IsDistinct, + sel: sel, + hasLimit: v.CTE.HasLimit, + limitBeg: v.CTE.LimitBeg, + limitEnd: v.CTE.LimitEnd, + } +} + +func (b *executorBuilder) buildCTETableReader(v *plannercore.PhysicalCTETable) Executor { + storageMap, ok := b.ctx.GetSessionVars().StmtCtx.CTEStorageMap.(map[int]*CTEStorages) + if !ok { + b.err = errors.New("type assertion for CTEStorageMap failed") + return nil + } + storages, ok := storageMap[v.IDForStorage] + if !ok { + b.err = errors.Errorf("iterInTbl should already be set up by CTEExec(id: %d)", v.IDForStorage) + return nil + } + return &CTETableReaderExec{ + baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()), + iterInTbl: storages.IterInTbl, + chkIdx: 0, + } +} + +func (b *executorBuilder) validCanReadTemporaryTable(tbl *model.TableInfo) error { + if tbl.TempTableType == model.TempTableNone { + return nil + } + + sessionVars := b.ctx.GetSessionVars() + if sessionVars.SnapshotTS != 0 { + return errors.New("can not read temporary table when 'tidb_snapshot' is set") + } + + if sessionVars.TxnCtx.IsStaleness || b.explicitStaleness { + return errors.New("can not stale read temporary table") + } + + return nil +} diff --git a/executor/checksum.go b/executor/checksum.go index 419c76042ad15..62543068820e9 100644 --- a/executor/checksum.go +++ b/executor/checksum.go @@ -240,6 +240,7 @@ func (c *checksumContext) buildTableRequest(ctx sessionctx.Context, tableID int6 } var builder distsql.RequestBuilder + builder.SetResourceGroupTag(ctx.GetSessionVars().StmtCtx) return builder.SetHandleRanges(ctx.GetSessionVars().StmtCtx, tableID, c.TableInfo.IsCommonHandle, ranges, nil). SetChecksumRequest(checksum). SetStartTS(c.StartTs). @@ -256,6 +257,7 @@ func (c *checksumContext) buildIndexRequest(ctx sessionctx.Context, tableID int6 ranges := ranger.FullRange() var builder distsql.RequestBuilder + builder.SetResourceGroupTag(ctx.GetSessionVars().StmtCtx) return builder.SetIndexRanges(ctx.GetSessionVars().StmtCtx, tableID, indexInfo.ID, ranges). SetChecksumRequest(checksum). SetStartTS(c.StartTs). @@ -269,7 +271,7 @@ func (c *checksumContext) HandleResponse(update *tipb.ChecksumResponse) { func getChecksumTableConcurrency(ctx sessionctx.Context) (int, error) { sessionVars := ctx.GetSessionVars() - concurrency, err := variable.GetSessionSystemVar(sessionVars, variable.TiDBChecksumTableConcurrency) + concurrency, err := variable.GetSessionOrGlobalSystemVar(sessionVars, variable.TiDBChecksumTableConcurrency) if err != nil { return 0, err } diff --git a/executor/compiler.go b/executor/compiler.go index bb0f5274a159e..c763c43067047 100644 --- a/executor/compiler.go +++ b/executor/compiler.go @@ -20,7 +20,6 @@ import ( "github.com/pingcap/parser/ast" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/config" - "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/planner" plannercore "github.com/pingcap/tidb/planner/core" @@ -53,13 +52,13 @@ func (c *Compiler) Compile(ctx context.Context, stmtNode ast.StmtNode) (*ExecStm ctx = opentracing.ContextWithSpan(ctx, span1) } - infoSchema := infoschema.GetInfoSchema(c.Ctx) - if err := plannercore.Preprocess(c.Ctx, stmtNode, infoSchema); err != nil { + ret := &plannercore.PreprocessorReturn{} + if err := plannercore.Preprocess(c.Ctx, stmtNode, plannercore.WithPreprocessorReturn(ret)); err != nil { return nil, err } stmtNode = plannercore.TryAddExtraLimit(c.Ctx, stmtNode) - finalPlan, names, err := planner.Optimize(ctx, c.Ctx, stmtNode, infoSchema) + finalPlan, names, err := planner.Optimize(ctx, c.Ctx, stmtNode, ret.InfoSchema) if err != nil { return nil, err } @@ -70,14 +69,17 @@ func (c *Compiler) Compile(ctx context.Context, stmtNode ast.StmtNode) (*ExecStm lowerPriority = needLowerPriority(finalPlan) } return &ExecStmt{ - GoCtx: ctx, - InfoSchema: infoSchema, - Plan: finalPlan, - LowerPriority: lowerPriority, - Text: stmtNode.Text(), - StmtNode: stmtNode, - Ctx: c.Ctx, - OutputNames: names, + GoCtx: ctx, + SnapshotTS: ret.SnapshotTS, + ExplicitStaleness: ret.ExplicitStaleness, + InfoSchema: ret.InfoSchema, + Plan: finalPlan, + LowerPriority: lowerPriority, + Text: stmtNode.Text(), + StmtNode: stmtNode, + Ctx: c.Ctx, + OutputNames: names, + Ti: &TelemetryInfo{}, }, nil } diff --git a/executor/concurrent_map.go b/executor/concurrent_map.go index 27f13a4f21dcb..3d6ef1082f605 100644 --- a/executor/concurrent_map.go +++ b/executor/concurrent_map.go @@ -56,7 +56,6 @@ func (m concurrentMap) Insert(key uint64, value *entry) { shard.items[key] = value } shard.Unlock() - return } // UpsertCb : Callback to return new element to be inserted into the map diff --git a/executor/coprocessor.go b/executor/coprocessor.go index 25959e5454655..04b90af2ad63b 100644 --- a/executor/coprocessor.go +++ b/executor/coprocessor.go @@ -159,7 +159,7 @@ func (h *CoprocessorDAGHandler) buildDAGExecutor(req *coprocessor.Request) (Exec return nil, errors.Trace(err) } h.dagReq = dagReq - is := h.sctx.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema) + is := h.sctx.GetInfoSchema().(infoschema.InfoSchema) // Build physical plan. bp := core.NewPBPlanBuilder(h.sctx, is, req.Ranges) plan, err := bp.Build(dagReq.Executors) @@ -168,7 +168,7 @@ func (h *CoprocessorDAGHandler) buildDAGExecutor(req *coprocessor.Request) (Exec } plan = core.InjectExtraProjection(plan) // Build executor. - b := newExecutorBuilder(h.sctx, is) + b := newExecutorBuilder(h.sctx, is, nil) return b.build(plan), nil } diff --git a/executor/cte.go b/executor/cte.go new file mode 100644 index 0000000000000..6ac26a50541b0 --- /dev/null +++ b/executor/cte.go @@ -0,0 +1,570 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor + +import ( + "context" + + "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/codec" + "github.com/pingcap/tidb/util/cteutil" + "github.com/pingcap/tidb/util/disk" + "github.com/pingcap/tidb/util/memory" +) + +var _ Executor = &CTEExec{} + +// CTEExec implements CTE. +// Following diagram describes how CTEExec works. +// +// `iterInTbl` is shared by `CTEExec` and `CTETableReaderExec`. +// `CTETableReaderExec` reads data from `iterInTbl`, +// and its output will be stored `iterOutTbl` by `CTEExec`. +// +// When an iteration ends, `CTEExec` will move all data from `iterOutTbl` into `iterInTbl`, +// which will be the input for new iteration. +// At the end of each iteration, data in `iterOutTbl` will also be added into `resTbl`. +// `resTbl` stores data of all iteration. +// +----------+ +// write |iterOutTbl| +// CTEExec ------------------->| | +// | +----+-----+ +// ------------- | write +// | | v +// other op other op +----------+ +// (seed) (recursive) | resTbl | +// ^ | | +// | +----------+ +// CTETableReaderExec +// ^ +// | read +----------+ +// +---------------+iterInTbl | +// | | +// +----------+ +type CTEExec struct { + baseExecutor + + seedExec Executor + recursiveExec Executor + + // `resTbl` and `iterInTbl` are shared by all CTEExec which reference to same the CTE. + // `iterInTbl` is also shared by CTETableReaderExec. + resTbl cteutil.Storage + iterInTbl cteutil.Storage + iterOutTbl cteutil.Storage + + hashTbl baseHashTable + + // Index of chunk to read from `resTbl`. + chkIdx int + + // UNION ALL or UNION DISTINCT. + isDistinct bool + curIter int + hCtx *hashContext + sel []int + + // Limit related info. + hasLimit bool + limitBeg uint64 + limitEnd uint64 + cursor uint64 + meetFirstBatch bool + + memTracker *memory.Tracker + diskTracker *disk.Tracker +} + +// Open implements the Executor interface. +func (e *CTEExec) Open(ctx context.Context) (err error) { + e.reset() + if err := e.baseExecutor.Open(ctx); err != nil { + return err + } + + if e.seedExec == nil { + return errors.New("seedExec for CTEExec is nil") + } + if err = e.seedExec.Open(ctx); err != nil { + return err + } + + e.memTracker = memory.NewTracker(e.id, -1) + e.diskTracker = disk.NewTracker(e.id, -1) + e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) + e.diskTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.DiskTracker) + + if e.recursiveExec != nil { + if err = e.recursiveExec.Open(ctx); err != nil { + return err + } + recursiveTypes := e.recursiveExec.base().retFieldTypes + e.iterOutTbl = cteutil.NewStorageRowContainer(recursiveTypes, e.maxChunkSize) + if err = e.iterOutTbl.OpenAndRef(); err != nil { + return err + } + } + + if e.isDistinct { + e.hashTbl = newConcurrentMapHashTable() + e.hCtx = &hashContext{ + allTypes: e.base().retFieldTypes, + } + // We use all columns to compute hash. + e.hCtx.keyColIdx = make([]int, len(e.hCtx.allTypes)) + for i := range e.hCtx.keyColIdx { + e.hCtx.keyColIdx[i] = i + } + } + return nil +} + +// Next implements the Executor interface. +func (e *CTEExec) Next(ctx context.Context, req *chunk.Chunk) (err error) { + req.Reset() + e.resTbl.Lock() + defer e.resTbl.Unlock() + if !e.resTbl.Done() { + resAction := setupCTEStorageTracker(e.resTbl, e.ctx, e.memTracker, e.diskTracker) + iterInAction := setupCTEStorageTracker(e.iterInTbl, e.ctx, e.memTracker, e.diskTracker) + iterOutAction := setupCTEStorageTracker(e.iterOutTbl, e.ctx, e.memTracker, e.diskTracker) + + failpoint.Inject("testCTEStorageSpill", func(val failpoint.Value) { + if val.(bool) && config.GetGlobalConfig().OOMUseTmpStorage { + defer resAction.WaitForTest() + defer iterInAction.WaitForTest() + defer iterOutAction.WaitForTest() + } + }) + + if err = e.computeSeedPart(ctx); err != nil { + // Don't put it in defer. + // Because it should be called only when the filling process is not completed. + if err1 := e.reopenTbls(); err1 != nil { + return err1 + } + return err + } + if err = e.computeRecursivePart(ctx); err != nil { + if err1 := e.reopenTbls(); err1 != nil { + return err1 + } + return err + } + e.resTbl.SetDone() + } + + if e.hasLimit { + return e.nextChunkLimit(req) + } + if e.chkIdx < e.resTbl.NumChunks() { + res, err := e.resTbl.GetChunk(e.chkIdx) + if err != nil { + return err + } + // Need to copy chunk to make sure upper operator will not change chunk in resTbl. + // Also we ignore copying rows not selected, because some operators like Projection + // doesn't support swap column if chunk.sel is no nil. + req.SwapColumns(res.CopyConstructSel()) + e.chkIdx++ + } + return nil +} + +// Close implements the Executor interface. +func (e *CTEExec) Close() (err error) { + e.reset() + if err = e.seedExec.Close(); err != nil { + return err + } + if e.recursiveExec != nil { + if err = e.recursiveExec.Close(); err != nil { + return err + } + } + + // `iterInTbl` and `resTbl` are shared by multiple operators, + // so will be closed when the SQL finishes. + if err = e.iterOutTbl.DerefAndClose(); err != nil { + return err + } + return e.baseExecutor.Close() +} + +func (e *CTEExec) computeSeedPart(ctx context.Context) (err error) { + e.curIter = 0 + e.iterInTbl.SetIter(e.curIter) + // This means iterInTbl's can be read. + defer close(e.iterInTbl.GetBegCh()) + chks := make([]*chunk.Chunk, 0, 10) + for { + if e.limitDone(e.iterInTbl) { + break + } + chk := newFirstChunk(e.seedExec) + if err = Next(ctx, e.seedExec, chk); err != nil { + return err + } + if chk.NumRows() == 0 { + break + } + if chk, err = e.tryDedupAndAdd(chk, e.iterInTbl, e.hashTbl); err != nil { + return err + } + chks = append(chks, chk) + } + // Initial resTbl is empty, so no need to deduplicate chk using resTbl. + // Just adding is ok. + for _, chk := range chks { + if err = e.resTbl.Add(chk); err != nil { + return err + } + } + e.curIter++ + e.iterInTbl.SetIter(e.curIter) + + return nil +} + +func (e *CTEExec) computeRecursivePart(ctx context.Context) (err error) { + if e.recursiveExec == nil || e.iterInTbl.NumChunks() == 0 { + return nil + } + + if e.curIter > e.ctx.GetSessionVars().CTEMaxRecursionDepth { + return ErrCTEMaxRecursionDepth.GenWithStackByArgs(e.curIter) + } + + if e.limitDone(e.resTbl) { + return nil + } + + for { + chk := newFirstChunk(e.recursiveExec) + if err = Next(ctx, e.recursiveExec, chk); err != nil { + return err + } + if chk.NumRows() == 0 { + if err = e.setupTblsForNewIteration(); err != nil { + return err + } + if e.limitDone(e.resTbl) { + break + } + if e.iterInTbl.NumChunks() == 0 { + break + } + // Next iteration begins. Need use iterOutTbl as input of next iteration. + e.curIter++ + e.iterInTbl.SetIter(e.curIter) + if e.curIter > e.ctx.GetSessionVars().CTEMaxRecursionDepth { + return ErrCTEMaxRecursionDepth.GenWithStackByArgs(e.curIter) + } + // Make sure iterInTbl is setup before Close/Open, + // because some executors will read iterInTbl in Open() (like IndexLookupJoin). + if err = e.recursiveExec.Close(); err != nil { + return err + } + if err = e.recursiveExec.Open(ctx); err != nil { + return err + } + } else { + if err = e.iterOutTbl.Add(chk); err != nil { + return err + } + } + } + return nil +} + +// Get next chunk from resTbl for limit. +func (e *CTEExec) nextChunkLimit(req *chunk.Chunk) error { + if !e.meetFirstBatch { + for e.chkIdx < e.resTbl.NumChunks() { + res, err := e.resTbl.GetChunk(e.chkIdx) + if err != nil { + return err + } + e.chkIdx++ + numRows := uint64(res.NumRows()) + if newCursor := e.cursor + numRows; newCursor >= e.limitBeg { + e.meetFirstBatch = true + begInChk, endInChk := e.limitBeg-e.cursor, numRows + if newCursor > e.limitEnd { + endInChk = e.limitEnd - e.cursor + } + e.cursor += endInChk + if begInChk == endInChk { + break + } + tmpChk := res.CopyConstructSel() + req.Append(tmpChk, int(begInChk), int(endInChk)) + return nil + } + e.cursor += numRows + } + } + if e.chkIdx < e.resTbl.NumChunks() && e.cursor < e.limitEnd { + res, err := e.resTbl.GetChunk(e.chkIdx) + if err != nil { + return err + } + e.chkIdx++ + numRows := uint64(res.NumRows()) + if e.cursor+numRows > e.limitEnd { + numRows = e.limitEnd - e.cursor + req.Append(res.CopyConstructSel(), 0, int(numRows)) + } else { + req.SwapColumns(res.CopyConstructSel()) + } + e.cursor += numRows + } + return nil +} + +func (e *CTEExec) setupTblsForNewIteration() (err error) { + num := e.iterOutTbl.NumChunks() + chks := make([]*chunk.Chunk, 0, num) + // Setup resTbl's data. + for i := 0; i < num; i++ { + chk, err := e.iterOutTbl.GetChunk(i) + if err != nil { + return err + } + // Data should be copied in UNION DISTINCT. + // Because deduplicate() will change data in iterOutTbl, + // which will cause panic when spilling data into disk concurrently. + if e.isDistinct { + chk = chk.CopyConstruct() + } + chk, err = e.tryDedupAndAdd(chk, e.resTbl, e.hashTbl) + if err != nil { + return err + } + chks = append(chks, chk) + } + + // Setup new iteration data in iterInTbl. + if err = e.iterInTbl.Reopen(); err != nil { + return err + } + defer close(e.iterInTbl.GetBegCh()) + if e.isDistinct { + // Already deduplicated by resTbl, adding directly is ok. + for _, chk := range chks { + if err = e.iterInTbl.Add(chk); err != nil { + return err + } + } + } else { + if err = e.iterInTbl.SwapData(e.iterOutTbl); err != nil { + return err + } + } + + // Clear data in iterOutTbl. + return e.iterOutTbl.Reopen() +} + +func (e *CTEExec) reset() { + e.curIter = 0 + e.chkIdx = 0 + e.hashTbl = nil + e.cursor = 0 + e.meetFirstBatch = false +} + +func (e *CTEExec) reopenTbls() (err error) { + e.hashTbl = newConcurrentMapHashTable() + if err := e.resTbl.Reopen(); err != nil { + return err + } + return e.iterInTbl.Reopen() +} + +// Check if tbl meets the requirement of limit. +func (e *CTEExec) limitDone(tbl cteutil.Storage) bool { + return e.hasLimit && uint64(tbl.NumRows()) >= e.limitEnd +} + +func setupCTEStorageTracker(tbl cteutil.Storage, ctx sessionctx.Context, parentMemTracker *memory.Tracker, + parentDiskTracker *disk.Tracker) (actionSpill *chunk.SpillDiskAction) { + memTracker := tbl.GetMemTracker() + memTracker.SetLabel(memory.LabelForCTEStorage) + memTracker.AttachTo(parentMemTracker) + + diskTracker := tbl.GetDiskTracker() + diskTracker.SetLabel(memory.LabelForCTEStorage) + diskTracker.AttachTo(parentDiskTracker) + + if config.GetGlobalConfig().OOMUseTmpStorage { + actionSpill = tbl.ActionSpill() + failpoint.Inject("testCTEStorageSpill", func(val failpoint.Value) { + if val.(bool) { + actionSpill = tbl.(*cteutil.StorageRC).ActionSpillForTest() + } + }) + ctx.GetSessionVars().StmtCtx.MemTracker.FallbackOldAndSetNewAction(actionSpill) + } + return actionSpill +} + +func (e *CTEExec) tryDedupAndAdd(chk *chunk.Chunk, + storage cteutil.Storage, + hashTbl baseHashTable) (res *chunk.Chunk, err error) { + if e.isDistinct { + if chk, err = e.deduplicate(chk, storage, hashTbl); err != nil { + return nil, err + } + } + return chk, storage.Add(chk) +} + +// Compute hash values in chk and put it in hCtx.hashVals. +// Use the returned sel to choose the computed hash values. +func (e *CTEExec) computeChunkHash(chk *chunk.Chunk) (sel []int, err error) { + numRows := chk.NumRows() + e.hCtx.initHash(numRows) + // Continue to reset to make sure all hasher is new. + for i := numRows; i < len(e.hCtx.hashVals); i++ { + e.hCtx.hashVals[i].Reset() + } + sel = chk.Sel() + var hashBitMap []bool + if sel != nil { + hashBitMap = make([]bool, chk.Capacity()) + for _, val := range sel { + hashBitMap[val] = true + } + } else { + // All rows is selected, sel will be [0....numRows). + // e.sel is setup when building executor. + sel = e.sel + } + + for i := 0; i < chk.NumCols(); i++ { + if err = codec.HashChunkSelected(e.ctx.GetSessionVars().StmtCtx, e.hCtx.hashVals, + chk, e.hCtx.allTypes[i], i, e.hCtx.buf, e.hCtx.hasNull, + hashBitMap, false); err != nil { + return nil, err + } + } + return sel, nil +} + +// Use hashTbl to deduplicate rows, and unique rows will be added to hashTbl. +// Duplicated rows are only marked to be removed by sel in Chunk, instead of really deleted. +func (e *CTEExec) deduplicate(chk *chunk.Chunk, + storage cteutil.Storage, + hashTbl baseHashTable) (chkNoDup *chunk.Chunk, err error) { + numRows := chk.NumRows() + if numRows == 0 { + return chk, nil + } + + // 1. Compute hash values for chunk. + chkHashTbl := newConcurrentMapHashTable() + selOri, err := e.computeChunkHash(chk) + if err != nil { + return nil, err + } + + // 2. Filter rows duplicated in input chunk. + // This sel is for filtering rows duplicated in cur chk. + selChk := make([]int, 0, numRows) + for i := 0; i < numRows; i++ { + key := e.hCtx.hashVals[selOri[i]].Sum64() + row := chk.GetRow(i) + + hasDup, err := e.checkHasDup(key, row, chk, storage, chkHashTbl) + if err != nil { + return nil, err + } + if hasDup { + continue + } + + selChk = append(selChk, selOri[i]) + + rowPtr := chunk.RowPtr{ChkIdx: uint32(0), RowIdx: uint32(i)} + chkHashTbl.Put(key, rowPtr) + } + chk.SetSel(selChk) + chkIdx := storage.NumChunks() + + // 3. Filter rows duplicated in RowContainer. + // This sel is for filtering rows duplicated in cteutil.Storage. + selStorage := make([]int, 0, len(selChk)) + for i := 0; i < len(selChk); i++ { + key := e.hCtx.hashVals[selChk[i]].Sum64() + row := chk.GetRow(i) + + hasDup, err := e.checkHasDup(key, row, nil, storage, hashTbl) + if err != nil { + return nil, err + } + if hasDup { + continue + } + + rowIdx := len(selStorage) + selStorage = append(selStorage, selChk[i]) + + rowPtr := chunk.RowPtr{ChkIdx: uint32(chkIdx), RowIdx: uint32(rowIdx)} + hashTbl.Put(key, rowPtr) + } + + chk.SetSel(selStorage) + return chk, nil +} + +// Use the row's probe key to check if it already exists in chk or storage. +// We also need to compare the row's real encoding value to avoid hash collision. +func (e *CTEExec) checkHasDup(probeKey uint64, + row chunk.Row, + curChk *chunk.Chunk, + storage cteutil.Storage, + hashTbl baseHashTable) (hasDup bool, err error) { + ptrs := hashTbl.Get(probeKey) + + if len(ptrs) == 0 { + return false, nil + } + + for _, ptr := range ptrs { + var matchedRow chunk.Row + if curChk != nil { + matchedRow = curChk.GetRow(int(ptr.RowIdx)) + } else { + matchedRow, err = storage.GetRow(ptr) + } + if err != nil { + return false, err + } + isEqual, err := codec.EqualChunkRow(e.ctx.GetSessionVars().StmtCtx, + row, e.hCtx.allTypes, e.hCtx.keyColIdx, + matchedRow, e.hCtx.allTypes, e.hCtx.keyColIdx) + if err != nil { + return false, err + } + if isEqual { + return true, nil + } + } + return false, nil +} diff --git a/executor/cte_table_reader.go b/executor/cte_table_reader.go new file mode 100644 index 0000000000000..94fedf01fd93e --- /dev/null +++ b/executor/cte_table_reader.go @@ -0,0 +1,78 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor + +import ( + "context" + + "github.com/pingcap/errors" + "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/cteutil" +) + +// CTETableReaderExec scans data in iterInTbl, which is filled by corresponding CTEExec. +type CTETableReaderExec struct { + baseExecutor + + iterInTbl cteutil.Storage + chkIdx int + curIter int +} + +// Open implements the Executor interface. +func (e *CTETableReaderExec) Open(ctx context.Context) error { + e.reset() + return e.baseExecutor.Open(ctx) +} + +// Next implements the Executor interface. +func (e *CTETableReaderExec) Next(ctx context.Context, req *chunk.Chunk) (err error) { + req.Reset() + + // Wait until iterInTbl can be read. This is controlled by corresponding CTEExec. + <-e.iterInTbl.GetBegCh() + + // We should read `iterInTbl` from the beginning when the next iteration starts. + // Can not directly judge whether to start the next iteration based on e.chkIdx, + // because some operators(Selection) may use forloop to read all data in `iterInTbl`. + if e.curIter != e.iterInTbl.GetIter() { + if e.curIter > e.iterInTbl.GetIter() { + return errors.Errorf("invalid iteration for CTETableReaderExec (e.curIter: %d, e.iterInTbl.GetIter(): %d)", + e.curIter, e.iterInTbl.GetIter()) + } + e.chkIdx = 0 + e.curIter = e.iterInTbl.GetIter() + } + if e.chkIdx < e.iterInTbl.NumChunks() { + res, err := e.iterInTbl.GetChunk(e.chkIdx) + if err != nil { + return err + } + // Need to copy chunk to make sure upper operators will not change chunk in iterInTbl. + req.SwapColumns(res.CopyConstructSel()) + e.chkIdx++ + } + return nil +} + +// Close implements the Executor interface. +func (e *CTETableReaderExec) Close() (err error) { + e.reset() + return e.baseExecutor.Close() +} + +func (e *CTETableReaderExec) reset() { + e.chkIdx = 0 + e.curIter = 0 +} diff --git a/executor/cte_test.go b/executor/cte_test.go new file mode 100644 index 0000000000000..4c6e1cf993975 --- /dev/null +++ b/executor/cte_test.go @@ -0,0 +1,447 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor_test + +import ( + "context" + "fmt" + "math/rand" + "sort" + + "github.com/pingcap/check" + + "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/sessionctx" + + "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/store/mockstore" + "github.com/pingcap/tidb/util/mock" + "github.com/pingcap/tidb/util/testkit" +) + +var _ = check.Suite(&CTETestSuite{&baseCTETestSuite{}}) +var _ = check.SerialSuites(&CTESerialTestSuite{&baseCTETestSuite{}}) + +type baseCTETestSuite struct { + store kv.Storage + dom *domain.Domain + sessionCtx sessionctx.Context + session session.Session + ctx context.Context +} + +type CTETestSuite struct { + *baseCTETestSuite +} + +type CTESerialTestSuite struct { + *baseCTETestSuite +} + +func (test *baseCTETestSuite) SetUpSuite(c *check.C) { + var err error + test.store, err = mockstore.NewMockStore() + c.Assert(err, check.IsNil) + + test.dom, err = session.BootstrapSession(test.store) + c.Assert(err, check.IsNil) + + test.sessionCtx = mock.NewContext() + + test.session, err = session.CreateSession4Test(test.store) + c.Assert(err, check.IsNil) + test.session.SetConnectionID(0) + + test.ctx = context.Background() +} + +func (test *baseCTETestSuite) TearDownSuite(c *check.C) { + test.dom.Close() + test.store.Close() +} + +func (test *CTETestSuite) TestBasicCTE(c *check.C) { + tk := testkit.NewTestKit(c, test.store) + tk.MustExec("use test") + + rows := tk.MustQuery("with recursive cte1 as (" + + "select 1 c1 " + + "union all " + + "select c1 + 1 c1 from cte1 where c1 < 5) " + + "select * from cte1") + rows.Check(testkit.Rows("1", "2", "3", "4", "5")) + + // Two seed parts. + rows = tk.MustQuery("with recursive cte1 as (" + + "select 1 c1 " + + "union all " + + "select 2 c1 " + + "union all " + + "select c1 + 1 c1 from cte1 where c1 < 10) " + + "select * from cte1 order by c1") + rows.Check(testkit.Rows("1", "2", "2", "3", "3", "4", "4", "5", "5", "6", "6", "7", "7", "8", "8", "9", "9", "10", "10")) + + // Two recursive parts. + rows = tk.MustQuery("with recursive cte1 as (" + + "select 1 c1 " + + "union all " + + "select 2 c1 " + + "union all " + + "select c1 + 1 c1 from cte1 where c1 < 3 " + + "union all " + + "select c1 + 2 c1 from cte1 where c1 < 5) " + + "select * from cte1 order by c1") + rows.Check(testkit.Rows("1", "2", "2", "3", "3", "3", "4", "4", "5", "5", "5", "6", "6")) + + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(a int);") + tk.MustExec("insert into t1 values(1);") + tk.MustExec("insert into t1 values(2);") + rows = tk.MustQuery("SELECT * FROM t1 dt WHERE EXISTS(WITH RECURSIVE qn AS (SELECT a*0 AS b UNION ALL SELECT b+1 FROM qn WHERE b=0) SELECT * FROM qn WHERE b=a);") + rows.Check(testkit.Rows("1")) + rows = tk.MustQuery("SELECT * FROM t1 dt WHERE EXISTS( WITH RECURSIVE qn AS (SELECT a*0 AS b UNION ALL SELECT b+1 FROM qn WHERE b=0 or b = 1) SELECT * FROM qn WHERE b=a );") + rows.Check(testkit.Rows("1", "2")) +} + +func (test *CTESerialTestSuite) TestSpillToDisk(c *check.C) { + defer config.RestoreFunc()() + config.UpdateGlobal(func(conf *config.Config) { + conf.OOMUseTmpStorage = true + }) + + tk := testkit.NewTestKit(c, test.store) + tk.MustExec("use test;") + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/testCTEStorageSpill", "return(true)"), check.IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/executor/testCTEStorageSpill"), check.IsNil) + tk.MustExec("set tidb_mem_quota_query = 1073741824;") + }() + c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/testSortedRowContainerSpill", "return(true)"), check.IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/executor/testSortedRowContainerSpill"), check.IsNil) + }() + + // Use duplicated rows to test UNION DISTINCT. + tk.MustExec("set tidb_mem_quota_query = 1073741824;") + insertStr := "insert into t1 values(0)" + rowNum := 1000 + vals := make([]int, rowNum) + vals[0] = 0 + for i := 1; i < rowNum; i++ { + v := rand.Intn(100) + vals[i] = v + insertStr += fmt.Sprintf(", (%d)", v) + } + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(c1 int);") + tk.MustExec(insertStr) + tk.MustExec("set tidb_mem_quota_query = 40000;") + tk.MustExec("set cte_max_recursion_depth = 500000;") + sql := fmt.Sprintf("with recursive cte1 as ( "+ + "select c1 from t1 "+ + "union "+ + "select c1 + 1 c1 from cte1 where c1 < %d) "+ + "select c1 from cte1 order by c1;", rowNum) + rows := tk.MustQuery(sql) + + memTracker := tk.Se.GetSessionVars().StmtCtx.MemTracker + diskTracker := tk.Se.GetSessionVars().StmtCtx.DiskTracker + c.Assert(memTracker.MaxConsumed(), check.Greater, int64(0)) + c.Assert(diskTracker.MaxConsumed(), check.Greater, int64(0)) + + sort.Ints(vals) + resRows := make([]string, 0, rowNum) + for i := vals[0]; i <= rowNum; i++ { + resRows = append(resRows, fmt.Sprintf("%d", i)) + } + rows.Check(testkit.Rows(resRows...)) +} + +func (test *CTETestSuite) TestUnionDistinct(c *check.C) { + tk := testkit.NewTestKit(c, test.store) + tk.MustExec("use test;") + + // Basic test. UNION/UNION ALL intersects. + rows := tk.MustQuery("with recursive cte1(c1) as (select 1 union select 1 union select 1 union all select c1 + 1 from cte1 where c1 < 3) select * from cte1 order by c1;") + rows.Check(testkit.Rows("1", "2", "3")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union all select 1 union select 1 union all select c1 + 1 from cte1 where c1 < 3) select * from cte1 order by c1;") + rows.Check(testkit.Rows("1", "2", "3")) + + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(c1 int, c2 int);") + tk.MustExec("insert into t1 values(1, 1), (1, 2), (2, 2);") + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from t1) select * from cte1 order by c1;") + rows.Check(testkit.Rows("1", "2", "3")) + + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(c1 int);") + tk.MustExec("insert into t1 values(1), (1), (1), (2), (2), (2);") + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 where c1 < 4) select * from cte1 order by c1;") + rows.Check(testkit.Rows("1", "2", "3", "4")) +} + +func (test *CTETestSuite) TestCTEMaxRecursionDepth(c *check.C) { + tk := testkit.NewTestKit(c, test.store) + tk.MustExec("use test;") + + tk.MustExec("set @@cte_max_recursion_depth = -1;") + err := tk.QueryToErr("with recursive cte1(c1) as (select 1 union select c1 + 1 c1 from cte1 where c1 < 100) select * from cte1;") + c.Assert(err, check.NotNil) + c.Assert(err.Error(), check.Equals, "[executor:3636]Recursive query aborted after 1 iterations. Try increasing @@cte_max_recursion_depth to a larger value") + // If there is no recursive part, query runs ok. + rows := tk.MustQuery("with recursive cte1(c1) as (select 1 union select 2) select * from cte1 order by c1;") + rows.Check(testkit.Rows("1", "2")) + rows = tk.MustQuery("with cte1(c1) as (select 1 union select 2) select * from cte1 order by c1;") + rows.Check(testkit.Rows("1", "2")) + + tk.MustExec("set @@cte_max_recursion_depth = 0;") + err = tk.QueryToErr("with recursive cte1(c1) as (select 1 union select c1 + 1 c1 from cte1 where c1 < 0) select * from cte1;") + c.Assert(err, check.NotNil) + c.Assert(err.Error(), check.Equals, "[executor:3636]Recursive query aborted after 1 iterations. Try increasing @@cte_max_recursion_depth to a larger value") + err = tk.QueryToErr("with recursive cte1(c1) as (select 1 union select c1 + 1 c1 from cte1 where c1 < 1) select * from cte1;") + c.Assert(err, check.NotNil) + c.Assert(err.Error(), check.Equals, "[executor:3636]Recursive query aborted after 1 iterations. Try increasing @@cte_max_recursion_depth to a larger value") + // If there is no recursive part, query runs ok. + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select 2) select * from cte1 order by c1;") + rows.Check(testkit.Rows("1", "2")) + rows = tk.MustQuery("with cte1(c1) as (select 1 union select 2) select * from cte1 order by c1;") + rows.Check(testkit.Rows("1", "2")) + + tk.MustExec("set @@cte_max_recursion_depth = 1;") + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select c1 + 1 c1 from cte1 where c1 < 0) select * from cte1;") + rows.Check(testkit.Rows("1")) + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select c1 + 1 c1 from cte1 where c1 < 1) select * from cte1;") + rows.Check(testkit.Rows("1")) + err = tk.QueryToErr("with recursive cte1(c1) as (select 1 union select c1 + 1 c1 from cte1 where c1 < 2) select * from cte1;") + c.Assert(err, check.NotNil) + c.Assert(err.Error(), check.Equals, "[executor:3636]Recursive query aborted after 2 iterations. Try increasing @@cte_max_recursion_depth to a larger value") + // If there is no recursive part, query runs ok. + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select 2) select * from cte1 order by c1;") + rows.Check(testkit.Rows("1", "2")) + rows = tk.MustQuery("with cte1(c1) as (select 1 union select 2) select * from cte1 order by c1;") + rows.Check(testkit.Rows("1", "2")) +} + +func (test *CTETestSuite) TestCTEWithLimit(c *check.C) { + tk := testkit.NewTestKit(c, test.store) + tk.MustExec("use test;") + + // Basic recursive tests. + rows := tk.MustQuery("with recursive cte1(c1) as (select 1 union select c1 + 1 from cte1 limit 5 offset 0) select * from cte1") + rows.Check(testkit.Rows("1", "2", "3", "4", "5")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select c1 + 1 from cte1 limit 5 offset 1) select * from cte1") + rows.Check(testkit.Rows("2", "3", "4", "5", "6")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select c1 + 1 from cte1 limit 5 offset 10) select * from cte1") + rows.Check(testkit.Rows("11", "12", "13", "14", "15")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select c1 + 1 from cte1 limit 5 offset 995) select * from cte1") + rows.Check(testkit.Rows("996", "997", "998", "999", "1000")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select c1 + 1 from cte1 limit 5 offset 6) select * from cte1;") + rows.Check(testkit.Rows("7", "8", "9", "10", "11")) + + // Test with cte_max_recursion_depth + tk.MustExec("set cte_max_recursion_depth=2;") + rows = tk.MustQuery("with recursive cte1(c1) as (select 0 union select c1 + 1 from cte1 limit 1 offset 2) select * from cte1;") + rows.Check(testkit.Rows("2")) + + err := tk.QueryToErr("with recursive cte1(c1) as (select 0 union select c1 + 1 from cte1 limit 1 offset 3) select * from cte1;") + c.Assert(err, check.NotNil) + c.Assert(err.Error(), check.Equals, "[executor:3636]Recursive query aborted after 3 iterations. Try increasing @@cte_max_recursion_depth to a larger value") + + tk.MustExec("set cte_max_recursion_depth=1000;") + rows = tk.MustQuery("with recursive cte1(c1) as (select 0 union select c1 + 1 from cte1 limit 5 offset 996) select * from cte1;") + rows.Check(testkit.Rows("996", "997", "998", "999", "1000")) + + err = tk.QueryToErr("with recursive cte1(c1) as (select 0 union select c1 + 1 from cte1 limit 5 offset 997) select * from cte1;") + c.Assert(err, check.NotNil) + c.Assert(err.Error(), check.Equals, "[executor:3636]Recursive query aborted after 1001 iterations. Try increasing @@cte_max_recursion_depth to a larger value") + + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select c1 + 1 from cte1 limit 0 offset 1) select * from cte1") + rows.Check(testkit.Rows()) + + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select c1 + 1 from cte1 limit 0 offset 10) select * from cte1") + rows.Check(testkit.Rows()) + + // Test join. + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select c1 + 1 from cte1 limit 2 offset 1) select * from cte1 dt1 join cte1 dt2 order by dt1.c1, dt2.c1;") + rows.Check(testkit.Rows("2 2", "2 3", "3 2", "3 3")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select c1 + 1 from cte1 limit 2 offset 1) select * from cte1 dt1 join cte1 dt2 on dt1.c1 = dt2.c1 order by dt1.c1, dt1.c1;") + rows.Check(testkit.Rows("2 2", "3 3")) + + // Test subquery. + // Different with mysql, maybe it's mysql bug?(https://bugs.mysql.com/bug.php?id=103890&thanks=4) + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select c1 + 1 from cte1 limit 2 offset 1) select c1 from cte1 where c1 in (select 2);") + rows.Check(testkit.Rows("2")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select c1 + 1 from cte1 limit 2 offset 1) select c1 from cte1 dt where c1 in (select c1 from cte1 where 1 = dt.c1 - 1);") + rows.Check(testkit.Rows("2")) + + // Test Apply. + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select c1 + 1 from cte1 limit 2 offset 1) select c1 from cte1 where cte1.c1 = (select dt1.c1 from cte1 dt1 where dt1.c1 = cte1.c1);") + rows.Check(testkit.Rows("2", "3")) + + // Recursive tests with table. + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(c1 int);") + tk.MustExec("insert into t1 values(1), (2), (3);") + + // Error: ERROR 1221 (HY000): Incorrect usage of UNION and LIMIT. + // Limit can only be at the end of SQL stmt. + err = tk.ExecToErr("with recursive cte1(c1) as (select c1 from t1 limit 1 offset 1 union select c1 + 1 from cte1 limit 0 offset 1) select * from cte1") + c.Assert(err.Error(), check.Equals, "[planner:1221]Incorrect usage of UNION and LIMIT") + + // Basic non-recusive tests. + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select 2 order by 1 limit 1 offset 1) select * from cte1") + rows.Check(testkit.Rows("2")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select 2 order by 1 limit 0 offset 1) select * from cte1") + rows.Check(testkit.Rows()) + + rows = tk.MustQuery("with recursive cte1(c1) as (select 1 union select 2 order by 1 limit 2 offset 0) select * from cte1") + rows.Check(testkit.Rows("1", "2")) + + // Test with table. + tk.MustExec("drop table if exists t1;") + insertStr := "insert into t1 values(0)" + for i := 1; i < 5000; i++ { + insertStr += fmt.Sprintf(", (%d)", i) + } + + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(c1 int);") + tk.MustExec(insertStr) + + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 1) select * from cte1") + rows.Check(testkit.Rows("0")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 1 offset 100) select * from cte1") + rows.Check(testkit.Rows("100")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 5 offset 100) select * from cte1") + rows.Check(testkit.Rows("100", "101", "102", "103", "104")) + + // Basic non-recursive tests. + rows = tk.MustQuery("with cte1 as (select c1 from t1 limit 2 offset 1) select * from cte1") + rows.Check(testkit.Rows("1", "2")) + + rows = tk.MustQuery("with cte1 as (select c1 from t1 limit 2 offset 1) select * from cte1 dt1 join cte1 dt2 on dt1.c1 = dt2.c1") + rows.Check(testkit.Rows("1 1", "2 2")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select 2 limit 0 offset 1) select * from cte1") + rows.Check(testkit.Rows()) + + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select 2 limit 0 offset 1) select * from cte1 dt1 join cte1 dt2 on dt1.c1 = dt2.c1") + rows.Check(testkit.Rows()) + + // rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select 2 limit 5 offset 100) select * from cte1") + // rows.Check(testkit.Rows("100", "101", "102", "103", "104")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 limit 3 offset 100) select * from cte1") + rows.Check(testkit.Rows("100", "101", "102")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 limit 3 offset 100) select * from cte1 dt1 join cte1 dt2 on dt1.c1 = dt2.c1") + rows.Check(testkit.Rows("100 100", "101 101", "102 102")) + + // Test limit 0. + tk.MustExec("set cte_max_recursion_depth = 0;") + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(c1 int);") + tk.MustExec("insert into t1 values(0);") + rows = tk.MustQuery("with recursive cte1 as (select 1/c1 c1 from t1 union select c1 + 1 c1 from cte1 where c1 < 2 limit 0) select * from cte1;") + rows.Check(testkit.Rows()) + // MySQL err: ERROR 1365 (22012): Division by 0. Because it gives error when computing 1/c1. + err = tk.QueryToErr("with recursive cte1 as (select 1/c1 c1 from t1 union select c1 + 1 c1 from cte1 where c1 < 2 limit 1) select * from cte1;") + c.Assert(err, check.NotNil) + c.Assert(err.Error(), check.Equals, "[executor:3636]Recursive query aborted after 1 iterations. Try increasing @@cte_max_recursion_depth to a larger value") + + tk.MustExec("set cte_max_recursion_depth = 1000;") + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(c1 int);") + tk.MustExec("insert into t1 values(1), (2), (3);") + + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 from cte1 limit 0 offset 2) select * from cte1;") + rows.Check(testkit.Rows()) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 from cte1 limit 1 offset 2) select * from cte1;") + rows.Check(testkit.Rows("3")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 from cte1 limit 2 offset 2) select * from cte1;") + rows.Check(testkit.Rows("3", "4")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 from cte1 limit 3 offset 2) select * from cte1;") + rows.Check(testkit.Rows("3", "4", "5")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 from cte1 limit 4 offset 2) select * from cte1;") + rows.Check(testkit.Rows("3", "4", "5", "6")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 from cte1 limit 0 offset 3) select * from cte1;") + rows.Check(testkit.Rows()) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 from cte1 limit 1 offset 3) select * from cte1;") + rows.Check(testkit.Rows("4")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 from cte1 limit 2 offset 3) select * from cte1;") + rows.Check(testkit.Rows("4", "5")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 from cte1 limit 3 offset 3) select * from cte1;") + rows.Check(testkit.Rows("4", "5", "6")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 from cte1 limit 4 offset 3) select * from cte1;") + rows.Check(testkit.Rows("4", "5", "6", "7")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 from cte1 limit 0 offset 4) select * from cte1;") + rows.Check(testkit.Rows()) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 from cte1 limit 1 offset 4) select * from cte1;") + rows.Check(testkit.Rows("5")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 from cte1 limit 2 offset 4) select * from cte1;") + rows.Check(testkit.Rows("5", "6")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 from cte1 limit 3 offset 4) select * from cte1;") + rows.Check(testkit.Rows("5", "6", "7")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 from cte1 limit 4 offset 4) select * from cte1;") + rows.Check(testkit.Rows("5", "6", "7", "8")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union all select c1 + 1 from cte1 limit 0 offset 2) select * from cte1;") + rows.Check(testkit.Rows()) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union all select c1 + 1 from cte1 limit 1 offset 2) select * from cte1;") + rows.Check(testkit.Rows("3")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union all select c1 + 1 from cte1 limit 2 offset 2) select * from cte1;") + rows.Check(testkit.Rows("3", "2")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union all select c1 + 1 from cte1 limit 3 offset 2) select * from cte1;") + rows.Check(testkit.Rows("3", "2", "3")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union all select c1 + 1 from cte1 limit 4 offset 2) select * from cte1;") + rows.Check(testkit.Rows("3", "2", "3", "4")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union all select c1 + 1 from cte1 limit 0 offset 3) select * from cte1;") + rows.Check(testkit.Rows()) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union all select c1 + 1 from cte1 limit 1 offset 3) select * from cte1;") + rows.Check(testkit.Rows("2")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union all select c1 + 1 from cte1 limit 2 offset 3) select * from cte1;") + rows.Check(testkit.Rows("2", "3")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union all select c1 + 1 from cte1 limit 3 offset 3) select * from cte1;") + rows.Check(testkit.Rows("2", "3", "4")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union all select c1 + 1 from cte1 limit 4 offset 3) select * from cte1;") + rows.Check(testkit.Rows("2", "3", "4", "3")) + + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union all select c1 + 1 from cte1 limit 0 offset 4) select * from cte1;") + rows.Check(testkit.Rows()) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union all select c1 + 1 from cte1 limit 1 offset 4) select * from cte1;") + rows.Check(testkit.Rows("3")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union all select c1 + 1 from cte1 limit 2 offset 4) select * from cte1;") + rows.Check(testkit.Rows("3", "4")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union all select c1 + 1 from cte1 limit 3 offset 4) select * from cte1;") + rows.Check(testkit.Rows("3", "4", "3")) + rows = tk.MustQuery("with recursive cte1(c1) as (select c1 from t1 union all select c1 + 1 from cte1 limit 4 offset 4) select * from cte1;") + rows.Check(testkit.Rows("3", "4", "3", "4")) +} diff --git a/executor/ddl.go b/executor/ddl.go index 64a597e2eb024..5058704ee2f9e 100644 --- a/executor/ddl.go +++ b/executor/ddl.go @@ -26,12 +26,14 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl" "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/util/admin" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/dbterror" "github.com/pingcap/tidb/util/gcutil" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/sqlexec" @@ -140,7 +142,6 @@ func (e *DDLExec) Next(ctx context.Context, req *chunk.Chunk) (err error) { is := dom.InfoSchema() txnCtx := e.ctx.GetSessionVars().TxnCtx txnCtx.InfoSchema = is - txnCtx.SchemaVersion = is.SchemaMetaVersion() // DDL will force commit old transaction, after DDL, in transaction status should be false. e.ctx.GetSessionVars().SetInTxn(false) return nil @@ -312,8 +313,12 @@ func (e *DDLExec) dropTableObject(objects []*ast.TableName, obt objectType, ifEx if isSystemTable(tn.Schema.L, tn.Name.L) { return errors.Errorf("Drop tidb system table '%s.%s' is forbidden", tn.Schema.L, tn.Name.L) } - - if obt == tableObject && config.CheckTableBeforeDrop { + tableInfo, err := e.is.TableByName(tn.Schema, tn.Name) + if err != nil { + return err + } + tempTableType := tableInfo.Meta().TempTableType + if obt == tableObject && config.CheckTableBeforeDrop && tempTableType == model.TempTableNone { logutil.BgLogger().Warn("admin check table before drop", zap.String("database", fullti.Schema.O), zap.String("table", fullti.Name.O), @@ -560,6 +565,11 @@ func (e *DDLExec) getRecoverTableByTableName(tableName *ast.TableName) (*model.J if tableInfo == nil || jobInfo == nil { return nil, nil, errors.Errorf("Can't find dropped/truncated table: %v in DDL history jobs", tableName.Name) } + // Dropping local temporary tables won't appear in DDL jobs. + if tableInfo.TempTableType == model.TempTableGlobal { + msg := mysql.Message("Recover/flashback table is not supported on temporary tables", nil) + return nil, nil, dbterror.ClassDDL.NewStdErr(errno.ErrUnsupportedDDLOperation, msg) + } return jobInfo, tableInfo, nil } diff --git a/executor/ddl_test.go b/executor/ddl_test.go index c55908066de62..922dd04ebdf2e 100644 --- a/executor/ddl_test.go +++ b/executor/ddl_test.go @@ -544,14 +544,14 @@ func (s *testSuite6) TestAlterTableModifyColumn(c *C) { c.Assert(err, NotNil) _, err = tk.Exec("alter table mc modify column c2 varchar(8)") - c.Assert(err, NotNil) + c.Assert(err, IsNil) tk.MustExec("alter table mc modify column c2 varchar(11)") - tk.MustGetErrCode("alter table mc modify column c2 text(13)", errno.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table mc modify column c2 text", errno.ErrUnsupportedDDLOperation) + tk.MustExec("alter table mc modify column c2 text(13)") + tk.MustExec("alter table mc modify column c2 text") tk.MustExec("alter table mc modify column c3 bit") result := tk.MustQuery("show create table mc") createSQL := result.Rows()[0][1] - expected := "CREATE TABLE `mc` (\n `c1` bigint(20) DEFAULT NULL,\n `c2` varchar(11) DEFAULT NULL,\n `c3` bit(1) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" + expected := "CREATE TABLE `mc` (\n `c1` bigint(20) DEFAULT NULL,\n `c2` text DEFAULT NULL,\n `c3` bit(1) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" c.Assert(createSQL, Equals, expected) tk.MustExec("create or replace view alter_view as select c1,c2 from mc") _, err = tk.Exec("alter table alter_view modify column c2 text") @@ -1526,3 +1526,14 @@ func (s *testRecoverTable) TestRenameMultiTables(c *C) { tk.MustExec("drop database rename2") tk.MustExec("drop database rename3") } + +// See https://github.com/pingcap/tidb/issues/24582 +func (s *testSuite6) TestDuplicatedEntryErr(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(a int, b varchar(20), primary key(a,b(3)) clustered);") + tk.MustExec("insert into t1 values(1,'aaaaa');") + _, err := tk.Exec("insert into t1 values(1,'aaaaa');") + c.Assert(err.Error(), Equals, "[kv:1062]Duplicate entry '1-aaa' for key 'PRIMARY'") +} diff --git a/executor/delete.go b/executor/delete.go index 1fe9c26b8ac82..16f0e9c421b19 100644 --- a/executor/delete.go +++ b/executor/delete.go @@ -188,10 +188,7 @@ func (e *DeleteExec) removeRowsInTblRowMap(tblRowMap tableRowMapType) error { var err error rowMap.Range(func(h kv.Handle, val interface{}) bool { err = e.removeRow(e.ctx, e.tblID2Table[id], h, val.([]types.Datum)) - if err != nil { - return false - } - return true + return err == nil }) if err != nil { return err diff --git a/executor/distsql.go b/executor/distsql.go index 6e3105a30142d..759ec6cb35e85 100644 --- a/executor/distsql.go +++ b/executor/distsql.go @@ -33,7 +33,6 @@ import ( "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/distsql" "github.com/pingcap/tidb/expression" - "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx" @@ -197,6 +196,10 @@ type IndexReaderExecutor struct { // Close clears all resources hold by current object. func (e *IndexReaderExecutor) Close() error { + if e.table != nil && e.table.Meta().TempTableType != model.TempTableNone { + return nil + } + err := e.result.Close() e.result = nil e.ctx.StoreQueryFeedback(e.feedback) @@ -205,6 +208,11 @@ func (e *IndexReaderExecutor) Close() error { // Next implements the Executor Next interface. func (e *IndexReaderExecutor) Next(ctx context.Context, req *chunk.Chunk) error { + if e.table != nil && e.table.Meta().TempTableType != model.TempTableNone { + req.Reset() + return nil + } + err := e.result.Next(ctx, req) if err != nil { e.feedback.Invalidate() @@ -267,20 +275,25 @@ func (e *IndexReaderExecutor) open(ctx context.Context, kvRanges []kv.KeyRange) e.dagPB.CollectExecutionSummaries = &collExec } e.kvRanges = kvRanges + // Treat temporary table as dummy table, avoid sending distsql request to TiKV. + // In a test case IndexReaderExecutor is mocked and e.table is nil. + if e.table != nil && e.table.Meta().TempTableType != model.TempTableNone { + return nil + } e.memTracker = memory.NewTracker(e.id, -1) e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) var builder distsql.RequestBuilder - kvReq, err := builder.SetKeyRanges(kvRanges). + builder.SetKeyRanges(kvRanges). SetDAGRequest(e.dagPB). SetStartTS(e.startTS). SetDesc(e.desc). SetKeepOrder(e.keepOrder). SetStreaming(e.streaming). SetFromSessionVars(e.ctx.GetSessionVars()). - SetMemTracker(e.memTracker). - SetFromInfoSchema(infoschema.GetInfoSchema(e.ctx)). - Build() + SetFromInfoSchema(e.ctx.GetInfoSchema()). + SetMemTracker(e.memTracker) + kvReq, err := builder.Build() if err != nil { e.feedback.Invalidate() return err @@ -382,6 +395,12 @@ func (e *IndexLookUpExecutor) Open(ctx context.Context) error { e.feedback.Invalidate() return err } + + // Treat temporary table as dummy table, avoid sending distsql request to TiKV. + if e.table.Meta().TempTableType == model.TempTableGlobal { + return nil + } + err = e.open(ctx) if err != nil { e.feedback.Invalidate() @@ -527,8 +546,8 @@ func (e *IndexLookUpExecutor) startIndexWorker(ctx context.Context, workCh chan< SetKeepOrder(e.keepOrder). SetStreaming(e.indexStreaming). SetFromSessionVars(e.ctx.GetSessionVars()). - SetMemTracker(tracker). - SetFromInfoSchema(infoschema.GetInfoSchema(e.ctx)) + SetFromInfoSchema(e.ctx.GetInfoSchema()). + SetMemTracker(tracker) for partTblIdx, kvRange := range kvRanges { // check if executor is closed @@ -640,6 +659,10 @@ func (e *IndexLookUpExecutor) buildTableReader(ctx context.Context, task *lookup // Close implements Exec Close interface. func (e *IndexLookUpExecutor) Close() error { + if e.table.Meta().TempTableType != model.TempTableNone { + return nil + } + if !e.workerStarted || e.finished == nil { return nil } @@ -660,6 +683,11 @@ func (e *IndexLookUpExecutor) Close() error { // Next implements Exec Next interface. func (e *IndexLookUpExecutor) Next(ctx context.Context, req *chunk.Chunk) error { + if e.table.Meta().TempTableType == model.TempTableGlobal { + req.Reset() + return nil + } + if !e.workerStarted { if err := e.startWorkers(ctx, req.RequiredRows()); err != nil { return err @@ -1088,7 +1116,7 @@ func (w *tableWorker) compareData(ctx context.Context, task *lookupTableTask, ta if chk.NumRows() == 0 { task.indexOrder.Range(func(h kv.Handle, val interface{}) bool { idxRow := task.idxRows.GetRow(val.(int)) - err = errors.Errorf("handle %#v, index:%#v != record:%#v", h, idxRow.GetDatum(0, w.idxColTps[0]), nil) + err = ErrDataInConsistentExtraIndex.GenWithStackByArgs(h, idxRow.GetDatum(0, w.idxColTps[0]), nil) return false }) if err != nil { @@ -1123,12 +1151,12 @@ func (w *tableWorker) compareData(ctx context.Context, task *lookupTableTask, ta tablecodec.TruncateIndexValue(&idxVal, w.idxLookup.index.Columns[i], col.ColumnInfo) cmpRes, err := idxVal.CompareDatum(sctx, &val) if err != nil { - return errors.Errorf("col %s, handle %#v, index:%#v != record:%#v, compare err:%#v", col.Name, + return ErrDataInConsistentMisMatchIndex.GenWithStackByArgs(col.Name, handle, idxRow.GetDatum(i, tp), val, err) } if cmpRes != 0 { - return errors.Errorf("col %s, handle %#v, index:%#v != record:%#v", col.Name, - handle, idxRow.GetDatum(i, tp), val) + return ErrDataInConsistentMisMatchIndex.GenWithStackByArgs(col.Name, + handle, idxRow.GetDatum(i, tp), val, err) } } } @@ -1206,10 +1234,20 @@ func (w *tableWorker) executeTask(ctx context.Context, task *lookupTableTask) er obtainedHandlesMap.Set(handle, true) } - logutil.Logger(ctx).Error("inconsistent index handles", zap.String("index", w.idxLookup.index.Name.O), - zap.Int("index_cnt", handleCnt), zap.Int("table_cnt", len(task.rows)), - zap.String("missing_handles", fmt.Sprint(GetLackHandles(task.handles, obtainedHandlesMap))), - zap.String("total_handles", fmt.Sprint(task.handles))) + if w.idxLookup.ctx.GetSessionVars().EnableRedactLog { + logutil.Logger(ctx).Error("inconsistent index handles", + zap.String("table_name", w.idxLookup.index.Table.O), + zap.String("index", w.idxLookup.index.Name.O), + zap.Int("index_cnt", handleCnt), + zap.Int("table_cnt", len(task.rows))) + } else { + logutil.Logger(ctx).Error("inconsistent index handles", + zap.String("table_name", w.idxLookup.index.Table.O), + zap.String("index", w.idxLookup.index.Name.O), + zap.Int("index_cnt", handleCnt), zap.Int("table_cnt", len(task.rows)), + zap.String("missing_handles", fmt.Sprint(GetLackHandles(task.handles, obtainedHandlesMap))), + zap.String("total_handles", fmt.Sprint(task.handles))) + } // table scan in double read can never has conditions according to convertToIndexScan. // if this table scan has no condition, the number of rows it returns must equal to the length of handles. diff --git a/executor/distsql_test.go b/executor/distsql_test.go index eca6e1d016b40..6f06fd550f0b4 100644 --- a/executor/distsql_test.go +++ b/executor/distsql_test.go @@ -241,7 +241,7 @@ func (s *testSuite3) TestInconsistentIndex(c *C) { for i := 0; i < 10; i++ { txn, err := s.store.Begin() c.Assert(err, IsNil) - err = idxOp.Delete(ctx.GetSessionVars().StmtCtx, txn.GetUnionStore(), types.MakeDatums(i+10), kv.IntHandle(100+i)) + err = idxOp.Delete(ctx.GetSessionVars().StmtCtx, txn, types.MakeDatums(i+10), kv.IntHandle(100+i)) c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) diff --git a/executor/errors.go b/executor/errors.go index 7f3345659e4f9..05f889e5c9ff8 100644 --- a/executor/errors.go +++ b/executor/errors.go @@ -42,6 +42,7 @@ var ( ErrTableaccessDenied = dbterror.ClassExecutor.NewStd(mysql.ErrTableaccessDenied) ErrBadDB = dbterror.ClassExecutor.NewStd(mysql.ErrBadDB) ErrWrongObject = dbterror.ClassExecutor.NewStd(mysql.ErrWrongObject) + ErrWrongUsage = dbterror.ClassExecutor.NewStd(mysql.ErrWrongUsage) ErrRoleNotGranted = dbterror.ClassPrivilege.NewStd(mysql.ErrRoleNotGranted) ErrDeadlock = dbterror.ClassExecutor.NewStd(mysql.ErrLockDeadlock) ErrQueryInterrupted = dbterror.ClassExecutor.NewStd(mysql.ErrQueryInterrupted) @@ -49,8 +50,11 @@ var ( ErrIllegalPrivilegeLevel = dbterror.ClassExecutor.NewStd(mysql.ErrIllegalPrivilegeLevel) ErrInvalidSplitRegionRanges = dbterror.ClassExecutor.NewStd(mysql.ErrInvalidSplitRegionRanges) - ErrBRIEBackupFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBRIEBackupFailed) - ErrBRIERestoreFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBRIERestoreFailed) - ErrBRIEImportFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBRIEImportFailed) - ErrBRIEExportFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBRIEExportFailed) + ErrBRIEBackupFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBRIEBackupFailed) + ErrBRIERestoreFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBRIERestoreFailed) + ErrBRIEImportFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBRIEImportFailed) + ErrBRIEExportFailed = dbterror.ClassExecutor.NewStd(mysql.ErrBRIEExportFailed) + ErrCTEMaxRecursionDepth = dbterror.ClassExecutor.NewStd(mysql.ErrCTEMaxRecursionDepth) + ErrDataInConsistentExtraIndex = dbterror.ClassExecutor.NewStd(mysql.ErrDataInConsistentExtraIndex) + ErrDataInConsistentMisMatchIndex = dbterror.ClassExecutor.NewStd(mysql.ErrDataInConsistentMisMatchIndex) ) diff --git a/executor/executor.go b/executor/executor.go index e5d5d44efefe3..bb5ba3b89fcac 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -18,6 +18,7 @@ import ( "fmt" "math" "runtime" + "runtime/pprof" "runtime/trace" "strconv" "strings" @@ -29,6 +30,7 @@ import ( "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/failpoint" + "github.com/pingcap/parser" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/auth" "github.com/pingcap/parser/model" @@ -49,6 +51,7 @@ import ( "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/tikv" + tikverr "github.com/pingcap/tidb/store/tikv/error" tikvstore "github.com/pingcap/tidb/store/tikv/kv" tikvutil "github.com/pingcap/tidb/store/tikv/util" "github.com/pingcap/tidb/table" @@ -58,10 +61,13 @@ import ( "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/admin" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/deadlockhistory" "github.com/pingcap/tidb/util/disk" "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/memory" + "github.com/pingcap/tidb/util/resourcegrouptag" + "github.com/pingcap/tidb/util/topsql" "go.uber.org/zap" ) @@ -971,6 +977,11 @@ func (e *SelectLockExec) Next(ctx context.Context, req *chunk.Chunk) error { } func newLockCtx(seVars *variable.SessionVars, lockWaitTime int64) *tikvstore.LockCtx { + var planDigest *parser.Digest + _, sqlDigest := seVars.StmtCtx.SQLDigest() + if variable.TopSQLEnabled() { + _, planDigest = seVars.StmtCtx.GetPlanDigest() + } return &tikvstore.LockCtx{ Killed: &seVars.Killed, ForUpdateTS: seVars.TxnCtx.GetForUpdateTS(), @@ -980,6 +991,14 @@ func newLockCtx(seVars *variable.SessionVars, lockWaitTime int64) *tikvstore.Loc LockKeysDuration: &seVars.StmtCtx.LockKeysDuration, LockKeysCount: &seVars.StmtCtx.LockKeysCount, LockExpired: &seVars.TxnCtx.LockExpire, + ResourceGroupTag: resourcegrouptag.EncodeResourceGroupTag(sqlDigest, planDigest), + OnDeadlock: func(deadlock *tikverr.ErrDeadlock) { + // TODO: Support collecting retryable deadlocks according to the config. + if !deadlock.IsRetryable { + rec := deadlockhistory.ErrDeadlockToDeadlockRecord(deadlock) + deadlockhistory.GlobalDeadlockHistory.Push(rec) + } + }, } } @@ -988,7 +1007,8 @@ func newLockCtx(seVars *variable.SessionVars, lockWaitTime int64) *tikvstore.Loc // locked by others. used for (select for update nowait) situation // except 0 means alwaysWait 1 means nowait func doLockKeys(ctx context.Context, se sessionctx.Context, lockCtx *tikvstore.LockCtx, keys ...kv.Key) error { - sctx := se.GetSessionVars().StmtCtx + sessVars := se.GetSessionVars() + sctx := sessVars.StmtCtx if !sctx.InUpdateStmt && !sctx.InDeleteStmt { atomic.StoreUint32(&se.GetSessionVars().TxnCtx.ForUpdate, 1) } @@ -997,6 +1017,10 @@ func doLockKeys(ctx context.Context, se sessionctx.Context, lockCtx *tikvstore.L if err != nil { return err } + + // Skip the temporary table keys. + keys = filterTemporaryTableKeys(sessVars, keys) + var lockKeyStats *tikvutil.LockKeysDetails ctx = context.WithValue(ctx, tikvutil.LockKeysDetailCtxKey, &lockKeyStats) err = txn.LockKeys(tikvutil.SetSessionID(ctx, se.GetSessionVars().ConnectionID), lockCtx, keys...) @@ -1006,6 +1030,22 @@ func doLockKeys(ctx context.Context, se sessionctx.Context, lockCtx *tikvstore.L return err } +func filterTemporaryTableKeys(vars *variable.SessionVars, keys []kv.Key) []kv.Key { + txnCtx := vars.TxnCtx + if txnCtx == nil || txnCtx.GlobalTemporaryTables == nil { + return keys + } + + newKeys := keys[:] + for _, key := range keys { + tblID := tablecodec.DecodeTableID(key) + if _, ok := txnCtx.GlobalTemporaryTables[tblID]; !ok { + newKeys = append(newKeys, key) + } + } + return newKeys +} + // LimitExec represents limit executor // It ignores 'Offset' rows from src, then returns 'Count' rows at maximum. type LimitExec struct { @@ -1591,10 +1631,11 @@ func (e *UnionExec) Close() error { func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) { vars := ctx.GetSessionVars() sc := &stmtctx.StatementContext{ - TimeZone: vars.Location(), - MemTracker: memory.NewTracker(memory.LabelForSQLText, vars.MemQuotaQuery), - DiskTracker: disk.NewTracker(memory.LabelForSQLText, -1), - TaskID: stmtctx.AllocateTaskID(), + TimeZone: vars.Location(), + MemTracker: memory.NewTracker(memory.LabelForSQLText, vars.MemQuotaQuery), + DiskTracker: disk.NewTracker(memory.LabelForSQLText, -1), + TaskID: stmtctx.AllocateTaskID(), + CTEStorageMap: map[int]*CTEStorages{}, } sc.MemTracker.AttachToGlobalTracker(GlobalMemoryUsageTracker) globalConfig := config.GetGlobalConfig() @@ -1614,9 +1655,20 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) { sc.MemTracker.SetActionOnExceed(action) } if execStmt, ok := s.(*ast.ExecuteStmt); ok { - s, err = planner.GetPreparedStmt(execStmt, vars) + prepareStmt, err := planner.GetPreparedStmt(execStmt, vars) if err != nil { - return + return err + } + s = prepareStmt.PreparedAst.Stmt + sc.InitSQLDigest(prepareStmt.NormalizedSQL, prepareStmt.SQLDigest) + // For `execute stmt` SQL, should reset the SQL digest with the prepare SQL digest. + goCtx := context.Background() + if variable.EnablePProfSQLCPU.Load() && len(prepareStmt.NormalizedSQL) > 0 { + goCtx = pprof.WithLabels(goCtx, pprof.Labels("sql", util.QueryStrForLog(prepareStmt.NormalizedSQL))) + pprof.SetGoroutineLabels(goCtx) + } + if variable.TopSQLEnabled() && prepareStmt.SQLDigest != nil { + topsql.AttachSQLInfo(goCtx, prepareStmt.NormalizedSQL, prepareStmt.SQLDigest, "", nil) } } // execute missed stmtID uses empty sql @@ -1782,3 +1834,9 @@ func FillVirtualColumnValue(virtualRetTypes []*types.FieldType, virtualColumnInd } return nil } + +func setResourceGroupTagForTxn(sc *stmtctx.StatementContext, snapshot kv.Snapshot) { + if snapshot != nil && variable.TopSQLEnabled() { + snapshot.SetOption(kv.ResourceGroupTag, sc.GetResourceGroupTag()) + } +} diff --git a/executor/executor_pkg_test.go b/executor/executor_pkg_test.go index 7cc5a8a69d66e..5591dcefde54d 100644 --- a/executor/executor_pkg_test.go +++ b/executor/executor_pkg_test.go @@ -30,6 +30,7 @@ import ( "github.com/pingcap/tidb/executor/aggfuncs" "github.com/pingcap/tidb/expression" plannerutil "github.com/pingcap/tidb/planner/util" + txninfo "github.com/pingcap/tidb/session/txninfo" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util" @@ -60,6 +61,10 @@ type mockSessionManager struct { serverID uint64 } +func (msm *mockSessionManager) ShowTxnList() []*txninfo.TxnInfo { + panic("unimplemented!") +} + // ShowProcessList implements the SessionManager.ShowProcessList interface. func (msm *mockSessionManager) ShowProcessList() map[uint64]*util.ProcessInfo { ret := make(map[uint64]*util.ProcessInfo) diff --git a/executor/executor_required_rows_test.go b/executor/executor_required_rows_test.go index 720b5300677c9..a0eb15ca77860 100644 --- a/executor/executor_required_rows_test.go +++ b/executor/executor_required_rows_test.go @@ -843,7 +843,7 @@ func buildMergeJoinExec(ctx sessionctx.Context, joinType plannercore.JoinType, i j.CompareFuncs = append(j.CompareFuncs, expression.GetCmpFunction(nil, j.LeftJoinKeys[i], j.RightJoinKeys[i])) } - b := newExecutorBuilder(ctx, nil) + b := newExecutorBuilder(ctx, nil, nil) return b.build(j) } diff --git a/executor/executor_test.go b/executor/executor_test.go index 80056439ec7c6..5cdd7516b8005 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -17,7 +17,6 @@ import ( "context" "flag" "fmt" - "io/ioutil" "math" "net" "os" @@ -32,7 +31,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/errors" "github.com/pingcap/failpoint" - pb "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/parser" "github.com/pingcap/parser/auth" "github.com/pingcap/parser/model" @@ -59,6 +58,7 @@ import ( "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/store/copr" "github.com/pingcap/tidb/store/mockstore" + "github.com/pingcap/tidb/store/mockstore/unistore" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/mockstore/cluster" "github.com/pingcap/tidb/store/tikv/oracle" @@ -70,7 +70,9 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/admin" + "github.com/pingcap/tidb/util/deadlockhistory" "github.com/pingcap/tidb/util/gcutil" + "github.com/pingcap/tidb/util/israce" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/mock" @@ -144,8 +146,10 @@ var _ = SerialSuites(&tiflashTestSuite{}) var _ = SerialSuites(&globalIndexSuite{&baseTestSuite{}}) var _ = SerialSuites(&testSerialSuite{&baseTestSuite{}}) var _ = SerialSuites(&testStaleTxnSerialSuite{&baseTestSuite{}}) +var _ = Suite(&testStaleTxnSuite{&baseTestSuite{}}) var _ = SerialSuites(&testCoprCache{}) var _ = SerialSuites(&testPrepareSuite{}) +var _ = SerialSuites(&testResourceTagSuite{&baseTestSuite{}}) type testSuite struct{ *baseTestSuite } type testSuiteP1 struct{ *baseTestSuite } @@ -160,12 +164,14 @@ type partitionTableSuite struct{ *baseTestSuite } type globalIndexSuite struct{ *baseTestSuite } type testSerialSuite struct{ *baseTestSuite } type testStaleTxnSerialSuite struct{ *baseTestSuite } +type testStaleTxnSuite struct{ *baseTestSuite } type testCoprCache struct { store kv.Storage dom *domain.Domain cls cluster.Cluster } type testPrepareSuite struct{ testData testutil.TestData } +type testResourceTagSuite struct{ *baseTestSuite } type baseTestSuite struct { cluster cluster.Cluster @@ -274,16 +280,6 @@ func (s *testSuiteP1) TestBind(c *C) { tk.MustExec("drop session binding for select * from testbind") } -func (s *testSuiteP1) TestChange(c *C) { - tk := testkit.NewTestKit(c, s.store) - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec("create table t(a int)") - tk.MustExec("alter table t change a b int") - tk.MustExec("alter table t change b c bigint") - c.Assert(tk.ExecToErr("alter table t change c d varchar(100)"), NotNil) -} - func (s *testSuiteP1) TestChangePumpAndDrainer(c *C) { tk := testkit.NewTestKit(c, s.store) // change pump or drainer's state need connect to etcd @@ -350,7 +346,17 @@ func (s *testSuiteP1) TestShow(c *C) { "Trigger Tables To use triggers", "Create tablespace Server Admin To create/alter/drop tablespaces", "Update Tables To update existing rows", - "Usage Server Admin No privileges - allow connect only")) + "Usage Server Admin No privileges - allow connect only", + "BACKUP_ADMIN Server Admin ", + "RESTORE_ADMIN Server Admin ", + "SYSTEM_VARIABLES_ADMIN Server Admin ", + "ROLE_ADMIN Server Admin ", + "CONNECTION_ADMIN Server Admin ", + "RESTRICTED_TABLES_ADMIN Server Admin ", + "RESTRICTED_STATUS_ADMIN Server Admin ", + "RESTRICTED_VARIABLES_ADMIN Server Admin ", + "RESTRICTED_USER_ADMIN Server Admin ", + )) c.Assert(len(tk.MustQuery("show table status").Rows()), Equals, 1) } @@ -2258,8 +2264,6 @@ func (s *testSuiteP2) TestSQLMode(c *C) { tk.MustExec("set sql_mode = 'STRICT_TRANS_TABLES'") tk.MustExec("set @@global.sql_mode = ''") - // Disable global variable cache, so load global session variable take effect immediate. - s.domain.GetGlobalVarsCache().Disable() tk2 := testkit.NewTestKit(c, s.store) tk2.MustExec("use test") tk2.MustExec("drop table if exists t2") @@ -2338,14 +2342,14 @@ func (s *testSuiteP2) TestIsPointGet(c *C) { "select * from help_topic where help_topic_id=1": true, "select * from help_topic where help_category_id=1": false, } - infoSchema := infoschema.GetInfoSchema(ctx) for sqlStr, result := range tests { stmtNode, err := s.ParseOneStmt(sqlStr, "", "") c.Check(err, IsNil) - err = plannercore.Preprocess(ctx, stmtNode, infoSchema) + preprocessorReturn := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(ctx, stmtNode, plannercore.WithPreprocessorReturn(preprocessorReturn)) c.Check(err, IsNil) - p, _, err := planner.Optimize(context.TODO(), ctx, stmtNode, infoSchema) + p, _, err := planner.Optimize(context.TODO(), ctx, stmtNode, preprocessorReturn.InfoSchema) c.Check(err, IsNil) ret, err := plannercore.IsPointGetWithPKOrUniqueKeyByAutoCommit(ctx, p) c.Assert(err, IsNil) @@ -2370,13 +2374,13 @@ func (s *testSuiteP2) TestClusteredIndexIsPointGet(c *C) { "select * from t where a='x' and c='x'": true, "select * from t where a='x' and c='x' and b=1": false, } - infoSchema := infoschema.GetInfoSchema(ctx) for sqlStr, result := range tests { stmtNode, err := s.ParseOneStmt(sqlStr, "", "") c.Check(err, IsNil) - err = plannercore.Preprocess(ctx, stmtNode, infoSchema) + preprocessorReturn := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(ctx, stmtNode, plannercore.WithPreprocessorReturn(preprocessorReturn)) c.Check(err, IsNil) - p, _, err := planner.Optimize(context.TODO(), ctx, stmtNode, infoSchema) + p, _, err := planner.Optimize(context.TODO(), ctx, stmtNode, preprocessorReturn.InfoSchema) c.Check(err, IsNil) ret, err := plannercore.IsPointGetWithPKOrUniqueKeyByAutoCommit(ctx, p) c.Assert(err, IsNil) @@ -2686,11 +2690,11 @@ func (s *testSuiteP2) TestHistoryRead(c *C) { // SnapshotTS Is not updated if check failed. c.Assert(tk.Se.GetSessionVars().SnapshotTS, Equals, uint64(0)) - curVer1, _ := s.store.CurrentVersion(oracle.GlobalTxnScope) + curVer1, _ := s.store.CurrentVersion(kv.GlobalTxnScope) time.Sleep(time.Millisecond) snapshotTime := time.Now() time.Sleep(time.Millisecond) - curVer2, _ := s.store.CurrentVersion(oracle.GlobalTxnScope) + curVer2, _ := s.store.CurrentVersion(kv.GlobalTxnScope) tk.MustExec("insert history_read values (2)") tk.MustQuery("select * from history_read").Check(testkit.Rows("1", "2")) tk.MustExec("set @@tidb_snapshot = '" + snapshotTime.Format("2006-01-02 15:04:05.999999") + "'") @@ -2719,7 +2723,7 @@ func (s *testSuiteP2) TestHistoryRead(c *C) { tk.MustQuery("select * from history_read order by a").Check(testkit.Rows("2 ", "4 ", "8 8", "9 9")) tk.MustExec("set @@tidb_snapshot = '" + snapshotTime.Format("2006-01-02 15:04:05.999999") + "'") tk.MustQuery("select * from history_read order by a").Check(testkit.Rows("2", "4")) - tsoStr := strconv.FormatUint(oracle.EncodeTSO(snapshotTime.UnixNano()/int64(time.Millisecond)), 10) + tsoStr := strconv.FormatUint(oracle.GoTimeToTS(snapshotTime), 10) tk.MustExec("set @@tidb_snapshot = '" + tsoStr + "'") tk.MustQuery("select * from history_read order by a").Check(testkit.Rows("2", "4")) @@ -3059,9 +3063,10 @@ func (s *testSerialSuite) TestTiDBLastTxnInfoCommitMode(c *C) { c.Assert(rows[0][1], Equals, "false") c.Assert(rows[0][2], Equals, "false") - config.UpdateGlobal(func(conf *config.Config) { - conf.TiKVClient.AsyncCommit.SafeWindow = 0 - }) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/invalidMaxCommitTS", "return"), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/invalidMaxCommitTS"), IsNil) + }() tk.MustExec("set @@tidb_enable_async_commit = 1") tk.MustExec("set @@tidb_enable_1pc = 0") @@ -3271,7 +3276,7 @@ const ( type checkRequestClient struct { tikv.Client - priority pb.CommandPri + priority kvrpcpb.CommandPri lowPriorityCnt uint32 mu struct { sync.RWMutex @@ -3280,12 +3285,12 @@ type checkRequestClient struct { } } -func (c *checkRequestClient) setCheckPriority(priority pb.CommandPri) { +func (c *checkRequestClient) setCheckPriority(priority kvrpcpb.CommandPri) { atomic.StoreInt32((*int32)(&c.priority), int32(priority)) } -func (c *checkRequestClient) getCheckPriority() pb.CommandPri { - return (pb.CommandPri)(atomic.LoadInt32((*int32)(&c.priority))) +func (c *checkRequestClient) getCheckPriority() kvrpcpb.CommandPri { + return (kvrpcpb.CommandPri)(atomic.LoadInt32((*int32)(&c.priority))) } func (c *checkRequestClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) { @@ -3309,7 +3314,7 @@ func (c *checkRequestClient) SendRequest(ctx context.Context, addr string, req * return nil, errors.New("fail to set priority") } } else if req.Type == tikvrpc.CmdPrewrite { - if c.getCheckPriority() == pb.CommandPri_Low { + if c.getCheckPriority() == kvrpcpb.CommandPri_Low { atomic.AddUint32(&c.lowPriorityCnt, 1) } } @@ -3397,7 +3402,7 @@ func (s *testSuite2) TestAddIndexPriority(c *C) { cli.mu.checkFlags = checkDDLAddIndexPriority cli.mu.Unlock() - cli.setCheckPriority(pb.CommandPri_Low) + cli.setCheckPriority(kvrpcpb.CommandPri_Low) tk.MustExec("alter table t1 add index t1_index (id);") c.Assert(atomic.LoadUint32(&cli.lowPriorityCnt) > 0, IsTrue) @@ -3413,7 +3418,7 @@ func (s *testSuite2) TestAddIndexPriority(c *C) { cli.mu.checkFlags = checkDDLAddIndexPriority cli.mu.Unlock() - cli.setCheckPriority(pb.CommandPri_Normal) + cli.setCheckPriority(kvrpcpb.CommandPri_Normal) tk.MustExec("alter table t1 add index t1_index (id);") cli.mu.Lock() @@ -3427,7 +3432,7 @@ func (s *testSuite2) TestAddIndexPriority(c *C) { cli.mu.checkFlags = checkDDLAddIndexPriority cli.mu.Unlock() - cli.setCheckPriority(pb.CommandPri_High) + cli.setCheckPriority(kvrpcpb.CommandPri_High) tk.MustExec("alter table t1 add index t1_index (id);") cli.mu.Lock() @@ -3833,7 +3838,7 @@ func (s *testSuite) TestCheckIndex(c *C) { c.Assert(err, IsNil) _, err = se.Execute(context.Background(), "admin check index t c") c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "handle 3, index:types.Datum{k:0x1, decimal:0x0, length:0x0, i:30, collation:\"\", b:[]uint8(nil), x:interface {}(nil)} != record:") + c.Assert(err.Error(), Equals, "[executor:8133]handle 3, index:types.Datum{k:0x1, decimal:0x0, length:0x0, i:30, collation:\"\", b:[]uint8(nil), x:interface {}(nil)} != record:") // set data to: // index data (handle, data): (1, 10), (2, 20), (3, 30), (4, 40) @@ -3852,9 +3857,9 @@ func (s *testSuite) TestCheckIndex(c *C) { // table data (handle, data): (1, 10), (2, 20), (4, 40) txn, err = s.store.Begin() c.Assert(err, IsNil) - err = idx.Delete(sc, txn.GetUnionStore(), types.MakeDatums(int64(30)), kv.IntHandle(3)) + err = idx.Delete(sc, txn, types.MakeDatums(int64(30)), kv.IntHandle(3)) c.Assert(err, IsNil) - err = idx.Delete(sc, txn.GetUnionStore(), types.MakeDatums(int64(20)), kv.IntHandle(2)) + err = idx.Delete(sc, txn, types.MakeDatums(int64(20)), kv.IntHandle(2)) c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) @@ -5186,6 +5191,24 @@ func (s *testSplitTable) TestShowTableRegion(c *C) { // Test show table regions. tk.MustQuery(`split table t_regions between (-10000) and (10000) regions 4;`).Check(testkit.Rows("4 1")) re := tk.MustQuery("show table t_regions regions") + + // Test show table regions and split table on temporary table. + tk.MustExec("drop table if exists t_regions_temporary_table") + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("create global temporary table t_regions_temporary_table (a int key, b int, c int, index idx(b), index idx2(c)) ON COMMIT DELETE ROWS;") + // Test show table regions. + _, err = tk.Exec("show table t_regions_temporary_table regions") + c.Assert(err.Error(), Equals, plannercore.ErrOptOnTemporaryTable.GenWithStackByArgs("show table regions").Error()) + // Test split table. + _, err = tk.Exec("split table t_regions_temporary_table between (-10000) and (10000) regions 4;") + c.Assert(err.Error(), Equals, plannercore.ErrOptOnTemporaryTable.GenWithStackByArgs("split table").Error()) + _, err = tk.Exec("split partition table t_regions_temporary_table partition (p1,p2) index idx between (0) and (20000) regions 2;") + c.Assert(err.Error(), Equals, plannercore.ErrOptOnTemporaryTable.GenWithStackByArgs("split table").Error()) + tk.MustExec("drop table if exists t_regions_temporary_table") + // Test pre split regions + _, err = tk.Exec("create global temporary table temporary_table_pre_split(id int ) pre_split_regions=2 ON COMMIT DELETE ROWS;") + c.Assert(err.Error(), Equals, ddl.ErrOptOnTemporaryTable.GenWithStackByArgs("pre split regions").Error()) + rows := re.Rows() // Table t_regions should have 5 regions now. // 4 regions to store record data. @@ -5697,24 +5720,15 @@ func (s *testRecoverTable) TearDownSuite(c *C) { s.dom.Close() } -func (s *testRecoverTable) TestRecoverTable(c *C) { - c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil) - defer func() { - err := failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange") - c.Assert(err, IsNil) - }() - tk := testkit.NewTestKit(c, s.store) - tk.MustExec("create database if not exists test_recover") - tk.MustExec("use test_recover") - tk.MustExec("drop table if exists t_recover") - tk.MustExec("create table t_recover (a int);") - defer func(originGC bool) { +func (s *testRecoverTable) mockGC(tk *testkit.TestKit) (string, string, string, func()) { + originGC := ddl.IsEmulatorGCEnable() + resetGC := func() { if originGC { ddl.EmulatorGCEnable() } else { ddl.EmulatorGCDisable() } - }(ddl.IsEmulatorGCEnable()) + } // disable emulator GC. // Otherwise emulator GC will delete table record as soon as possible after execute drop table ddl. @@ -5727,6 +5741,23 @@ func (s *testRecoverTable) TestRecoverTable(c *C) { UPDATE variable_value = '%[1]s'` // clear GC variables first. tk.MustExec("delete from mysql.tidb where variable_name in ( 'tikv_gc_safe_point','tikv_gc_enable' )") + return timeBeforeDrop, timeAfterDrop, safePointSQL, resetGC +} + +func (s *testRecoverTable) TestRecoverTable(c *C) { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil) + defer func() { + err := failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange") + c.Assert(err, IsNil) + }() + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database if not exists test_recover") + tk.MustExec("use test_recover") + tk.MustExec("drop table if exists t_recover") + tk.MustExec("create table t_recover (a int);") + + timeBeforeDrop, timeAfterDrop, safePointSQL, resetGC := s.mockGC(tk) + defer resetGC() tk.MustExec("insert into t_recover values (1),(2),(3)") tk.MustExec("drop table t_recover") @@ -5801,7 +5832,7 @@ func (s *testRecoverTable) TestRecoverTable(c *C) { // Test for recover one table multiple time. tk.MustExec("drop table t_recover") tk.MustExec("flashback table t_recover to t_recover_tmp") - _, err = tk.Exec(fmt.Sprintf("recover table t_recover")) + _, err = tk.Exec("recover table t_recover") c.Assert(infoschema.ErrTableExists.Equal(err), IsTrue) gcEnable, err := gcutil.CheckGCEnable(tk.Se) @@ -5819,24 +5850,10 @@ func (s *testRecoverTable) TestFlashbackTable(c *C) { tk.MustExec("use test_flashback") tk.MustExec("drop table if exists t_flashback") tk.MustExec("create table t_flashback (a int);") - defer func(originGC bool) { - if originGC { - ddl.EmulatorGCEnable() - } else { - ddl.EmulatorGCDisable() - } - }(ddl.IsEmulatorGCEnable()) - // Disable emulator GC. - // Otherwise emulator GC will delete table record as soon as possible after execute drop table ddl. - ddl.EmulatorGCDisable() - gcTimeFormat := "20060102-15:04:05 -0700 MST" - timeBeforeDrop := time.Now().Add(0 - 48*60*60*time.Second).Format(gcTimeFormat) - safePointSQL := `INSERT HIGH_PRIORITY INTO mysql.tidb VALUES ('tikv_gc_safe_point', '%[1]s', '') - ON DUPLICATE KEY - UPDATE variable_value = '%[1]s'` - // Clear GC variables first. - tk.MustExec("delete from mysql.tidb where variable_name in ( 'tikv_gc_safe_point','tikv_gc_enable' )") + timeBeforeDrop, _, safePointSQL, resetGC := s.mockGC(tk) + defer resetGC() + // Set GC safe point tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop)) // Set GC enable. @@ -5882,7 +5899,7 @@ func (s *testRecoverTable) TestFlashbackTable(c *C) { tk.MustQuery("select a,_tidb_rowid from t_flashback2;").Check(testkit.Rows("1 1", "2 2", "3 3", "4 5001", "5 5002", "6 5003", "7 10001", "8 10002", "9 10003")) // Test for flashback one table multiple time. - _, err = tk.Exec(fmt.Sprintf("flashback table t_flashback to t_flashback4")) + _, err = tk.Exec("flashback table t_flashback to t_flashback4") c.Assert(infoschema.ErrTableExists.Equal(err), IsTrue) // Test for flashback truncated table to new table. @@ -5939,6 +5956,24 @@ func (s *testRecoverTable) TestFlashbackTable(c *C) { tk.MustQuery("select a from t order by a").Check(testkit.Rows("1", "2", "3")) } +func (s *testRecoverTable) TestRecoverTempTable(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database if not exists test_recover") + tk.MustExec("use test_recover") + tk.MustExec("drop table if exists t_recover") + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("create global temporary table t_recover (a int) on commit delete rows;") + + timeBeforeDrop, _, safePointSQL, resetGC := s.mockGC(tk) + defer resetGC() + // Set GC safe point + tk.MustExec(fmt.Sprintf(safePointSQL, timeBeforeDrop)) + + tk.MustExec("drop table t_recover") + tk.MustGetErrCode("recover table t_recover;", errno.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("flashback table t_recover;", errno.ErrUnsupportedDDLOperation) +} + func (s *testSuiteP2) TestPointGetPreparedPlan(c *C) { tk1 := testkit.NewTestKit(c, s.store) tk1.MustExec("drop database if exists ps_text") @@ -6978,7 +7013,7 @@ func (s *testSlowQuery) TestSlowQuerySensitiveQuery(c *C) { originCfg := config.GetGlobalConfig() newCfg := *originCfg - f, err := ioutil.TempFile("", "tidb-slow-*.log") + f, err := os.CreateTemp("", "tidb-slow-*.log") c.Assert(err, IsNil) f.Close() newCfg.Log.SlowQueryFile = f.Name() @@ -7012,7 +7047,7 @@ func (s *testSlowQuery) TestSlowQueryPrepared(c *C) { originCfg := config.GetGlobalConfig() newCfg := *originCfg - f, err := ioutil.TempFile("", "tidb-slow-*.log") + f, err := os.CreateTemp("", "tidb-slow-*.log") c.Assert(err, IsNil) f.Close() newCfg.Log.SlowQueryFile = f.Name() @@ -7048,7 +7083,7 @@ func (s *testSlowQuery) TestSlowQueryPrepared(c *C) { func (s *testSlowQuery) TestLogSlowLogIndex(c *C) { tk := testkit.NewTestKit(c, s.store) - f, err := ioutil.TempFile("", "tidb-slow-*.log") + f, err := os.CreateTemp("", "tidb-slow-*.log") c.Assert(err, IsNil) f.Close() @@ -7074,7 +7109,7 @@ func (s *testSlowQuery) TestLogSlowLogIndex(c *C) { func (s *testSlowQuery) TestSlowQuery(c *C) { tk := testkit.NewTestKit(c, s.store) - f, err := ioutil.TempFile("", "tidb-slow-*.log") + f, err := os.CreateTemp("", "tidb-slow-*.log") c.Assert(err, IsNil) _, err = f.WriteString(` # Time: 2020-10-13T20:08:13.970563+08:00 @@ -8128,10 +8163,454 @@ func (s *testSerialSuite) TestIssue24210(c *C) { // for SelectionExec c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/mockSelectionExecBaseExecutorOpenReturnedError", `return(true)`), IsNil) - _, err = tk.Exec("select * from (select 1 as a) t where a > 0") + _, err = tk.Exec("select * from (select rand() as a) t where a > 0") c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "mock SelectionExec.baseExecutor.Open returned error") err = failpoint.Disable("github.com/pingcap/tidb/executor/mockSelectionExecBaseExecutorOpenReturnedError") c.Assert(err, IsNil) +} + +func (s *testSerialSuite) TestDeadlockTable(c *C) { + deadlockhistory.GlobalDeadlockHistory.Clear() + deadlockhistory.GlobalDeadlockHistory.Resize(10) + occurTime := time.Date(2021, 5, 10, 1, 2, 3, 456789000, time.Local) + rec := &deadlockhistory.DeadlockRecord{ + OccurTime: occurTime, + IsRetryable: false, + WaitChain: []deadlockhistory.WaitChainItem{ + { + TryLockTxn: 101, + SQLDigest: "aabbccdd", + Key: []byte("k1"), + AllSQLDigests: nil, + TxnHoldingLock: 102, + }, + { + TryLockTxn: 102, + SQLDigest: "ddccbbaa", + Key: []byte("k2"), + AllSQLDigests: []string{"sql1"}, + TxnHoldingLock: 101, + }, + }, + } + deadlockhistory.GlobalDeadlockHistory.Push(rec) + + occurTime2 := time.Date(2022, 6, 11, 2, 3, 4, 987654000, time.Local) + rec2 := &deadlockhistory.DeadlockRecord{ + OccurTime: occurTime2, + IsRetryable: true, + WaitChain: []deadlockhistory.WaitChainItem{ + { + TryLockTxn: 201, + AllSQLDigests: []string{}, + TxnHoldingLock: 202, + }, + { + TryLockTxn: 202, + AllSQLDigests: []string{"sql1", "sql2, sql3"}, + TxnHoldingLock: 203, + }, + { + TryLockTxn: 203, + TxnHoldingLock: 201, + }, + }, + } + deadlockhistory.GlobalDeadlockHistory.Push(rec2) + + // `Push` sets the record's ID, and ID in a single DeadlockHistory is monotonically increasing. We must get it here + // to know what it is. + id1 := strconv.FormatUint(rec.ID, 10) + id2 := strconv.FormatUint(rec2.ID, 10) + + tk := testkit.NewTestKit(c, s.store) + tk.MustQuery("select * from information_schema.deadlocks").Check( + testutil.RowsWithSep("/", + id1+"/2021-05-10 01:02:03.456789/0/101/aabbccdd/6B31/102", + id1+"/2021-05-10 01:02:03.456789/0/102/ddccbbaa/6B32/101", + id2+"/2022-06-11 02:03:04.987654/1/201///202", + id2+"/2022-06-11 02:03:04.987654/1/202///203", + id2+"/2022-06-11 02:03:04.987654/1/203///201", + )) +} + +func (s *testSuite1) TestTemporaryTableNoPessimisticLock(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("use test") + tk.MustExec("drop table if exists t;") + tk.MustExec("create global temporary table t (a int primary key, b int) on commit delete rows") + tk.MustExec("insert into t values (1, 1)") + + // Do something on the temporary table, pessimistic transaction mode. + tk.MustExec("begin pessimistic") + tk.MustExec("insert into t values (2, 2)") + tk.MustExec("update t set b = b + 1 where a = 1") + tk.MustExec("delete from t where a > 1") + tk.MustQuery("select count(*) from t where b >= 2 for update") + + // Get the temporary table ID. + schema := tk.Se.GetInfoSchema().(infoschema.InfoSchema) + tbl, err := schema.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + c.Assert(err, IsNil) + meta := tbl.Meta() + c.Assert(meta.TempTableType, Equals, model.TempTableGlobal) + + // Scan the table range to check there is no lock. + // It's better to use the rawkv client, but the txnkv client should also works. + // If there is a lock, the txnkv client should have reported the lock error. + txn, err := s.store.Begin() + c.Assert(err, IsNil) + seekKey := tablecodec.EncodeTablePrefix(meta.ID) + endKey := tablecodec.EncodeTablePrefix(meta.ID + 1) + scanner, err := txn.Iter(seekKey, endKey) + c.Assert(err, IsNil) + for scanner.Valid() { + // No lock written to TiKV here. + c.FailNow() + } + + tk.MustExec("rollback") +} + +func (s testSerialSuite) TestExprBlackListForEnum(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("use test") + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(a enum('a','b','c'), b enum('a','b','c'), c int, index idx(b,a));") + tk.MustExec("insert into t values(1,1,1),(2,2,2),(3,3,3);") + + checkFuncPushDown := func(rows [][]interface{}, keyWord string) bool { + for _, line := range rows { + // Agg/Expr push down + if line[2].(string) == "cop[tikv]" && strings.Contains(line[4].(string), keyWord) { + return true + } + // access index + if line[2].(string) == "cop[tikv]" && strings.Contains(line[3].(string), keyWord) { + return true + } + } + return false + } + + // Test agg(enum) push down + tk.MustExec("insert into mysql.expr_pushdown_blacklist(name) values('enum');") + tk.MustExec("admin reload expr_pushdown_blacklist;") + rows := tk.MustQuery("desc format='brief' select /*+ HASH_AGG() */ max(a) from t;").Rows() + c.Assert(checkFuncPushDown(rows, "max"), IsFalse) + rows = tk.MustQuery("desc format='brief' select /*+ STREAM_AGG() */ max(a) from t;").Rows() + c.Assert(checkFuncPushDown(rows, "max"), IsFalse) + + tk.MustExec("delete from mysql.expr_pushdown_blacklist;") + tk.MustExec("admin reload expr_pushdown_blacklist;") + rows = tk.MustQuery("desc format='brief' select /*+ HASH_AGG() */ max(a) from t;").Rows() + c.Assert(checkFuncPushDown(rows, "max"), IsTrue) + rows = tk.MustQuery("desc format='brief' select /*+ STREAM_AGG() */ max(a) from t;").Rows() + c.Assert(checkFuncPushDown(rows, "max"), IsTrue) + + // Test expr(enum) push down + tk.MustExec("insert into mysql.expr_pushdown_blacklist(name) values('enum');") + tk.MustExec("admin reload expr_pushdown_blacklist;") + rows = tk.MustQuery("desc format='brief' select * from t where a + b;").Rows() + c.Assert(checkFuncPushDown(rows, "plus"), IsFalse) + rows = tk.MustQuery("desc format='brief' select * from t where a + b;").Rows() + c.Assert(checkFuncPushDown(rows, "plus"), IsFalse) + + tk.MustExec("delete from mysql.expr_pushdown_blacklist;") + tk.MustExec("admin reload expr_pushdown_blacklist;") + rows = tk.MustQuery("desc format='brief' select * from t where a + b;").Rows() + c.Assert(checkFuncPushDown(rows, "plus"), IsTrue) + rows = tk.MustQuery("desc format='brief' select * from t where a + b;").Rows() + c.Assert(checkFuncPushDown(rows, "plus"), IsTrue) + + // Test enum index + tk.MustExec("insert into mysql.expr_pushdown_blacklist(name) values('enum');") + tk.MustExec("admin reload expr_pushdown_blacklist;") + rows = tk.MustQuery("desc format='brief' select * from t where b = 1;").Rows() + c.Assert(checkFuncPushDown(rows, "index:idx(b)"), IsFalse) + rows = tk.MustQuery("desc format='brief' select * from t where b = 'a';").Rows() + c.Assert(checkFuncPushDown(rows, "index:idx(b)"), IsFalse) + rows = tk.MustQuery("desc format='brief' select * from t where b > 1;").Rows() + c.Assert(checkFuncPushDown(rows, "index:idx(b)"), IsFalse) + rows = tk.MustQuery("desc format='brief' select * from t where b > 'a';").Rows() + c.Assert(checkFuncPushDown(rows, "index:idx(b)"), IsFalse) + + tk.MustExec("delete from mysql.expr_pushdown_blacklist;") + tk.MustExec("admin reload expr_pushdown_blacklist;") + rows = tk.MustQuery("desc format='brief' select * from t where b = 1 and a = 1;").Rows() + c.Assert(checkFuncPushDown(rows, "index:idx(b, a)"), IsTrue) + rows = tk.MustQuery("desc format='brief' select * from t where b = 'a' and a = 'a';").Rows() + c.Assert(checkFuncPushDown(rows, "index:idx(b, a)"), IsTrue) + rows = tk.MustQuery("desc format='brief' select * from t where b = 1 and a > 1;").Rows() + c.Assert(checkFuncPushDown(rows, "index:idx(b, a)"), IsTrue) + rows = tk.MustQuery("desc format='brief' select * from t where b = 1 and a > 'a'").Rows() + c.Assert(checkFuncPushDown(rows, "index:idx(b, a)"), IsTrue) +} + +func (s testSerialSuite) TestTemporaryTableNoNetwork(c *C) { + // Test that table reader/index reader/index lookup on the temporary table do not need to visit TiKV. + tk := testkit.NewTestKit(c, s.store) + tk1 := testkit.NewTestKit(c, s.store) + + tk.MustExec("use test") + tk1.MustExec("use test") + tk.MustExec("create table normal (id int, a int, index(a))") + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("create global temporary table tmp_t (id int, a int, index(a)) on commit delete rows") + + tk.MustExec("begin") + tk.MustExec("insert into tmp_t values (1, 1)") + tk.MustExec("insert into tmp_t values (2, 2)") + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/unistore/rpcServerBusy", "return(true)"), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/unistore/rpcServerBusy"), IsNil) + }() + + // Make sure the fail point works. + // With that failpoint, all requests to the TiKV is discard. + rs, err := tk1.Exec("select * from normal") + c.Assert(err, IsNil) + blocked := make(chan struct{}) + ctx, cancelFunc := context.WithCancel(context.Background()) + go func() { + _, err := session.ResultSetToStringSlice(ctx, tk1.Se, rs) + blocked <- struct{}{} + c.Assert(err, NotNil) + }() + select { + case <-blocked: + c.Error("The query should block when the failpoint is enabled") + case <-time.After(200 * time.Millisecond): + } + cancelFunc() + + // Check the temporary table do not send request to TiKV. + // Table reader + tk.HasPlan("select * from tmp_t", "TableReader") + tk.MustQuery("select * from tmp_t").Check(testkit.Rows("1 1", "2 2")) + // Index reader + tk.HasPlan("select /*+ USE_INDEX(tmp_t, a) */ a from tmp_t", "IndexReader") + tk.MustQuery("select /*+ USE_INDEX(tmp_t, a) */ a from tmp_t").Check(testkit.Rows("1", "2")) + // Index lookup + tk.HasPlan("select id from tmp_t where a = 1", "IndexLookUp") + tk.MustQuery("select id from tmp_t where a = 1").Check(testkit.Rows("1")) + + tk.MustExec("rollback") +} + +func (s *testResourceTagSuite) TestResourceGroupTag(c *C) { + if israce.RaceEnabled { + c.Skip("unstable, skip it and fix it before 20210622") + } + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(a int, b int, unique index idx(a));") + tbInfo := testGetTableByName(c, tk.Se, "test", "t") + + // Enable Top SQL + variable.TopSQLVariable.Enable.Store(true) + variable.TopSQLVariable.AgentAddress.Store("mock-agent") + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/unistore/unistoreRPCClientSendHook", `return(true)`), IsNil) + defer failpoint.Disable("github.com/pingcap/tidb/store/mockstore/unistore/unistoreRPCClientSendHook") + + var sqlDigest, planDigest *parser.Digest + checkFn := func() {} + unistore.UnistoreRPCClientSendHook = func(req *tikvrpc.Request) { + var startKey []byte + var ctx *kvrpcpb.Context + switch req.Type { + case tikvrpc.CmdGet: + request := req.Get() + startKey = request.Key + ctx = request.Context + case tikvrpc.CmdBatchGet: + request := req.BatchGet() + startKey = request.Keys[0] + ctx = request.Context + case tikvrpc.CmdPrewrite: + request := req.Prewrite() + startKey = request.Mutations[0].Key + ctx = request.Context + case tikvrpc.CmdCommit: + request := req.Commit() + startKey = request.Keys[0] + ctx = request.Context + case tikvrpc.CmdCop: + request := req.Cop() + startKey = request.Ranges[0].Start + ctx = request.Context + case tikvrpc.CmdPessimisticLock: + request := req.PessimisticLock() + startKey = request.PrimaryLock + ctx = request.Context + } + tid := tablecodec.DecodeTableID(startKey) + if tid != tbInfo.Meta().ID { + return + } + if ctx == nil { + return + } + tag := &tipb.ResourceGroupTag{} + err := tag.Unmarshal(ctx.ResourceGroupTag) + c.Assert(err, IsNil) + sqlDigest = parser.NewDigest(tag.SqlDigest) + planDigest = parser.NewDigest(tag.PlanDigest) + checkFn() + } + + resetVars := func() { + sqlDigest = parser.NewDigest(nil) + planDigest = parser.NewDigest(nil) + } + + cases := []struct { + sql string + ignore bool + }{ + {sql: "insert into t values(1,1),(2,2),(3,3)"}, + {sql: "select * from t use index (idx) where a=1"}, + {sql: "select * from t use index (idx) where a in (1,2,3)"}, + {sql: "select * from t use index (idx) where a>1"}, + {sql: "select * from t where b>1"}, + {sql: "begin pessimistic", ignore: true}, + {sql: "insert into t values(4,4)"}, + {sql: "commit", ignore: true}, + {sql: "update t set a=5,b=5 where a=5"}, + {sql: "replace into t values(6,6)"}, + } + for _, ca := range cases { + resetVars() + commentf := Commentf("%v", ca.sql) + + _, expectSQLDigest := parser.NormalizeDigest(ca.sql) + var expectPlanDigest *parser.Digest + checkCnt := 0 + checkFn = func() { + if ca.ignore { + return + } + if expectPlanDigest == nil { + info := tk.Se.ShowProcess() + c.Assert(info, NotNil) + p, ok := info.Plan.(plannercore.Plan) + c.Assert(ok, IsTrue) + _, expectPlanDigest = plannercore.NormalizePlan(p) + } + c.Assert(sqlDigest.String(), Equals, expectSQLDigest.String(), commentf) + c.Assert(planDigest.String(), Equals, expectPlanDigest.String()) + checkCnt++ + } + + if strings.HasPrefix(ca.sql, "select") { + tk.MustQuery(ca.sql) + } else { + tk.MustExec(ca.sql) + } + if ca.ignore { + continue + } + c.Assert(checkCnt > 0, IsTrue, commentf) + } +} + +func (s *testStaleTxnSuite) TestInvalidReadTemporaryTable(c *C) { + tk := testkit.NewTestKit(c, s.store) + // For mocktikv, safe point is not initialized, we manually insert it for snapshot to use. + safePointName := "tikv_gc_safe_point" + safePointValue := "20160102-15:04:05 -0700" + safePointComment := "All versions after safe point can be accessed. (DO NOT EDIT)" + updateSafePoint := fmt.Sprintf(`INSERT INTO mysql.tidb VALUES ('%[1]s', '%[2]s', '%[3]s') + ON DUPLICATE KEY + UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment) + tk.MustExec(updateSafePoint) + + tk.MustExec("set @@tidb_enable_global_temporary_table=1") + tk.MustExec("use test") + tk.MustExec("drop table if exists tmp1") + tk.MustExec("create global temporary table tmp1 " + + "(id int not null primary key, code int not null, value int default null, unique key code(code))" + + "on commit delete rows") + + // sleep 1us to make test stale + time.Sleep(time.Microsecond) + + tk.MustGetErrMsg("select * from tmp1 tablesample regions()", "TABLESAMPLE clause can not be applied to temporary tables") + + queries := []struct { + sql string + }{ + { + sql: "select * from tmp1 where id=1", + }, + { + sql: "select * from tmp1 where code=1", + }, + { + sql: "select * from tmp1 where id in (1, 2, 3)", + }, + { + sql: "select * from tmp1 where code in (1, 2, 3)", + }, + { + sql: "select * from tmp1 where id > 1", + }, + { + sql: "select /*+use_index(tmp1, code)*/ * from tmp1 where code > 1", + }, + { + sql: "select /*+use_index(tmp1, code)*/ code from tmp1 where code > 1", + }, + { + sql: "select /*+ use_index_merge(tmp1, primary, code) */ * from tmp1 where id > 1 or code > 2", + }, + } + + addStaleReadToSQL := func(sql string) string { + idx := strings.Index(sql, " where ") + if idx < 0 { + return "" + } + return sql[0:idx] + " as of timestamp NOW(6)" + sql[idx:] + } + + for _, query := range queries { + sql := addStaleReadToSQL(query.sql) + if sql != "" { + tk.MustGetErrMsg(sql, "can not stale read temporary table") + } + } + + tk.MustExec("start transaction read only as of timestamp NOW(6)") + for _, query := range queries { + tk.MustGetErrMsg(query.sql, "can not stale read temporary table") + } + tk.MustExec("commit") + + for _, query := range queries { + tk.MustExec(query.sql) + } + + tk.MustExec("set transaction read only as of timestamp NOW(6)") + tk.MustExec("start transaction") + for _, query := range queries { + tk.MustGetErrMsg(query.sql, "can not stale read temporary table") + } + tk.MustExec("commit") + + for _, query := range queries { + tk.MustExec(query.sql) + } + + tk.MustExec("set @@tidb_snapshot=NOW(6)") + for _, query := range queries { + tk.MustGetErrMsg(query.sql, "can not read temporary table when 'tidb_snapshot' is set") + } } diff --git a/executor/explain_test.go b/executor/explain_test.go index 8a0b062ef68cb..a0ce1a1eb2e0c 100644 --- a/executor/explain_test.go +++ b/executor/explain_test.go @@ -207,6 +207,7 @@ func (s *testSuite2) TestExplainAnalyzeExecutionInfo(c *C) { s.checkExecutionInfo(c, tk, "explain analyze select * from t") s.checkExecutionInfo(c, tk, "explain analyze select k from t use index(k)") s.checkExecutionInfo(c, tk, "explain analyze select * from t use index(k)") + s.checkExecutionInfo(c, tk, "explain analyze with recursive cte(a) as (select 1 union select a + 1 from cte where a < 1000) select * from cte;") tk.MustExec("CREATE TABLE IF NOT EXISTS nation ( N_NATIONKEY BIGINT NOT NULL,N_NAME CHAR(25) NOT NULL,N_REGIONKEY BIGINT NOT NULL,N_COMMENT VARCHAR(152),PRIMARY KEY (N_NATIONKEY));") tk.MustExec("CREATE TABLE IF NOT EXISTS part ( P_PARTKEY BIGINT NOT NULL,P_NAME VARCHAR(55) NOT NULL,P_MFGR CHAR(25) NOT NULL,P_BRAND CHAR(10) NOT NULL,P_TYPE VARCHAR(25) NOT NULL,P_SIZE BIGINT NOT NULL,P_CONTAINER CHAR(10) NOT NULL,P_RETAILPRICE DECIMAL(15,2) NOT NULL,P_COMMENT VARCHAR(23) NOT NULL,PRIMARY KEY (P_PARTKEY));") @@ -320,9 +321,33 @@ func (s *testSuite1) TestCheckActRowsWithUnistore(c *C) { sql: "select count(*) from t_unistore_act_rows group by b", expected: []string{"2", "2", "2", "4"}, }, + { + sql: "with cte(a) as (select a from t_unistore_act_rows) select (select 1 from cte limit 1) from cte;", + expected: []string{"4", "4", "4", "4", "4"}, + }, } for _, test := range tests { checkActRows(c, tk, test.sql, test.expected) } } + +func (s *testSuite2) TestExplainAnalyzeCTEMemoryAndDiskInfo(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int)") + tk.MustExec("insert into t with recursive cte(a) as (select 1 union select a + 1 from cte where a < 1000) select * from cte;") + + rows := tk.MustQuery("explain analyze with recursive cte(a) as (select 1 union select a + 1 from cte where a < 1000)" + + " select * from cte, t;").Rows() + + c.Assert(rows[4][7].(string), Not(Equals), "N/A") + c.Assert(rows[4][8].(string), Equals, "0 Bytes") + + tk.MustExec("set @@tidb_mem_quota_query=10240;") + rows = tk.MustQuery("explain analyze with recursive cte(a) as (select 1 union select a + 1 from cte where a < 1000)" + + " select * from cte, t;").Rows() + + c.Assert(rows[4][7].(string), Not(Equals), "N/A") + c.Assert(rows[4][8].(string), Not(Equals), "N/A") +} diff --git a/executor/explainfor_test.go b/executor/explainfor_test.go index a113200a925d8..46df545b1ff47 100644 --- a/executor/explainfor_test.go +++ b/executor/explainfor_test.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/parser/auth" "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" + txninfo "github.com/pingcap/tidb/session/txninfo" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/israce" @@ -38,6 +39,10 @@ type mockSessionManager1 struct { PS []*util.ProcessInfo } +func (msm *mockSessionManager1) ShowTxnList() []*txninfo.TxnInfo { + return nil +} + // ShowProcessList implements the SessionManager.ShowProcessList interface. func (msm *mockSessionManager1) ShowProcessList() map[uint64]*util.ProcessInfo { ret := make(map[uint64]*util.ProcessInfo) @@ -176,11 +181,11 @@ func (s *testSuite) TestExplainMemTablePredicate(c *C) { func (s *testSuite) TestExplainClusterTable(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustQuery(fmt.Sprintf("desc select * from information_schema.cluster_config where type in ('tikv', 'tidb')")).Check(testkit.Rows( + tk.MustQuery("desc select * from information_schema.cluster_config where type in ('tikv', 'tidb')").Check(testkit.Rows( `MemTableScan_5 10000.00 root table:CLUSTER_CONFIG node_types:["tidb","tikv"]`)) - tk.MustQuery(fmt.Sprintf("desc select * from information_schema.cluster_config where instance='192.168.1.7:2379'")).Check(testkit.Rows( + tk.MustQuery("desc select * from information_schema.cluster_config where instance='192.168.1.7:2379'").Check(testkit.Rows( `MemTableScan_5 10000.00 root table:CLUSTER_CONFIG instances:["192.168.1.7:2379"]`)) - tk.MustQuery(fmt.Sprintf("desc select * from information_schema.cluster_config where type='tidb' and instance='192.168.1.7:2379'")).Check(testkit.Rows( + tk.MustQuery("desc select * from information_schema.cluster_config where type='tidb' and instance='192.168.1.7:2379'").Check(testkit.Rows( `MemTableScan_5 10000.00 root table:CLUSTER_CONFIG node_types:["tidb"], instances:["192.168.1.7:2379"]`)) } @@ -198,11 +203,11 @@ func (s *testSuite) TestInspectionResultTable(c *C) { func (s *testSuite) TestInspectionRuleTable(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustQuery(fmt.Sprintf("desc select * from information_schema.inspection_rules where type='inspection'")).Check(testkit.Rows( + tk.MustQuery("desc select * from information_schema.inspection_rules where type='inspection'").Check(testkit.Rows( `MemTableScan_5 10000.00 root table:INSPECTION_RULES node_types:["inspection"]`)) - tk.MustQuery(fmt.Sprintf("desc select * from information_schema.inspection_rules where type='inspection' or type='summary'")).Check(testkit.Rows( + tk.MustQuery("desc select * from information_schema.inspection_rules where type='inspection' or type='summary'").Check(testkit.Rows( `MemTableScan_5 10000.00 root table:INSPECTION_RULES node_types:["inspection","summary"]`)) - tk.MustQuery(fmt.Sprintf("desc select * from information_schema.inspection_rules where type='inspection' and type='summary'")).Check(testkit.Rows( + tk.MustQuery("desc select * from information_schema.inspection_rules where type='inspection' and type='summary'").Check(testkit.Rows( `MemTableScan_5 10000.00 root table:INSPECTION_RULES skip_request: true`)) } @@ -350,12 +355,12 @@ func (s *testPrepareSerialSuite) TestExplainDotForQuery(c *C) { func (s *testSuite) TestExplainTableStorage(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustQuery(fmt.Sprintf("desc select * from information_schema.TABLE_STORAGE_STATS where TABLE_SCHEMA = 'information_schema'")).Check(testkit.Rows( - fmt.Sprintf("MemTableScan_5 10000.00 root table:TABLE_STORAGE_STATS schema:[\"information_schema\"]"))) - tk.MustQuery(fmt.Sprintf("desc select * from information_schema.TABLE_STORAGE_STATS where TABLE_NAME = 'schemata'")).Check(testkit.Rows( - fmt.Sprintf("MemTableScan_5 10000.00 root table:TABLE_STORAGE_STATS table:[\"schemata\"]"))) - tk.MustQuery(fmt.Sprintf("desc select * from information_schema.TABLE_STORAGE_STATS where TABLE_SCHEMA = 'information_schema' and TABLE_NAME = 'schemata'")).Check(testkit.Rows( - fmt.Sprintf("MemTableScan_5 10000.00 root table:TABLE_STORAGE_STATS schema:[\"information_schema\"], table:[\"schemata\"]"))) + tk.MustQuery("desc select * from information_schema.TABLE_STORAGE_STATS where TABLE_SCHEMA = 'information_schema'").Check(testkit.Rows( + "MemTableScan_5 10000.00 root table:TABLE_STORAGE_STATS schema:[\"information_schema\"]")) + tk.MustQuery("desc select * from information_schema.TABLE_STORAGE_STATS where TABLE_NAME = 'schemata'").Check(testkit.Rows( + "MemTableScan_5 10000.00 root table:TABLE_STORAGE_STATS table:[\"schemata\"]")) + tk.MustQuery("desc select * from information_schema.TABLE_STORAGE_STATS where TABLE_SCHEMA = 'information_schema' and TABLE_NAME = 'schemata'").Check(testkit.Rows( + "MemTableScan_5 10000.00 root table:TABLE_STORAGE_STATS schema:[\"information_schema\"], table:[\"schemata\"]")) } func (s *testSuite) TestInspectionSummaryTable(c *C) { diff --git a/executor/grant.go b/executor/grant.go index f6a8453ba72dd..49536dc79aa86 100644 --- a/executor/grant.go +++ b/executor/grant.go @@ -16,7 +16,6 @@ package executor import ( "context" "encoding/json" - "fmt" "strings" "github.com/pingcap/errors" @@ -74,7 +73,7 @@ func (e *GrantExec) Next(ctx context.Context, req *chunk.Chunk) error { // Make sure the table exist. if e.Level.Level == ast.GrantLevelTable { dbNameStr := model.NewCIStr(dbName) - schema := infoschema.GetInfoSchema(e.ctx) + schema := e.ctx.GetInfoSchema().(infoschema.InfoSchema) tbl, err := schema.TableByName(dbNameStr, model.NewCIStr(e.Level.TableName)) if err != nil { return err @@ -137,7 +136,7 @@ func (e *GrantExec) Next(ctx context.Context, req *chunk.Chunk) error { if !ok { return errors.Trace(ErrPasswordFormat) } - _, err := internalSession.(sqlexec.SQLExecutor).ExecuteInternal(ctx, `INSERT INTO %n.%n (Host, User, authentication_string) VALUES (%?, %?, %?);`, mysql.SystemDB, mysql.UserTable, user.User.Hostname, user.User.Username, pwd) + _, err := internalSession.(sqlexec.SQLExecutor).ExecuteInternal(ctx, `INSERT INTO %n.%n (Host, User, authentication_string, plugin) VALUES (%?, %?, %?, %?);`, mysql.SystemDB, mysql.UserTable, user.User.Hostname, user.User.Username, pwd, mysql.AuthNativePassword) if err != nil { return err } @@ -429,9 +428,6 @@ func (e *GrantExec) grantLevelPriv(priv *ast.PrivElem, user *ast.UserSpec, inter func (e *GrantExec) grantDynamicPriv(privName string, user *ast.UserSpec, internalSession sessionctx.Context) error { privName = strings.ToUpper(privName) - if !e.ctx.GetSessionVars().EnableDynamicPrivileges { - return fmt.Errorf("dynamic privileges is an experimental feature. Run 'SET tidb_enable_dynamic_privileges=1'") - } if e.Level.Level != ast.GrantLevelGlobal { // DYNAMIC can only be *.* return ErrIllegalPrivilegeLevel.GenWithStackByArgs(privName) } @@ -473,6 +469,12 @@ func (e *GrantExec) grantDBLevel(priv *ast.PrivElem, user *ast.UserSpec, interna if priv.Priv == mysql.UsagePriv { return nil } + for _, v := range mysql.StaticGlobalOnlyPrivs { + if v == priv.Priv { + return ErrWrongUsage.GenWithStackByArgs("DB GRANT", "GLOBAL PRIVILEGES") + } + } + dbName := e.Level.DBName if len(dbName) == 0 { dbName = e.ctx.GetSessionVars().CurrentDB diff --git a/executor/grant_test.go b/executor/grant_test.go index bb720b48b730b..8ccfac3533fb6 100644 --- a/executor/grant_test.go +++ b/executor/grant_test.go @@ -68,7 +68,7 @@ func (s *testSuite3) TestGrantDBScope(c *C) { createUserSQL := `CREATE USER 'testDB'@'localhost' IDENTIFIED BY '123';` tk.MustExec(createUserSQL) // Make sure all the db privs for new user is empty. - sql := fmt.Sprintf("SELECT * FROM mysql.db WHERE User=\"testDB\" and host=\"localhost\"") + sql := `SELECT * FROM mysql.db WHERE User="testDB" and host="localhost"` tk.MustQuery(sql).Check(testkit.Rows()) // Grant each priv to the user. @@ -89,6 +89,10 @@ func (s *testSuite3) TestGrantDBScope(c *C) { sql := fmt.Sprintf("SELECT %s FROM mysql.DB WHERE User=\"testDB1\" and host=\"localhost\" and db=\"test\";", mysql.Priv2UserCol[v]) tk.MustQuery(sql).Check(testkit.Rows("Y")) } + + // Grant in wrong scope. + _, err := tk.Exec(` grant create user on test.* to 'testDB1'@'localhost';`) + c.Assert(terror.ErrorEqual(err, executor.ErrWrongUsage.GenWithStackByArgs("DB GRANT", "GLOBAL PRIVILEGES")), IsTrue) } func (s *testSuite3) TestWithGrantOption(c *C) { @@ -97,7 +101,7 @@ func (s *testSuite3) TestWithGrantOption(c *C) { createUserSQL := `CREATE USER 'testWithGrant'@'localhost' IDENTIFIED BY '123';` tk.MustExec(createUserSQL) // Make sure all the db privs for new user is empty. - sql := fmt.Sprintf("SELECT * FROM mysql.db WHERE User=\"testWithGrant\" and host=\"localhost\"") + sql := `SELECT * FROM mysql.db WHERE User="testWithGrant" and host="localhost"` tk.MustQuery(sql).Check(testkit.Rows()) // Grant select priv to the user, with grant option. @@ -399,12 +403,8 @@ func (s *testSuite3) TestIssue22721(c *C) { func (s *testSuite3) TestGrantDynamicPrivs(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("create user dyn") - tk.MustExec("SET tidb_enable_dynamic_privileges=0") - _, err := tk.Exec("GRANT BACKUP_ADMIN ON *.* TO dyn") - c.Assert(err.Error(), Equals, "dynamic privileges is an experimental feature. Run 'SET tidb_enable_dynamic_privileges=1'") - tk.MustExec("SET tidb_enable_dynamic_privileges=1") - _, err = tk.Exec("GRANT BACKUP_ADMIN ON test.* TO dyn") + _, err := tk.Exec("GRANT BACKUP_ADMIN ON test.* TO dyn") c.Assert(terror.ErrorEqual(err, executor.ErrIllegalPrivilegeLevel), IsTrue) _, err = tk.Exec("GRANT BOGUS_GRANT ON *.* TO dyn") c.Assert(terror.ErrorEqual(err, executor.ErrDynamicPrivilegeNotRegistered), IsTrue) diff --git a/executor/index_lookup_join.go b/executor/index_lookup_join.go index 9bec8e118515a..95f7d19f3616a 100644 --- a/executor/index_lookup_join.go +++ b/executor/index_lookup_join.go @@ -95,6 +95,7 @@ type innerCtx struct { readerBuilder *dataReaderBuilder rowTypes []*types.FieldType keyCols []int + keyColIDs []int64 // the original ID in its table, used by dynamic partition pruning hashCols []int colLens []int hasPrefixCol bool @@ -472,9 +473,10 @@ func (iw *innerWorker) run(ctx context.Context, wg *sync.WaitGroup) { } type indexJoinLookUpContent struct { - keys []types.Datum - row chunk.Row - keyCols []int + keys []types.Datum + row chunk.Row + keyCols []int + keyColIDs []int64 // the original ID in its table, used by dynamic partition pruning } func (iw *innerWorker) handleTask(ctx context.Context, task *lookUpJoinTask) error { @@ -535,7 +537,7 @@ func (iw *innerWorker) constructLookupContent(task *lookUpJoinTask) ([]*indexJoi // Store the encoded lookup key in chunk, so we can use it to lookup the matched inners directly. task.encodedLookUpKeys[chkIdx].AppendBytes(0, keyBuf) if iw.hasPrefixCol { - for i, outerOffset := range iw.outerCtx.keyCols { + for i, outerOffset := range iw.keyOff2IdxOff { // If it's a prefix column. Try to fix it. joinKeyColPrefixLen := iw.colLens[outerOffset] if joinKeyColPrefixLen != types.UnspecifiedLength { @@ -545,7 +547,7 @@ func (iw *innerWorker) constructLookupContent(task *lookUpJoinTask) ([]*indexJoi // dLookUpKey is sorted and deduplicated at sortAndDedupLookUpContents. // So we don't need to do it here. } - lookUpContents = append(lookUpContents, &indexJoinLookUpContent{keys: dLookUpKey, row: chk.GetRow(rowIdx), keyCols: iw.keyCols}) + lookUpContents = append(lookUpContents, &indexJoinLookUpContent{keys: dLookUpKey, row: chk.GetRow(rowIdx), keyCols: iw.keyCols, keyColIDs: iw.keyColIDs}) } } @@ -575,14 +577,11 @@ func (iw *innerWorker) constructDatumLookupKey(task *lookUpJoinTask, chkIdx, row } innerColType := iw.rowTypes[iw.hashCols[i]] innerValue, err := outerValue.ConvertTo(sc, innerColType) - if err != nil { + if err != nil && !(terror.ErrorEqual(err, types.ErrTruncated) && (innerColType.Tp == mysql.TypeSet || innerColType.Tp == mysql.TypeEnum)) { // If the converted outerValue overflows, we don't need to lookup it. if terror.ErrorEqual(err, types.ErrOverflow) { return nil, nil, nil } - if terror.ErrorEqual(err, types.ErrTruncated) && (innerColType.Tp == mysql.TypeSet || innerColType.Tp == mysql.TypeEnum) { - return nil, nil, nil - } return nil, nil, err } cmp, err := outerValue.CompareDatum(sc, &innerValue) diff --git a/executor/index_lookup_join_test.go b/executor/index_lookup_join_test.go index 9515907b8eba6..4a3923298ae43 100644 --- a/executor/index_lookup_join_test.go +++ b/executor/index_lookup_join_test.go @@ -248,7 +248,8 @@ func (s *testSuite5) TestIndexJoinEnumSetIssue19233(c *C) { tk.MustExec(`insert into p1 values('HOST_PORT');`) tk.MustExec(`insert into p2 values('HOST_PORT');`) for _, table := range []string{"p1", "p2"} { - for _, hint := range []string{"INL_HASH_JOIN", "INL_MERGE_JOIN", "INL_JOIN"} { + // INL_MERGE_JOIN do not support enum type. ref: https://github.com/pingcap/tidb/issues/24473 + for _, hint := range []string{"INL_HASH_JOIN", "INL_JOIN"} { sql := fmt.Sprintf(`select /*+ %s(%s) */ * from i, %s where i.objectType = %s.type;`, hint, table, table, table) rows := tk.MustQuery(sql).Rows() c.Assert(len(rows), Equals, 64) @@ -335,6 +336,18 @@ func (s *testSuite5) TestIssue23722(c *C) { "order by col_15 , col_16 , col_17 , col_18 , col_19;").Check(testkit.Rows("38799.400 20301 KETeFZhkoxnwMAhA Charlie zyhXEppZdqyqNV")) } +func (s *testSuite5) TestIssue24547(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists a") + tk.MustExec("drop table if exists b") + tk.MustExec("CREATE TABLE `a` (\n `v` varchar(100) DEFAULT NULL,\n `k1` varchar(100) NOT NULL,\n `k2` varchar(100) NOT NULL,\n PRIMARY KEY (`k1`(3),`k2`(3)) /*T![clustered_index] CLUSTERED */,\n KEY `kk2` (`k2`(3)),\n UNIQUE KEY `uk1` (`v`)\n)") + tk.MustExec("CREATE TABLE `b` (\n `v` varchar(100) DEFAULT NULL,\n `k1` varchar(100) NOT NULL,\n `k2` varchar(100) NOT NULL,\n PRIMARY KEY (`k1`(3),`k2`(3)) /*T![clustered_index] CLUSTERED */,\n KEY `kk2` (`k2`(3))\n)") + tk.MustExec("insert into a(v, k1, k2) values('1', '1', '1'), ('22', '22', '22'), ('333', '333', '333'), ('3444', '3444', '3444'), ('444', '444', '444')") + tk.MustExec("insert into b(v, k1, k2) values('1', '1', '1'), ('22', '22', '22'), ('333', '333', '333'), ('2333', '2333', '2333'), ('555', '555', '555')") + tk.MustExec("delete a from a inner join b on a.k1 = b.k1 and a.k2 = b.k2 where b.k2 <> '333'") +} + func (s *testSuite5) TestPartitionTableIndexJoinAndIndexReader(c *C) { if israce.RaceEnabled { c.Skip("exhaustive types test, skip race test") diff --git a/executor/index_lookup_merge_join_test.go b/executor/index_lookup_merge_join_test.go index 37943c8ec5f11..2158026f1966f 100644 --- a/executor/index_lookup_merge_join_test.go +++ b/executor/index_lookup_merge_join_test.go @@ -158,3 +158,14 @@ func (s *testSuite9) TestIssue20549(c *C) { tk.MustQuery("SELECT /*+ HASH_JOIN(t1,t2) */ 1 from t1 left outer join t2 on t1.t2id=t2.id;\n").Check( testkit.Rows("1")) } + +func (s *testSuite9) TestIssue24473(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists x, t2, t3") + tk.MustExec("CREATE TABLE `x` ( `a` enum('y','b','1','x','0','null') DEFAULT NULL, KEY `a` (`a`));") + tk.MustExec("insert into x values(\"x\"),(\"x\"),(\"b\"),(\"y\");") + tk.MustQuery("SELECT /*+ merge_join (t2,t3) */ t2.a,t3.a FROM x t2 inner join x t3 on t2.a = t3.a;").Check( + testkit.Rows("y y", "b b", "x x", "x x", "x x", "x x")) + tk.MustQuery("SELECT /*+ inl_merge_join (t2,t3) */ t2.a,t3.a FROM x t2 inner join x t3 on t2.a = t3.a;").Check( + testkit.Rows("y y", "b b", "x x", "x x", "x x", "x x")) +} diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go index c769d2705c44b..343ee4cb06105 100644 --- a/executor/index_merge_reader.go +++ b/executor/index_merge_reader.go @@ -29,7 +29,6 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/distsql" - "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx" @@ -245,7 +244,7 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context, SetStreaming(e.partialStreamings[workID]). SetFromSessionVars(e.ctx.GetSessionVars()). SetMemTracker(e.memTracker). - SetFromInfoSchema(infoschema.GetInfoSchema(e.ctx)) + SetFromInfoSchema(e.ctx.GetInfoSchema()) worker := &partialIndexWorker{ stats: e.stats, diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index 368d8838a777e..b01972726991a 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -16,9 +16,10 @@ package executor import ( "bytes" "context" + "encoding/hex" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "sort" "strconv" @@ -52,13 +53,17 @@ import ( "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/collate" + "github.com/pingcap/tidb/util/deadlockhistory" + "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/pdapi" + "github.com/pingcap/tidb/util/resourcegrouptag" "github.com/pingcap/tidb/util/sem" "github.com/pingcap/tidb/util/set" "github.com/pingcap/tidb/util/sqlexec" "github.com/pingcap/tidb/util/stmtsummary" "github.com/pingcap/tidb/util/stringutil" "go.etcd.io/etcd/clientv3" + "go.uber.org/zap" ) type memtableRetriever struct { @@ -79,7 +84,7 @@ func (e *memtableRetriever) retrieve(ctx context.Context, sctx sessionctx.Contex // Cache the ret full rows in schemataRetriever if !e.initialized { - is := infoschema.GetInfoSchema(sctx) + is := sctx.GetInfoSchema().(infoschema.InfoSchema) dbs := is.AllSchemas() sort.Sort(infoschema.SchemasSorter(dbs)) var err error @@ -143,12 +148,24 @@ func (e *memtableRetriever) retrieve(ctx context.Context, sctx sessionctx.Contex infoschema.ClusterTableStatementsSummary, infoschema.ClusterTableStatementsSummaryHistory: err = e.setDataForStatementsSummary(sctx, e.table.Name.O) + case infoschema.TableStatementsSummaryEvicted: + e.setDataForStatementsSummaryEvicted(sctx) case infoschema.TablePlacementPolicy: err = e.setDataForPlacementPolicy(sctx) case infoschema.TableClientErrorsSummaryGlobal, infoschema.TableClientErrorsSummaryByUser, infoschema.TableClientErrorsSummaryByHost: err = e.setDataForClientErrorsSummary(sctx, e.table.Name.O) + case infoschema.TableTiDBTrx: + e.setDataForTiDBTrx(sctx) + case infoschema.ClusterTableTiDBTrx: + err = e.setDataForClusterTiDBTrx(sctx) + case infoschema.TableDeadlocks: + err = e.setDataForDeadlock(sctx) + case infoschema.ClusterTableDeadlocks: + err = e.setDataForClusterDeadlock(sctx) + case infoschema.TableDataLockWaits: + err = e.setDataForTableDataLockWaits(sctx) } if err != nil { return nil, err @@ -295,7 +312,7 @@ func (c *statsCache) get(ctx sessionctx.Context) (map[int64]uint64, map[tableHis } func getAutoIncrementID(ctx sessionctx.Context, schema *model.DBInfo, tblInfo *model.TableInfo) (int64, error) { - is := infoschema.GetInfoSchema(ctx) + is := ctx.GetInfoSchema().(infoschema.InfoSchema) tbl, err := is.TableByName(schema.Name, tblInfo.Name) if err != nil { return 0, err @@ -583,7 +600,7 @@ func (e *hugeMemTableRetriever) setDataForColumns(ctx context.Context, sctx sess } func (e *hugeMemTableRetriever) dataForColumnsInTable(ctx context.Context, sctx sessionctx.Context, schema *model.DBInfo, tbl *model.TableInfo) { - if err := tryFillViewColumnType(ctx, sctx, infoschema.GetInfoSchema(sctx), schema.Name, tbl); err != nil { + if err := tryFillViewColumnType(ctx, sctx, sctx.GetInfoSchema().(infoschema.InfoSchema), schema.Name, tbl); err != nil { sctx.GetSessionVars().StmtCtx.AppendWarning(err) return } @@ -996,6 +1013,40 @@ func (e *memtableRetriever) dataForTiKVStoreStatus(ctx sessionctx.Context) (err return nil } +func hasPriv(ctx sessionctx.Context, priv mysql.PrivilegeType) bool { + if pm := privilege.GetPrivilegeManager(ctx); pm != nil { + return pm.RequestVerification(ctx.GetSessionVars().ActiveRoles, "", "", "", priv) + } + return false +} + +func (e *memtableRetriever) setDataForTableDataLockWaits(ctx sessionctx.Context) error { + if !hasPriv(ctx, mysql.ProcessPriv) { + return plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS") + } + waits, err := ctx.GetStore().GetLockWaits() + if err != nil { + return err + } + for _, wait := range waits { + var digestStr interface{} + digest, err := resourcegrouptag.DecodeResourceGroupTag(wait.ResourceGroupTag) + if err != nil { + logutil.BgLogger().Warn("failed to decode resource group tag", zap.Error(err)) + digestStr = nil + } else { + digestStr = hex.EncodeToString(digest) + } + e.rows = append(e.rows, types.MakeDatums( + hex.EncodeToString(wait.Key), + wait.Txn, + wait.WaitForTxn, + digestStr, + )) + } + return nil +} + // DDLJobsReaderExec executes DDLJobs information retrieving. type DDLJobsReaderExec struct { baseExecutor @@ -1180,13 +1231,7 @@ func (e *memtableRetriever) setDataForProcessList(ctx sessionctx.Context) { } loginUser := ctx.GetSessionVars().User - var hasProcessPriv bool - if pm := privilege.GetPrivilegeManager(ctx); pm != nil { - if pm.RequestVerification(ctx.GetSessionVars().ActiveRoles, "", "", "", mysql.ProcessPriv) { - hasProcessPriv = true - } - } - + hasProcessPriv := hasPriv(ctx, mysql.ProcessPriv) pl := sm.ShowProcessList() records := make([][]types.Datum, 0, len(pl)) @@ -1330,7 +1375,7 @@ func (e *memtableRetriever) setDataForTiKVRegionStatus(ctx sessionctx.Context) e if err != nil { return err } - allSchemas := ctx.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema).AllSchemas() + allSchemas := ctx.GetInfoSchema().(infoschema.InfoSchema).AllSchemas() tableInfos := tikvHelper.GetRegionsTableInfo(regionsInfo, allSchemas) for _, region := range regionsInfo.Regions { tableList := tableInfos[region.ID] @@ -1401,7 +1446,7 @@ func (e *memtableRetriever) setNewTiKVRegionPeersCols(region *helper.RegionInfo) } downPeerMap := make(map[int64]int64, len(region.DownPeers)) for _, peerStat := range region.DownPeers { - downPeerMap[peerStat.ID] = peerStat.DownSec + downPeerMap[peerStat.Peer.ID] = peerStat.DownSec } for _, peer := range region.Peers { row := make([]types.Datum, len(infoschema.TableTiKVRegionPeersCols)) @@ -1418,11 +1463,11 @@ func (e *memtableRetriever) setNewTiKVRegionPeersCols(region *helper.RegionInfo) } else { row[4].SetInt64(0) } - if pendingPeerIDSet.Exist(peer.ID) { - row[5].SetString(pendingPeer, mysql.DefaultCollationName) - } else if downSec, ok := downPeerMap[peer.ID]; ok { + if downSec, ok := downPeerMap[peer.ID]; ok { row[5].SetString(downPeer, mysql.DefaultCollationName) row[6].SetInt64(downSec) + } else if pendingPeerIDSet.Exist(peer.ID) { + row[5].SetString(pendingPeer, mysql.DefaultCollationName) } else { row[5].SetString(normalPeer, mysql.DefaultCollationName) } @@ -1442,7 +1487,7 @@ func (e *memtableRetriever) setDataForTiDBHotRegions(ctx sessionctx.Context) err if !ok { return errors.New("Information about hot region can be gotten only when the storage is TiKV") } - allSchemas := ctx.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema).AllSchemas() + allSchemas := ctx.GetInfoSchema().(infoschema.InfoSchema).AllSchemas() tikvHelper := &helper.Helper{ Store: tikvStore, RegionCache: tikvStore.GetRegionCache(), @@ -1591,7 +1636,7 @@ type initialTable struct { } func (e *tableStorageStatsRetriever) initialize(sctx sessionctx.Context) error { - is := infoschema.GetInfoSchema(sctx) + is := sctx.GetInfoSchema().(infoschema.InfoSchema) var databases []string schemas := e.extractor.TableSchema tables := e.extractor.TableName @@ -1675,7 +1720,7 @@ func (e *memtableRetriever) setDataFromSessionVar(ctx sessionctx.Context) error sessionVars := ctx.GetSessionVars() for _, v := range variable.GetSysVars() { var value string - value, err = variable.GetSessionSystemVar(sessionVars, v.Name) + value, err = variable.GetSessionOrGlobalSystemVar(sessionVars, v.Name) if err != nil { return err } @@ -1852,7 +1897,6 @@ func (e *memtableRetriever) dataForTableTiFlashReplica(ctx sessionctx.Context, s } } e.rows = rows - return } func (e *memtableRetriever) setDataForStatementsSummary(ctx sessionctx.Context, tableName string) error { @@ -1883,7 +1927,7 @@ func (e *memtableRetriever) setDataForStatementsSummary(ctx sessionctx.Context, func (e *memtableRetriever) setDataForPlacementPolicy(ctx sessionctx.Context) error { checker := privilege.GetPrivilegeManager(ctx) - is := infoschema.GetInfoSchema(ctx) + is := ctx.GetInfoSchema().(infoschema.InfoSchema) var rows [][]types.Datum for _, bundle := range is.RuleBundles() { id, err := placement.ObjectIDFromGroupID(bundle.ID) @@ -1912,7 +1956,7 @@ func (e *memtableRetriever) setDataForPlacementPolicy(ctx sessionctx.Context) er continue } for _, rule := range bundle.Rules { - constraint, err := rule.LabelConstraints.Restore() + constraint, err := rule.Constraints.Restore() if err != nil { return errors.Wrapf(err, "Restore rule %s in bundle %s failed", rule.ID, bundle.ID) } @@ -1938,13 +1982,8 @@ func (e *memtableRetriever) setDataForPlacementPolicy(ctx sessionctx.Context) er func (e *memtableRetriever) setDataForClientErrorsSummary(ctx sessionctx.Context, tableName string) error { // Seeing client errors should require the PROCESS privilege, with the exception of errors for your own user. // This is similar to information_schema.processlist, which is the closest comparison. - var hasProcessPriv bool + hasProcessPriv := hasPriv(ctx, mysql.ProcessPriv) loginUser := ctx.GetSessionVars().User - if pm := privilege.GetPrivilegeManager(ctx); pm != nil { - if pm.RequestVerification(ctx.GetSessionVars().ActiveRoles, "", "", "", mysql.ProcessPriv) { - hasProcessPriv = true - } - } var rows [][]types.Datum switch tableName { @@ -2011,6 +2050,61 @@ func (e *memtableRetriever) setDataForClientErrorsSummary(ctx sessionctx.Context return nil } +func (e *memtableRetriever) setDataForTiDBTrx(ctx sessionctx.Context) { + sm := ctx.GetSessionManager() + if sm == nil { + return + } + + loginUser := ctx.GetSessionVars().User + hasProcessPriv := hasPriv(ctx, mysql.ProcessPriv) + infoList := sm.ShowTxnList() + for _, info := range infoList { + // If you have the PROCESS privilege, you can see all running transactions. + // Otherwise, you can see only your own transactions. + if !hasProcessPriv && loginUser != nil && info.Username != loginUser.Username { + continue + } + e.rows = append(e.rows, info.ToDatum()) + } +} + +func (e *memtableRetriever) setDataForClusterTiDBTrx(ctx sessionctx.Context) error { + e.setDataForTiDBTrx(ctx) + rows, err := infoschema.AppendHostInfoToRows(ctx, e.rows) + if err != nil { + return err + } + e.rows = rows + return nil +} + +func (e *memtableRetriever) setDataForDeadlock(ctx sessionctx.Context) error { + if !hasPriv(ctx, mysql.ProcessPriv) { + return plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS") + } + + e.rows = deadlockhistory.GlobalDeadlockHistory.GetAllDatum() + return nil +} + +func (e *memtableRetriever) setDataForClusterDeadlock(ctx sessionctx.Context) error { + err := e.setDataForDeadlock(ctx) + if err != nil { + return err + } + rows, err := infoschema.AppendHostInfoToRows(ctx, e.rows) + if err != nil { + return err + } + e.rows = rows + return nil +} + +func (e *memtableRetriever) setDataForStatementsSummaryEvicted(ctx sessionctx.Context) { + e.rows = stmtsummary.StmtSummaryByDigestMap.ToEvictedCountDatum() +} + type hugeMemTableRetriever struct { dummyCloser table *model.TableInfo @@ -2030,7 +2124,7 @@ func (e *hugeMemTableRetriever) retrieve(ctx context.Context, sctx sessionctx.Co } if !e.initialized { - is := infoschema.GetInfoSchema(sctx) + is := sctx.GetInfoSchema().(infoschema.InfoSchema) dbs := is.AllSchemas() sort.Sort(infoschema.SchemasSorter(dbs)) e.dbs = dbs @@ -2197,7 +2291,7 @@ func (e *TiFlashSystemTableRetriever) dataForTiFlashSystemTables(ctx sessionctx. if err != nil { return nil, errors.Trace(err) } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) terror.Log(resp.Body.Close()) if err != nil { return nil, errors.Trace(err) diff --git a/executor/infoschema_reader_test.go b/executor/infoschema_reader_test.go index c3e125824873d..1170453d368b0 100644 --- a/executor/infoschema_reader_test.go +++ b/executor/infoschema_reader_test.go @@ -36,6 +36,7 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/server" "github.com/pingcap/tidb/session" + txninfo "github.com/pingcap/tidb/session/txninfo" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/statistics/handle" @@ -369,7 +370,6 @@ func (s *testInfoschemaTableSuite) TestUserPrivilegesTable(c *C) { tk.MustExec("GRANT SELECT ON *.* to usageuser WITH GRANT OPTION") tk.MustQuery(`SELECT * FROM information_schema.user_privileges WHERE grantee="'usageuser'@'%'"`).Check(testkit.Rows("'usageuser'@'%' def Select YES")) // test DYNAMIC privs - tk.MustExec("SET tidb_enable_dynamic_privileges=1") tk.MustExec("GRANT BACKUP_ADMIN ON *.* to usageuser") tk.MustQuery(`SELECT * FROM information_schema.user_privileges WHERE grantee="'usageuser'@'%'" ORDER BY privilege_type`).Check(testkit.Rows("'usageuser'@'%' def BACKUP_ADMIN NO", "'usageuser'@'%' def Select YES")) } @@ -728,6 +728,10 @@ type mockSessionManager struct { serverID uint64 } +func (sm *mockSessionManager) ShowTxnList() []*txninfo.TxnInfo { + panic("unimplemented!") +} + func (sm *mockSessionManager) ShowProcessList() map[uint64]*util.ProcessInfo { return sm.processInfoMap } diff --git a/executor/insert.go b/executor/insert.go index e8fdb9da3444e..c6195ccef34c9 100644 --- a/executor/insert.go +++ b/executor/insert.go @@ -24,7 +24,6 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/kv" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" @@ -64,6 +63,7 @@ func (e *InsertExec) exec(ctx context.Context, rows [][]types.Datum) error { if err != nil { return err } + setResourceGroupTagForTxn(sessVars.StmtCtx, txn) txnSize := txn.Size() sessVars.StmtCtx.AddRecordRows(uint64(len(rows))) // If you use the IGNORE keyword, duplicate-key error that occurs while executing the INSERT statement are ignored. @@ -215,8 +215,8 @@ func (e *InsertExec) batchUpdateDupRows(ctx context.Context, newRows [][]types.D if e.collectRuntimeStatsEnabled() { if snapshot := txn.GetSnapshot(); snapshot != nil { - snapshot.SetOption(tikvstore.CollectRuntimeStats, e.stats.SnapshotRuntimeStats) - defer snapshot.DelOption(tikvstore.CollectRuntimeStats) + snapshot.SetOption(kv.CollectRuntimeStats, e.stats.SnapshotRuntimeStats) + defer snapshot.SetOption(kv.CollectRuntimeStats, nil) } } prefetchStart := time.Now() diff --git a/executor/insert_common.go b/executor/insert_common.go index 10fc6cb9edc59..5782c4189522a 100644 --- a/executor/insert_common.go +++ b/executor/insert_common.go @@ -34,7 +34,6 @@ import ( "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/store/tikv" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/types" @@ -1049,8 +1048,8 @@ func (e *InsertValues) batchCheckAndInsert(ctx context.Context, rows [][]types.D } if e.collectRuntimeStatsEnabled() { if snapshot := txn.GetSnapshot(); snapshot != nil { - snapshot.SetOption(tikvstore.CollectRuntimeStats, e.stats.SnapshotRuntimeStats) - defer snapshot.DelOption(tikvstore.CollectRuntimeStats) + snapshot.SetOption(kv.CollectRuntimeStats, e.stats.SnapshotRuntimeStats) + defer snapshot.SetOption(kv.CollectRuntimeStats, nil) } } prefetchStart := time.Now() diff --git a/executor/insert_test.go b/executor/insert_test.go index bee38e51c0fea..1e7a5bb36ef87 100644 --- a/executor/insert_test.go +++ b/executor/insert_test.go @@ -426,7 +426,7 @@ func (s *testSuite3) TestInsertDateTimeWithTimeZone(c *C) { tk.MustExec("drop table if exists t") tk.MustExec("create table t (ts timestamp)") tk.MustExec("insert into t values ('2020-10-22T12:00:00Z'), ('2020-10-22T13:00:00Z'), ('2020-10-22T14:00:00Z')") - tk.MustQuery(fmt.Sprintf("select count(*) from t where ts > '2020-10-22T12:00:00Z'")).Check(testkit.Rows("2")) + tk.MustQuery("select count(*) from t where ts > '2020-10-22T12:00:00Z'").Check(testkit.Rows("2")) // test for datetime with fsp fspCases := []struct { @@ -1590,3 +1590,123 @@ func (s *testSuite10) TestBinaryLiteralInsertToSet(c *C) { tk.MustExec("insert into bintest(h) values(0x61)") tk.MustQuery("select * from bintest").Check(testkit.Rows("a")) } + +var _ = SerialSuites(&testSuite13{&baseTestSuite{}}) + +type testSuite13 struct { + *baseTestSuite +} + +func (s *testSuite13) TestGlobalTempTableAutoInc(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec(`use test`) + tk.MustExec("drop table if exists temp_test") + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("create global temporary table temp_test(id int primary key auto_increment) on commit delete rows") + defer tk.MustExec("drop table if exists temp_test") + + // Data is cleared after transaction auto commits. + tk.MustExec("insert into temp_test(id) values(0)") + tk.MustQuery("select * from temp_test").Check(testkit.Rows()) + + // Data is not cleared inside a transaction. + tk.MustExec("begin") + tk.MustExec("insert into temp_test(id) values(0)") + tk.MustQuery("select * from temp_test").Check(testkit.Rows("1")) + tk.MustExec("commit") + + // AutoID allocator is cleared. + tk.MustExec("begin") + tk.MustExec("insert into temp_test(id) values(0)") + tk.MustQuery("select * from temp_test").Check(testkit.Rows("1")) + // Test whether auto-inc is incremental + tk.MustExec("insert into temp_test(id) values(0)") + tk.MustQuery("select id from temp_test order by id").Check(testkit.Rows("1", "2")) + tk.MustExec("commit") + + // multi-value insert + tk.MustExec("begin") + tk.MustExec("insert into temp_test(id) values(0), (0)") + tk.MustQuery("select id from temp_test order by id").Check(testkit.Rows("1", "2")) + tk.MustExec("insert into temp_test(id) values(0), (0)") + tk.MustQuery("select id from temp_test order by id").Check(testkit.Rows("1", "2", "3", "4")) + tk.MustExec("commit") + + // rebase + tk.MustExec("begin") + tk.MustExec("insert into temp_test(id) values(10)") + tk.MustExec("insert into temp_test(id) values(0)") + tk.MustQuery("select id from temp_test order by id").Check(testkit.Rows("10", "11")) + tk.MustExec("insert into temp_test(id) values(20), (30)") + tk.MustExec("insert into temp_test(id) values(0), (0)") + tk.MustQuery("select id from temp_test order by id").Check(testkit.Rows("10", "11", "20", "30", "31", "32")) + tk.MustExec("commit") +} + +func (s *testSuite13) TestGlobalTempTableRowID(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec(`use test`) + tk.MustExec("drop table if exists temp_test") + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("create global temporary table temp_test(id int) on commit delete rows") + defer tk.MustExec("drop table if exists temp_test") + + // Data is cleared after transaction auto commits. + tk.MustExec("insert into temp_test(id) values(0)") + tk.MustQuery("select _tidb_rowid from temp_test").Check(testkit.Rows()) + + // Data is not cleared inside a transaction. + tk.MustExec("begin") + tk.MustExec("insert into temp_test(id) values(0)") + tk.MustQuery("select _tidb_rowid from temp_test").Check(testkit.Rows("1")) + tk.MustExec("commit") + + // AutoID allocator is cleared. + tk.MustExec("begin") + tk.MustExec("insert into temp_test(id) values(0)") + tk.MustQuery("select _tidb_rowid from temp_test").Check(testkit.Rows("1")) + // Test whether row id is incremental + tk.MustExec("insert into temp_test(id) values(0)") + tk.MustQuery("select _tidb_rowid from temp_test order by _tidb_rowid").Check(testkit.Rows("1", "2")) + tk.MustExec("commit") + + // multi-value insert + tk.MustExec("begin") + tk.MustExec("insert into temp_test(id) values(0), (0)") + tk.MustQuery("select _tidb_rowid from temp_test order by _tidb_rowid").Check(testkit.Rows("1", "2")) + tk.MustExec("insert into temp_test(id) values(0), (0)") + tk.MustQuery("select _tidb_rowid from temp_test order by _tidb_rowid").Check(testkit.Rows("1", "2", "3", "4")) + tk.MustExec("commit") +} + +func (s *testSuite13) TestGlobalTempTableParallel(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec(`use test`) + tk.MustExec("drop table if exists temp_test") + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("create global temporary table temp_test(id int primary key auto_increment) on commit delete rows") + defer tk.MustExec("drop table if exists temp_test") + + threads := 8 + loops := 1 + wg := sync.WaitGroup{} + wg.Add(threads) + + insertFunc := func() { + defer wg.Done() + newTk := testkit.NewTestKitWithInit(c, s.store) + newTk.MustExec("begin") + for i := 0; i < loops; i++ { + newTk.MustExec("insert temp_test value(0)") + newTk.MustExec("insert temp_test value(0), (0)") + } + maxID := strconv.Itoa(loops * 3) + newTk.MustQuery("select max(id) from temp_test").Check(testkit.Rows(maxID)) + newTk.MustExec("commit") + } + + for i := 0; i < threads; i++ { + go insertFunc() + } + wg.Wait() +} diff --git a/executor/join.go b/executor/join.go index c1a8045aba9a3..1a3f62de47ac1 100644 --- a/executor/join.go +++ b/executor/join.go @@ -1078,7 +1078,7 @@ func (e *joinRuntimeStats) String() string { if e.cache.useCache { buf.WriteString(fmt.Sprintf(", cache:ON, cacheHitRatio:%.3f%%", e.cache.hitRatio*100)) } else { - buf.WriteString(fmt.Sprintf(", cache:OFF")) + buf.WriteString(", cache:OFF") } } if e.hasHashStat { diff --git a/executor/join_test.go b/executor/join_test.go index bfd0048a63b3d..3519b14893491 100644 --- a/executor/join_test.go +++ b/executor/join_test.go @@ -1182,6 +1182,7 @@ func (s *testSuiteJoin1) TestIssue15850JoinNullValue(c *C) { } func (s *testSuiteJoin1) TestIndexLookupJoin(c *C) { + c.Skip("unstable, skip it and fix it before 20210622") tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("set @@tidb_init_chunk_size=2") diff --git a/executor/load_stats.go b/executor/load_stats.go index 83fbb3ad188f7..473a14150e711 100644 --- a/executor/load_stats.go +++ b/executor/load_stats.go @@ -86,5 +86,5 @@ func (e *LoadStatsInfo) Update(data []byte) error { if h == nil { return errors.New("Load Stats: handle is nil") } - return h.LoadStatsFromJSON(infoschema.GetInfoSchema(e.Ctx), jsonTbl) + return h.LoadStatsFromJSON(e.Ctx.GetInfoSchema().(infoschema.InfoSchema), jsonTbl) } diff --git a/executor/mem_reader.go b/executor/mem_reader.go index f6023c93c5b1a..4a2ab2f4b191c 100644 --- a/executor/mem_reader.go +++ b/executor/mem_reader.go @@ -379,6 +379,11 @@ type memIndexLookUpReader struct { retFieldTypes []*types.FieldType idxReader *memIndexReader + + // partition mode + partitionMode bool // if it is accessing a partition table + partitionTables []table.PhysicalTable // partition tables to access + partitionKVRanges [][]kv.KeyRange // kv ranges for these partition tables } func buildMemIndexLookUpReader(us *UnionScanExec, idxLookUpReader *IndexLookUpExecutor) *memIndexLookUpReader { @@ -404,16 +409,43 @@ func buildMemIndexLookUpReader(us *UnionScanExec, idxLookUpReader *IndexLookUpEx conditions: us.conditions, retFieldTypes: retTypes(us), idxReader: memIdxReader, + + partitionMode: idxLookUpReader.partitionTableMode, + partitionKVRanges: idxLookUpReader.partitionKVRanges, + partitionTables: idxLookUpReader.prunedPartitions, } } func (m *memIndexLookUpReader) getMemRows() ([][]types.Datum, error) { - handles, err := m.idxReader.getMemRowsHandle() - if err != nil || len(handles) == 0 { - return nil, err + kvRanges := [][]kv.KeyRange{m.idxReader.kvRanges} + tbls := []table.Table{m.table} + if m.partitionMode { + m.idxReader.desc = false // keep-order if always false for IndexLookUp reading partitions so this parameter makes no sense + kvRanges = m.partitionKVRanges + tbls = tbls[:0] + for _, p := range m.partitionTables { + tbls = append(tbls, p) + } + } + + tblKVRanges := make([]kv.KeyRange, 0, 16) + numHandles := 0 + for i, tbl := range tbls { + m.idxReader.kvRanges = kvRanges[i] + handles, err := m.idxReader.getMemRowsHandle() + if err != nil { + return nil, err + } + if len(handles) == 0 { + continue + } + numHandles += len(handles) + tblKVRanges = append(tblKVRanges, distsql.TableHandlesToKVRanges(getPhysicalTableID(tbl), handles)...) + } + if numHandles == 0 { + return nil, nil } - tblKVRanges := distsql.TableHandlesToKVRanges(getPhysicalTableID(m.table), handles) colIDs := make(map[int64]int, len(m.columns)) for i, col := range m.columns { colIDs[col.ID] = i @@ -440,7 +472,7 @@ func (m *memIndexLookUpReader) getMemRows() ([][]types.Datum, error) { columns: m.columns, kvRanges: tblKVRanges, conditions: m.conditions, - addedRows: make([][]types.Datum, 0, len(handles)), + addedRows: make([][]types.Datum, 0, numHandles), retFieldTypes: m.retFieldTypes, colIDs: colIDs, pkColIDs: pkColIDs, diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 24e2001131580..f98306b5da51d 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -191,6 +191,9 @@ func fetchClusterConfig(sctx sessionctx.Context, nodeTypes, nodeAddrs set.String url = fmt.Sprintf("%s://%s%s", util.InternalHTTPSchema(), statusAddr, pdapi.Config) case "tikv", "tidb": url = fmt.Sprintf("%s://%s/config", util.InternalHTTPSchema(), statusAddr) + case "tiflash": + // TODO: support show tiflash config once tiflash supports it + return default: ch <- result{err: errors.Errorf("unknown node type: %s(%s)", typ, address)} return @@ -230,9 +233,9 @@ func fetchClusterConfig(sctx sessionctx.Context, nodeTypes, nodeAddrs set.String continue } var str string - switch val.(type) { + switch val := val.(type) { case string: // remove quotes - str = val.(string) + str = val default: tmp, err := json.Marshal(val) if err != nil { diff --git a/executor/memtable_reader_test.go b/executor/memtable_reader_test.go index 53607dd80ef03..94a8b49ca26bb 100644 --- a/executor/memtable_reader_test.go +++ b/executor/memtable_reader_test.go @@ -16,7 +16,6 @@ package executor_test import ( "context" "fmt" - "io/ioutil" "log" "net" "net/http/httptest" @@ -170,7 +169,7 @@ func (s *testMemTableReaderSuite) TestTiDBClusterConfig(c *C) { // mock servers servers := []string{} - for _, typ := range []string{"tidb", "tikv", "pd"} { + for _, typ := range []string{"tidb", "tikv", "tiflash", "pd"} { for _, server := range testServers { servers = append(servers, strings.Join([]string{typ, server.address, server.address}, ",")) } @@ -431,7 +430,7 @@ func (s *testMemTableReaderSuite) TestTiDBClusterConfig(c *C) { } func (s *testClusterTableBase) writeTmpFile(c *C, dir, filename string, lines []string) { - err := ioutil.WriteFile(filepath.Join(dir, filename), []byte(strings.Join(lines, "\n")), os.ModePerm) + err := os.WriteFile(filepath.Join(dir, filename), []byte(strings.Join(lines, "\n")), os.ModePerm) c.Assert(err, IsNil, Commentf("write tmp file %s failed", filename)) } @@ -449,7 +448,7 @@ func (s *testClusterTableBase) setupClusterGRPCServer(c *C) map[string]*testServ // create gRPC servers for _, typ := range []string{"tidb", "tikv", "pd"} { - tmpDir, err := ioutil.TempDir("", typ) + tmpDir, err := os.MkdirTemp("", typ) c.Assert(err, IsNil) server := grpc.NewServer() diff --git a/executor/merge_join_test.go b/executor/merge_join_test.go index 94b9c33a5a9d7..c58eb89181ef5 100644 --- a/executor/merge_join_test.go +++ b/executor/merge_join_test.go @@ -277,9 +277,10 @@ func (s *testSerialSuite1) TestShuffleMergeJoinInDisk(c *C) { c.Assert(tk.Se.GetSessionVars().StmtCtx.MemTracker.MaxConsumed(), Greater, int64(0)) c.Assert(tk.Se.GetSessionVars().StmtCtx.DiskTracker.BytesConsumed(), Equals, int64(0)) c.Assert(tk.Se.GetSessionVars().StmtCtx.DiskTracker.MaxConsumed(), Greater, int64(0)) - return } func (s *testSerialSuite1) TestMergeJoinInDisk(c *C) { + c.Skip("unstable, skip it and fix it before 20210618") + defer config.RestoreFunc()() config.UpdateGlobal(func(conf *config.Config) { conf.OOMUseTmpStorage = true @@ -313,7 +314,6 @@ func (s *testSerialSuite1) TestMergeJoinInDisk(c *C) { c.Assert(tk.Se.GetSessionVars().StmtCtx.MemTracker.MaxConsumed(), Greater, int64(0)) c.Assert(tk.Se.GetSessionVars().StmtCtx.DiskTracker.BytesConsumed(), Equals, int64(0)) c.Assert(tk.Se.GetSessionVars().StmtCtx.DiskTracker.MaxConsumed(), Greater, int64(0)) - return } func (s *testSuite2) TestMergeJoin(c *C) { @@ -726,6 +726,7 @@ func (s *testSuite2) TestMergeJoinDifferentTypes(c *C) { } // TestVectorizedMergeJoin is used to test vectorized merge join with some corner cases. +//nolint:gosimple // generates false positive fmt.Sprintf warnings which keep aligned func (s *testSuiteJoin3) TestVectorizedMergeJoin(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -841,6 +842,7 @@ func (s *testSuiteJoin3) TestVectorizedMergeJoin(c *C) { } // TestVectorizedShuffleMergeJoin is used to test vectorized shuffle merge join with some corner cases. +//nolint:gosimple // generates false positive fmt.Sprintf warnings which keep aligned func (s *testSuiteJoin3) TestVectorizedShuffleMergeJoin(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("set @@session.tidb_merge_join_concurrency = 4;") diff --git a/executor/metrics_reader_test.go b/executor/metrics_reader_test.go index 19000b4faee5d..8d75ac41fd96e 100644 --- a/executor/metrics_reader_test.go +++ b/executor/metrics_reader_test.go @@ -20,10 +20,8 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/parser" "github.com/pingcap/tidb/executor" - "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/planner" plannercore "github.com/pingcap/tidb/planner/core" - "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/util/testkit" ) @@ -62,10 +60,11 @@ func (s *testSuite7) TestStmtLabel(c *C) { for _, tt := range tests { stmtNode, err := parser.New().ParseOneStmt(tt.sql, "", "") c.Check(err, IsNil) - is := infoschema.GetInfoSchema(tk.Se) - err = plannercore.Preprocess(tk.Se.(sessionctx.Context), stmtNode, is) + preprocessorReturn := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(tk.Se, stmtNode, plannercore.WithPreprocessorReturn(preprocessorReturn)) + c.Check(err, IsNil) c.Assert(err, IsNil) - _, _, err = planner.Optimize(context.TODO(), tk.Se, stmtNode, is) + _, _, err = planner.Optimize(context.TODO(), tk.Se, stmtNode, preprocessorReturn.InfoSchema) c.Assert(err, IsNil) c.Assert(executor.GetStmtLabel(stmtNode), Equals, tt.label) } diff --git a/executor/mpp_gather.go b/executor/mpp_gather.go index 64236558af94e..e517a0130ec4f 100644 --- a/executor/mpp_gather.go +++ b/executor/mpp_gather.go @@ -30,7 +30,7 @@ import ( ) func useMPPExecution(ctx sessionctx.Context, tr *plannercore.PhysicalTableReader) bool { - if !ctx.GetSessionVars().AllowMPPExecution { + if !ctx.GetSessionVars().IsMPPAllowed() { return false } _, ok := tr.GetTablePlan().(*plannercore.PhysicalExchangeSender) @@ -50,7 +50,7 @@ type MPPGather struct { respIter distsql.SelectResult } -func (e *MPPGather) appendMPPDispatchReq(pf *plannercore.Fragment, tasks []*kv.MPPTask, isRoot bool) error { +func (e *MPPGather) appendMPPDispatchReq(pf *plannercore.Fragment) error { dagReq, _, err := constructDAGReq(e.ctx, []plannercore.PhysicalPlan{pf.ExchangeSender}, kv.TiFlash) if err != nil { return errors.Trace(err) @@ -58,12 +58,12 @@ func (e *MPPGather) appendMPPDispatchReq(pf *plannercore.Fragment, tasks []*kv.M for i := range pf.ExchangeSender.Schema().Columns { dagReq.OutputOffsets = append(dagReq.OutputOffsets, uint32(i)) } - if !isRoot { + if !pf.IsRoot { dagReq.EncodeType = tipb.EncodeType_TypeCHBlock } else { dagReq.EncodeType = tipb.EncodeType_TypeChunk } - for _, mppTask := range tasks { + for _, mppTask := range pf.ExchangeSender.Tasks { err := updateExecutorTableID(context.Background(), dagReq.RootExecutor, mppTask.TableID, true) if err != nil { return errors.Trace(err) @@ -77,7 +77,7 @@ func (e *MPPGather) appendMPPDispatchReq(pf *plannercore.Fragment, tasks []*kv.M Data: pbData, Meta: mppTask.Meta, ID: mppTask.ID, - IsRoot: isRoot, + IsRoot: pf.IsRoot, Timeout: 10, SchemaVar: e.is.SchemaMetaVersion(), StartTs: e.startTS, @@ -85,12 +85,6 @@ func (e *MPPGather) appendMPPDispatchReq(pf *plannercore.Fragment, tasks []*kv.M } e.mppReqs = append(e.mppReqs, req) } - for _, r := range pf.ExchangeReceivers { - err = e.appendMPPDispatchReq(r.GetExchangeSender().Fragment, r.Tasks, false) - if err != nil { - return errors.Trace(err) - } - } return nil } @@ -108,13 +102,15 @@ func (e *MPPGather) Open(ctx context.Context) (err error) { // TODO: Move the construct tasks logic to planner, so we can see the explain results. sender := e.originalPlan.(*plannercore.PhysicalExchangeSender) planIDs := collectPlanIDS(e.originalPlan, nil) - rootTasks, err := plannercore.GenerateRootMPPTasks(e.ctx, e.startTS, sender, e.is) + frags, err := plannercore.GenerateRootMPPTasks(e.ctx, e.startTS, sender, e.is) if err != nil { return errors.Trace(err) } - err = e.appendMPPDispatchReq(sender.Fragment, rootTasks, true) - if err != nil { - return errors.Trace(err) + for _, frag := range frags { + err = e.appendMPPDispatchReq(frag) + if err != nil { + return errors.Trace(err) + } } failpoint.Inject("checkTotalMPPTasks", func(val failpoint.Value) { if val.(int) != len(e.mppReqs) { diff --git a/executor/parallel_apply.go b/executor/parallel_apply.go index d02ebac9e5349..636ec96ad2868 100644 --- a/executor/parallel_apply.go +++ b/executor/parallel_apply.go @@ -68,6 +68,7 @@ type ParallelNestedLoopApplyExec struct { // fields about concurrency control concurrency int started uint32 + drained uint32 // drained == true indicates there is no more data freeChkCh chan *chunk.Chunk resultChkCh chan result outerRowCh chan outerRow @@ -130,6 +131,11 @@ func (e *ParallelNestedLoopApplyExec) Open(ctx context.Context) error { // Next implements the Executor interface. func (e *ParallelNestedLoopApplyExec) Next(ctx context.Context, req *chunk.Chunk) (err error) { + if atomic.LoadUint32(&e.drained) == 1 { + req.Reset() + return nil + } + if atomic.CompareAndSwapUint32(&e.started, 0, 1) { e.workerWg.Add(1) go e.outerWorker(ctx) @@ -147,6 +153,7 @@ func (e *ParallelNestedLoopApplyExec) Next(ctx context.Context, req *chunk.Chunk } if result.chk == nil { // no more data req.Reset() + atomic.StoreUint32(&e.drained, 1) return nil } req.SwapColumns(result.chk) diff --git a/executor/parallel_apply_test.go b/executor/parallel_apply_test.go index b849d3d961043..93448f0e3f92f 100644 --- a/executor/parallel_apply_test.go +++ b/executor/parallel_apply_test.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/util/collate" + "github.com/pingcap/tidb/util/israce" "github.com/pingcap/tidb/util/testkit" ) @@ -43,7 +44,6 @@ func checkApplyPlan(c *C, tk *testkit.TestKit, sql string, parallel int) { } } c.Assert(containApply, IsTrue) - return } func (s *testSuite) TestParallelApply(c *C) { @@ -570,6 +570,10 @@ func (s *testSuite) TestApplyCacheRatio(c *C) { } func (s *testSuite) TestApplyGoroutinePanic(c *C) { + if israce.RaceEnabled { + c.Skip("race detected, skip it temporarily and fix it before 20210619") + } + tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("set tidb_enable_parallel_apply=true") tk.MustExec("drop table if exists t1, t2") @@ -597,3 +601,14 @@ func (s *testSuite) TestApplyGoroutinePanic(c *C) { c.Assert(failpoint.Disable(panicPath), IsNil) } } + +func (s *testSuite) TestIssue24930(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("set tidb_enable_parallel_apply=true") + tk.MustExec("drop table if exists t1, t2") + tk.MustExec("create table t1(a int)") + tk.MustExec("create table t2(a int)") + tk.MustQuery(`select case when t1.a is null + then (select t2.a from t2 where t2.a = t1.a limit 1) else t1.a end a + from t1 where t1.a=1 order by a limit 1`).Check(testkit.Rows()) // can return an empty result instead of hanging forever +} diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go index ee7d807ef4e8f..11ec1b2a097e3 100644 --- a/executor/partition_table_test.go +++ b/executor/partition_table_test.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/util/israce" "github.com/pingcap/tidb/util/testkit" + "github.com/pingcap/tidb/util/testutil" ) func (s *partitionTableSuite) TestFourReader(c *C) { @@ -110,6 +111,80 @@ func (s *partitionTableSuite) TestPartitionUnionScanIndexJoin(c *C) { tk.MustExec("commit") } +func (s *partitionTableSuite) TestPointGetwithRangeAndListPartitionTable(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_pointget_list_hash") + tk.MustExec("use test_pointget_list_hash") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + tk.MustExec("set @@session.tidb_enable_list_partition = ON") + + // list partition table + tk.MustExec(`create table tlist(a int, b int, unique index idx_a(a), index idx_b(b)) partition by list(a)( + partition p0 values in (NULL, 1, 2, 3, 4), + partition p1 values in (5, 6, 7, 8), + partition p2 values in (9, 10, 11, 12));`) + + // range partition table + tk.MustExec(`create table trange1(a int, unique key(a)) partition by range(a) ( + partition p0 values less than (30), + partition p1 values less than (60), + partition p2 values less than (90), + partition p3 values less than (120));`) + + // range partition table + unsigned int + tk.MustExec(`create table trange2(a int unsigned, unique key(a)) partition by range(a) ( + partition p0 values less than (30), + partition p1 values less than (60), + partition p2 values less than (90), + partition p3 values less than (120));`) + + // insert data into list partition table + tk.MustExec("insert into tlist values(1,1), (2,2), (3, 3), (4, 4), (5,5), (6, 6), (7,7), (8, 8), (9, 9), (10, 10), (11, 11), (12, 12), (NULL, NULL);") + + vals := make([]string, 0, 100) + // insert data into range partition table and hash partition table + for i := 0; i < 100; i++ { + vals = append(vals, fmt.Sprintf("(%v)", i+1)) + } + tk.MustExec("insert into trange1 values " + strings.Join(vals, ",")) + tk.MustExec("insert into trange2 values " + strings.Join(vals, ",")) + + // test PointGet + for i := 0; i < 100; i++ { + // explain select a from t where a = {x}; // x >= 1 and x <= 100 Check if PointGet is used + // select a from t where a={x}; // the result is {x} + x := rand.Intn(100) + 1 + queryRange1 := fmt.Sprintf("select a from trange1 where a=%v", x) + c.Assert(tk.HasPlan(queryRange1, "Point_Get"), IsTrue) // check if PointGet is used + tk.MustQuery(queryRange1).Check(testkit.Rows(fmt.Sprintf("%v", x))) + + queryRange2 := fmt.Sprintf("select a from trange1 where a=%v", x) + c.Assert(tk.HasPlan(queryRange2, "Point_Get"), IsTrue) // check if PointGet is used + tk.MustQuery(queryRange2).Check(testkit.Rows(fmt.Sprintf("%v", x))) + + y := rand.Intn(12) + 1 + queryList := fmt.Sprintf("select a from tlist where a=%v", y) + c.Assert(tk.HasPlan(queryList, "Point_Get"), IsTrue) // check if PointGet is used + tk.MustQuery(queryList).Check(testkit.Rows(fmt.Sprintf("%v", y))) + } + + // test table dual + queryRange1 := "select a from trange1 where a=200" + c.Assert(tk.HasPlan(queryRange1, "TableDual"), IsTrue) // check if TableDual is used + tk.MustQuery(queryRange1).Check(testkit.Rows()) + + queryRange2 := "select a from trange2 where a=200" + c.Assert(tk.HasPlan(queryRange2, "TableDual"), IsTrue) // check if TableDual is used + tk.MustQuery(queryRange2).Check(testkit.Rows()) + + queryList := "select a from tlist where a=200" + c.Assert(tk.HasPlan(queryList, "TableDual"), IsTrue) // check if TableDual is used + tk.MustQuery(queryList).Check(testkit.Rows()) +} + func (s *partitionTableSuite) TestPartitionReaderUnderApply(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("use test") @@ -209,7 +284,7 @@ func (s *partitionTableSuite) TestPartitionInfoDisable(c *C) { PARTITION p202010 VALUES LESS THAN ("2020-11-01"), PARTITION p202011 VALUES LESS THAN ("2020-12-01") )`) - is := infoschema.GetInfoSchema(tk.Se) + is := tk.Se.GetInfoSchema().(infoschema.InfoSchema) tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t_info_null")) c.Assert(err, IsNil) @@ -227,6 +302,489 @@ func (s *partitionTableSuite) TestPartitionInfoDisable(c *C) { tk.MustQuery("select * from t_info_null where (date = '2020-10-02' or date = '2020-10-06') and app = 'xxx' and media = '19003006'").Check(testkit.Rows()) } +func (s *partitionTableSuite) TestOrderByandLimit(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_orderby_limit") + tk.MustExec("use test_orderby_limit") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + // range partition table + tk.MustExec(`create table trange(a int, b int, index idx_a(a)) partition by range(a) ( + partition p0 values less than(300), + partition p1 values less than (500), + partition p2 values less than(1100));`) + + // hash partition table + tk.MustExec("create table thash(a int, b int, index idx_a(a), index idx_b(b)) partition by hash(a) partitions 4;") + + // regular table + tk.MustExec("create table tregular(a int, b int, index idx_a(a))") + + // generate some random data to be inserted + vals := make([]string, 0, 2000) + for i := 0; i < 2000; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(1100), rand.Intn(2000))) + } + tk.MustExec("insert into trange values " + strings.Join(vals, ",")) + tk.MustExec("insert into thash values " + strings.Join(vals, ",")) + tk.MustExec("insert into tregular values " + strings.Join(vals, ",")) + + // test indexLookUp + for i := 0; i < 100; i++ { + // explain select * from t where a > {y} use index(idx_a) order by a limit {x}; // check if IndexLookUp is used + // select * from t where a > {y} use index(idx_a) order by a limit {x}; // it can return the correct result + x := rand.Intn(1099) + y := rand.Intn(2000) + 1 + queryPartition := fmt.Sprintf("select * from trange use index(idx_a) where a > %v order by a, b limit %v;", x, y) + queryRegular := fmt.Sprintf("select * from tregular use index(idx_a) where a > %v order by a, b limit %v;", x, y) + c.Assert(tk.HasPlan(queryPartition, "IndexLookUp"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + } + + // test tableReader + for i := 0; i < 100; i++ { + // explain select * from t where a > {y} ignore index(idx_a) order by a limit {x}; // check if IndexLookUp is used + // select * from t where a > {y} ignore index(idx_a) order by a limit {x}; // it can return the correct result + x := rand.Intn(1099) + y := rand.Intn(2000) + 1 + queryPartition := fmt.Sprintf("select * from trange ignore index(idx_a) where a > %v order by a, b limit %v;", x, y) + queryRegular := fmt.Sprintf("select * from tregular ignore index(idx_a) where a > %v order by a, b limit %v;", x, y) + c.Assert(tk.HasPlan(queryPartition, "TableReader"), IsTrue) // check if tableReader is used + tk.MustQuery(queryPartition).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + } + + // test indexReader + for i := 0; i < 100; i++ { + // explain select a from t where a > {y} use index(idx_a) order by a limit {x}; // check if IndexLookUp is used + // select a from t where a > {y} use index(idx_a) order by a limit {x}; // it can return the correct result + x := rand.Intn(1099) + y := rand.Intn(2000) + 1 + queryPartition := fmt.Sprintf("select a from trange use index(idx_a) where a > %v order by a limit %v;", x, y) + queryRegular := fmt.Sprintf("select a from tregular use index(idx_a) where a > %v order by a limit %v;", x, y) + c.Assert(tk.HasPlan(queryPartition, "IndexReader"), IsTrue) // check if indexReader is used + tk.MustQuery(queryPartition).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + } + + // test indexMerge + for i := 0; i < 100; i++ { + // explain select /*+ use_index_merge(t) */ * from t where a > 2 or b < 5 order by a limit {x}; // check if IndexMerge is used + // select /*+ use_index_merge(t) */ * from t where a > 2 or b < 5 order by a limit {x}; // can return the correct value + y := rand.Intn(2000) + 1 + queryPartition := fmt.Sprintf("select /*+ use_index_merge(thash) */ * from thash where a > 2 or b < 5 order by a, b limit %v;", y) + queryRegular := fmt.Sprintf("select * from tregular where a > 2 or b < 5 order by a, b limit %v;", y) + c.Assert(tk.HasPlan(queryPartition, "IndexMerge"), IsTrue) // check if indexMerge is used + tk.MustQuery(queryPartition).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + } +} + +func (s *partitionTableSuite) TestBatchGetandPointGetwithHashPartition(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_batchget_pointget") + tk.MustExec("use test_batchget_pointget") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + // hash partition table + tk.MustExec("create table thash(a int, unique key(a)) partition by hash(a) partitions 4;") + + // regular partition table + tk.MustExec("create table tregular(a int, unique key(a));") + + vals := make([]string, 0, 100) + // insert data into range partition table and hash partition table + for i := 0; i < 100; i++ { + vals = append(vals, fmt.Sprintf("(%v)", i+1)) + } + tk.MustExec("insert into thash values " + strings.Join(vals, ",")) + tk.MustExec("insert into tregular values " + strings.Join(vals, ",")) + + // test PointGet + for i := 0; i < 100; i++ { + // explain select a from t where a = {x}; // x >= 1 and x <= 100 Check if PointGet is used + // select a from t where a={x}; // the result is {x} + x := rand.Intn(100) + 1 + queryHash := fmt.Sprintf("select a from thash where a=%v", x) + queryRegular := fmt.Sprintf("select a from thash where a=%v", x) + c.Assert(tk.HasPlan(queryHash, "Point_Get"), IsTrue) // check if PointGet is used + tk.MustQuery(queryHash).Check(tk.MustQuery(queryRegular).Rows()) + } + + // test empty PointGet + queryHash := "select a from thash where a=200" + c.Assert(tk.HasPlan(queryHash, "Point_Get"), IsTrue) // check if PointGet is used + tk.MustQuery(queryHash).Check(testkit.Rows()) + + // test BatchGet + for i := 0; i < 100; i++ { + // explain select a from t where a in ({x1}, {x2}, ... {x10}); // BatchGet is used + // select a from t where where a in ({x1}, {x2}, ... {x10}); + points := make([]string, 0, 10) + for i := 0; i < 10; i++ { + x := rand.Intn(100) + 1 + points = append(points, fmt.Sprintf("%v", x)) + } + + queryHash := fmt.Sprintf("select a from thash where a in (%v)", strings.Join(points, ",")) + queryRegular := fmt.Sprintf("select a from tregular where a in (%v)", strings.Join(points, ",")) + c.Assert(tk.HasPlan(queryHash, "Point_Get"), IsTrue) // check if PointGet is used + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + } +} + +func (s *partitionTableSuite) TestView(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_view") + tk.MustExec("use test_view") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + tk.MustExec(`create table thash (a int, b int, key(a)) partition by hash(a) partitions 4`) + tk.MustExec(`create table trange (a varchar(10), b varchar(10), key(a)) partition by range columns(a) ( + partition p0 values less than ('300'), + partition p1 values less than ('600'), + partition p2 values less than ('900'), + partition p3 values less than ('9999'))`) + tk.MustExec(`create table t1 (a int, b int, key(a))`) + tk.MustExec(`create table t2 (a varchar(10), b varchar(10), key(a))`) + + // insert the same data into thash and t1 + vals := make([]string, 0, 3000) + for i := 0; i < 3000; i++ { + vals = append(vals, fmt.Sprintf(`(%v, %v)`, rand.Intn(10000), rand.Intn(10000))) + } + tk.MustExec(fmt.Sprintf(`insert into thash values %v`, strings.Join(vals, ", "))) + tk.MustExec(fmt.Sprintf(`insert into t1 values %v`, strings.Join(vals, ", "))) + + // insert the same data into trange and t2 + vals = vals[:0] + for i := 0; i < 2000; i++ { + vals = append(vals, fmt.Sprintf(`("%v", "%v")`, rand.Intn(1000), rand.Intn(1000))) + } + tk.MustExec(fmt.Sprintf(`insert into trange values %v`, strings.Join(vals, ", "))) + tk.MustExec(fmt.Sprintf(`insert into t2 values %v`, strings.Join(vals, ", "))) + + // test views on a single table + tk.MustExec(`create definer='root'@'localhost' view vhash as select a*2 as a, a+b as b from thash`) + tk.MustExec(`create definer='root'@'localhost' view v1 as select a*2 as a, a+b as b from t1`) + tk.MustExec(`create definer='root'@'localhost' view vrange as select concat(a, b) as a, a+b as b from trange`) + tk.MustExec(`create definer='root'@'localhost' view v2 as select concat(a, b) as a, a+b as b from t2`) + for i := 0; i < 100; i++ { + xhash := rand.Intn(10000) + tk.MustQuery(fmt.Sprintf(`select * from vhash where a>=%v`, xhash)).Sort().Check( + tk.MustQuery(fmt.Sprintf(`select * from v1 where a>=%v`, xhash)).Sort().Rows()) + tk.MustQuery(fmt.Sprintf(`select * from vhash where b>=%v`, xhash)).Sort().Check( + tk.MustQuery(fmt.Sprintf(`select * from v1 where b>=%v`, xhash)).Sort().Rows()) + tk.MustQuery(fmt.Sprintf(`select * from vhash where a>=%v and b>=%v`, xhash, xhash)).Sort().Check( + tk.MustQuery(fmt.Sprintf(`select * from v1 where a>=%v and b>=%v`, xhash, xhash)).Sort().Rows()) + + xrange := fmt.Sprintf(`"%v"`, rand.Intn(1000)) + tk.MustQuery(fmt.Sprintf(`select * from vrange where a>=%v`, xrange)).Sort().Check( + tk.MustQuery(fmt.Sprintf(`select * from v2 where a>=%v`, xrange)).Sort().Rows()) + tk.MustQuery(fmt.Sprintf(`select * from vrange where b>=%v`, xrange)).Sort().Check( + tk.MustQuery(fmt.Sprintf(`select * from v2 where b>=%v`, xrange)).Sort().Rows()) + tk.MustQuery(fmt.Sprintf(`select * from vrange where a>=%v and b<=%v`, xrange, xrange)).Sort().Check( + tk.MustQuery(fmt.Sprintf(`select * from v2 where a>=%v and b<=%v`, xrange, xrange)).Sort().Rows()) + } + + // test views on both tables + tk.MustExec(`create definer='root'@'localhost' view vboth as select thash.a+trange.a as a, thash.b+trange.b as b from thash, trange where thash.a=trange.a`) + tk.MustExec(`create definer='root'@'localhost' view vt as select t1.a+t2.a as a, t1.b+t2.b as b from t1, t2 where t1.a=t2.a`) + for i := 0; i < 100; i++ { + x := rand.Intn(10000) + tk.MustQuery(fmt.Sprintf(`select * from vboth where a>=%v`, x)).Sort().Check( + tk.MustQuery(fmt.Sprintf(`select * from vt where a>=%v`, x)).Sort().Rows()) + tk.MustQuery(fmt.Sprintf(`select * from vboth where b>=%v`, x)).Sort().Check( + tk.MustQuery(fmt.Sprintf(`select * from vt where b>=%v`, x)).Sort().Rows()) + tk.MustQuery(fmt.Sprintf(`select * from vboth where a>=%v and b>=%v`, x, x)).Sort().Check( + tk.MustQuery(fmt.Sprintf(`select * from vt where a>=%v and b>=%v`, x, x)).Sort().Rows()) + } +} + +func (s *partitionTableSuite) TestDirectReadingwithIndexJoin(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_dr_join") + tk.MustExec("use test_dr_join") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + // hash and range partition + tk.MustExec("create table thash (a int, b int, c int, primary key(a), index idx_b(b)) partition by hash(a) partitions 4;") + tk.MustExec(`create table trange (a int, b int, c int, primary key(a), index idx_b(b)) partition by range(a) ( +   partition p0 values less than(1000), +   partition p1 values less than(2000), +   partition p2 values less than(3000), +   partition p3 values less than(4000));`) + + // regualr table + tk.MustExec(`create table tnormal (a int, b int, c int, primary key(a), index idx_b(b));`) + tk.MustExec(`create table touter (a int, b int, c int);`) + + // generate some random data to be inserted + vals := make([]string, 0, 2000) + for i := 0; i < 2000; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v, %v)", rand.Intn(4000), rand.Intn(4000), rand.Intn(4000))) + } + tk.MustExec("insert ignore into trange values " + strings.Join(vals, ",")) + tk.MustExec("insert ignore into thash values " + strings.Join(vals, ",")) + tk.MustExec("insert ignore into tnormal values " + strings.Join(vals, ",")) + tk.MustExec("insert ignore into touter values " + strings.Join(vals, ",")) + + // test indexLookUp + hash + queryPartition := "select /*+ INL_JOIN(touter, thash) */ * from touter join thash use index(idx_b) on touter.b = thash.b" + queryRegular := "select /*+ INL_JOIN(touter, tnormal) */ * from touter join tnormal use index(idx_b) on touter.b = tnormal.b" + tk.MustQuery("explain format = 'brief' " + queryPartition).Check(testkit.Rows( + "IndexJoin 12487.50 root inner join, inner:IndexLookUp, outer key:test_dr_join.touter.b, inner key:test_dr_join.thash.b, equal cond:eq(test_dr_join.touter.b, test_dr_join.thash.b)", + "├─TableReader(Build) 9990.00 root data:Selection", + "│ └─Selection 9990.00 cop[tikv] not(isnull(test_dr_join.touter.b))", + "│ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo", + "└─IndexLookUp(Probe) 1.25 root partition:all ", + " ├─Selection(Build) 1.25 cop[tikv] not(isnull(test_dr_join.thash.b))", + " │ └─IndexRangeScan 1.25 cop[tikv] table:thash, index:idx_b(b) range: decided by [eq(test_dr_join.thash.b, test_dr_join.touter.b)], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 1.25 cop[tikv] table:thash keep order:false, stats:pseudo")) // check if IndexLookUp is used + tk.MustQuery(queryPartition).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + // test tableReader + hash + queryPartition = "select /*+ INL_JOIN(touter, thash) */ * from touter join thash on touter.a = thash.a" + queryRegular = "select /*+ INL_JOIN(touter, tnormal) */ * from touter join tnormal on touter.a = tnormal.a" + tk.MustQuery("explain format = 'brief' " + queryPartition).Check(testkit.Rows( + "IndexJoin 12487.50 root inner join, inner:TableReader, outer key:test_dr_join.touter.a, inner key:test_dr_join.thash.a, equal cond:eq(test_dr_join.touter.a, test_dr_join.thash.a)", + "├─TableReader(Build) 9990.00 root data:Selection", + "│ └─Selection 9990.00 cop[tikv] not(isnull(test_dr_join.touter.a))", + "│ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo", + "└─TableReader(Probe) 1.00 root partition:all data:TableRangeScan", + " └─TableRangeScan 1.00 cop[tikv] table:thash range: decided by [test_dr_join.touter.a], keep order:false, stats:pseudo")) // check if tableReader is used + tk.MustQuery(queryPartition).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + // test indexReader + hash + queryPartition = "select /*+ INL_JOIN(touter, thash) */ thash.b from touter join thash use index(idx_b) on touter.b = thash.b;" + queryRegular = "select /*+ INL_JOIN(touter, tnormal) */ tnormal.b from touter join tnormal use index(idx_b) on touter.b = tnormal.b;" + tk.MustQuery("explain format = 'brief' " + queryPartition).Check(testkit.Rows( + "IndexJoin 12487.50 root inner join, inner:IndexReader, outer key:test_dr_join.touter.b, inner key:test_dr_join.thash.b, equal cond:eq(test_dr_join.touter.b, test_dr_join.thash.b)", + "├─TableReader(Build) 9990.00 root data:Selection", + "│ └─Selection 9990.00 cop[tikv] not(isnull(test_dr_join.touter.b))", + "│ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo", + "└─IndexReader(Probe) 1.25 root partition:all index:Selection", + " └─Selection 1.25 cop[tikv] not(isnull(test_dr_join.thash.b))", + " └─IndexRangeScan 1.25 cop[tikv] table:thash, index:idx_b(b) range: decided by [eq(test_dr_join.thash.b, test_dr_join.touter.b)], keep order:false, stats:pseudo")) // check if indexReader is used + tk.MustQuery(queryPartition).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + // test indexLookUp + range + // explain select /*+ INL_JOIN(touter, tinner) */ * from touter join tinner use index(a) on touter.a = tinner.a; + queryPartition = "select /*+ INL_JOIN(touter, trange) */ * from touter join trange use index(idx_b) on touter.b = trange.b;" + queryRegular = "select /*+ INL_JOIN(touter, tnormal) */ * from touter join tnormal use index(idx_b) on touter.b = tnormal.b;" + tk.MustQuery("explain format = 'brief' " + queryPartition).Check(testkit.Rows( + "IndexJoin 12487.50 root inner join, inner:IndexLookUp, outer key:test_dr_join.touter.b, inner key:test_dr_join.trange.b, equal cond:eq(test_dr_join.touter.b, test_dr_join.trange.b)", + "├─TableReader(Build) 9990.00 root data:Selection", + "│ └─Selection 9990.00 cop[tikv] not(isnull(test_dr_join.touter.b))", + "│ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo", + "└─IndexLookUp(Probe) 1.25 root partition:all ", + " ├─Selection(Build) 1.25 cop[tikv] not(isnull(test_dr_join.trange.b))", + " │ └─IndexRangeScan 1.25 cop[tikv] table:trange, index:idx_b(b) range: decided by [eq(test_dr_join.trange.b, test_dr_join.touter.b)], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 1.25 cop[tikv] table:trange keep order:false, stats:pseudo")) // check if IndexLookUp is used + tk.MustQuery(queryPartition).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + // test tableReader + range + queryPartition = "select /*+ INL_JOIN(touter, trange) */ * from touter join trange on touter.a = trange.a;" + queryRegular = "select /*+ INL_JOIN(touter, tnormal) */ * from touter join tnormal on touter.a = tnormal.a;" + tk.MustQuery("explain format = 'brief' " + queryPartition).Check(testkit.Rows( + "IndexJoin 12487.50 root inner join, inner:TableReader, outer key:test_dr_join.touter.a, inner key:test_dr_join.trange.a, equal cond:eq(test_dr_join.touter.a, test_dr_join.trange.a)", + "├─TableReader(Build) 9990.00 root data:Selection", + "│ └─Selection 9990.00 cop[tikv] not(isnull(test_dr_join.touter.a))", + "│ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo", + "└─TableReader(Probe) 1.00 root partition:all data:TableRangeScan", + " └─TableRangeScan 1.00 cop[tikv] table:trange range: decided by [test_dr_join.touter.a], keep order:false, stats:pseudo")) // check if tableReader is used + tk.MustQuery(queryPartition).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + // test indexReader + range + // explain select /*+ INL_JOIN(touter, tinner) */ tinner.a from touter join tinner on touter.a = tinner.a; + queryPartition = "select /*+ INL_JOIN(touter, trange) */ trange.b from touter join trange use index(idx_b) on touter.b = trange.b;" + queryRegular = "select /*+ INL_JOIN(touter, tnormal) */ tnormal.b from touter join tnormal use index(idx_b) on touter.b = tnormal.b;" + tk.MustQuery("explain format = 'brief' " + queryPartition).Check(testkit.Rows( + "IndexJoin 12487.50 root inner join, inner:IndexReader, outer key:test_dr_join.touter.b, inner key:test_dr_join.trange.b, equal cond:eq(test_dr_join.touter.b, test_dr_join.trange.b)", + "├─TableReader(Build) 9990.00 root data:Selection", + "│ └─Selection 9990.00 cop[tikv] not(isnull(test_dr_join.touter.b))", + "│ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo", + "└─IndexReader(Probe) 1.25 root partition:all index:Selection", + " └─Selection 1.25 cop[tikv] not(isnull(test_dr_join.trange.b))", + " └─IndexRangeScan 1.25 cop[tikv] table:trange, index:idx_b(b) range: decided by [eq(test_dr_join.trange.b, test_dr_join.touter.b)], keep order:false, stats:pseudo")) // check if indexReader is used + tk.MustQuery(queryPartition).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) +} + +func (s *partitionTableSuite) TestDynamicPruningUnderIndexJoin(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + + tk.MustExec("create database pruing_under_index_join") + tk.MustExec("use pruing_under_index_join") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + tk.MustExec(`create table tnormal (a int, b int, c int, primary key(a), index idx_b(b))`) + tk.MustExec(`create table thash (a int, b int, c int, primary key(a), index idx_b(b)) partition by hash(a) partitions 4`) + tk.MustExec(`create table touter (a int, b int, c int)`) + + vals := make([]string, 0, 2000) + for i := 0; i < 2000; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v, %v)", i, rand.Intn(10000), rand.Intn(10000))) + } + tk.MustExec(`insert into tnormal values ` + strings.Join(vals, ", ")) + tk.MustExec(`insert into thash values ` + strings.Join(vals, ", ")) + tk.MustExec(`insert into touter values ` + strings.Join(vals, ", ")) + + // case 1: IndexReader in the inner side + tk.MustQuery(`explain format='brief' select /*+ INL_JOIN(touter, thash) */ thash.b from touter join thash use index(idx_b) on touter.b = thash.b`).Check(testkit.Rows( + `IndexJoin 12487.50 root inner join, inner:IndexReader, outer key:pruing_under_index_join.touter.b, inner key:pruing_under_index_join.thash.b, equal cond:eq(pruing_under_index_join.touter.b, pruing_under_index_join.thash.b)`, + `├─TableReader(Build) 9990.00 root data:Selection`, + `│ └─Selection 9990.00 cop[tikv] not(isnull(pruing_under_index_join.touter.b))`, + `│ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo`, + `└─IndexReader(Probe) 1.25 root partition:all index:Selection`, + ` └─Selection 1.25 cop[tikv] not(isnull(pruing_under_index_join.thash.b))`, + ` └─IndexRangeScan 1.25 cop[tikv] table:thash, index:idx_b(b) range: decided by [eq(pruing_under_index_join.thash.b, pruing_under_index_join.touter.b)], keep order:false, stats:pseudo`)) + tk.MustQuery(`select /*+ INL_JOIN(touter, thash) */ thash.b from touter join thash use index(idx_b) on touter.b = thash.b`).Sort().Check( + tk.MustQuery(`select /*+ INL_JOIN(touter, tnormal) */ tnormal.b from touter join tnormal use index(idx_b) on touter.b = tnormal.b`).Sort().Rows()) + + // case 2: TableReader in the inner side + tk.MustQuery(`explain format='brief' select /*+ INL_JOIN(touter, thash) */ thash.* from touter join thash use index(primary) on touter.b = thash.a`).Check(testkit.Rows( + `IndexJoin 12487.50 root inner join, inner:TableReader, outer key:pruing_under_index_join.touter.b, inner key:pruing_under_index_join.thash.a, equal cond:eq(pruing_under_index_join.touter.b, pruing_under_index_join.thash.a)`, + `├─TableReader(Build) 9990.00 root data:Selection`, + `│ └─Selection 9990.00 cop[tikv] not(isnull(pruing_under_index_join.touter.b))`, + `│ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo`, + `└─TableReader(Probe) 1.00 root partition:all data:TableRangeScan`, + ` └─TableRangeScan 1.00 cop[tikv] table:thash range: decided by [pruing_under_index_join.touter.b], keep order:false, stats:pseudo`)) + tk.MustQuery(`select /*+ INL_JOIN(touter, thash) */ thash.* from touter join thash use index(primary) on touter.b = thash.a`).Sort().Check( + tk.MustQuery(`select /*+ INL_JOIN(touter, tnormal) */ tnormal.* from touter join tnormal use index(primary) on touter.b = tnormal.a`).Sort().Rows()) + + // case 3: IndexLookUp in the inner side + read all inner columns + tk.MustQuery(`explain format='brief' select /*+ INL_JOIN(touter, thash) */ thash.* from touter join thash use index(idx_b) on touter.b = thash.b`).Check(testkit.Rows( + `IndexJoin 12487.50 root inner join, inner:IndexLookUp, outer key:pruing_under_index_join.touter.b, inner key:pruing_under_index_join.thash.b, equal cond:eq(pruing_under_index_join.touter.b, pruing_under_index_join.thash.b)`, + `├─TableReader(Build) 9990.00 root data:Selection`, + `│ └─Selection 9990.00 cop[tikv] not(isnull(pruing_under_index_join.touter.b))`, + `│ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo`, + `└─IndexLookUp(Probe) 1.25 root partition:all `, + ` ├─Selection(Build) 1.25 cop[tikv] not(isnull(pruing_under_index_join.thash.b))`, + ` │ └─IndexRangeScan 1.25 cop[tikv] table:thash, index:idx_b(b) range: decided by [eq(pruing_under_index_join.thash.b, pruing_under_index_join.touter.b)], keep order:false, stats:pseudo`, + ` └─TableRowIDScan(Probe) 1.25 cop[tikv] table:thash keep order:false, stats:pseudo`)) + tk.MustQuery(`select /*+ INL_JOIN(touter, thash) */ thash.* from touter join thash use index(idx_b) on touter.b = thash.b`).Sort().Check( + tk.MustQuery(`select /*+ INL_JOIN(touter, tnormal) */ tnormal.* from touter join tnormal use index(idx_b) on touter.b = tnormal.b`).Sort().Rows()) +} + +func (s *partitionTableSuite) TestBatchGetforRangeandListPartitionTable(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_pointget") + tk.MustExec("use test_pointget") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + tk.MustExec("set @@session.tidb_enable_list_partition = ON") + + // list partition table + tk.MustExec(`create table tlist(a int, b int, unique index idx_a(a), index idx_b(b)) partition by list(a)( + partition p0 values in (1, 2, 3, 4), + partition p1 values in (5, 6, 7, 8), + partition p2 values in (9, 10, 11, 12));`) + + // range partition table + tk.MustExec(`create table trange(a int, unique key(a)) partition by range(a) ( + partition p0 values less than (30), + partition p1 values less than (60), + partition p2 values less than (90), + partition p3 values less than (120));`) + + // hash partition table + tk.MustExec("create table thash(a int unsigned, unique key(a)) partition by hash(a) partitions 4;") + + // insert data into list partition table + tk.MustExec("insert into tlist values(1,1), (2,2), (3, 3), (4, 4), (5,5), (6, 6), (7,7), (8, 8), (9, 9), (10, 10), (11, 11), (12, 12);") + // regular partition table + tk.MustExec("create table tregular1(a int, unique key(a));") + tk.MustExec("create table tregular2(a int, unique key(a));") + + vals := make([]string, 0, 100) + // insert data into range partition table and hash partition table + for i := 0; i < 100; i++ { + vals = append(vals, fmt.Sprintf("(%v)", i+1)) + } + tk.MustExec("insert into trange values " + strings.Join(vals, ",")) + tk.MustExec("insert into thash values " + strings.Join(vals, ",")) + tk.MustExec("insert into tregular1 values " + strings.Join(vals, ",")) + tk.MustExec("insert into tregular2 values (1), (2), (3), (4), (5), (6), (7), (8), (9), (10), (11), (12)") + + // test BatchGet + for i := 0; i < 100; i++ { + // explain select a from t where a in ({x1}, {x2}, ... {x10}); // BatchGet is used + // select a from t where where a in ({x1}, {x2}, ... {x10}); + points := make([]string, 0, 10) + for i := 0; i < 10; i++ { + x := rand.Intn(100) + 1 + points = append(points, fmt.Sprintf("%v", x)) + } + queryRegular1 := fmt.Sprintf("select a from tregular1 where a in (%v)", strings.Join(points, ",")) + + queryHash := fmt.Sprintf("select a from thash where a in (%v)", strings.Join(points, ",")) + c.Assert(tk.HasPlan(queryHash, "Batch_Point_Get"), IsTrue) // check if BatchGet is used + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular1).Sort().Rows()) + + queryRange := fmt.Sprintf("select a from trange where a in (%v)", strings.Join(points, ",")) + c.Assert(tk.HasPlan(queryRange, "Batch_Point_Get"), IsTrue) // check if BatchGet is used + tk.MustQuery(queryRange).Sort().Check(tk.MustQuery(queryRegular1).Sort().Rows()) + + points = make([]string, 0, 10) + for i := 0; i < 10; i++ { + x := rand.Intn(12) + 1 + points = append(points, fmt.Sprintf("%v", x)) + } + queryRegular2 := fmt.Sprintf("select a from tregular2 where a in (%v)", strings.Join(points, ",")) + queryList := fmt.Sprintf("select a from tlist where a in (%v)", strings.Join(points, ",")) + c.Assert(tk.HasPlan(queryList, "Batch_Point_Get"), IsTrue) // check if BatchGet is used + tk.MustQuery(queryList).Sort().Check(tk.MustQuery(queryRegular2).Sort().Rows()) + } + + // test different data type + // unsigned flag + // partition table and reguar table pair + tk.MustExec(`create table trange3(a int unsigned, unique key(a)) partition by range(a) ( + partition p0 values less than (30), + partition p1 values less than (60), + partition p2 values less than (90), + partition p3 values less than (120));`) + tk.MustExec("create table tregular3(a int unsigned, unique key(a));") + vals = make([]string, 0, 100) + // insert data into range partition table and hash partition table + for i := 0; i < 100; i++ { + vals = append(vals, fmt.Sprintf("(%v)", i+1)) + } + tk.MustExec("insert into trange3 values " + strings.Join(vals, ",")) + tk.MustExec("insert into tregular3 values " + strings.Join(vals, ",")) + // test BatchGet + // explain select a from t where a in ({x1}, {x2}, ... {x10}); // BatchGet is used + // select a from t where where a in ({x1}, {x2}, ... {x10}); + points := make([]string, 0, 10) + for i := 0; i < 10; i++ { + x := rand.Intn(100) + 1 + points = append(points, fmt.Sprintf("%v", x)) + } + queryRegular := fmt.Sprintf("select a from tregular3 where a in (%v)", strings.Join(points, ",")) + queryRange := fmt.Sprintf("select a from trange3 where a in (%v)", strings.Join(points, ",")) + c.Assert(tk.HasPlan(queryRange, "Batch_Point_Get"), IsTrue) // check if BatchGet is used + tk.MustQuery(queryRange).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) +} + func (s *partitionTableSuite) TestGlobalStatsAndSQLBinding(c *C) { if israce.RaceEnabled { c.Skip("exhaustive types test, skip race test") @@ -305,32 +863,1892 @@ func (s *partitionTableSuite) TestGlobalStatsAndSQLBinding(c *C) { tk.MustIndexLookup("select * from tlist where a<1") } -func (s *globalIndexSuite) TestGlobalIndexScan(c *C) { +func (s *partitionTableSuite) TestPartitionTableWithDifferentJoin(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustExec("drop table if exists p") - tk.MustExec(`create table p (id int, c int) partition by range (c) ( -partition p0 values less than (4), -partition p1 values less than (7), -partition p2 values less than (10))`) - tk.MustExec("alter table p add unique idx(id)") - tk.MustExec("insert into p values (1,3), (3,4), (5,6), (7,9)") - tk.MustQuery("select id from p use index (idx)").Check(testkit.Rows("1", "3", "5", "7")) + tk.MustExec("create database test_partition_joins") + tk.MustExec("use test_partition_joins") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + // hash and range partition + tk.MustExec("create table thash(a int, b int, key(a)) partition by hash(a) partitions 4") + tk.MustExec("create table tregular1(a int, b int, key(a))") + + tk.MustExec(`create table trange(a int, b int, key(a)) partition by range(a) ( + partition p0 values less than (200), + partition p1 values less than (400), + partition p2 values less than (600), + partition p3 values less than (800), + partition p4 values less than (1001))`) + tk.MustExec("create table tregular2(a int, b int, key(a))") + + vals := make([]string, 0, 2000) + for i := 0; i < 2000; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(1000), rand.Intn(1000))) + } + tk.MustExec("insert into thash values " + strings.Join(vals, ",")) + tk.MustExec("insert into tregular1 values " + strings.Join(vals, ",")) + + vals = make([]string, 0, 2000) + for i := 0; i < 2000; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(1000), rand.Intn(1000))) + } + tk.MustExec("insert into trange values " + strings.Join(vals, ",")) + tk.MustExec("insert into tregular2 values " + strings.Join(vals, ",")) + + // random params + x1 := rand.Intn(1000) + x2 := rand.Intn(1000) + x3 := rand.Intn(1000) + x4 := rand.Intn(1000) + + // group 1 + // hash_join range partition and hash partition + queryHash := fmt.Sprintf("select /*+ hash_join(trange, thash) */ * from trange, thash where trange.b=thash.b and thash.a = %v and trange.a > %v;", x1, x2) + queryRegular := fmt.Sprintf("select /*+ hash_join(tregular2, tregular1) */ * from tregular2, tregular1 where tregular2.b=tregular1.b and tregular1.a = %v and tregular2.a > %v;", x1, x2) + c.Assert(tk.HasPlan(queryHash, "HashJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ hash_join(trange, thash) */ * from trange, thash where trange.a=thash.a and thash.a > %v;", x1) + queryRegular = fmt.Sprintf("select /*+ hash_join(tregular2, tregular1) */ * from tregular2, tregular1 where tregular2.a=tregular1.a and tregular1.a > %v;", x1) + c.Assert(tk.HasPlan(queryHash, "HashJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ hash_join(trange, thash) */ * from trange, thash where trange.a=thash.a and trange.b = thash.b and thash.a > %v;", x1) + queryRegular = fmt.Sprintf("select /*+ hash_join(tregular2, tregular1) */ * from tregular2, tregular1 where tregular2.a=tregular1.a and tregular1.b = tregular2.b and tregular1.a > %v;", x1) + c.Assert(tk.HasPlan(queryHash, "HashJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ hash_join(trange, thash) */ * from trange, thash where trange.a=thash.a and thash.a = %v;", x1) + queryRegular = fmt.Sprintf("select /*+ hash_join(tregular2, tregular1) */ * from tregular2, tregular1 where tregular2.a=tregular1.a and tregular1.a = %v;", x1) + c.Assert(tk.HasPlan(queryHash, "HashJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + // group 2 + // hash_join range partition and regular table + queryHash = fmt.Sprintf("select /*+ hash_join(trange, tregular1) */ * from trange, tregular1 where trange.a = tregular1.a and trange.a >= %v and tregular1.a > %v;", x1, x2) + queryRegular = fmt.Sprintf("select /*+ hash_join(tregular2, tregular1) */ * from tregular2, tregular1 where tregular2.a = tregular1.a and tregular2.a >= %v and tregular1.a > %v;", x1, x2) + c.Assert(tk.HasPlan(queryHash, "HashJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ hash_join(trange, tregular1) */ * from trange, tregular1 where trange.a = tregular1.a and trange.a in (%v, %v, %v);", x1, x2, x3) + queryRegular = fmt.Sprintf("select /*+ hash_join(tregular2, tregular1) */ * from tregular2, tregular1 where tregular2.a = tregular1.a and tregular2.a in (%v, %v, %v);", x1, x2, x3) + c.Assert(tk.HasPlan(queryHash, "HashJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ hash_join(trange, tregular1) */ * from trange, tregular1 where trange.a = tregular1.a and tregular1.a >= %v;", x1) + queryRegular = fmt.Sprintf("select /*+ hash_join(tregular2, tregular1) */ * from tregular2, tregular1 where tregular2.a = tregular1.a and tregular1.a >= %v;", x1) + c.Assert(tk.HasPlan(queryHash, "HashJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + // group 3 + // merge_join range partition and hash partition + queryHash = fmt.Sprintf("select /*+ merge_join(trange, thash) */ * from trange, thash where trange.b=thash.b and thash.a = %v and trange.a > %v;", x1, x2) + queryRegular = fmt.Sprintf("select /*+ merge_join(tregular2, tregular1) */ * from tregular2, tregular1 where tregular2.b=tregular1.b and tregular1.a = %v and tregular2.a > %v;", x1, x2) + c.Assert(tk.HasPlan(queryHash, "MergeJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ merge_join(trange, thash) */ * from trange, thash where trange.a=thash.a and thash.a > %v;", x1) + queryRegular = fmt.Sprintf("select /*+ merge_join(tregular2, tregular1) */ * from tregular2, tregular1 where tregular2.a=tregular1.a and tregular1.a > %v;", x1) + c.Assert(tk.HasPlan(queryHash, "MergeJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ merge_join(trange, thash) */ * from trange, thash where trange.a=thash.a and trange.b = thash.b and thash.a > %v;", x1) + queryRegular = fmt.Sprintf("select /*+ merge_join(tregular2, tregular1) */ * from tregular2, tregular1 where tregular2.a=tregular1.a and tregular1.b = tregular2.b and tregular1.a > %v;", x1) + c.Assert(tk.HasPlan(queryHash, "MergeJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ merge_join(trange, thash) */ * from trange, thash where trange.a=thash.a and thash.a = %v;", x1) + queryRegular = fmt.Sprintf("select /*+ merge_join(tregular2, tregular1) */ * from tregular2, tregular1 where tregular2.a=tregular1.a and tregular1.a = %v;", x1) + c.Assert(tk.HasPlan(queryHash, "MergeJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + // group 4 + // merge_join range partition and regular table + queryHash = fmt.Sprintf("select /*+ merge_join(trange, tregular1) */ * from trange, tregular1 where trange.a = tregular1.a and trange.a >= %v and tregular1.a > %v;", x1, x2) + queryRegular = fmt.Sprintf("select /*+ merge_join(tregular2, tregular1) */ * from tregular2, tregular1 where tregular2.a = tregular1.a and tregular2.a >= %v and tregular1.a > %v;", x1, x2) + c.Assert(tk.HasPlan(queryHash, "MergeJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ merge_join(trange, tregular1) */ * from trange, tregular1 where trange.a = tregular1.a and trange.a in (%v, %v, %v);", x1, x2, x3) + queryRegular = fmt.Sprintf("select /*+ merge_join(tregular2, tregular1) */ * from tregular2, tregular1 where tregular2.a = tregular1.a and tregular2.a in (%v, %v, %v);", x1, x2, x3) + c.Assert(tk.HasPlan(queryHash, "MergeJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ merge_join(trange, tregular1) */ * from trange, tregular1 where trange.a = tregular1.a and tregular1.a >= %v;", x1) + queryRegular = fmt.Sprintf("select /*+ merge_join(tregular2, tregular1) */ * from tregular2, tregular1 where tregular2.a = tregular1.a and tregular1.a >= %v;", x1) + c.Assert(tk.HasPlan(queryHash, "MergeJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + // new table instances + tk.MustExec("create table thash2(a int, b int, index idx(a)) partition by hash(a) partitions 4") + tk.MustExec("create table tregular3(a int, b int, index idx(a))") + + tk.MustExec(`create table trange2(a int, b int, index idx(a)) partition by range(a) ( + partition p0 values less than (200), + partition p1 values less than (400), + partition p2 values less than (600), + partition p3 values less than (800), + partition p4 values less than (1001))`) + tk.MustExec("create table tregular4(a int, b int, index idx(a))") + + vals = make([]string, 0, 2000) + for i := 0; i < 2000; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(1000), rand.Intn(1000))) + } + tk.MustExec("insert into thash2 values " + strings.Join(vals, ",")) + tk.MustExec("insert into tregular3 values " + strings.Join(vals, ",")) + + vals = make([]string, 0, 2000) + for i := 0; i < 2000; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(1000), rand.Intn(1000))) + } + tk.MustExec("insert into trange2 values " + strings.Join(vals, ",")) + tk.MustExec("insert into tregular4 values " + strings.Join(vals, ",")) + + // group 5 + // index_merge_join range partition and range partition + // Currently don't support index merge join on two partition tables. Only test warning. + queryHash = fmt.Sprintf("select /*+ inl_merge_join(trange, trange2) */ * from trange, trange2 where trange.a=trange2.a and trange.a > %v;", x1) + // queryRegular = fmt.Sprintf("select /*+ inl_merge_join(tregular2, tregular4) */ * from tregular2, tregular4 where tregular2.a=tregular4.a and tregular2.a > %v;", x1) + // c.Assert(tk.HasPlan(queryHash, "IndexMergeJoin"), IsTrue) + // tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + tk.MustQuery(queryHash) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1815|Optimizer Hint /*+ INL_MERGE_JOIN(trange, trange2) */ is inapplicable")) + + queryHash = fmt.Sprintf("select /*+ inl_merge_join(trange, trange2) */ * from trange, trange2 where trange.a=trange2.a and trange.a > %v and trange2.a > %v;", x1, x2) + // queryRegular = fmt.Sprintf("select /*+ inl_merge_join(tregular2, tregular4) */ * from tregular2, tregular4 where tregular2.a=tregular4.a and tregular2.a > %v and tregular4.a > %v;", x1, x2) + // c.Assert(tk.HasPlan(queryHash, "IndexMergeJoin"), IsTrue) + // tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + tk.MustQuery(queryHash) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1815|Optimizer Hint /*+ INL_MERGE_JOIN(trange, trange2) */ is inapplicable")) + + queryHash = fmt.Sprintf("select /*+ inl_merge_join(trange, trange2) */ * from trange, trange2 where trange.a=trange2.a and trange.a > %v and trange.b > %v;", x1, x2) + // queryRegular = fmt.Sprintf("select /*+ inl_merge_join(tregular2, tregular4) */ * from tregular2, tregular4 where tregular2.a=tregular4.a and tregular2.a > %v and tregular2.b > %v;", x1, x2) + // c.Assert(tk.HasPlan(queryHash, "IndexMergeJoin"), IsTrue) + // tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + tk.MustQuery(queryHash) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1815|Optimizer Hint /*+ INL_MERGE_JOIN(trange, trange2) */ is inapplicable")) + + queryHash = fmt.Sprintf("select /*+ inl_merge_join(trange, trange2) */ * from trange, trange2 where trange.a=trange2.a and trange.a > %v and trange2.b > %v;", x1, x2) + // queryRegular = fmt.Sprintf("select /*+ inl_merge_join(tregular2, tregular4) */ * from tregular2, tregular4 where tregular2.a=tregular4.a and tregular2.a > %v and tregular4.b > %v;", x1, x2) + // c.Assert(tk.HasPlan(queryHash, "IndexMergeJoin"), IsTrue) + // tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + tk.MustQuery(queryHash) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1815|Optimizer Hint /*+ INL_MERGE_JOIN(trange, trange2) */ is inapplicable")) + + // group 6 + // index_merge_join range partition and regualr table + queryHash = fmt.Sprintf("select /*+ inl_merge_join(trange, tregular4) */ * from trange, tregular4 where trange.a=tregular4.a and trange.a > %v;", x1) + queryRegular = fmt.Sprintf("select /*+ inl_merge_join(tregular2, tregular4) */ * from tregular2, tregular4 where tregular2.a=tregular4.a and tregular2.a > %v;", x1) + c.Assert(tk.HasPlan(queryHash, "IndexMergeJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ inl_merge_join(trange, tregular4) */ * from trange, tregular4 where trange.a=tregular4.a and trange.a > %v and tregular4.a > %v;", x1, x2) + queryRegular = fmt.Sprintf("select /*+ inl_merge_join(tregular2, tregular4) */ * from tregular2, tregular4 where tregular2.a=tregular4.a and tregular2.a > %v and tregular4.a > %v;", x1, x2) + c.Assert(tk.HasPlan(queryHash, "IndexMergeJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ inl_merge_join(trange, tregular4) */ * from trange, tregular4 where trange.a=tregular4.a and trange.a > %v and trange.b > %v;", x1, x2) + queryRegular = fmt.Sprintf("select /*+ inl_merge_join(tregular2, tregular4) */ * from tregular2, tregular4 where tregular2.a=tregular4.a and tregular2.a > %v and tregular2.b > %v;", x1, x2) + c.Assert(tk.HasPlan(queryHash, "IndexMergeJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ inl_merge_join(trange, tregular4) */ * from trange, tregular4 where trange.a=tregular4.a and trange.a > %v and tregular4.b > %v;", x1, x2) + queryRegular = fmt.Sprintf("select /*+ inl_merge_join(tregular2, tregular4) */ * from tregular2, tregular4 where tregular2.a=tregular4.a and tregular2.a > %v and tregular4.b > %v;", x1, x2) + c.Assert(tk.HasPlan(queryHash, "IndexMergeJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + // group 7 + // index_hash_join hash partition and hash partition + queryHash = fmt.Sprintf("select /*+ inl_hash_join(thash, thash2) */ * from thash, thash2 where thash.a = thash2.a and thash.a in (%v, %v);", x1, x2) + queryRegular = fmt.Sprintf("select /*+ inl_hash_join(tregular1, tregular3) */ * from tregular1, tregular3 where tregular1.a = tregular3.a and tregular1.a in (%v, %v);", x1, x2) + c.Assert(tk.HasPlan(queryHash, "IndexHashJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ inl_hash_join(thash, thash2) */ * from thash, thash2 where thash.a = thash2.a and thash.a in (%v, %v) and thash2.a in (%v, %v);", x1, x2, x3, x4) + queryRegular = fmt.Sprintf("select /*+ inl_hash_join(tregular1, tregular3) */ * from tregular1, tregular3 where tregular1.a = tregular3.a and tregular1.a in (%v, %v) and tregular3.a in (%v, %v);", x1, x2, x3, x4) + c.Assert(tk.HasPlan(queryHash, "IndexHashJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ inl_hash_join(thash, thash2) */ * from thash, thash2 where thash.a = thash2.a and thash.a > %v and thash2.b > %v;", x1, x2) + queryRegular = fmt.Sprintf("select /*+ inl_hash_join(tregular1, tregular3) */ * from tregular1, tregular3 where tregular1.a = tregular3.a and tregular1.a > %v and tregular3.b > %v;", x1, x2) + c.Assert(tk.HasPlan(queryHash, "IndexHashJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + // group 8 + // index_hash_join hash partition and hash partition + queryHash = fmt.Sprintf("select /*+ inl_hash_join(thash, tregular3) */ * from thash, tregular3 where thash.a = tregular3.a and thash.a in (%v, %v);", x1, x2) + queryRegular = fmt.Sprintf("select /*+ inl_hash_join(tregular1, tregular3) */ * from tregular1, tregular3 where tregular1.a = tregular3.a and tregular1.a in (%v, %v);", x1, x2) + c.Assert(tk.HasPlan(queryHash, "IndexHashJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ inl_hash_join(thash, tregular3) */ * from thash, tregular3 where thash.a = tregular3.a and thash.a in (%v, %v) and tregular3.a in (%v, %v);", x1, x2, x3, x4) + queryRegular = fmt.Sprintf("select /*+ inl_hash_join(tregular1, tregular3) */ * from tregular1, tregular3 where tregular1.a = tregular3.a and tregular1.a in (%v, %v) and tregular3.a in (%v, %v);", x1, x2, x3, x4) + c.Assert(tk.HasPlan(queryHash, "IndexHashJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) + + queryHash = fmt.Sprintf("select /*+ inl_hash_join(thash, tregular3) */ * from thash, tregular3 where thash.a = tregular3.a and thash.a > %v and tregular3.b > %v;", x1, x2) + queryRegular = fmt.Sprintf("select /*+ inl_hash_join(tregular1, tregular3) */ * from tregular1, tregular3 where tregular1.a = tregular3.a and tregular1.a > %v and tregular3.b > %v;", x1, x2) + c.Assert(tk.HasPlan(queryHash, "IndexHashJoin"), IsTrue) + tk.MustQuery(queryHash).Sort().Check(tk.MustQuery(queryRegular).Sort().Rows()) } -func (s *globalIndexSuite) TestGlobalIndexDoubleRead(c *C) { +func createTable4DynamicPruneModeTestWithExpression(tk *testkit.TestKit) { + tk.MustExec("create table trange(a int, b int) partition by range(a) (partition p0 values less than(3), partition p1 values less than (5), partition p2 values less than(11));") + tk.MustExec("create table thash(a int, b int) partition by hash(a) partitions 4;") + tk.MustExec("create table t(a int, b int)") + tk.MustExec("insert into trange values(1, NULL), (1, NULL), (1, 1), (2, 1), (3, 2), (4, 3), (5, 5), (6, 7), (7, 7), (7, 7), (10, NULL), (NULL, NULL), (NULL, 1);") + tk.MustExec("insert into thash values(1, NULL), (1, NULL), (1, 1), (2, 1), (3, 2), (4, 3), (5, 5), (6, 7), (7, 7), (7, 7), (10, NULL), (NULL, NULL), (NULL, 1);") + tk.MustExec("insert into t values(1, NULL), (1, NULL), (1, 1), (2, 1), (3, 2), (4, 3), (5, 5), (6, 7), (7, 7), (7, 7), (10, NULL), (NULL, NULL), (NULL, 1);") + tk.MustExec("set session tidb_partition_prune_mode='dynamic'") + tk.MustExec("analyze table trange") + tk.MustExec("analyze table thash") + tk.MustExec("analyze table t") +} + +type testData4Expression struct { + sql string + partitions []string +} + +func (s *partitionTableSuite) TestDateColWithUnequalExpression(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustExec("drop table if exists p") - tk.MustExec(`create table p (id int, c int) partition by range (c) ( -partition p0 values less than (4), -partition p1 values less than (7), -partition p2 values less than (10))`) - tk.MustExec("alter table p add unique idx(id)") - tk.MustExec("insert into p values (1,3), (3,4), (5,6), (7,9)") - tk.MustQuery("select * from p use index (idx)").Check(testkit.Rows("1 3", "3 4", "5 6", "7 9")) + tk.MustExec("drop database if exists db_datetime_unequal_expression") + tk.MustExec("create database db_datetime_unequal_expression") + tk.MustExec("use db_datetime_unequal_expression") + tk.MustExec("set tidb_partition_prune_mode='dynamic'") + tk.MustExec(`create table tp(a datetime, b int) partition by range columns (a) (partition p0 values less than("2012-12-10 00:00:00"), partition p1 values less than("2022-12-30 00:00:00"), partition p2 values less than("2025-12-12 00:00:00"))`) + tk.MustExec(`create table t(a datetime, b int) partition by range columns (a) (partition p0 values less than("2012-12-10 00:00:00"), partition p1 values less than("2022-12-30 00:00:00"), partition p2 values less than("2025-12-12 00:00:00"))`) + tk.MustExec(`insert into tp values("2015-09-09 00:00:00", 1), ("2020-08-08 19:00:01", 2), ("2024-01-01 01:01:01", 3)`) + tk.MustExec(`insert into t values("2015-09-09 00:00:00", 1), ("2020-08-08 19:00:01", 2), ("2024-01-01 01:01:01", 3)`) + tk.MustExec("analyze table tp") + tk.MustExec("analyze table t") + + tests := []testData4Expression{ + { + sql: "select * from %s where a != '2024-01-01 01:01:01'", + partitions: []string{"all"}, + }, + { + sql: "select * from %s where a != '2024-01-01 01:01:01' and a > '2015-09-09 00:00:00'", + partitions: []string{"p1,p2"}, + }, + } + + for _, t := range tests { + tpSQL := fmt.Sprintf(t.sql, "tp") + tSQL := fmt.Sprintf(t.sql, "t") + tk.MustPartition(tpSQL, t.partitions[0]).Sort().Check(tk.MustQuery(tSQL).Sort().Rows()) + } } -func (s *globalIndexSuite) TestIssue21731(c *C) { +func (s *partitionTableSuite) TestToDaysColWithExpression(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustExec("drop table if exists p, t") - tk.MustExec("create table t (a int, b int, unique index idx(a)) partition by list columns(b) (partition p0 values in (1), partition p1 values in (2));") + tk.MustExec("drop database if exists db_to_days_expression") + tk.MustExec("create database db_to_days_expression") + tk.MustExec("use db_to_days_expression") + tk.MustExec("set tidb_partition_prune_mode='dynamic'") + tk.MustExec("create table tp(a date, b int) partition by range(to_days(a)) (partition p0 values less than (737822), partition p1 values less than (738019), partition p2 values less than (738154))") + tk.MustExec("create table t(a date, b int)") + tk.MustExec("insert into tp values('2020-01-01', 1), ('2020-03-02', 2), ('2020-05-05', 3), ('2020-11-11', 4)") + tk.MustExec("insert into t values('2020-01-01', 1), ('2020-03-02', 2), ('2020-05-05', 3), ('2020-11-11', 4)") + tk.MustExec("analyze table tp") + tk.MustExec("analyze table t") + + tests := []testData4Expression{ + { + sql: "select * from %s where a < '2020-08-16'", + partitions: []string{"p0,p1"}, + }, + { + sql: "select * from %s where a between '2020-05-01' and '2020-10-01'", + partitions: []string{"p1,p2"}, + }, + } + + for _, t := range tests { + tpSQL := fmt.Sprintf(t.sql, "tp") + tSQL := fmt.Sprintf(t.sql, "t") + tk.MustPartition(tpSQL, t.partitions[0]).Sort().Check(tk.MustQuery(tSQL).Sort().Rows()) + } +} + +func (s *partitionTableSuite) TestWeekdayWithExpression(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop database if exists db_weekday_expression") + tk.MustExec("create database db_weekday_expression") + tk.MustExec("use db_weekday_expression") + tk.MustExec("set tidb_partition_prune_mode='dynamic'") + tk.MustExec("create table tp(a datetime, b int) partition by range(weekday(a)) (partition p0 values less than(3), partition p1 values less than(5), partition p2 values less than(8))") + tk.MustExec("create table t(a datetime, b int)") + tk.MustExec(`insert into tp values("2020-08-17 00:00:00", 1), ("2020-08-18 00:00:00", 2), ("2020-08-19 00:00:00", 4), ("2020-08-20 00:00:00", 5), ("2020-08-21 00:00:00", 6), ("2020-08-22 00:00:00", 0)`) + tk.MustExec(`insert into t values("2020-08-17 00:00:00", 1), ("2020-08-18 00:00:00", 2), ("2020-08-19 00:00:00", 4), ("2020-08-20 00:00:00", 5), ("2020-08-21 00:00:00", 6), ("2020-08-22 00:00:00", 0)`) + tk.MustExec("analyze table tp") + tk.MustExec("analyze table t") + + tests := []testData4Expression{ + { + sql: "select * from %s where a = '2020-08-17 00:00:00'", + partitions: []string{"p0"}, + }, + { + sql: "select * from %s where a= '2020-08-20 00:00:00' and a < '2020-08-22 00:00:00'", + partitions: []string{"p1"}, + }, + { + sql: " select * from %s where a < '2020-08-19 00:00:00'", + partitions: []string{"all"}, + }, + } + + for _, t := range tests { + tpSQL := fmt.Sprintf(t.sql, "tp") + tSQL := fmt.Sprintf(t.sql, "t") + tk.MustPartition(tpSQL, t.partitions[0]).Sort().Check(tk.MustQuery(tSQL).Sort().Rows()) + } +} + +func (s *partitionTableSuite) TestFloorUnixTimestampAndIntColWithExpression(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop database if exists db_floor_unix_timestamp_int_expression") + tk.MustExec("create database db_floor_unix_timestamp_int_expression") + tk.MustExec("use db_floor_unix_timestamp_int_expression") + tk.MustExec("set tidb_partition_prune_mode='dynamic'") + tk.MustExec("create table tp(a timestamp, b int) partition by range(floor(unix_timestamp(a))) (partition p0 values less than(1580670000), partition p1 values less than(1597622400), partition p2 values less than(1629158400))") + tk.MustExec("create table t(a timestamp, b int)") + tk.MustExec("insert into tp values('2020-01-01 19:00:00', 1),('2020-08-15 00:00:00', -1), ('2020-08-18 05:00:01', 2), ('2020-10-01 14:13:15', 3)") + tk.MustExec("insert into t values('2020-01-01 19:00:00', 1),('2020-08-15 00:00:00', -1), ('2020-08-18 05:00:01', 2), ('2020-10-01 14:13:15', 3)") + tk.MustExec("analyze table tp") + tk.MustExec("analyze table t") + + tests := []testData4Expression{ + { + sql: "select * from %s where a > '2020-09-11 00:00:00'", + partitions: []string{"p2"}, + }, + { + sql: "select * from %s where a < '2020-07-07 01:00:00'", + partitions: []string{"p0,p1"}, + }, + } + + for _, t := range tests { + tpSQL := fmt.Sprintf(t.sql, "tp") + tSQL := fmt.Sprintf(t.sql, "t") + tk.MustPartition(tpSQL, t.partitions[0]).Sort().Check(tk.MustQuery(tSQL).Sort().Rows()) + } +} + +func (s *partitionTableSuite) TestUnixTimestampAndIntColWithExpression(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop database if exists db_unix_timestamp_int_expression") + tk.MustExec("create database db_unix_timestamp_int_expression") + tk.MustExec("use db_unix_timestamp_int_expression") + tk.MustExec("set tidb_partition_prune_mode='dynamic'") + tk.MustExec("create table tp(a timestamp, b int) partition by range(unix_timestamp(a)) (partition p0 values less than(1580670000), partition p1 values less than(1597622400), partition p2 values less than(1629158400))") + tk.MustExec("create table t(a timestamp, b int)") + tk.MustExec("insert into tp values('2020-01-01 19:00:00', 1),('2020-08-15 00:00:00', -1), ('2020-08-18 05:00:01', 2), ('2020-10-01 14:13:15', 3)") + tk.MustExec("insert into t values('2020-01-01 19:00:00', 1),('2020-08-15 00:00:00', -1), ('2020-08-18 05:00:01', 2), ('2020-10-01 14:13:15', 3)") + tk.MustExec("analyze table tp") + tk.MustExec("analyze table t") + + tests := []testData4Expression{ + { + sql: "select * from %s where a > '2020-09-11 00:00:00'", + partitions: []string{"p2"}, + }, + { + sql: "select * from %s where a < '2020-07-07 01:00:00'", + partitions: []string{"p0,p1"}, + }, + } + + for _, t := range tests { + tpSQL := fmt.Sprintf(t.sql, "tp") + tSQL := fmt.Sprintf(t.sql, "t") + tk.MustPartition(tpSQL, t.partitions[0]).Sort().Check(tk.MustQuery(tSQL).Sort().Rows()) + } +} + +func (s *partitionTableSuite) TestDatetimeColAndIntColWithExpression(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop database if exists db_datetime_int_expression") + tk.MustExec("create database db_datetime_int_expression") + tk.MustExec("use db_datetime_int_expression") + tk.MustExec("set tidb_partition_prune_mode='dynamic'") + tk.MustExec("create table tp(a datetime, b int) partition by range columns(a) (partition p0 values less than('2020-02-02 00:00:00'), partition p1 values less than('2020-09-01 00:00:00'), partition p2 values less than('2020-12-20 00:00:00'))") + tk.MustExec("create table t(a datetime, b int)") + tk.MustExec("insert into tp values('2020-01-01 12:00:00', 1), ('2020-08-22 10:00:00', 2), ('2020-09-09 11:00:00', 3), ('2020-10-01 00:00:00', 4)") + tk.MustExec("insert into t values('2020-01-01 12:00:00', 1), ('2020-08-22 10:00:00', 2), ('2020-09-09 11:00:00', 3), ('2020-10-01 00:00:00', 4)") + tk.MustExec("analyze table tp") + tk.MustExec("analyze table t") + + tests := []testData4Expression{ + { + sql: "select * from %s where a < '2020-09-01 00:00:00'", + partitions: []string{"p0,p1"}, + }, + { + sql: "select * from %s where a > '2020-07-07 01:00:00'", + partitions: []string{"p1,p2"}, + }, + } + + for _, t := range tests { + tpSQL := fmt.Sprintf(t.sql, "tp") + tSQL := fmt.Sprintf(t.sql, "t") + tk.MustPartition(tpSQL, t.partitions[0]).Sort().Check(tk.MustQuery(tSQL).Sort().Rows()) + } +} + +func (s *partitionTableSuite) TestVarcharColAndIntColWithExpression(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop database if exists db_varchar_int_expression") + tk.MustExec("create database db_varchar_int_expression") + tk.MustExec("use db_varchar_int_expression") + tk.MustExec("set tidb_partition_prune_mode='dynamic'") + tk.MustExec("create table tp(a varchar(255), b int) partition by range columns(a) (partition p0 values less than('ddd'), partition p1 values less than('ggggg'), partition p2 values less than('mmmmmm'))") + tk.MustExec("create table t(a varchar(255), b int)") + tk.MustExec("insert into tp values('aaa', 1), ('bbbb', 2), ('ccc', 3), ('dfg', 4), ('kkkk', 5), ('10', 6)") + tk.MustExec("insert into t values('aaa', 1), ('bbbb', 2), ('ccc', 3), ('dfg', 4), ('kkkk', 5), ('10', 6)") + tk.MustExec("analyze table tp") + tk.MustExec("analyze table t") + + tests := []testData4Expression{ + { + sql: "select * from %s where a < '10'", + partitions: []string{"p0"}, + }, + { + sql: "select * from %s where a > 0", + partitions: []string{"all"}, + }, + { + sql: "select * from %s where a < 0", + partitions: []string{"all"}, + }, + } + + for _, t := range tests { + tpSQL := fmt.Sprintf(t.sql, "tp") + tSQL := fmt.Sprintf(t.sql, "t") + tk.MustPartition(tpSQL, t.partitions[0]).Sort().Check(tk.MustQuery(tSQL).Sort().Rows()) + } +} + +func (s *partitionTableSuite) TestDynamicPruneModeWithExpression(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop database if exists db_equal_expression") + tk.MustExec("create database db_equal_expression") + tk.MustExec("use db_equal_expression") + createTable4DynamicPruneModeTestWithExpression(tk) + + tables := []string{"trange", "thash"} + tests := []testData4Expression{ + { + sql: "select * from %s where a = 2", + partitions: []string{ + "p0", + "p2", + }, + }, + { + sql: "select * from %s where a = 4 or a = 1", + partitions: []string{ + "p0,p1", + "p0,p1", + }, + }, + { + sql: "select * from %s where a = -1", + partitions: []string{ + "p0", + "p1", + }, + }, + { + sql: "select * from %s where a is NULL", + partitions: []string{ + "p0", + "p0", + }, + }, + { + sql: "select * from %s where b is NULL", + partitions: []string{ + "all", + "all", + }, + }, + { + sql: "select * from %s where a > -1", + partitions: []string{ + "all", + "all", + }, + }, + { + sql: "select * from %s where a >= 4 and a <= 5", + partitions: []string{ + "p1,p2", + "p0,p1", + }, + }, + { + sql: "select * from %s where a > 10", + partitions: []string{ + "dual", + "all", + }, + }, + { + sql: "select * from %s where a >=2 and a <= 3", + partitions: []string{ + "p0,p1", + "p2,p3", + }, + }, + { + sql: "select * from %s where a between 2 and 3", + partitions: []string{ + "p0,p1", + "p2,p3", + }, + }, + { + sql: "select * from %s where a < 2", + partitions: []string{ + "p0", + "all", + }, + }, + { + sql: "select * from %s where a <= 3", + partitions: []string{ + "p0,p1", + "all", + }, + }, + { + sql: "select * from %s where a in (2, 3)", + partitions: []string{ + "p0,p1", + "p2,p3", + }, + }, + { + sql: "select * from %s where a in (1, 5)", + partitions: []string{ + "p0,p2", + "p1", + }, + }, + { + sql: "select * from %s where a not in (1, 5)", + partitions: []string{ + "all", + "all", + }, + }, + { + sql: "select * from %s where a = 2 and a = 2", + partitions: []string{ + "p0", + "p2", + }, + }, + { + sql: "select * from %s where a = 2 and a = 3", + partitions: []string{ + // This means that we have no partition-read plan + "", + "", + }, + }, + { + sql: "select * from %s where a < 2 and a > 0", + partitions: []string{ + "p0", + "p1", + }, + }, + { + sql: "select * from %s where a < 2 and a < 3", + partitions: []string{ + "p0", + "all", + }, + }, + { + sql: "select * from %s where a > 1 and a > 2", + partitions: []string{ + "p1,p2", + "all", + }, + }, + { + sql: "select * from %s where a = 2 or a = 3", + partitions: []string{ + "p0,p1", + "p2,p3", + }, + }, + { + sql: "select * from %s where a = 2 or a in (3)", + partitions: []string{ + "p0,p1", + "p2,p3", + }, + }, + { + sql: "select * from %s where a = 2 or a > 3", + partitions: []string{ + "all", + "all", + }, + }, + { + sql: "select * from %s where a = 2 or a <= 1", + partitions: []string{ + "p0", + "all", + }, + }, + { + sql: "select * from %s where a = 2 or a between 2 and 2", + partitions: []string{ + "p0", + "p2", + }, + }, + { + sql: "select * from %s where a != 2", + partitions: []string{ + "all", + "all", + }, + }, + { + sql: "select * from %s where a != 2 and a > 4", + partitions: []string{ + "p2", + "all", + }, + }, + { + sql: "select * from %s where a != 2 and a != 3", + partitions: []string{ + "all", + "all", + }, + }, + { + sql: "select * from %s where a != 2 and a = 3", + partitions: []string{ + "p1", + "p3", + }, + }, + { + sql: "select * from %s where not (a = 2)", + partitions: []string{ + "all", + "all", + }, + }, + { + sql: "select * from %s where not (a > 2)", + partitions: []string{ + "p0", + "all", + }, + }, + { + sql: "select * from %s where not (a < 2)", + partitions: []string{ + "all", + "all", + }, + }, + // cases that partition pruning can not work + { + sql: "select * from %s where a + 1 > 4", + partitions: []string{ + "all", + "all", + }, + }, + { + sql: "select * from %s where a - 1 > 0", + partitions: []string{ + "all", + "all", + }, + }, + { + sql: "select * from %s where a * 2 < 0", + partitions: []string{ + "all", + "all", + }, + }, + { + sql: "select * from %s where a << 1 < 0", + partitions: []string{ + "all", + "all", + }, + }, + // comparison between int column and string column + { + sql: "select * from %s where a > '10'", + partitions: []string{ + "dual", + "all", + }, + }, + { + sql: "select * from %s where a > '10ab'", + partitions: []string{ + "dual", + "all", + }, + }, + } + + for _, t := range tests { + for i := range t.partitions { + sql := fmt.Sprintf(t.sql, tables[i]) + tk.MustPartition(sql, t.partitions[i]).Sort().Check(tk.MustQuery(fmt.Sprintf(t.sql, "t")).Sort().Rows()) + } + } +} + +func (s *partitionTableSuite) TestAddDropPartitions(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_add_drop_partition") + tk.MustExec("use test_add_drop_partition") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + tk.MustExec(`create table t(a int) partition by range(a) ( + partition p0 values less than (5), + partition p1 values less than (10), + partition p2 values less than (15))`) + tk.MustExec(`insert into t values (2), (7), (12)`) + tk.MustPartition(`select * from t where a < 3`, "p0").Sort().Check(testkit.Rows("2")) + tk.MustPartition(`select * from t where a < 8`, "p0,p1").Sort().Check(testkit.Rows("2", "7")) + tk.MustPartition(`select * from t where a < 20`, "all").Sort().Check(testkit.Rows("12", "2", "7")) + + // remove p0 + tk.MustExec(`alter table t drop partition p0`) + tk.MustPartition(`select * from t where a < 3`, "p1").Sort().Check(testkit.Rows()) + tk.MustPartition(`select * from t where a < 8`, "p1").Sort().Check(testkit.Rows("7")) + tk.MustPartition(`select * from t where a < 20`, "all").Sort().Check(testkit.Rows("12", "7")) + + // add 2 more partitions + tk.MustExec(`alter table t add partition (partition p3 values less than (20))`) + tk.MustExec(`alter table t add partition (partition p4 values less than (40))`) + tk.MustExec(`insert into t values (15), (25)`) + tk.MustPartition(`select * from t where a < 3`, "p1").Sort().Check(testkit.Rows()) + tk.MustPartition(`select * from t where a < 8`, "p1").Sort().Check(testkit.Rows("7")) + tk.MustPartition(`select * from t where a < 20`, "p1,p2,p3").Sort().Check(testkit.Rows("12", "15", "7")) +} + +func (s *partitionTableSuite) PartitionPruningInTransaction(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_pruning_transaction") + defer tk.MustExec(`drop database test_pruning_transaction`) + tk.MustExec("use test_pruning_transaction") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + tk.MustExec(`create table t(a int, b int) partition by range(a) (partition p0 values less than(3), partition p1 values less than (5), partition p2 values less than(11))`) + tk.MustExec(`begin`) + tk.MustPartition(`select * from t`, "all") + tk.MustPartition(`select * from t where a > 4`, "p1,p2") // partition pruning can work in transactions + tk.MustPartition(`select * from t where a > 7`, "p2") + tk.MustExec(`rollback`) +} + +func (s *partitionTableSuite) TestDML(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_DML") + defer tk.MustExec(`drop database test_DML`) + tk.MustExec("use test_DML") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + tk.MustExec(`create table tinner (a int, b int)`) + tk.MustExec(`create table thash (a int, b int) partition by hash(a) partitions 4`) + tk.MustExec(`create table trange (a int, b int) partition by range(a) ( + partition p0 values less than(10000), + partition p1 values less than(20000), + partition p2 values less than(30000), + partition p3 values less than(40000), + partition p4 values less than MAXVALUE)`) + + vals := make([]string, 0, 50) + for i := 0; i < 50; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(40000), rand.Intn(40000))) + } + tk.MustExec(`insert into tinner values ` + strings.Join(vals, ", ")) + tk.MustExec(`insert into thash values ` + strings.Join(vals, ", ")) + tk.MustExec(`insert into trange values ` + strings.Join(vals, ", ")) + + // delete, insert, replace, update + for i := 0; i < 200; i++ { + var pattern string + switch rand.Intn(4) { + case 0: // delete + col := []string{"a", "b"}[rand.Intn(2)] + l := rand.Intn(40000) + r := l + rand.Intn(5000) + pattern = fmt.Sprintf(`delete from %%v where %v>%v and %v<%v`, col, l, col, r) + case 1: // insert + a, b := rand.Intn(40000), rand.Intn(40000) + pattern = fmt.Sprintf(`insert into %%v values (%v, %v)`, a, b) + case 2: // replace + a, b := rand.Intn(40000), rand.Intn(40000) + pattern = fmt.Sprintf(`replace into %%v(a, b) values (%v, %v)`, a, b) + case 3: // update + col := []string{"a", "b"}[rand.Intn(2)] + l := rand.Intn(40000) + r := l + rand.Intn(5000) + x := rand.Intn(1000) - 500 + pattern = fmt.Sprintf(`update %%v set %v=%v+%v where %v>%v and %v<%v`, col, col, x, col, l, col, r) + } + for _, tbl := range []string{"tinner", "thash", "trange"} { + tk.MustExec(fmt.Sprintf(pattern, tbl)) + } + + // check + r := tk.MustQuery(`select * from tinner`).Sort().Rows() + tk.MustQuery(`select * from thash`).Sort().Check(r) + tk.MustQuery(`select * from trange`).Sort().Check(r) + } +} + +func (s *partitionTableSuite) TestUnion(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_union") + defer tk.MustExec(`drop database test_union`) + tk.MustExec("use test_union") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + tk.MustExec(`create table t(a int, b int, key(a))`) + tk.MustExec(`create table thash (a int, b int, key(a)) partition by hash(a) partitions 4`) + tk.MustExec(`create table trange (a int, b int, key(a)) partition by range(a) ( + partition p0 values less than (10000), + partition p1 values less than (20000), + partition p2 values less than (30000), + partition p3 values less than (40000))`) + + vals := make([]string, 0, 1000) + for i := 0; i < 1000; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(40000), rand.Intn(40000))) + } + tk.MustExec(`insert into t values ` + strings.Join(vals, ", ")) + tk.MustExec(`insert into thash values ` + strings.Join(vals, ", ")) + tk.MustExec(`insert into trange values ` + strings.Join(vals, ", ")) + + randRange := func() (int, int) { + l, r := rand.Intn(40000), rand.Intn(40000) + if l > r { + l, r = r, l + } + return l, r + } + + for i := 0; i < 100; i++ { + a1l, a1r := randRange() + a2l, a2r := randRange() + b1l, b1r := randRange() + b2l, b2r := randRange() + for _, utype := range []string{"union all", "union distinct"} { + pattern := fmt.Sprintf(`select * from %%v where a>=%v and a<=%v and b>=%v and b<=%v + %v select * from %%v where a>=%v and a<=%v and b>=%v and b<=%v`, a1l, a1r, b1l, b1r, utype, a2l, a2r, b2l, b2r) + r := tk.MustQuery(fmt.Sprintf(pattern, "t", "t")).Sort().Rows() + tk.MustQuery(fmt.Sprintf(pattern, "thash", "thash")).Sort().Check(r) // hash + hash + tk.MustQuery(fmt.Sprintf(pattern, "trange", "trange")).Sort().Check(r) // range + range + tk.MustQuery(fmt.Sprintf(pattern, "trange", "thash")).Sort().Check(r) // range + hash + } + } +} + +func (s *partitionTableSuite) TestSubqueries(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_subquery") + defer tk.MustExec(`drop database test_subquery`) + tk.MustExec("use test_subquery") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + tk.MustExec(`create table touter (a int, b int, index(a))`) + tk.MustExec(`create table tinner (a int, b int, c int, index(a))`) + tk.MustExec(`create table thash (a int, b int, c int, index(a)) partition by hash(a) partitions 4`) + tk.MustExec(`create table trange (a int, b int, c int, index(a)) partition by range(a) ( + partition p0 values less than(10000), + partition p1 values less than(20000), + partition p2 values less than(30000), + partition p3 values less than(40000))`) + + outerVals := make([]string, 0, 100) + for i := 0; i < 100; i++ { + outerVals = append(outerVals, fmt.Sprintf(`(%v, %v)`, rand.Intn(40000), rand.Intn(40000))) + } + tk.MustExec(`insert into touter values ` + strings.Join(outerVals, ", ")) + vals := make([]string, 0, 2000) + for i := 0; i < 2000; i++ { + vals = append(vals, fmt.Sprintf(`(%v, %v, %v)`, rand.Intn(40000), rand.Intn(40000), rand.Intn(40000))) + } + tk.MustExec(`insert into tinner values ` + strings.Join(vals, ", ")) + tk.MustExec(`insert into thash values ` + strings.Join(vals, ", ")) + tk.MustExec(`insert into trange values ` + strings.Join(vals, ", ")) + + // in + for i := 0; i < 50; i++ { + for _, op := range []string{"in", "not in"} { + x := rand.Intn(40000) + var r [][]interface{} + for _, t := range []string{"tinner", "thash", "trange"} { + q := fmt.Sprintf(`select * from touter where touter.a %v (select %v.b from %v where %v.a > touter.b and %v.c > %v)`, op, t, t, t, t, x) + if r == nil { + r = tk.MustQuery(q).Sort().Rows() + } else { + tk.MustQuery(q).Sort().Check(r) + } + } + } + } + + // exist + for i := 0; i < 50; i++ { + for _, op := range []string{"exists", "not exists"} { + x := rand.Intn(40000) + var r [][]interface{} + for _, t := range []string{"tinner", "thash", "trange"} { + q := fmt.Sprintf(`select * from touter where %v (select %v.b from %v where %v.a > touter.b and %v.c > %v)`, op, t, t, t, t, x) + if r == nil { + r = tk.MustQuery(q).Sort().Rows() + } else { + tk.MustQuery(q).Sort().Check(r) + } + } + } + } +} + +func (s *partitionTableSuite) TestSplitRegion(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_split_region") + tk.MustExec("use test_split_region") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + tk.MustExec(`create table tnormal (a int, b int)`) + tk.MustExec(`create table thash (a int, b int, index(a)) partition by hash(a) partitions 4`) + tk.MustExec(`create table trange (a int, b int, index(a)) partition by range(a) ( + partition p0 values less than (10000), + partition p1 values less than (20000), + partition p2 values less than (30000), + partition p3 values less than (40000))`) + vals := make([]string, 0, 1000) + for i := 0; i < 1000; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(40000), rand.Intn(40000))) + } + tk.MustExec(`insert into tnormal values ` + strings.Join(vals, ", ")) + tk.MustExec(`insert into thash values ` + strings.Join(vals, ", ")) + tk.MustExec(`insert into trange values ` + strings.Join(vals, ", ")) + + tk.MustExec(`SPLIT TABLE thash INDEX a BETWEEN (1) AND (25000) REGIONS 10`) + tk.MustExec(`SPLIT TABLE trange INDEX a BETWEEN (1) AND (25000) REGIONS 10`) + + result := tk.MustQuery(`select * from tnormal where a>=1 and a<=15000`).Sort().Rows() + tk.MustPartition(`select * from trange where a>=1 and a<=15000`, "p0,p1").Sort().Check(result) + tk.MustPartition(`select * from thash where a>=1 and a<=15000`, "all").Sort().Check(result) + + result = tk.MustQuery(`select * from tnormal where a in (1, 10001, 20001)`).Sort().Rows() + tk.MustPartition(`select * from trange where a in (1, 10001, 20001)`, "p0,p1,p2").Sort().Check(result) + tk.MustPartition(`select * from thash where a in (1, 10001, 20001)`, "p1").Sort().Check(result) +} + +func (s *partitionTableSuite) TestParallelApply(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_parallel_apply") + tk.MustExec("use test_parallel_apply") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + tk.MustExec("set tidb_enable_parallel_apply=true") + + tk.MustExec(`create table touter (a int, b int)`) + tk.MustExec(`create table tinner (a int, b int, key(a))`) + tk.MustExec(`create table thash (a int, b int, key(a)) partition by hash(a) partitions 4`) + tk.MustExec(`create table trange (a int, b int, key(a)) partition by range(a) ( + partition p0 values less than(10000), + partition p1 values less than(20000), + partition p2 values less than(30000), + partition p3 values less than(40000))`) + + vouter := make([]string, 0, 100) + for i := 0; i < 100; i++ { + vouter = append(vouter, fmt.Sprintf("(%v, %v)", rand.Intn(40000), rand.Intn(40000))) + } + tk.MustExec("insert into touter values " + strings.Join(vouter, ", ")) + + vals := make([]string, 0, 2000) + for i := 0; i < 100; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(40000), rand.Intn(40000))) + } + tk.MustExec("insert into tinner values " + strings.Join(vals, ", ")) + tk.MustExec("insert into thash values " + strings.Join(vals, ", ")) + tk.MustExec("insert into trange values " + strings.Join(vals, ", ")) + + // parallel apply + hash partition + IndexReader as its inner child + tk.MustQuery(`explain format='brief' select * from touter where touter.a > (select sum(thash.a) from thash use index(a) where thash.a>touter.b)`).Check(testkit.Rows( + `Projection 10000.00 root test_parallel_apply.touter.a, test_parallel_apply.touter.b`, + `└─Apply 10000.00 root CARTESIAN inner join, other cond:gt(cast(test_parallel_apply.touter.a, decimal(20,0) BINARY), Column#7)`, + ` ├─TableReader(Build) 10000.00 root data:TableFullScan`, + ` │ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo`, + ` └─StreamAgg(Probe) 1.00 root funcs:sum(Column#9)->Column#7`, + ` └─IndexReader 1.00 root partition:all index:StreamAgg`, // IndexReader is a inner child of Apply + ` └─StreamAgg 1.00 cop[tikv] funcs:sum(test_parallel_apply.thash.a)->Column#9`, + ` └─Selection 8000.00 cop[tikv] gt(test_parallel_apply.thash.a, test_parallel_apply.touter.b)`, + ` └─IndexFullScan 10000.00 cop[tikv] table:thash, index:a(a) keep order:false, stats:pseudo`)) + tk.MustQuery(`select * from touter where touter.a > (select sum(thash.a) from thash use index(a) where thash.a>touter.b)`).Sort().Check( + tk.MustQuery(`select * from touter where touter.a > (select sum(tinner.a) from tinner use index(a) where tinner.a>touter.b)`).Sort().Rows()) + + // parallel apply + hash partition + TableReader as its inner child + tk.MustQuery(`explain format='brief' select * from touter where touter.a > (select sum(thash.b) from thash ignore index(a) where thash.a>touter.b)`).Check(testkit.Rows( + `Projection 10000.00 root test_parallel_apply.touter.a, test_parallel_apply.touter.b`, + `└─Apply 10000.00 root CARTESIAN inner join, other cond:gt(cast(test_parallel_apply.touter.a, decimal(20,0) BINARY), Column#7)`, + ` ├─TableReader(Build) 10000.00 root data:TableFullScan`, + ` │ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo`, + ` └─StreamAgg(Probe) 1.00 root funcs:sum(Column#9)->Column#7`, + ` └─TableReader 1.00 root partition:all data:StreamAgg`, // TableReader is a inner child of Apply + ` └─StreamAgg 1.00 cop[tikv] funcs:sum(test_parallel_apply.thash.b)->Column#9`, + ` └─Selection 8000.00 cop[tikv] gt(test_parallel_apply.thash.a, test_parallel_apply.touter.b)`, + ` └─TableFullScan 10000.00 cop[tikv] table:thash keep order:false, stats:pseudo`)) + tk.MustQuery(`select * from touter where touter.a > (select sum(thash.b) from thash ignore index(a) where thash.a>touter.b)`).Sort().Check( + tk.MustQuery(`select * from touter where touter.a > (select sum(tinner.b) from tinner ignore index(a) where tinner.a>touter.b)`).Sort().Rows()) + + // parallel apply + hash partition + IndexLookUp as its inner child + tk.MustQuery(`explain format='brief' select * from touter where touter.a > (select sum(tinner.b) from tinner use index(a) where tinner.a>touter.b)`).Check(testkit.Rows( + `Projection 10000.00 root test_parallel_apply.touter.a, test_parallel_apply.touter.b`, + `└─Apply 10000.00 root CARTESIAN inner join, other cond:gt(cast(test_parallel_apply.touter.a, decimal(20,0) BINARY), Column#7)`, + ` ├─TableReader(Build) 10000.00 root data:TableFullScan`, + ` │ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo`, + ` └─HashAgg(Probe) 1.00 root funcs:sum(Column#9)->Column#7`, + ` └─IndexLookUp 1.00 root `, // IndexLookUp is a inner child of Apply + ` ├─Selection(Build) 8000.00 cop[tikv] gt(test_parallel_apply.tinner.a, test_parallel_apply.touter.b)`, + ` │ └─IndexFullScan 10000.00 cop[tikv] table:tinner, index:a(a) keep order:false, stats:pseudo`, + ` └─HashAgg(Probe) 1.00 cop[tikv] funcs:sum(test_parallel_apply.tinner.b)->Column#9`, + ` └─TableRowIDScan 8000.00 cop[tikv] table:tinner keep order:false, stats:pseudo`)) + tk.MustQuery(`select * from touter where touter.a > (select sum(thash.b) from thash use index(a) where thash.a>touter.b)`).Sort().Check( + tk.MustQuery(`select * from touter where touter.a > (select sum(tinner.b) from tinner use index(a) where tinner.a>touter.b)`).Sort().Rows()) + + // parallel apply + range partition + IndexReader as its inner child + tk.MustQuery(`explain format='brief' select * from touter where touter.a > (select sum(trange.a) from trange use index(a) where trange.a>touter.b)`).Check(testkit.Rows( + `Projection 10000.00 root test_parallel_apply.touter.a, test_parallel_apply.touter.b`, + `└─Apply 10000.00 root CARTESIAN inner join, other cond:gt(cast(test_parallel_apply.touter.a, decimal(20,0) BINARY), Column#7)`, + ` ├─TableReader(Build) 10000.00 root data:TableFullScan`, + ` │ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo`, + ` └─StreamAgg(Probe) 1.00 root funcs:sum(Column#9)->Column#7`, + ` └─IndexReader 1.00 root partition:all index:StreamAgg`, // IndexReader is a inner child of Apply + ` └─StreamAgg 1.00 cop[tikv] funcs:sum(test_parallel_apply.trange.a)->Column#9`, + ` └─Selection 8000.00 cop[tikv] gt(test_parallel_apply.trange.a, test_parallel_apply.touter.b)`, + ` └─IndexFullScan 10000.00 cop[tikv] table:trange, index:a(a) keep order:false, stats:pseudo`)) + tk.MustQuery(`select * from touter where touter.a > (select sum(trange.a) from trange use index(a) where trange.a>touter.b)`).Sort().Check( + tk.MustQuery(`select * from touter where touter.a > (select sum(tinner.a) from tinner use index(a) where tinner.a>touter.b)`).Sort().Rows()) + + // parallel apply + range partition + TableReader as its inner child + tk.MustQuery(`explain format='brief' select * from touter where touter.a > (select sum(trange.b) from trange ignore index(a) where trange.a>touter.b)`).Check(testkit.Rows( + `Projection 10000.00 root test_parallel_apply.touter.a, test_parallel_apply.touter.b`, + `└─Apply 10000.00 root CARTESIAN inner join, other cond:gt(cast(test_parallel_apply.touter.a, decimal(20,0) BINARY), Column#7)`, + ` ├─TableReader(Build) 10000.00 root data:TableFullScan`, + ` │ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo`, + ` └─StreamAgg(Probe) 1.00 root funcs:sum(Column#9)->Column#7`, + ` └─TableReader 1.00 root partition:all data:StreamAgg`, // TableReader is a inner child of Apply + ` └─StreamAgg 1.00 cop[tikv] funcs:sum(test_parallel_apply.trange.b)->Column#9`, + ` └─Selection 8000.00 cop[tikv] gt(test_parallel_apply.trange.a, test_parallel_apply.touter.b)`, + ` └─TableFullScan 10000.00 cop[tikv] table:trange keep order:false, stats:pseudo`)) + tk.MustQuery(`select * from touter where touter.a > (select sum(trange.b) from trange ignore index(a) where trange.a>touter.b)`).Sort().Check( + tk.MustQuery(`select * from touter where touter.a > (select sum(tinner.b) from tinner ignore index(a) where tinner.a>touter.b)`).Sort().Rows()) + + // parallel apply + range partition + IndexLookUp as its inner child + tk.MustQuery(`explain format='brief' select * from touter where touter.a > (select sum(tinner.b) from tinner use index(a) where tinner.a>touter.b)`).Check(testkit.Rows( + `Projection 10000.00 root test_parallel_apply.touter.a, test_parallel_apply.touter.b`, + `└─Apply 10000.00 root CARTESIAN inner join, other cond:gt(cast(test_parallel_apply.touter.a, decimal(20,0) BINARY), Column#7)`, + ` ├─TableReader(Build) 10000.00 root data:TableFullScan`, + ` │ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo`, + ` └─HashAgg(Probe) 1.00 root funcs:sum(Column#9)->Column#7`, + ` └─IndexLookUp 1.00 root `, // IndexLookUp is a inner child of Apply + ` ├─Selection(Build) 8000.00 cop[tikv] gt(test_parallel_apply.tinner.a, test_parallel_apply.touter.b)`, + ` │ └─IndexFullScan 10000.00 cop[tikv] table:tinner, index:a(a) keep order:false, stats:pseudo`, + ` └─HashAgg(Probe) 1.00 cop[tikv] funcs:sum(test_parallel_apply.tinner.b)->Column#9`, + ` └─TableRowIDScan 8000.00 cop[tikv] table:tinner keep order:false, stats:pseudo`)) + tk.MustQuery(`select * from touter where touter.a > (select sum(trange.b) from trange use index(a) where trange.a>touter.b)`).Sort().Check( + tk.MustQuery(`select * from touter where touter.a > (select sum(tinner.b) from tinner use index(a) where tinner.a>touter.b)`).Sort().Rows()) + + // random queries + ops := []string{"!=", ">", "<", ">=", "<="} + aggFuncs := []string{"sum", "count", "max", "min"} + tbls := []string{"tinner", "thash", "trange"} + for i := 0; i < 50; i++ { + var r [][]interface{} + op := ops[rand.Intn(len(ops))] + agg := aggFuncs[rand.Intn(len(aggFuncs))] + x := rand.Intn(10000) + for _, tbl := range tbls { + q := fmt.Sprintf(`select * from touter where touter.a > (select %v(%v.b) from %v where %v.a%vtouter.b-%v)`, agg, tbl, tbl, tbl, op, x) + if r == nil { + r = tk.MustQuery(q).Sort().Rows() + } else { + tk.MustQuery(q).Sort().Check(r) + } + } + } +} + +func (s *partitionTableSuite) TestDirectReadingWithUnionScan(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_unionscan") + defer tk.MustExec(`drop database test_unionscan`) + tk.MustExec("use test_unionscan") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + tk.MustExec(`create table trange(a int, b int, index idx_a(a)) partition by range(a) ( + partition p0 values less than (10), + partition p1 values less than (30), + partition p2 values less than (50))`) + tk.MustExec(`create table thash(a int, b int, index idx_a(a)) partition by hash(a) partitions 4`) + tk.MustExec(`create table tnormal(a int, b int, index idx_a(a))`) + + vals := make([]string, 0, 1000) + for i := 0; i < 1000; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(50), rand.Intn(50))) + } + for _, tb := range []string{`trange`, `tnormal`, `thash`} { + sql := fmt.Sprintf(`insert into %v values `+strings.Join(vals, ", "), tb) + tk.MustExec(sql) + } + + randCond := func(col string) string { + la, ra := rand.Intn(50), rand.Intn(50) + if la > ra { + la, ra = ra, la + } + return fmt.Sprintf(`%v>=%v and %v<=%v`, col, la, col, ra) + } + + tk.MustExec(`begin`) + for i := 0; i < 1000; i++ { + if i == 0 || rand.Intn(2) == 0 { // insert some inflight rows + val := fmt.Sprintf("(%v, %v)", rand.Intn(50), rand.Intn(50)) + for _, tb := range []string{`trange`, `tnormal`, `thash`} { + sql := fmt.Sprintf(`insert into %v values `+val, tb) + tk.MustExec(sql) + } + } else { + var sql string + switch rand.Intn(3) { + case 0: // table scan + sql = `select * from %v ignore index(idx_a) where ` + randCond(`b`) + case 1: // index reader + sql = `select a from %v use index(idx_a) where ` + randCond(`a`) + case 2: // index lookup + sql = `select * from %v use index(idx_a) where ` + randCond(`a`) + ` and ` + randCond(`b`) + } + switch rand.Intn(2) { + case 0: // order by a + sql += ` order by a` + case 1: // order by b + sql += ` order by b` + } + + var result [][]interface{} + for _, tb := range []string{`trange`, `tnormal`, `thash`} { + q := fmt.Sprintf(sql, tb) + tk.HasPlan(q, `UnionScan`) + if result == nil { + result = tk.MustQuery(q).Sort().Rows() + } else { + tk.MustQuery(q).Sort().Check(result) + } + } + } + } + tk.MustExec(`rollback`) +} + +func (s *partitionTableSuite) TestIssue25030(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_issue_25030") + tk.MustExec("use test_issue_25030") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + tk.MustExec(`CREATE TABLE tbl_936 ( + col_5410 smallint NOT NULL, + col_5411 double, + col_5412 boolean NOT NULL DEFAULT 1, + col_5413 set('Alice', 'Bob', 'Charlie', 'David') NOT NULL DEFAULT 'Charlie', + col_5414 varbinary(147) COLLATE 'binary' DEFAULT 'bvpKgYWLfyuTiOYSkj', + col_5415 timestamp NOT NULL DEFAULT '2021-07-06', + col_5416 decimal(6, 6) DEFAULT 0.49, + col_5417 text COLLATE utf8_bin, + col_5418 float DEFAULT 2048.0762299371554, + col_5419 int UNSIGNED NOT NULL DEFAULT 3152326370, + PRIMARY KEY (col_5419) ) + PARTITION BY HASH (col_5419) PARTITIONS 3`) + tk.MustQuery(`SELECT last_value(col_5414) OVER w FROM tbl_936 + WINDOW w AS (ORDER BY col_5410, col_5411, col_5412, col_5413, col_5414, col_5415, col_5416, col_5417, col_5418, col_5419) + ORDER BY col_5410, col_5411, col_5412, col_5413, col_5414, col_5415, col_5416, col_5417, col_5418, col_5419, nth_value(col_5412, 5) OVER w`). + Check(testkit.Rows()) // can work properly without any error or panic +} + +func (s *partitionTableSuite) TestUnsignedPartitionColumn(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_unsigned_partition") + tk.MustExec("use test_unsigned_partition") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + tk.MustExec(`create table thash_pk (a int unsigned, b int, primary key(a)) partition by hash (a) partitions 3`) + tk.MustExec(`create table trange_pk (a int unsigned, b int, primary key(a)) partition by range (a) ( + partition p1 values less than (100000), + partition p2 values less than (200000), + partition p3 values less than (300000), + partition p4 values less than (400000))`) + tk.MustExec(`create table tnormal_pk (a int unsigned, b int, primary key(a))`) + tk.MustExec(`create table thash_uniq (a int unsigned, b int, unique key(a)) partition by hash (a) partitions 3`) + tk.MustExec(`create table trange_uniq (a int unsigned, b int, unique key(a)) partition by range (a) ( + partition p1 values less than (100000), + partition p2 values less than (200000), + partition p3 values less than (300000), + partition p4 values less than (400000))`) + tk.MustExec(`create table tnormal_uniq (a int unsigned, b int, unique key(a))`) + + valColA := make(map[int]struct{}, 1000) + vals := make([]string, 0, 1000) + for len(vals) < 1000 { + a := rand.Intn(400000) + if _, ok := valColA[a]; ok { + continue + } + valColA[a] = struct{}{} + vals = append(vals, fmt.Sprintf("(%v, %v)", a, rand.Intn(400000))) + } + valStr := strings.Join(vals, ", ") + for _, tbl := range []string{"thash_pk", "trange_pk", "tnormal_pk", "thash_uniq", "trange_uniq", "tnormal_uniq"} { + tk.MustExec(fmt.Sprintf("insert into %v values %v", tbl, valStr)) + } + + for i := 0; i < 100; i++ { + scanCond := fmt.Sprintf("a %v %v", []string{">", "<"}[rand.Intn(2)], rand.Intn(400000)) + pointCond := fmt.Sprintf("a = %v", rand.Intn(400000)) + batchCond := fmt.Sprintf("a in (%v, %v, %v)", rand.Intn(400000), rand.Intn(400000), rand.Intn(400000)) + + var rScan, rPoint, rBatch [][]interface{} + for tid, tbl := range []string{"tnormal_pk", "trange_pk", "thash_pk"} { + // unsigned + TableReader + scanSQL := fmt.Sprintf("select * from %v use index(primary) where %v", tbl, scanCond) + c.Assert(tk.HasPlan(scanSQL, "TableReader"), IsTrue) + r := tk.MustQuery(scanSQL).Sort() + if tid == 0 { + rScan = r.Rows() + } else { + r.Check(rScan) + } + + // unsigned + PointGet on PK + pointSQL := fmt.Sprintf("select * from %v use index(primary) where %v", tbl, pointCond) + tk.MustPointGet(pointSQL) + r = tk.MustQuery(pointSQL).Sort() + if tid == 0 { + rPoint = r.Rows() + } else { + r.Check(rPoint) + } + + // unsigned + BatchGet on PK + batchSQL := fmt.Sprintf("select * from %v where %v", tbl, batchCond) + c.Assert(tk.HasPlan(batchSQL, "Batch_Point_Get"), IsTrue) + r = tk.MustQuery(batchSQL).Sort() + if tid == 0 { + rBatch = r.Rows() + } else { + r.Check(rBatch) + } + } + + lookupCond := fmt.Sprintf("a %v %v", []string{">", "<"}[rand.Intn(2)], rand.Intn(400000)) + var rLookup [][]interface{} + for tid, tbl := range []string{"tnormal_uniq", "trange_uniq", "thash_uniq"} { + // unsigned + IndexReader + scanSQL := fmt.Sprintf("select a from %v use index(a) where %v", tbl, scanCond) + c.Assert(tk.HasPlan(scanSQL, "IndexReader"), IsTrue) + r := tk.MustQuery(scanSQL).Sort() + if tid == 0 { + rScan = r.Rows() + } else { + r.Check(rScan) + } + + // unsigned + IndexLookUp + lookupSQL := fmt.Sprintf("select * from %v use index(a) where %v", tbl, lookupCond) + tk.MustIndexLookup(lookupSQL) + r = tk.MustQuery(lookupSQL).Sort() + if tid == 0 { + rLookup = r.Rows() + } else { + r.Check(rLookup) + } + + // unsigned + PointGet on UniqueIndex + pointSQL := fmt.Sprintf("select * from %v use index(a) where %v", tbl, pointCond) + tk.MustPointGet(pointSQL) + r = tk.MustQuery(pointSQL).Sort() + if tid == 0 { + rPoint = r.Rows() + } else { + r.Check(rPoint) + } + + // unsigned + BatchGet on UniqueIndex + batchSQL := fmt.Sprintf("select * from %v where %v", tbl, batchCond) + c.Assert(tk.HasPlan(batchSQL, "Batch_Point_Get"), IsTrue) + r = tk.MustQuery(batchSQL).Sort() + if tid == 0 { + rBatch = r.Rows() + } else { + r.Check(rBatch) + } + } + } +} + +func (s *partitionTableSuite) TestDirectReadingWithAgg(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_dr_agg") + tk.MustExec("use test_dr_agg") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + // list partition table + tk.MustExec(`create table tlist(a int, b int, index idx_a(a), index idx_b(b)) partition by list(a)( + partition p0 values in (1, 2, 3, 4), + partition p1 values in (5, 6, 7, 8), + partition p2 values in (9, 10, 11, 12));`) + + // range partition table + tk.MustExec(`create table trange(a int, b int, index idx_a(a), index idx_b(b)) partition by range(a) ( + partition p0 values less than(300), + partition p1 values less than (500), + partition p2 values less than(1100));`) + + // hash partition table + tk.MustExec(`create table thash(a int, b int) partition by hash(a) partitions 4;`) + + // regular table + tk.MustExec("create table tregular1(a int, b int, index idx_a(a))") + tk.MustExec("create table tregular2(a int, b int, index idx_a(a))") + + // generate some random data to be inserted + vals := make([]string, 0, 2000) + for i := 0; i < 2000; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(1100), rand.Intn(2000))) + } + + tk.MustExec("insert into trange values " + strings.Join(vals, ",")) + tk.MustExec("insert into thash values " + strings.Join(vals, ",")) + tk.MustExec("insert into tregular1 values " + strings.Join(vals, ",")) + + vals = make([]string, 0, 2000) + for i := 0; i < 2000; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(12)+1, rand.Intn(20))) + } + + tk.MustExec("insert into tlist values " + strings.Join(vals, ",")) + tk.MustExec("insert into tregular2 values " + strings.Join(vals, ",")) + + // test range partition + for i := 0; i < 200; i++ { + // select /*+ stream_agg() */ a from t where a > ? group by a; + // select /*+ hash_agg() */ a from t where a > ? group by a; + // select /*+ stream_agg() */ a from t where a in(?, ?, ?) group by a; + // select /*+ hash_agg() */ a from t where a in (?, ?, ?) group by a; + x := rand.Intn(1099) + + queryPartition1 := fmt.Sprintf("select /*+ stream_agg() */ count(*), sum(b), max(b), a from trange where a > %v group by a;", x) + queryRegular1 := fmt.Sprintf("select /*+ stream_agg() */ count(*), sum(b), max(b), a from tregular1 where a > %v group by a;", x) + c.Assert(tk.HasPlan(queryPartition1, "StreamAgg"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition1).Sort().Check(tk.MustQuery(queryRegular1).Sort().Rows()) + + queryPartition2 := fmt.Sprintf("select /*+ hash_agg() */ count(*), sum(b), max(b), a from trange where a > %v group by a;", x) + queryRegular2 := fmt.Sprintf("select /*+ hash_agg() */ count(*), sum(b), max(b), a from tregular1 where a > %v group by a;", x) + c.Assert(tk.HasPlan(queryPartition2, "HashAgg"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition2).Sort().Check(tk.MustQuery(queryRegular2).Sort().Rows()) + + y := rand.Intn(1099) + z := rand.Intn(1099) + + queryPartition3 := fmt.Sprintf("select /*+ stream_agg() */ count(*), sum(b), max(b), a from trange where a in(%v, %v, %v) group by a;", x, y, z) + queryRegular3 := fmt.Sprintf("select /*+ stream_agg() */ count(*), sum(b), max(b), a from tregular1 where a in(%v, %v, %v) group by a;", x, y, z) + c.Assert(tk.HasPlan(queryPartition3, "StreamAgg"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition3).Sort().Check(tk.MustQuery(queryRegular3).Sort().Rows()) + + queryPartition4 := fmt.Sprintf("select /*+ hash_agg() */ count(*), sum(b), max(b), a from trange where a in (%v, %v, %v) group by a;", x, y, z) + queryRegular4 := fmt.Sprintf("select /*+ hash_agg() */ count(*), sum(b), max(b), a from tregular1 where a in (%v, %v, %v) group by a;", x, y, z) + c.Assert(tk.HasPlan(queryPartition4, "HashAgg"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition4).Sort().Check(tk.MustQuery(queryRegular4).Sort().Rows()) + } + + // test hash partition + for i := 0; i < 200; i++ { + // select /*+ stream_agg() */ a from t where a > ? group by a; + // select /*+ hash_agg() */ a from t where a > ? group by a; + // select /*+ stream_agg() */ a from t where a in(?, ?, ?) group by a; + // select /*+ hash_agg() */ a from t where a in (?, ?, ?) group by a; + x := rand.Intn(1099) + + queryPartition1 := fmt.Sprintf("select /*+ stream_agg() */ count(*), sum(b), max(b), a from thash where a > %v group by a;", x) + queryRegular1 := fmt.Sprintf("select /*+ stream_agg() */ count(*), sum(b), max(b), a from tregular1 where a > %v group by a;", x) + c.Assert(tk.HasPlan(queryPartition1, "StreamAgg"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition1).Sort().Check(tk.MustQuery(queryRegular1).Sort().Rows()) + + queryPartition2 := fmt.Sprintf("select /*+ hash_agg() */ count(*), sum(b), max(b), a from thash where a > %v group by a;", x) + queryRegular2 := fmt.Sprintf("select /*+ hash_agg() */ count(*), sum(b), max(b), a from tregular1 where a > %v group by a;", x) + c.Assert(tk.HasPlan(queryPartition2, "HashAgg"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition2).Sort().Check(tk.MustQuery(queryRegular2).Sort().Rows()) + + y := rand.Intn(1099) + z := rand.Intn(1099) + + queryPartition3 := fmt.Sprintf("select /*+ stream_agg() */ count(*), sum(b), max(b), a from thash where a in(%v, %v, %v) group by a;", x, y, z) + queryRegular3 := fmt.Sprintf("select /*+ stream_agg() */ count(*), sum(b), max(b), a from tregular1 where a in(%v, %v, %v) group by a;", x, y, z) + c.Assert(tk.HasPlan(queryPartition3, "StreamAgg"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition3).Sort().Check(tk.MustQuery(queryRegular3).Sort().Rows()) + + queryPartition4 := fmt.Sprintf("select /*+ hash_agg() */ count(*), sum(b), max(b), a from thash where a in (%v, %v, %v) group by a;", x, y, z) + queryRegular4 := fmt.Sprintf("select /*+ hash_agg() */ count(*), sum(b), max(b), a from tregular1 where a in (%v, %v, %v) group by a;", x, y, z) + c.Assert(tk.HasPlan(queryPartition4, "HashAgg"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition4).Sort().Check(tk.MustQuery(queryRegular4).Sort().Rows()) + } + + // test list partition + for i := 0; i < 200; i++ { + // select /*+ stream_agg() */ a from t where a > ? group by a; + // select /*+ hash_agg() */ a from t where a > ? group by a; + // select /*+ stream_agg() */ a from t where a in(?, ?, ?) group by a; + // select /*+ hash_agg() */ a from t where a in (?, ?, ?) group by a; + x := rand.Intn(12) + 1 + + queryPartition1 := fmt.Sprintf("select /*+ stream_agg() */ count(*), sum(b), max(b), a from tlist where a > %v group by a;", x) + queryRegular1 := fmt.Sprintf("select /*+ stream_agg() */ count(*), sum(b), max(b), a from tregular2 where a > %v group by a;", x) + c.Assert(tk.HasPlan(queryPartition1, "StreamAgg"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition1).Sort().Check(tk.MustQuery(queryRegular1).Sort().Rows()) + + queryPartition2 := fmt.Sprintf("select /*+ hash_agg() */ count(*), sum(b), max(b), a from tlist where a > %v group by a;", x) + queryRegular2 := fmt.Sprintf("select /*+ hash_agg() */ count(*), sum(b), max(b), a from tregular2 where a > %v group by a;", x) + c.Assert(tk.HasPlan(queryPartition2, "HashAgg"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition2).Sort().Check(tk.MustQuery(queryRegular2).Sort().Rows()) + + y := rand.Intn(12) + 1 + z := rand.Intn(12) + 1 + + queryPartition3 := fmt.Sprintf("select /*+ stream_agg() */ count(*), sum(b), max(b), a from tlist where a in(%v, %v, %v) group by a;", x, y, z) + queryRegular3 := fmt.Sprintf("select /*+ stream_agg() */ count(*), sum(b), max(b), a from tregular2 where a in(%v, %v, %v) group by a;", x, y, z) + c.Assert(tk.HasPlan(queryPartition3, "StreamAgg"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition3).Sort().Check(tk.MustQuery(queryRegular3).Sort().Rows()) + + queryPartition4 := fmt.Sprintf("select /*+ hash_agg() */ count(*), sum(b), max(b), a from tlist where a in (%v, %v, %v) group by a;", x, y, z) + queryRegular4 := fmt.Sprintf("select /*+ hash_agg() */ count(*), sum(b), max(b), a from tregular2 where a in (%v, %v, %v) group by a;", x, y, z) + c.Assert(tk.HasPlan(queryPartition4, "HashAgg"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition4).Sort().Check(tk.MustQuery(queryRegular4).Sort().Rows()) + } +} + +func (s *partitionTableSuite) TestDynamicModeByDefault(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_dynamic_by_default") + + tk.MustExec(`create table trange(a int, b int, primary key(a) clustered, index idx_b(b)) partition by range(a) ( + partition p0 values less than(300), + partition p1 values less than(500), + partition p2 values less than(1100));`) + tk.MustExec(`create table thash(a int, b int, primary key(a) clustered, index idx_b(b)) partition by hash(a) partitions 4;`) + + for _, q := range []string{ + "explain select * from trange where a>400", + "explain select * from thash where a>=100", + } { + for _, r := range tk.MustQuery(q).Rows() { + c.Assert(strings.Contains(strings.ToLower(r[0].(string)), "partitionunion"), IsFalse) + } + } +} + +func (s *partitionTableSuite) TestIssue24636(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_issue_24636") + tk.MustExec("use test_issue_24636") + + tk.MustExec(`CREATE TABLE t (a int, b date, c int, PRIMARY KEY (a,b)) + PARTITION BY RANGE ( TO_DAYS(b) ) ( + PARTITION p0 VALUES LESS THAN (737821), + PARTITION p1 VALUES LESS THAN (738289) + )`) + tk.MustExec(`INSERT INTO t (a, b, c) VALUES(0, '2021-05-05', 0)`) + tk.MustQuery(`select c from t use index(primary) where a=0 limit 1`).Check(testkit.Rows("0")) + + tk.MustExec(` + CREATE TABLE test_partition ( + a varchar(100) NOT NULL, + b date NOT NULL, + c varchar(100) NOT NULL, + d datetime DEFAULT NULL, + e datetime DEFAULT NULL, + f bigint(20) DEFAULT NULL, + g bigint(20) DEFAULT NULL, + h bigint(20) DEFAULT NULL, + i bigint(20) DEFAULT NULL, + j bigint(20) DEFAULT NULL, + k bigint(20) DEFAULT NULL, + l bigint(20) DEFAULT NULL, + PRIMARY KEY (a,b,c) /*T![clustered_index] NONCLUSTERED */ + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin + PARTITION BY RANGE ( TO_DAYS(b) ) ( + PARTITION pmin VALUES LESS THAN (737821), + PARTITION p20200601 VALUES LESS THAN (738289))`) + tk.MustExec(`INSERT INTO test_partition (a, b, c, d, e, f, g, h, i, j, k, l) VALUES('aaa', '2021-05-05', '428ff6a1-bb37-42ac-9883-33d7a29961e6', '2021-05-06 08:13:38', '2021-05-06 13:28:08', 0, 8, 3, 0, 9, 1, 0)`) + tk.MustQuery(`select c,j,l from test_partition where c='428ff6a1-bb37-42ac-9883-33d7a29961e6' and a='aaa' limit 0, 200`).Check(testkit.Rows("428ff6a1-bb37-42ac-9883-33d7a29961e6 9 0")) +} + +func (s *partitionTableSuite) TestIdexMerge(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_idx_merge") + tk.MustExec("use test_idx_merge") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + // list partition table + tk.MustExec(`create table tlist(a int, b int, primary key(a) clustered, index idx_b(b)) partition by list(a)( + partition p0 values in (1, 2, 3, 4), + partition p1 values in (5, 6, 7, 8), + partition p2 values in (9, 10, 11, 12));`) + + // range partition table + tk.MustExec(`create table trange(a int, b int, primary key(a) clustered, index idx_b(b)) partition by range(a) ( + partition p0 values less than(300), + partition p1 values less than (500), + partition p2 values less than(1100));`) + + // hash partition table + tk.MustExec(`create table thash(a int, b int, primary key(a) clustered, index idx_b(b)) partition by hash(a) partitions 4;`) + + // regular table + tk.MustExec("create table tregular1(a int, b int, primary key(a) clustered)") + tk.MustExec("create table tregular2(a int, b int, primary key(a) clustered)") + + // generate some random data to be inserted + vals := make([]string, 0, 2000) + for i := 0; i < 2000; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(1100), rand.Intn(2000))) + } + + tk.MustExec("insert ignore into trange values " + strings.Join(vals, ",")) + tk.MustExec("insert ignore into thash values " + strings.Join(vals, ",")) + tk.MustExec("insert ignore into tregular1 values " + strings.Join(vals, ",")) + + vals = make([]string, 0, 2000) + for i := 0; i < 2000; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(12)+1, rand.Intn(20))) + } + + tk.MustExec("insert ignore into tlist values " + strings.Join(vals, ",")) + tk.MustExec("insert ignore into tregular2 values " + strings.Join(vals, ",")) + + // test range partition + for i := 0; i < 100; i++ { + x1 := rand.Intn(1099) + x2 := rand.Intn(1099) + + queryPartition1 := fmt.Sprintf("select /*+ use_index_merge(trange) */ * from trange where a > %v or b < %v;", x1, x2) + queryRegular1 := fmt.Sprintf("select /*+ use_index_merge(tregular1) */ * from tregular1 where a > %v or b < %v;", x1, x2) + c.Assert(tk.HasPlan(queryPartition1, "IndexMerge"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition1).Sort().Check(tk.MustQuery(queryRegular1).Sort().Rows()) + + queryPartition2 := fmt.Sprintf("select /*+ use_index_merge(trange) */ * from trange where a > %v or b > %v;", x1, x2) + queryRegular2 := fmt.Sprintf("select /*+ use_index_merge(tregular1) */ * from tregular1 where a > %v or b > %v;", x1, x2) + c.Assert(tk.HasPlan(queryPartition2, "IndexMerge"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition2).Sort().Check(tk.MustQuery(queryRegular2).Sort().Rows()) + } + + // test hash partition + for i := 0; i < 100; i++ { + x1 := rand.Intn(1099) + x2 := rand.Intn(1099) + + queryPartition1 := fmt.Sprintf("select /*+ use_index_merge(thash) */ * from thash where a > %v or b < %v;", x1, x2) + queryRegular1 := fmt.Sprintf("select /*+ use_index_merge(tregualr1) */ * from tregular1 where a > %v or b < %v;", x1, x2) + c.Assert(tk.HasPlan(queryPartition1, "IndexMerge"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition1).Sort().Check(tk.MustQuery(queryRegular1).Sort().Rows()) + + queryPartition2 := fmt.Sprintf("select /*+ use_index_merge(thash) */ * from thash where a > %v or b > %v;", x1, x2) + queryRegular2 := fmt.Sprintf("select /*+ use_index_merge(tregular1) */ * from tregular1 where a > %v or b > %v;", x1, x2) + c.Assert(tk.HasPlan(queryPartition2, "IndexMerge"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition2).Sort().Check(tk.MustQuery(queryRegular2).Sort().Rows()) + } + + // test list partition + for i := 0; i < 100; i++ { + x1 := rand.Intn(12) + 1 + x2 := rand.Intn(12) + 1 + queryPartition1 := fmt.Sprintf("select /*+ use_index_merge(tlist) */ * from tlist where a > %v or b < %v;", x1, x2) + queryRegular1 := fmt.Sprintf("select /*+ use_index_merge(tregular2) */ * from tregular2 where a > %v or b < %v;", x1, x2) + c.Assert(tk.HasPlan(queryPartition1, "IndexMerge"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition1).Sort().Check(tk.MustQuery(queryRegular1).Sort().Rows()) + + queryPartition2 := fmt.Sprintf("select /*+ use_index_merge(tlist) */ * from tlist where a > %v or b > %v;", x1, x2) + queryRegular2 := fmt.Sprintf("select /*+ use_index_merge(tregular2) */ * from tregular2 where a > %v or b > %v;", x1, x2) + c.Assert(tk.HasPlan(queryPartition2, "IndexMerge"), IsTrue) // check if IndexLookUp is used + tk.MustQuery(queryPartition2).Sort().Check(tk.MustQuery(queryRegular2).Sort().Rows()) + } +} + +func (s *globalIndexSuite) TestGlobalIndexScan(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists p") + tk.MustExec(`create table p (id int, c int) partition by range (c) ( +partition p0 values less than (4), +partition p1 values less than (7), +partition p2 values less than (10))`) + tk.MustExec("alter table p add unique idx(id)") + tk.MustExec("insert into p values (1,3), (3,4), (5,6), (7,9)") + tk.MustQuery("select id from p use index (idx)").Check(testkit.Rows("1", "3", "5", "7")) +} + +func (s *globalIndexSuite) TestGlobalIndexDoubleRead(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists p") + tk.MustExec(`create table p (id int, c int) partition by range (c) ( +partition p0 values less than (4), +partition p1 values less than (7), +partition p2 values less than (10))`) + tk.MustExec("alter table p add unique idx(id)") + tk.MustExec("insert into p values (1,3), (3,4), (5,6), (7,9)") + tk.MustQuery("select * from p use index (idx)").Check(testkit.Rows("1 3", "3 4", "5 6", "7 9")) +} + +func (s *globalIndexSuite) TestIssue21731(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists p, t") + tk.MustExec("create table t (a int, b int, unique index idx(a)) partition by list columns(b) (partition p0 values in (1), partition p1 values in (2));") +} + +type testOutput struct { + SQL string + Plan []string + Res []string +} + +func (s *testSuiteWithData) verifyPartitionResult(tk *testkit.TestKit, input []string, output []testOutput) { + for i, tt := range input { + var isSelect bool = false + if strings.HasPrefix(strings.ToLower(tt), "select ") { + isSelect = true + } + s.testData.OnRecord(func() { + output[i].SQL = tt + if isSelect { + output[i].Plan = s.testData.ConvertRowsToStrings(tk.UsedPartitions(tt).Rows()) + output[i].Res = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Sort().Rows()) + } else { + // Just verify SELECT (also avoid double INSERTs during record) + output[i].Res = nil + output[i].Plan = nil + } + }) + if isSelect { + tk.UsedPartitions(tt).Check(testkit.Rows(output[i].Plan...)) + tk.MustQuery(tt).Sort().Check(testkit.Rows(output[i].Res...)) + } else { + tk.MustExec(tt) + } + } +} + +func (s *testSuiteWithData) TestRangePartitionBoundariesEq(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("SET @@tidb_partition_prune_mode = 'dynamic'") + tk.MustExec("CREATE DATABASE TestRangePartitionBoundaries") + defer tk.MustExec("DROP DATABASE TestRangePartitionBoundaries") + tk.MustExec("USE TestRangePartitionBoundaries") + tk.MustExec("DROP TABLE IF EXISTS t") + tk.MustExec(`CREATE TABLE t +(a INT, b varchar(255)) +PARTITION BY RANGE (a) ( + PARTITION p0 VALUES LESS THAN (1000000), + PARTITION p1 VALUES LESS THAN (2000000), + PARTITION p2 VALUES LESS THAN (3000000)); +`) + + var input []string + var output []testOutput + s.testData.GetTestCases(c, &input, &output) + s.verifyPartitionResult(tk, input, output) +} + +func (s *testSuiteWithData) TestRangePartitionBoundariesNe(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("SET @@tidb_partition_prune_mode = 'dynamic'") + tk.MustExec("CREATE DATABASE TestRangePartitionBoundariesNe") + defer tk.MustExec("DROP DATABASE TestRangePartitionBoundariesNe") + tk.MustExec("USE TestRangePartitionBoundariesNe") + tk.MustExec("DROP TABLE IF EXISTS t") + tk.MustExec(`CREATE TABLE t +(a INT, b varchar(255)) +PARTITION BY RANGE (a) ( + PARTITION p0 VALUES LESS THAN (1), + PARTITION p1 VALUES LESS THAN (2), + PARTITION p2 VALUES LESS THAN (3), + PARTITION p3 VALUES LESS THAN (4), + PARTITION p4 VALUES LESS THAN (5), + PARTITION p5 VALUES LESS THAN (6), + PARTITION p6 VALUES LESS THAN (7))`) + + var input []string + var output []testOutput + s.testData.GetTestCases(c, &input, &output) + s.verifyPartitionResult(tk, input, output) +} + +func (s *testSuiteWithData) TestRangePartitionBoundariesBetweenM(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("CREATE DATABASE IF NOT EXISTS TestRangePartitionBoundariesBetweenM") + defer tk.MustExec("DROP DATABASE TestRangePartitionBoundariesBetweenM") + tk.MustExec("USE TestRangePartitionBoundariesBetweenM") + tk.MustExec("DROP TABLE IF EXISTS t") + tk.MustExec(`CREATE TABLE t +(a INT, b varchar(255)) +PARTITION BY RANGE (a) ( + PARTITION p0 VALUES LESS THAN (1000000), + PARTITION p1 VALUES LESS THAN (2000000), + PARTITION p2 VALUES LESS THAN (3000000))`) + + var input []string + var output []testOutput + s.testData.GetTestCases(c, &input, &output) + s.verifyPartitionResult(tk, input, output) +} + +func (s *testSuiteWithData) TestRangePartitionBoundariesBetweenS(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("CREATE DATABASE IF NOT EXISTS TestRangePartitionBoundariesBetweenS") + defer tk.MustExec("DROP DATABASE TestRangePartitionBoundariesBetweenS") + tk.MustExec("USE TestRangePartitionBoundariesBetweenS") + tk.MustExec("DROP TABLE IF EXISTS t") + tk.MustExec(`CREATE TABLE t +(a INT, b varchar(255)) +PARTITION BY RANGE (a) ( + PARTITION p0 VALUES LESS THAN (1), + PARTITION p1 VALUES LESS THAN (2), + PARTITION p2 VALUES LESS THAN (3), + PARTITION p3 VALUES LESS THAN (4), + PARTITION p4 VALUES LESS THAN (5), + PARTITION p5 VALUES LESS THAN (6), + PARTITION p6 VALUES LESS THAN (7))`) + + var input []string + var output []testOutput + s.testData.GetTestCases(c, &input, &output) + s.verifyPartitionResult(tk, input, output) +} + +func (s *testSuiteWithData) TestRangePartitionBoundariesLtM(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + tk.MustExec("create database TestRangePartitionBoundariesLtM") + defer tk.MustExec("drop database TestRangePartitionBoundariesLtM") + tk.MustExec("use TestRangePartitionBoundariesLtM") + tk.MustExec("drop table if exists t") + tk.MustExec(`CREATE TABLE t +(a INT, b varchar(255)) +PARTITION BY RANGE (a) ( + PARTITION p0 VALUES LESS THAN (1000000), + PARTITION p1 VALUES LESS THAN (2000000), + PARTITION p2 VALUES LESS THAN (3000000))`) + + var input []string + var output []testOutput + s.testData.GetTestCases(c, &input, &output) + s.verifyPartitionResult(tk, input, output) +} + +func (s *testSuiteWithData) TestRangePartitionBoundariesLtS(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + tk.MustExec("create database TestRangePartitionBoundariesLtS") + defer tk.MustExec("drop database TestRangePartitionBoundariesLtS") + tk.MustExec("use TestRangePartitionBoundariesLtS") + tk.MustExec("drop table if exists t") + tk.MustExec(`CREATE TABLE t +(a INT, b varchar(255)) +PARTITION BY RANGE (a) ( + PARTITION p0 VALUES LESS THAN (1), + PARTITION p1 VALUES LESS THAN (2), + PARTITION p2 VALUES LESS THAN (3), + PARTITION p3 VALUES LESS THAN (4), + PARTITION p4 VALUES LESS THAN (5), + PARTITION p5 VALUES LESS THAN (6), + PARTITION p6 VALUES LESS THAN (7))`) + + var input []string + var output []testOutput + s.testData.GetTestCases(c, &input, &output) + s.verifyPartitionResult(tk, input, output) } diff --git a/executor/pipelined_window.go b/executor/pipelined_window.go new file mode 100644 index 0000000000000..3f016ff32860c --- /dev/null +++ b/executor/pipelined_window.go @@ -0,0 +1,457 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor + +import ( + "context" + + "github.com/cznic/mathutil" + "github.com/pingcap/errors" + "github.com/pingcap/parser/ast" + "github.com/pingcap/tidb/executor/aggfuncs" + "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/planner/core" + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/util/chunk" +) + +type dataInfo struct { + chk *chunk.Chunk + remaining uint64 + accumulated uint64 +} + +// PipelinedWindowExec is the executor for window functions. +type PipelinedWindowExec struct { + baseExecutor + numWindowFuncs int + windowFuncs []aggfuncs.AggFunc + slidingWindowFuncs []aggfuncs.SlidingWindowAggFunc + partialResults []aggfuncs.PartialResult + start *core.FrameBound + end *core.FrameBound + groupChecker *vecGroupChecker + + // childResult stores the child chunk. Note that even if remaining is 0, e.rows might still references rows in data[0].chk after returned it to upper executor, since there is no guarantee what the upper executor will do to the returned chunk, it might destroy the data (as in the benchmark test, it reused the chunk to pull data, and it will be chk.Reset(), causing panicking). So dataIdx, accumulated and dropped are added to ensure that chunk will only be returned if there is no row reference. + childResult *chunk.Chunk + data []dataInfo + dataIdx int + + // done indicates the child executor is drained or something unexpected happened. + done bool + accumulated uint64 + dropped uint64 + rowToConsume uint64 + newPartition bool + + curRowIdx uint64 + // curStartRow and curEndRow defines the current frame range + lastStartRow uint64 + lastEndRow uint64 + stagedStartRow uint64 + stagedEndRow uint64 + rowStart uint64 + orderByCols []*expression.Column + // expectedCmpResult is used to decide if one value is included in the frame. + expectedCmpResult int64 + + // rows keeps rows starting from curStartRow + rows []chunk.Row + rowCnt uint64 + whole bool + isRangeFrame bool + emptyFrame bool + initializedSlidingWindow bool +} + +// Close implements the Executor Close interface. +func (e *PipelinedWindowExec) Close() error { + return errors.Trace(e.baseExecutor.Close()) +} + +// Open implements the Executor Open interface +func (e *PipelinedWindowExec) Open(ctx context.Context) (err error) { + e.rowToConsume = 0 + e.done = false + e.accumulated = 0 + e.dropped = 0 + e.data = make([]dataInfo, 0) + e.dataIdx = 0 + e.slidingWindowFuncs = make([]aggfuncs.SlidingWindowAggFunc, len(e.windowFuncs)) + for i, windowFunc := range e.windowFuncs { + if slidingWindowAggFunc, ok := windowFunc.(aggfuncs.SlidingWindowAggFunc); ok { + e.slidingWindowFuncs[i] = slidingWindowAggFunc + } + } + e.rows = make([]chunk.Row, 0) + return e.baseExecutor.Open(ctx) +} + +func (e *PipelinedWindowExec) firstResultChunkNotReady() bool { + // chunk can't be ready unless, 1. all of the rows in the chunk is filled, 2. e.rows doesn't contain rows in the chunk + return len(e.data) > 0 && (e.data[0].remaining != 0 || e.data[0].accumulated > e.dropped) +} + +// Next implements the Executor Next interface. +func (e *PipelinedWindowExec) Next(ctx context.Context, chk *chunk.Chunk) (err error) { + chk.Reset() + + for !e.done || e.firstResultChunkNotReady() { + // we firstly gathering enough rows and consume them, until we are able to produce. + // for unbounded frame, it needs consume the whole partition before being able to produce, in this case + // e.p.enoughToProduce will be false until so. + var enough bool + enough, err = e.enoughToProduce(e.ctx) + if err != nil { + return + } + if !enough { + if !e.done && e.rowToConsume == 0 { + err = e.getRowsInPartition(ctx) + if err != nil { + return err + } + } + if e.done || e.newPartition { + e.finish() + // if we continued, the rows will not be consumed, so next time we should consume it instead of calling e.getRowsInPartition + enough, err = e.enoughToProduce(e.ctx) + if err != nil { + return + } + if enough { + continue + } + e.newPartition = false + e.reset() + if e.rowToConsume == 0 { + // no more data + break + } + } + e.rowCnt += e.rowToConsume + e.rowToConsume = 0 + } + + // e.p is ready to produce data + if len(e.data) > e.dataIdx && e.data[e.dataIdx].remaining != 0 { + produced, err := e.produce(e.ctx, e.data[e.dataIdx].chk, e.data[e.dataIdx].remaining) + if err != nil { + return err + } + e.data[e.dataIdx].remaining -= produced + if e.data[e.dataIdx].remaining == 0 { + e.dataIdx++ + } + } + } + if len(e.data) > 0 { + chk.SwapColumns(e.data[0].chk) + e.data = e.data[1:] + e.dataIdx-- + } + return nil +} + +func (e *PipelinedWindowExec) getRowsInPartition(ctx context.Context) (err error) { + e.newPartition = true + if len(e.rows) == 0 { + // if getRowsInPartition is called for the first time, we ignore it as a new partition + e.newPartition = false + } + + if e.groupChecker.isExhausted() { + var drained, samePartition bool + drained, err = e.fetchChild(ctx) + if err != nil { + err = errors.Trace(err) + } + // we return immediately to use a combination of true newPartition but 0 in e.rowToConsume to indicate the data source is drained, + if drained { + e.done = true + return nil + } + samePartition, err = e.groupChecker.splitIntoGroups(e.childResult) + if samePartition { + // the only case that when getRowsInPartition gets called, it is not a new partition. + e.newPartition = false + } + if err != nil { + return errors.Trace(err) + } + } + begin, end := e.groupChecker.getNextGroup() + e.rowToConsume += uint64(end - begin) + for i := begin; i < end; i++ { + e.rows = append(e.rows, e.childResult.GetRow(i)) + } + return +} + +func (e *PipelinedWindowExec) fetchChild(ctx context.Context) (EOF bool, err error) { + // TODO: reuse chunks + childResult := newFirstChunk(e.children[0]) + err = Next(ctx, e.children[0], childResult) + if err != nil { + return false, errors.Trace(err) + } + // No more data. + numRows := childResult.NumRows() + if numRows == 0 { + return true, nil + } + + // TODO: reuse chunks + resultChk := chunk.New(e.retFieldTypes, 0, numRows) + err = e.copyChk(childResult, resultChk) + if err != nil { + return false, err + } + e.accumulated += uint64(numRows) + e.data = append(e.data, dataInfo{chk: resultChk, remaining: uint64(numRows), accumulated: e.accumulated}) + + e.childResult = childResult + return false, nil +} + +func (e *PipelinedWindowExec) copyChk(src, dst *chunk.Chunk) error { + columns := e.Schema().Columns[:len(e.Schema().Columns)-e.numWindowFuncs] + for i, col := range columns { + if err := dst.MakeRefTo(i, src, col.Index); err != nil { + return err + } + } + return nil +} + +func (e *PipelinedWindowExec) getRow(i uint64) chunk.Row { + return e.rows[i-e.rowStart] +} + +func (e *PipelinedWindowExec) getRows(start, end uint64) []chunk.Row { + return e.rows[start-e.rowStart : end-e.rowStart] +} + +// finish is called upon a whole partition is consumed +func (e *PipelinedWindowExec) finish() { + e.whole = true +} + +func (e *PipelinedWindowExec) getStart(ctx sessionctx.Context) (uint64, error) { + if e.start.UnBounded { + return 0, nil + } + if e.isRangeFrame { + var start uint64 + for start = mathutil.MaxUint64(e.lastStartRow, e.stagedStartRow); start < e.rowCnt; start++ { + var res int64 + var err error + for i := range e.orderByCols { + res, _, err = e.start.CmpFuncs[i](ctx, e.orderByCols[i], e.start.CalcFuncs[i], e.getRow(start), e.getRow(e.curRowIdx)) + if err != nil { + return 0, err + } + if res != 0 { + break + } + } + // For asc, break when the calculated result is greater than the current value. + // For desc, break when the calculated result is less than the current value. + if res != e.expectedCmpResult { + break + } + } + e.stagedStartRow = start + return start, nil + } + switch e.start.Type { + case ast.Preceding: + if e.curRowIdx > e.start.Num { + return e.curRowIdx - e.start.Num, nil + } + return 0, nil + case ast.Following: + return e.curRowIdx + e.start.Num, nil + default: // ast.CurrentRow + return e.curRowIdx, nil + } +} + +func (e *PipelinedWindowExec) getEnd(ctx sessionctx.Context) (uint64, error) { + if e.end.UnBounded { + return e.rowCnt, nil + } + if e.isRangeFrame { + var end uint64 + for end = mathutil.MaxUint64(e.lastEndRow, e.stagedEndRow); end < e.rowCnt; end++ { + var res int64 + var err error + for i := range e.orderByCols { + res, _, err = e.end.CmpFuncs[i](ctx, e.end.CalcFuncs[i], e.orderByCols[i], e.getRow(e.curRowIdx), e.getRow(end)) + if err != nil { + return 0, err + } + if res != 0 { + break + } + } + // For asc, break when the calculated result is greater than the current value. + // For desc, break when the calculated result is less than the current value. + if res == e.expectedCmpResult { + break + } + } + e.stagedEndRow = end + return end, nil + } + switch e.end.Type { + case ast.Preceding: + if e.curRowIdx >= e.end.Num { + return e.curRowIdx - e.end.Num + 1, nil + } + return 0, nil + case ast.Following: + return e.curRowIdx + e.end.Num + 1, nil + default: // ast.CurrentRow: + return e.curRowIdx + 1, nil + } +} + +// produce produces rows and append it to chk, return produced means number of rows appended into chunk, available means +// number of rows processed but not fetched +func (e *PipelinedWindowExec) produce(ctx sessionctx.Context, chk *chunk.Chunk, remained uint64) (produced uint64, err error) { + var ( + start uint64 + end uint64 + enough bool + ) + for remained > 0 { + enough, err = e.enoughToProduce(ctx) + if err != nil { + return + } + if !enough { + break + } + start, err = e.getStart(ctx) + if err != nil { + return + } + end, err = e.getEnd(ctx) + if err != nil { + return + } + if end > e.rowCnt { + end = e.rowCnt + } + if start >= e.rowCnt { + start = e.rowCnt + } + // if start >= end, we should return a default value, and we reset the frame to empty. + if start >= end { + for i, wf := range e.windowFuncs { + if !e.emptyFrame { + wf.ResetPartialResult(e.partialResults[i]) + } + err = wf.AppendFinalResult2Chunk(ctx, e.partialResults[i], chk) + if err != nil { + return + } + } + if !e.emptyFrame { + e.emptyFrame = true + e.initializedSlidingWindow = false + } + } else { + e.emptyFrame = false + for i, wf := range e.windowFuncs { + slidingWindowAggFunc := e.slidingWindowFuncs[i] + if e.lastStartRow != start || e.lastEndRow != end { + if slidingWindowAggFunc != nil && e.initializedSlidingWindow { + err = slidingWindowAggFunc.Slide(ctx, e.getRow, e.lastStartRow, e.lastEndRow, start-e.lastStartRow, end-e.lastEndRow, e.partialResults[i]) + } else { + // For MinMaxSlidingWindowAggFuncs, it needs the absolute value of each start of window, to compare + // whether elements inside deque are out of current window. + if minMaxSlidingWindowAggFunc, ok := wf.(aggfuncs.MaxMinSlidingWindowAggFunc); ok { + // Store start inside MaxMinSlidingWindowAggFunc.windowInfo + minMaxSlidingWindowAggFunc.SetWindowStart(start) + } + // TODO(zhifeng): track memory usage here + wf.ResetPartialResult(e.partialResults[i]) + _, err = wf.UpdatePartialResult(ctx, e.getRows(start, end), e.partialResults[i]) + } + } + if err != nil { + return + } + err = wf.AppendFinalResult2Chunk(ctx, e.partialResults[i], chk) + if err != nil { + return + } + } + e.initializedSlidingWindow = true + } + e.curRowIdx++ + e.lastStartRow, e.lastEndRow = start, end + + produced++ + remained-- + } + extend := mathutil.MinUint64Val(e.curRowIdx, e.lastEndRow, e.lastStartRow) + if extend > e.rowStart { + numDrop := extend - e.rowStart + e.dropped += numDrop + e.rows = e.rows[numDrop:] + e.rowStart = extend + } + return +} + +func (e *PipelinedWindowExec) enoughToProduce(ctx sessionctx.Context) (enough bool, err error) { + if e.curRowIdx >= e.rowCnt { + return false, nil + } + if e.whole { + return true, nil + } + start, err := e.getStart(ctx) + if err != nil { + return + } + end, err := e.getEnd(ctx) + if err != nil { + return + } + return end < e.rowCnt && start < e.rowCnt, nil +} + +// reset resets the processor +func (e *PipelinedWindowExec) reset() { + e.lastStartRow = 0 + e.lastEndRow = 0 + e.stagedStartRow = 0 + e.stagedEndRow = 0 + e.emptyFrame = false + e.curRowIdx = 0 + e.whole = false + numDrop := e.rowCnt - e.rowStart + e.dropped += numDrop + e.rows = e.rows[numDrop:] + e.rowStart = 0 + e.rowCnt = 0 + e.initializedSlidingWindow = false + for i, windowFunc := range e.windowFuncs { + windowFunc.ResetPartialResult(e.partialResults[i]) + } +} diff --git a/executor/point_get.go b/executor/point_get.go index 241f52d421344..51c9bc5877273 100644 --- a/executor/point_get.go +++ b/executor/point_get.go @@ -31,8 +31,6 @@ import ( plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/store/tikv" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" - "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/tablecodec" @@ -44,6 +42,11 @@ import ( ) func (b *executorBuilder) buildPointGet(p *plannercore.PointGetPlan) Executor { + if err := b.validCanReadTemporaryTable(p.TblInfo); err != nil { + b.err = err + return nil + } + startTS, err := b.getSnapshotTS() if err != nil { b.err = err @@ -144,30 +147,32 @@ func (e *PointGetExecutor) Open(context.Context) error { e.stats = &runtimeStatsWithSnapshot{ SnapshotRuntimeStats: snapshotStats, } - e.snapshot.SetOption(tikvstore.CollectRuntimeStats, snapshotStats) + e.snapshot.SetOption(kv.CollectRuntimeStats, snapshotStats) e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats) } if e.ctx.GetSessionVars().GetReplicaRead().IsFollowerRead() { - e.snapshot.SetOption(tikvstore.ReplicaRead, tikvstore.ReplicaReadFollower) + e.snapshot.SetOption(kv.ReplicaRead, kv.ReplicaReadFollower) } - e.snapshot.SetOption(tikvstore.TaskID, e.ctx.GetSessionVars().StmtCtx.TaskID) + e.snapshot.SetOption(kv.TaskID, e.ctx.GetSessionVars().StmtCtx.TaskID) + e.snapshot.SetOption(kv.TxnScope, e.ctx.GetSessionVars().TxnCtx.TxnScope) isStaleness := e.ctx.GetSessionVars().TxnCtx.IsStaleness - e.snapshot.SetOption(tikvstore.IsStalenessReadOnly, isStaleness) - if isStaleness && e.ctx.GetSessionVars().TxnCtx.TxnScope != oracle.GlobalTxnScope { - e.snapshot.SetOption(tikvstore.MatchStoreLabels, []*metapb.StoreLabel{ + e.snapshot.SetOption(kv.IsStalenessReadOnly, isStaleness) + if isStaleness && e.ctx.GetSessionVars().TxnCtx.TxnScope != kv.GlobalTxnScope { + e.snapshot.SetOption(kv.MatchStoreLabels, []*metapb.StoreLabel{ { Key: placement.DCLabelKey, Value: e.ctx.GetSessionVars().TxnCtx.TxnScope, }, }) } + setResourceGroupTagForTxn(e.ctx.GetSessionVars().StmtCtx, e.snapshot) return nil } // Close implements the Executor interface. func (e *PointGetExecutor) Close() error { if e.runtimeStats != nil && e.snapshot != nil { - e.snapshot.DelOption(tikvstore.CollectRuntimeStats) + e.snapshot.SetOption(kv.CollectRuntimeStats, nil) } if e.idxInfo != nil && e.tblInfo != nil { actRows := int64(0) @@ -322,19 +327,14 @@ func (e *PointGetExecutor) lockKeyIfNeeded(ctx context.Context, key []byte) erro if e.lock { seVars := e.ctx.GetSessionVars() lockCtx := newLockCtx(seVars, e.lockWaitTime) - lockCtx.ReturnValues = true - lockCtx.Values = map[string]tikvstore.ReturnedValue{} + lockCtx.InitReturnValues(1) err := doLockKeys(ctx, e.ctx, lockCtx, key) if err != nil { return err } - lockCtx.ValuesLock.Lock() - defer lockCtx.ValuesLock.Unlock() - for key, val := range lockCtx.Values { - if !val.AlreadyLocked { - seVars.TxnCtx.SetPessimisticLockCache(kv.Key(key), val.Value) - } - } + lockCtx.IterateValuesNotLocked(func(k, v []byte) { + seVars.TxnCtx.SetPessimisticLockCache(kv.Key(k), v) + }) if len(e.handleVal) > 0 { seVars.TxnCtx.SetPessimisticLockCache(e.idxKey, e.handleVal) } @@ -375,6 +375,10 @@ func (e *PointGetExecutor) get(ctx context.Context, key kv.Key) ([]byte, error) // fallthrough to snapshot get. } + // Global temporary table is always empty, so no need to send the request. + if e.tblInfo.TempTableType == model.TempTableGlobal { + return nil, nil + } lock := e.tblInfo.Lock if lock != nil && (lock.Tp == model.TableLockRead || lock.Tp == model.TableLockReadOnly) { if e.ctx.GetSessionVars().EnablePointGetCache { @@ -391,14 +395,14 @@ func (e *PointGetExecutor) get(ctx context.Context, key kv.Key) ([]byte, error) } func (e *PointGetExecutor) verifyTxnScope() error { - txnScope := e.txn.GetOption(tikvstore.TxnScope).(string) - if txnScope == "" || txnScope == oracle.GlobalTxnScope { + txnScope := e.txn.GetOption(kv.TxnScope).(string) + if txnScope == "" || txnScope == kv.GlobalTxnScope { return nil } var tblID int64 var tblName string var partName string - is := infoschema.GetInfoSchema(e.ctx) + is := e.ctx.GetInfoSchema().(infoschema.InfoSchema) if e.partInfo != nil { tblID = e.partInfo.ID tblInfo, _, partInfo := is.FindTableByPartitionID(tblID) @@ -442,6 +446,18 @@ func EncodeUniqueIndexValuesForKey(ctx sessionctx.Context, tblInfo *model.TableI var str string str, err = idxVals[i].ToString() idxVals[i].SetString(str, colInfo.FieldType.Collate) + } else if colInfo.Tp == mysql.TypeEnum && (idxVals[i].Kind() == types.KindString || idxVals[i].Kind() == types.KindBytes || idxVals[i].Kind() == types.KindBinaryLiteral) { + var str string + var e types.Enum + str, err = idxVals[i].ToString() + if err != nil { + return nil, kv.ErrNotExist + } + e, err = types.ParseEnumName(colInfo.FieldType.Elems, str, colInfo.FieldType.Collate) + if err != nil { + return nil, kv.ErrNotExist + } + idxVals[i].SetMysqlEnum(e, colInfo.FieldType.Collate) } else { // If a truncated error or an overflow error is thrown when converting the type of `idxVal[i]` to // the type of `colInfo`, the `idxVal` does not exist in the `idxInfo` for sure. @@ -531,6 +547,9 @@ func tryDecodeFromHandle(tblInfo *model.TableInfo, schemaColIdx int, col *expres chk.AppendInt64(schemaColIdx, handle.IntValue()) return true, nil } + if types.NeedRestoredData(col.RetType) { + return false, nil + } // Try to decode common handle. if mysql.HasPriKeyFlag(col.RetType.Flag) { for i, hid := range pkCols { diff --git a/executor/point_get_test.go b/executor/point_get_test.go index 846b6f1628fe1..f66446a6bef83 100644 --- a/executor/point_get_test.go +++ b/executor/point_get_test.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx/variable" - txndriver "github.com/pingcap/tidb/store/driver/txn" + storeerr "github.com/pingcap/tidb/store/driver/error" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/tablecodec" @@ -536,15 +536,15 @@ func (s *testPointGetSuite) TestSelectCheckVisibility(c *C) { c.Assert(expectErr.Equal(err), IsTrue) } // Test point get. - checkSelectResultError("select * from t where a='1'", txndriver.ErrGCTooEarly) + checkSelectResultError("select * from t where a='1'", storeerr.ErrGCTooEarly) // Test batch point get. - checkSelectResultError("select * from t where a in ('1','2')", txndriver.ErrGCTooEarly) + checkSelectResultError("select * from t where a in ('1','2')", storeerr.ErrGCTooEarly) // Test Index look up read. - checkSelectResultError("select * from t where b > 0 ", txndriver.ErrGCTooEarly) + checkSelectResultError("select * from t where b > 0 ", storeerr.ErrGCTooEarly) // Test Index read. - checkSelectResultError("select b from t where b > 0 ", txndriver.ErrGCTooEarly) + checkSelectResultError("select b from t where b > 0 ", storeerr.ErrGCTooEarly) // Test table read. - checkSelectResultError("select * from t", txndriver.ErrGCTooEarly) + checkSelectResultError("select * from t", storeerr.ErrGCTooEarly) } func (s *testPointGetSuite) TestReturnValues(c *C) { diff --git a/executor/prepared.go b/executor/prepared.go index c5fdd5c1bf404..a5fd131d1a87b 100644 --- a/executor/prepared.go +++ b/executor/prepared.go @@ -29,12 +29,14 @@ import ( "github.com/pingcap/tidb/planner" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" driver "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/hint" "github.com/pingcap/tidb/util/sqlexec" + "github.com/pingcap/tidb/util/topsql" "go.uber.org/zap" ) @@ -79,7 +81,6 @@ func (e *paramMarkerExtractor) Leave(in ast.Node) (ast.Node, bool) { type PrepareExec struct { baseExecutor - is infoschema.InfoSchema name string sqlText string @@ -89,12 +90,11 @@ type PrepareExec struct { } // NewPrepareExec creates a new PrepareExec. -func NewPrepareExec(ctx sessionctx.Context, is infoschema.InfoSchema, sqlTxt string) *PrepareExec { +func NewPrepareExec(ctx sessionctx.Context, sqlTxt string) *PrepareExec { base := newBaseExecutor(ctx, nil, 0) base.initCap = chunk.ZeroCapacity return &PrepareExec{ baseExecutor: base, - is: is, sqlText: sqlTxt, } } @@ -117,7 +117,7 @@ func (e *PrepareExec) Next(ctx context.Context, req *chunk.Chunk) error { ) if sqlParser, ok := e.ctx.(sqlexec.SQLParser); ok { // FIXME: ok... yet another parse API, may need some api interface clean. - stmts, err = sqlParser.ParseSQL(e.sqlText, charset, collation) + stmts, _, err = sqlParser.ParseSQL(ctx, e.sqlText, charset, collation) } else { p := parser.New() p.SetParserConfig(vars.BuildParserConfig()) @@ -159,7 +159,8 @@ func (e *PrepareExec) Next(ctx context.Context, req *chunk.Chunk) error { return ErrPsManyParam } - err = plannercore.Preprocess(e.ctx, stmt, e.is, plannercore.InPrepare) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(e.ctx, stmt, plannercore.InPrepare, plannercore.WithPreprocessorReturn(ret)) if err != nil { return err } @@ -177,14 +178,18 @@ func (e *PrepareExec) Next(ctx context.Context, req *chunk.Chunk) error { Stmt: stmt, StmtType: GetStmtLabel(stmt), Params: sorter.markers, - SchemaVersion: e.is.SchemaMetaVersion(), + SchemaVersion: ret.InfoSchema.SchemaMetaVersion(), + } + normalizedSQL, digest := parser.NormalizeDigest(prepared.Stmt.Text()) + if variable.TopSQLEnabled() { + ctx = topsql.AttachSQLInfo(ctx, normalizedSQL, digest, "", nil) } if !plannercore.PreparedPlanCacheEnabled() { prepared.UseCache = false } else { if !e.ctx.GetSessionVars().UseDynamicPartitionPrune() { - prepared.UseCache = plannercore.Cacheable(stmt, e.is) + prepared.UseCache = plannercore.Cacheable(stmt, ret.InfoSchema) } else { prepared.UseCache = plannercore.Cacheable(stmt, nil) } @@ -199,7 +204,7 @@ func (e *PrepareExec) Next(ctx context.Context, req *chunk.Chunk) error { var p plannercore.Plan e.ctx.GetSessionVars().PlanID = 0 e.ctx.GetSessionVars().PlanColumnID = 0 - destBuilder, _ := plannercore.NewPlanBuilder(e.ctx, e.is, &hint.BlockHintProcessor{}) + destBuilder, _ := plannercore.NewPlanBuilder(e.ctx, ret.InfoSchema, &hint.BlockHintProcessor{}) p, err = destBuilder.Build(ctx, stmt) if err != nil { return err @@ -214,11 +219,10 @@ func (e *PrepareExec) Next(ctx context.Context, req *chunk.Chunk) error { vars.PreparedStmtNameToID[e.name] = e.ID } - normalized, digest := parser.NormalizeDigest(prepared.Stmt.Text()) preparedObj := &plannercore.CachedPrepareStmt{ PreparedAst: prepared, VisitInfos: destBuilder.GetVisitInfo(), - NormalizedSQL: normalized, + NormalizedSQL: normalizedSQL, SQLDigest: digest, ForUpdateRead: destBuilder.GetIsForUpdateRead(), } @@ -320,7 +324,7 @@ func CompileExecutePreparedStmt(ctx context.Context, sctx sessionctx.Context, return nil, false, false, err } execStmt.BinaryArgs = args - is := infoschema.GetInfoSchema(sctx) + is := sctx.GetInfoSchema().(infoschema.InfoSchema) execPlan, names, err := planner.Optimize(ctx, sctx, execStmt, is) if err != nil { return nil, false, false, err @@ -333,6 +337,7 @@ func CompileExecutePreparedStmt(ctx context.Context, sctx sessionctx.Context, StmtNode: execStmt, Ctx: sctx, OutputNames: names, + Ti: &TelemetryInfo{}, } if preparedPointer, ok := sctx.GetSessionVars().PreparedStmts[ID]; ok { preparedObj, ok := preparedPointer.(*plannercore.CachedPrepareStmt) diff --git a/executor/prepared_test.go b/executor/prepared_test.go index 1f8edf79d942e..e0e2c19ee0f22 100644 --- a/executor/prepared_test.go +++ b/executor/prepared_test.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb/domain" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" + txninfo "github.com/pingcap/tidb/session/txninfo" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/israce" @@ -135,6 +136,10 @@ type mockSessionManager2 struct { killed bool } +func (sm *mockSessionManager2) ShowTxnList() []*txninfo.TxnInfo { + panic("unimplemented!") +} + func (sm *mockSessionManager2) ShowProcessList() map[uint64]*util.ProcessInfo { pl := make(map[uint64]*util.ProcessInfo) if pi, ok := sm.GetProcessInfo(0); ok { diff --git a/executor/replace.go b/executor/replace.go index 20af75fe4a0ae..83df806489524 100644 --- a/executor/replace.go +++ b/executor/replace.go @@ -24,7 +24,6 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx/stmtctx" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" @@ -221,10 +220,11 @@ func (e *ReplaceExec) exec(ctx context.Context, newRows [][]types.Datum) error { if e.collectRuntimeStatsEnabled() { if snapshot := txn.GetSnapshot(); snapshot != nil { - snapshot.SetOption(tikvstore.CollectRuntimeStats, e.stats.SnapshotRuntimeStats) - defer snapshot.DelOption(tikvstore.CollectRuntimeStats) + snapshot.SetOption(kv.CollectRuntimeStats, e.stats.SnapshotRuntimeStats) + defer snapshot.SetOption(kv.CollectRuntimeStats, nil) } } + setResourceGroupTagForTxn(e.ctx.GetSessionVars().StmtCtx, txn) prefetchStart := time.Now() // Use BatchGet to fill cache. // It's an optimization and could be removed without affecting correctness. diff --git a/executor/revoke.go b/executor/revoke.go index 1477534962fbe..f84bbac9676da 100644 --- a/executor/revoke.go +++ b/executor/revoke.go @@ -15,7 +15,6 @@ package executor import ( "context" - "fmt" "strings" "github.com/pingcap/errors" @@ -88,8 +87,14 @@ func (e *RevokeExec) Next(ctx context.Context, req *chunk.Chunk) error { return err } + sessVars := e.ctx.GetSessionVars() // Revoke for each user. for _, user := range e.Users { + if user.User.CurrentUser { + user.User.Username = sessVars.User.AuthUsername + user.User.Hostname = sessVars.User.AuthHostname + } + // Check if user exists. exists, err := userExists(e.ctx, user.User.Username, user.User.Hostname) if err != nil { @@ -187,9 +192,6 @@ func (e *RevokeExec) revokePriv(internalSession sessionctx.Context, priv *ast.Pr func (e *RevokeExec) revokeDynamicPriv(internalSession sessionctx.Context, privName string, user, host string) error { privName = strings.ToUpper(privName) - if !e.ctx.GetSessionVars().EnableDynamicPrivileges { - return fmt.Errorf("dynamic privileges is an experimental feature. Run 'SET tidb_enable_dynamic_privileges=1'") - } if !privilege.GetPrivilegeManager(e.ctx).IsDynamicPrivilege(privName) { // for MySQL compatibility e.ctx.GetSessionVars().StmtCtx.AppendWarning(ErrDynamicPrivilegeNotRegistered.GenWithStackByArgs(privName)) } diff --git a/executor/revoke_test.go b/executor/revoke_test.go index 20cfc8b0250f1..f4a9ea8df231b 100644 --- a/executor/revoke_test.go +++ b/executor/revoke_test.go @@ -159,17 +159,11 @@ func (s *testSuite1) TestRevokeDynamicPrivs(c *C) { tk.MustExec("DROP USER if exists dyn") tk.MustExec("create user dyn") - tk.MustExec("SET tidb_enable_dynamic_privileges=0") - _, err := tk.Exec("GRANT BACKUP_ADMIN ON *.* TO dyn") - c.Assert(err.Error(), Equals, "dynamic privileges is an experimental feature. Run 'SET tidb_enable_dynamic_privileges=1'") - - tk.MustExec("SET tidb_enable_dynamic_privileges=1") - tk.MustExec("GRANT BACKUP_Admin ON *.* TO dyn") // grant one priv tk.MustQuery("SELECT * FROM mysql.global_grants WHERE `Host` = '%' AND `User` = 'dyn' ORDER BY user,host,priv,with_grant_option").Check(testkit.Rows("dyn % BACKUP_ADMIN N")) // try revoking only on test.* - should fail: - _, err = tk.Exec("REVOKE BACKUP_Admin,system_variables_admin ON test.* FROM dyn") + _, err := tk.Exec("REVOKE BACKUP_Admin,system_variables_admin ON test.* FROM dyn") c.Assert(terror.ErrorEqual(err, executor.ErrIllegalPrivilegeLevel), IsTrue) // privs should still be intact: diff --git a/executor/select_into_test.go b/executor/select_into_test.go index c499db9bdd322..e9b268703ff26 100644 --- a/executor/select_into_test.go +++ b/executor/select_into_test.go @@ -15,7 +15,6 @@ package executor_test import ( "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -29,7 +28,7 @@ import ( ) func cmpAndRm(expected, outfile string, c *C) { - content, err := ioutil.ReadFile(outfile) + content, err := os.ReadFile(outfile) c.Assert(err, IsNil) c.Assert(string(content), Equals, expected) c.Assert(os.Remove(outfile), IsNil) diff --git a/executor/seqtest/prepared_test.go b/executor/seqtest/prepared_test.go index 916f218db1f9d..bb8f05e5eff54 100644 --- a/executor/seqtest/prepared_test.go +++ b/executor/seqtest/prepared_test.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/tidb/metrics" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/session/txninfo" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/kvcache" @@ -796,6 +797,10 @@ type mockSessionManager1 struct { Se session.Session } +func (msm *mockSessionManager1) ShowTxnList() []*txninfo.TxnInfo { + panic("unimplemented!") +} + // ShowProcessList implements the SessionManager.ShowProcessList interface. func (msm *mockSessionManager1) ShowProcessList() map[uint64]*util.ProcessInfo { ret := make(map[uint64]*util.ProcessInfo) diff --git a/executor/seqtest/seq_executor_test.go b/executor/seqtest/seq_executor_test.go index 061e09dcc1315..34199a149794f 100644 --- a/executor/seqtest/seq_executor_test.go +++ b/executor/seqtest/seq_executor_test.go @@ -33,7 +33,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/errors" "github.com/pingcap/failpoint" - pb "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/parser" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" @@ -1095,19 +1095,19 @@ func (s *seqTestSuite) TestBatchInsertDelete(c *C) { type checkPrioClient struct { tikv.Client - priority pb.CommandPri + priority kvrpcpb.CommandPri mu struct { sync.RWMutex checkPrio bool } } -func (c *checkPrioClient) setCheckPriority(priority pb.CommandPri) { +func (c *checkPrioClient) setCheckPriority(priority kvrpcpb.CommandPri) { atomic.StoreInt32((*int32)(&c.priority), int32(priority)) } -func (c *checkPrioClient) getCheckPriority() pb.CommandPri { - return (pb.CommandPri)(atomic.LoadInt32((*int32)(&c.priority))) +func (c *checkPrioClient) getCheckPriority() kvrpcpb.CommandPri { + return (kvrpcpb.CommandPri)(atomic.LoadInt32((*int32)(&c.priority))) } func (c *checkPrioClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) { @@ -1174,11 +1174,11 @@ func (s *seqTestSuite1) TestCoprocessorPriority(c *C) { cli.mu.checkPrio = true cli.mu.Unlock() - cli.setCheckPriority(pb.CommandPri_High) + cli.setCheckPriority(kvrpcpb.CommandPri_High) tk.MustQuery("select id from t where id = 1") tk.MustQuery("select * from t1 where id = 1") - cli.setCheckPriority(pb.CommandPri_Normal) + cli.setCheckPriority(kvrpcpb.CommandPri_Normal) tk.MustQuery("select count(*) from t") tk.MustExec("update t set id = 3") tk.MustExec("delete from t") @@ -1193,43 +1193,43 @@ func (s *seqTestSuite1) TestCoprocessorPriority(c *C) { conf.Log.ExpensiveThreshold = 0 }) - cli.setCheckPriority(pb.CommandPri_High) + cli.setCheckPriority(kvrpcpb.CommandPri_High) tk.MustQuery("select id from t where id = 1") tk.MustQuery("select * from t1 where id = 1") tk.MustExec("delete from t where id = 2") tk.MustExec("update t set id = 2 where id = 1") - cli.setCheckPriority(pb.CommandPri_Low) + cli.setCheckPriority(kvrpcpb.CommandPri_Low) tk.MustQuery("select count(*) from t") tk.MustExec("delete from t") tk.MustExec("insert into t values (3)") // Test priority specified by SQL statement. - cli.setCheckPriority(pb.CommandPri_High) + cli.setCheckPriority(kvrpcpb.CommandPri_High) tk.MustQuery("select HIGH_PRIORITY * from t") - cli.setCheckPriority(pb.CommandPri_Low) + cli.setCheckPriority(kvrpcpb.CommandPri_Low) tk.MustQuery("select LOW_PRIORITY id from t where id = 1") - cli.setCheckPriority(pb.CommandPri_High) + cli.setCheckPriority(kvrpcpb.CommandPri_High) tk.MustExec("set tidb_force_priority = 'HIGH_PRIORITY'") tk.MustQuery("select * from t").Check(testkit.Rows("3")) tk.MustExec("update t set id = id + 1") tk.MustQuery("select v from t1 where id = 0 or id = 1").Check(testkit.Rows("0", "1")) - cli.setCheckPriority(pb.CommandPri_Low) + cli.setCheckPriority(kvrpcpb.CommandPri_Low) tk.MustExec("set tidb_force_priority = 'LOW_PRIORITY'") tk.MustQuery("select * from t").Check(testkit.Rows("4")) tk.MustExec("update t set id = id + 1") tk.MustQuery("select v from t1 where id = 0 or id = 1").Check(testkit.Rows("0", "1")) - cli.setCheckPriority(pb.CommandPri_Normal) + cli.setCheckPriority(kvrpcpb.CommandPri_Normal) tk.MustExec("set tidb_force_priority = 'DELAYED'") tk.MustQuery("select * from t").Check(testkit.Rows("5")) tk.MustExec("update t set id = id + 1") tk.MustQuery("select v from t1 where id = 0 or id = 1").Check(testkit.Rows("0", "1")) - cli.setCheckPriority(pb.CommandPri_Low) + cli.setCheckPriority(kvrpcpb.CommandPri_Low) tk.MustExec("set tidb_force_priority = 'NO_PRIORITY'") tk.MustQuery("select * from t").Check(testkit.Rows("6")) tk.MustExec("update t set id = id + 1") @@ -1473,8 +1473,6 @@ func (s *seqTestSuite) TestMaxDeltaSchemaCount(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") c.Assert(variable.GetMaxDeltaSchemaCount(), Equals, int64(variable.DefTiDBMaxDeltaSchemaCount)) - gvc := domain.GetDomain(tk.Se).GetGlobalVarsCache() - gvc.Disable() tk.MustExec("set @@global.tidb_max_delta_schema_count= -1") tk.MustQuery("show warnings;").Check(testkit.Rows("Warning 1292 Truncated incorrect tidb_max_delta_schema_count value: '-1'")) diff --git a/executor/set.go b/executor/set.go index 1223f973f5e39..825f79e985fc9 100644 --- a/executor/set.go +++ b/executor/set.go @@ -29,7 +29,6 @@ import ( "github.com/pingcap/tidb/util/collate" "github.com/pingcap/tidb/util/gcutil" "github.com/pingcap/tidb/util/logutil" - "github.com/pingcap/tidb/util/stmtsummary" "go.uber.org/zap" ) @@ -107,10 +106,8 @@ func (e *SetExecutor) setSysVariable(name string, v *expression.VarAssignment) e if sysVar == nil { return variable.ErrUnknownSystemVar.GenWithStackByArgs(name) } - var valStr string - var err error if v.IsGlobal { - valStr, err = e.getVarValue(v, sysVar) + valStr, err := e.getVarValue(v, sysVar) if err != nil { return err } @@ -125,64 +122,54 @@ func (e *SetExecutor) setSysVariable(name string, v *expression.VarAssignment) e } return nil }) - if err != nil { - return err - } logutil.BgLogger().Info("set global var", zap.Uint64("conn", sessionVars.ConnectionID), zap.String("name", name), zap.String("val", valStr)) - } else { - valStr, err = e.getVarValue(v, nil) - if err != nil { - return err - } - oldSnapshotTS := sessionVars.SnapshotTS - if name == variable.TxnIsolationOneShot && sessionVars.InTxn() { - return errors.Trace(ErrCantChangeTxCharacteristics) - } - err = variable.SetSessionSystemVar(sessionVars, name, valStr) - if err != nil { - return err + return err + } + // Set session variable + valStr, err := e.getVarValue(v, nil) + if err != nil { + return err + } + getSnapshotTSByName := func() uint64 { + if name == variable.TiDBSnapshot { + return sessionVars.SnapshotTS + } else if name == variable.TiDBTxnReadTS { + return sessionVars.TxnReadTS.PeakTxnReadTS() } - newSnapshotIsSet := sessionVars.SnapshotTS > 0 && sessionVars.SnapshotTS != oldSnapshotTS - if newSnapshotIsSet { - err = gcutil.ValidateSnapshot(e.ctx, sessionVars.SnapshotTS) - if err != nil { - sessionVars.SnapshotTS = oldSnapshotTS - return err - } + return 0 + } + oldSnapshotTS := getSnapshotTSByName() + fallbackOldSnapshotTS := func() { + if name == variable.TiDBSnapshot { + sessionVars.SnapshotTS = oldSnapshotTS + } else if name == variable.TiDBTxnReadTS { + sessionVars.TxnReadTS.SetTxnReadTS(oldSnapshotTS) } - err = e.loadSnapshotInfoSchemaIfNeeded(name) + } + if name == variable.TxnIsolationOneShot && sessionVars.InTxn() { + return errors.Trace(ErrCantChangeTxCharacteristics) + } + err = variable.SetSessionSystemVar(sessionVars, name, valStr) + if err != nil { + return err + } + newSnapshotTS := getSnapshotTSByName() + newSnapshotIsSet := newSnapshotTS > 0 && newSnapshotTS != oldSnapshotTS + if newSnapshotIsSet { + err = gcutil.ValidateSnapshot(e.ctx, newSnapshotTS) if err != nil { - sessionVars.SnapshotTS = oldSnapshotTS + fallbackOldSnapshotTS() return err } - // Clients are often noisy in setting session variables such as - // autocommit, timezone, query cache - logutil.BgLogger().Debug("set session var", zap.Uint64("conn", sessionVars.ConnectionID), zap.String("name", name), zap.String("val", valStr)) } - - // These are server instance scoped variables, and have special semantics. - // i.e. after SET SESSION, other users sessions will reflect the new value. - // TODO: in future these could be better managed as a post-set hook. - - valStrToBoolStr := variable.BoolToOnOff(variable.TiDBOptOn(valStr)) - - switch name { - case variable.TiDBEnableStmtSummary: - return stmtsummary.StmtSummaryByDigestMap.SetEnabled(valStr, !v.IsGlobal) - case variable.TiDBStmtSummaryInternalQuery: - return stmtsummary.StmtSummaryByDigestMap.SetEnabledInternalQuery(valStr, !v.IsGlobal) - case variable.TiDBStmtSummaryRefreshInterval: - return stmtsummary.StmtSummaryByDigestMap.SetRefreshInterval(valStr, !v.IsGlobal) - case variable.TiDBStmtSummaryHistorySize: - return stmtsummary.StmtSummaryByDigestMap.SetHistorySize(valStr, !v.IsGlobal) - case variable.TiDBStmtSummaryMaxStmtCount: - return stmtsummary.StmtSummaryByDigestMap.SetMaxStmtCount(valStr, !v.IsGlobal) - case variable.TiDBStmtSummaryMaxSQLLength: - return stmtsummary.StmtSummaryByDigestMap.SetMaxSQLLength(valStr, !v.IsGlobal) - case variable.TiDBCapturePlanBaseline: - variable.CapturePlanBaseline.Set(valStrToBoolStr, !v.IsGlobal) + err = e.loadSnapshotInfoSchemaIfNeeded(newSnapshotTS) + if err != nil { + fallbackOldSnapshotTS() + return err } - + // Clients are often noisy in setting session variables such as + // autocommit, timezone, query cache + logutil.BgLogger().Debug("set session var", zap.Uint64("conn", sessionVars.ConnectionID), zap.String("name", name), zap.String("val", valStr)) return nil } @@ -248,18 +235,17 @@ func (e *SetExecutor) getVarValue(v *expression.VarAssignment, sysVar *variable. return nativeVal.ToString() } -func (e *SetExecutor) loadSnapshotInfoSchemaIfNeeded(name string) error { - if name != variable.TiDBSnapshot { - return nil - } +func (e *SetExecutor) loadSnapshotInfoSchemaIfNeeded(snapshotTS uint64) error { vars := e.ctx.GetSessionVars() - if vars.SnapshotTS == 0 { + if snapshotTS == 0 { vars.SnapshotInfoschema = nil return nil } - logutil.BgLogger().Info("load snapshot info schema", zap.Uint64("conn", vars.ConnectionID), zap.Uint64("SnapshotTS", vars.SnapshotTS)) + logutil.BgLogger().Info("load snapshot info schema", + zap.Uint64("conn", vars.ConnectionID), + zap.Uint64("SnapshotTS", snapshotTS)) dom := domain.GetDomain(e.ctx) - snapInfo, err := dom.GetSnapshotInfoSchema(vars.SnapshotTS) + snapInfo, err := dom.GetSnapshotInfoSchema(snapshotTS) if err != nil { return err } diff --git a/executor/set_config.go b/executor/set_config.go index 05064b614d7a6..e31bab643cbb8 100644 --- a/executor/set_config.go +++ b/executor/set_config.go @@ -17,7 +17,7 @@ import ( "bytes" "context" "fmt" - "io/ioutil" + "io" "net" "net/http" "strings" @@ -143,7 +143,7 @@ func (s *SetConfigExec) doRequest(url string) (retErr error) { if resp.StatusCode == http.StatusOK { return nil } else if resp.StatusCode >= 400 && resp.StatusCode < 600 { - message, err := ioutil.ReadAll(resp.Body) + message, err := io.ReadAll(resp.Body) if err != nil { return err } diff --git a/executor/set_test.go b/executor/set_test.go index ec97914cc12a5..c4dcc1b608e46 100644 --- a/executor/set_test.go +++ b/executor/set_test.go @@ -18,7 +18,7 @@ import ( "context" "errors" "fmt" - "io/ioutil" + "io" "net/http" "strconv" @@ -374,18 +374,6 @@ func (s *testSerialSuite1) TestSetVar(c *C) { tk.MustQuery("select @@session.tidb_store_limit;").Check(testkit.Rows("0")) tk.MustQuery("select @@global.tidb_store_limit;").Check(testkit.Rows("100")) - tk.MustQuery("select @@global.tidb_enable_change_column_type;").Check(testkit.Rows("0")) - tk.MustExec("set global tidb_enable_change_column_type = 1") - tk.MustQuery("select @@global.tidb_enable_change_column_type;").Check(testkit.Rows("1")) - tk.MustExec("set global tidb_enable_change_column_type = off") - tk.MustQuery("select @@global.tidb_enable_change_column_type;").Check(testkit.Rows("0")) - // test tidb_enable_change_column_type in session scope. - tk.MustQuery("select @@session.tidb_enable_change_column_type;").Check(testkit.Rows("0")) - tk.MustExec("set @@session.tidb_enable_change_column_type = 1") - tk.MustQuery("select @@session.tidb_enable_change_column_type;").Check(testkit.Rows("1")) - tk.MustExec("set @@session.tidb_enable_change_column_type = off") - tk.MustQuery("select @@session.tidb_enable_change_column_type;").Check(testkit.Rows("0")) - tk.MustQuery("select @@session.tidb_metric_query_step;").Check(testkit.Rows("60")) tk.MustExec("set @@session.tidb_metric_query_step = 120") _, err = tk.Exec("set @@session.tidb_metric_query_step = 9") @@ -570,7 +558,7 @@ func (s *testSuite5) TestSetCharset(c *C) { check := func(args ...string) { for i, v := range characterSetVariables { - sVar, err := variable.GetSessionSystemVar(sessionVars, v) + sVar, err := variable.GetSessionOrGlobalSystemVar(sessionVars, v) c.Assert(err, IsNil) c.Assert(sVar, Equals, args[i], Commentf("%d: %s", i, characterSetVariables[i])) } @@ -896,9 +884,9 @@ func (s *testSuite5) TestValidateSetVar(c *C) { result = tk.MustQuery("select @@tmp_table_size;") result.Check(testkit.Rows("167772161")) - tk.MustExec("set @@tmp_table_size=18446744073709551615") + tk.MustExec("set @@tmp_table_size=9223372036854775807") result = tk.MustQuery("select @@tmp_table_size;") - result.Check(testkit.Rows("18446744073709551615")) + result.Check(testkit.Rows("9223372036854775807")) _, err = tk.Exec("set @@tmp_table_size=18446744073709551616") c.Assert(terror.ErrorEqual(err, variable.ErrWrongTypeForVar), IsTrue) @@ -1320,7 +1308,7 @@ func (s *testSuite5) TestSetClusterConfig(c *C) { httpCnt := 0 tk.Se.SetValue(executor.TestSetConfigHTTPHandlerKey, func(*http.Request) (*http.Response, error) { httpCnt++ - return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(nil)}, nil + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(nil)}, nil }) tk.MustExec("set config tikv log.level='info'") c.Assert(httpCnt, Equals, 2) @@ -1338,7 +1326,7 @@ func (s *testSuite5) TestSetClusterConfig(c *C) { "Warning 1105 something wrong", "Warning 1105 something wrong")) tk.Se.SetValue(executor.TestSetConfigHTTPHandlerKey, func(*http.Request) (*http.Response, error) { - return &http.Response{StatusCode: http.StatusBadRequest, Body: ioutil.NopCloser(bytes.NewBufferString("WRONG"))}, nil + return &http.Response{StatusCode: http.StatusBadRequest, Body: io.NopCloser(bytes.NewBufferString("WRONG"))}, nil }) tk.MustExec("set config tikv log.level='info'") tk.MustQuery("show warnings").Check(testkit.Rows( @@ -1376,3 +1364,91 @@ func (s *testSuite5) TestSetClusterConfigJSONData(c *C) { } } } + +func (s *testSerialSuite) TestSetTopSQLVariables(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("set @@tidb_enable_top_sql='On';") + tk.MustQuery("select @@tidb_enable_top_sql;").Check(testkit.Rows("1")) + c.Assert(variable.TopSQLVariable.Enable.Load(), IsTrue) + tk.MustExec("set @@tidb_enable_top_sql='off';") + tk.MustQuery("select @@tidb_enable_top_sql;").Check(testkit.Rows("0")) + c.Assert(variable.TopSQLVariable.Enable.Load(), IsFalse) + tk.MustExec("set @@global.tidb_enable_top_sql='On';") + tk.MustQuery("select @@global.tidb_enable_top_sql;").Check(testkit.Rows("1")) + c.Assert(variable.TopSQLVariable.Enable.Load(), IsTrue) + tk.MustExec("set @@global.tidb_enable_top_sql='off';") + tk.MustQuery("select @@global.tidb_enable_top_sql;").Check(testkit.Rows("0")) + c.Assert(variable.TopSQLVariable.Enable.Load(), IsFalse) + + tk.MustExec("set @@tidb_top_sql_agent_address='127.0.0.1:4001';") + tk.MustQuery("select @@tidb_top_sql_agent_address;").Check(testkit.Rows("127.0.0.1:4001")) + c.Assert(variable.TopSQLVariable.AgentAddress.Load(), Equals, "127.0.0.1:4001") + tk.MustExec("set @@tidb_top_sql_agent_address='';") + tk.MustQuery("select @@tidb_top_sql_agent_address;").Check(testkit.Rows("")) + c.Assert(variable.TopSQLVariable.AgentAddress.Load(), Equals, "") + tk.MustExec("set @@global.tidb_top_sql_agent_address='127.0.0.1:4001';") + tk.MustQuery("select @@global.tidb_top_sql_agent_address;").Check(testkit.Rows("127.0.0.1:4001")) + c.Assert(variable.TopSQLVariable.AgentAddress.Load(), Equals, "127.0.0.1:4001") + tk.MustExec("set @@global.tidb_top_sql_agent_address='';") + tk.MustQuery("select @@global.tidb_top_sql_agent_address;").Check(testkit.Rows("")) + c.Assert(variable.TopSQLVariable.AgentAddress.Load(), Equals, "") + + tk.MustExec("set @@tidb_top_sql_precision_seconds=60;") + tk.MustQuery("select @@tidb_top_sql_precision_seconds;").Check(testkit.Rows("60")) + c.Assert(variable.TopSQLVariable.PrecisionSeconds.Load(), Equals, int64(60)) + _, err := tk.Exec("set @@tidb_top_sql_precision_seconds='abc';") + c.Assert(err.Error(), Equals, "[variable:1232]Incorrect argument type to variable 'tidb_top_sql_precision_seconds'") + _, err = tk.Exec("set @@tidb_top_sql_precision_seconds='-1';") + c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_precision_seconds' can't be set to the value of '-1'") + tk.MustQuery("select @@tidb_top_sql_precision_seconds;").Check(testkit.Rows("60")) + c.Assert(variable.TopSQLVariable.PrecisionSeconds.Load(), Equals, int64(60)) + tk.MustExec("set @@global.tidb_top_sql_precision_seconds=2;") + tk.MustQuery("select @@global.tidb_top_sql_precision_seconds;").Check(testkit.Rows("2")) + c.Assert(variable.TopSQLVariable.PrecisionSeconds.Load(), Equals, int64(2)) + _, err = tk.Exec("set @@global.tidb_top_sql_precision_seconds='abc';") + c.Assert(err.Error(), Equals, "[variable:1232]Incorrect argument type to variable 'tidb_top_sql_precision_seconds'") + _, err = tk.Exec("set @@global.tidb_top_sql_precision_seconds='-1';") + c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_precision_seconds' can't be set to the value of '-1'") + tk.MustQuery("select @@global.tidb_top_sql_precision_seconds;").Check(testkit.Rows("2")) + c.Assert(variable.TopSQLVariable.PrecisionSeconds.Load(), Equals, int64(2)) + + tk.MustExec("set @@tidb_top_sql_max_statement_count=5000;") + tk.MustQuery("select @@tidb_top_sql_max_statement_count;").Check(testkit.Rows("5000")) + c.Assert(variable.TopSQLVariable.MaxStatementCount.Load(), Equals, int64(5000)) + _, err = tk.Exec("set @@tidb_top_sql_max_statement_count='abc';") + c.Assert(err.Error(), Equals, "[variable:1232]Incorrect argument type to variable 'tidb_top_sql_max_statement_count'") + _, err = tk.Exec("set @@tidb_top_sql_max_statement_count='-1';") + c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_max_statement_count' can't be set to the value of '-1'") + tk.MustQuery("select @@tidb_top_sql_max_statement_count;").Check(testkit.Rows("5000")) + c.Assert(variable.TopSQLVariable.MaxStatementCount.Load(), Equals, int64(5000)) + tk.MustExec("set @@global.tidb_top_sql_max_statement_count=2;") + tk.MustQuery("select @@global.tidb_top_sql_max_statement_count;").Check(testkit.Rows("2")) + c.Assert(variable.TopSQLVariable.MaxStatementCount.Load(), Equals, int64(2)) + _, err = tk.Exec("set @@global.tidb_top_sql_max_statement_count='abc';") + c.Assert(err.Error(), Equals, "[variable:1232]Incorrect argument type to variable 'tidb_top_sql_max_statement_count'") + _, err = tk.Exec("set @@global.tidb_top_sql_max_statement_count='-1';") + c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_max_statement_count' can't be set to the value of '-1'") + _, err = tk.Exec("set @@global.tidb_top_sql_max_statement_count='5001';") + c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_max_statement_count' can't be set to the value of '5001'") + tk.MustQuery("select @@global.tidb_top_sql_precision_seconds;").Check(testkit.Rows("2")) + c.Assert(variable.TopSQLVariable.MaxStatementCount.Load(), Equals, int64(2)) + + tk.MustExec("set @@tidb_top_sql_report_interval_seconds=10;") + tk.MustQuery("select @@tidb_top_sql_report_interval_seconds;").Check(testkit.Rows("10")) + c.Assert(variable.TopSQLVariable.ReportIntervalSeconds.Load(), Equals, int64(10)) + _, err = tk.Exec("set @@tidb_top_sql_report_interval_seconds='abc';") + c.Assert(err.Error(), Equals, "[variable:1232]Incorrect argument type to variable 'tidb_top_sql_report_interval_seconds'") + _, err = tk.Exec("set @@tidb_top_sql_report_interval_seconds='5000';") + c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_report_interval_seconds' can't be set to the value of '5000'") + tk.MustQuery("select @@tidb_top_sql_report_interval_seconds;").Check(testkit.Rows("10")) + c.Assert(variable.TopSQLVariable.ReportIntervalSeconds.Load(), Equals, int64(10)) + tk.MustExec("set @@global.tidb_top_sql_report_interval_seconds=120;") + tk.MustQuery("select @@global.tidb_top_sql_report_interval_seconds;").Check(testkit.Rows("120")) + c.Assert(variable.TopSQLVariable.ReportIntervalSeconds.Load(), Equals, int64(120)) + _, err = tk.Exec("set @@global.tidb_top_sql_report_interval_seconds='abc';") + c.Assert(err.Error(), Equals, "[variable:1232]Incorrect argument type to variable 'tidb_top_sql_report_interval_seconds'") + _, err = tk.Exec("set @@global.tidb_top_sql_report_interval_seconds='5000';") + c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_report_interval_seconds' can't be set to the value of '5000'") + tk.MustQuery("select @@global.tidb_top_sql_report_interval_seconds;").Check(testkit.Rows("120")) + c.Assert(variable.TopSQLVariable.ReportIntervalSeconds.Load(), Equals, int64(120)) +} diff --git a/executor/show.go b/executor/show.go index 725c0e0e28ec2..2c7d922981784 100644 --- a/executor/show.go +++ b/executor/show.go @@ -661,6 +661,17 @@ func (e *ShowExec) fetchShowMasterStatus() error { return nil } +func (e *ShowExec) sysVarHiddenForSem(sysVarNameInLower string) bool { + if !sem.IsEnabled() || !sem.IsInvisibleSysVar(sysVarNameInLower) { + return false + } + checker := privilege.GetPrivilegeManager(e.ctx) + if checker == nil || checker.RequestDynamicVerification(e.ctx.GetSessionVars().ActiveRoles, "RESTRICTED_VARIABLES_ADMIN", false) { + return false + } + return true +} + func (e *ShowExec) fetchShowVariables() (err error) { var ( value string @@ -673,7 +684,7 @@ func (e *ShowExec) fetchShowVariables() (err error) { // otherwise, fetch the value from table `mysql.Global_Variables`. for _, v := range variable.GetSysVars() { if v.Scope != variable.ScopeSession { - if v.Hidden { + if v.Hidden || e.sysVarHiddenForSem(v.Name) { continue } value, err = variable.GetGlobalSystemVar(sessionVars, v.Name) @@ -690,10 +701,10 @@ func (e *ShowExec) fetchShowVariables() (err error) { // If it is a session only variable, use the default value defined in code, // otherwise, fetch the value from table `mysql.Global_Variables`. for _, v := range variable.GetSysVars() { - if v.Hidden { + if v.Hidden || e.sysVarHiddenForSem(v.Name) { continue } - value, err = variable.GetSessionSystemVar(sessionVars, v.Name) + value, err = variable.GetSessionOrGlobalSystemVar(sessionVars, v.Name) if err != nil { return errors.Trace(err) } @@ -764,7 +775,13 @@ func ConstructResultOfShowCreateTable(ctx sessionctx.Context, tableInfo *model.T } sqlMode := ctx.GetSessionVars().SQLMode - fmt.Fprintf(buf, "CREATE TABLE %s (\n", stringutil.Escape(tableInfo.Name.O, sqlMode)) + tableName := stringutil.Escape(tableInfo.Name.O, sqlMode) + switch tableInfo.TempTableType { + case model.TempTableGlobal: + fmt.Fprintf(buf, "CREATE GLOBAL TEMPORARY TABLE %s (\n", tableName) + default: + fmt.Fprintf(buf, "CREATE TABLE %s (\n", tableName) + } var pkCol *model.ColumnInfo var hasAutoIncID bool needAddComma := false @@ -862,10 +879,10 @@ func ConstructResultOfShowCreateTable(ctx sessionctx.Context, tableInfo *model.T } if pkCol != nil { - // If PKIsHanle, pk info is not in tb.Indices(). We should handle it here. + // If PKIsHandle, pk info is not in tb.Indices(). We should handle it here. buf.WriteString(",\n") fmt.Fprintf(buf, " PRIMARY KEY (%s)", stringutil.Escape(pkCol.Name.O, sqlMode)) - buf.WriteString(fmt.Sprintf(" /*T![clustered_index] CLUSTERED */")) + buf.WriteString(" /*T![clustered_index] CLUSTERED */") } publicIndices := make([]*model.IndexInfo, 0, len(tableInfo.Indices)) @@ -906,9 +923,9 @@ func ConstructResultOfShowCreateTable(ctx sessionctx.Context, tableInfo *model.T } if idxInfo.Primary { if tableInfo.PKIsHandle || tableInfo.IsCommonHandle { - buf.WriteString(fmt.Sprintf(" /*T![clustered_index] CLUSTERED */")) + buf.WriteString(" /*T![clustered_index] CLUSTERED */") } else { - buf.WriteString(fmt.Sprintf(" /*T![clustered_index] NONCLUSTERED */")) + buf.WriteString(" /*T![clustered_index] NONCLUSTERED */") } } if i != len(publicIndices)-1 { @@ -941,7 +958,14 @@ func ConstructResultOfShowCreateTable(ctx sessionctx.Context, tableInfo *model.T buf.WriteString("\n") - buf.WriteString(") ENGINE=InnoDB") + switch tableInfo.TempTableType { + case model.TempTableNone: + buf.WriteString(") ENGINE=InnoDB") + default: + // For now the only supported engine for temporary table is memory. + buf.WriteString(") ENGINE=memory") + } + // We need to explicitly set the default charset and collation // to make it work on MySQL server which has default collate utf8_general_ci. if len(tblCollate) == 0 || tblCollate == "binary" { @@ -998,6 +1022,11 @@ func ConstructResultOfShowCreateTable(ctx sessionctx.Context, tableInfo *model.T if len(tableInfo.Comment) > 0 { fmt.Fprintf(buf, " COMMENT='%s'", format.OutputFormat(tableInfo.Comment)) } + + if tableInfo.TempTableType == model.TempTableGlobal { + fmt.Fprintf(buf, " ON COMMIT DELETE ROWS") + } + // add partition info here. appendPartitionInfo(tableInfo.Partition, buf) return nil @@ -1302,7 +1331,7 @@ func (e *ShowExec) fetchShowCreateUser() error { exec := e.ctx.(sqlexec.RestrictedSQLExecutor) - stmt, err := exec.ParseWithParams(context.TODO(), `SELECT * FROM %n.%n WHERE User=%? AND Host=%?`, mysql.SystemDB, mysql.UserTable, userName, hostName) + stmt, err := exec.ParseWithParams(context.TODO(), `SELECT plugin FROM %n.%n WHERE User=%? AND Host=%?`, mysql.SystemDB, mysql.UserTable, userName, hostName) if err != nil { return errors.Trace(err) } @@ -1317,6 +1346,11 @@ func (e *ShowExec) fetchShowCreateUser() error { fmt.Sprintf("'%s'@'%s'", e.User.Username, e.User.Hostname)) } + authplugin := mysql.AuthNativePassword + if len(rows) == 1 && rows[0].GetString(0) != "" { + authplugin = rows[0].GetString(0) + } + stmt, err = exec.ParseWithParams(context.TODO(), `SELECT Priv FROM %n.%n WHERE User=%? AND Host=%?`, mysql.SystemDB, mysql.GlobalPrivTable, userName, hostName) if err != nil { return errors.Trace(err) @@ -1337,8 +1371,8 @@ func (e *ShowExec) fetchShowCreateUser() error { require = privValue.RequireStr() } // FIXME: the returned string is not escaped safely - showStr := fmt.Sprintf("CREATE USER '%s'@'%s' IDENTIFIED WITH 'mysql_native_password' AS '%s' REQUIRE %s PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK", - e.User.Username, e.User.Hostname, checker.GetEncodedPassword(e.User.Username, e.User.Hostname), require) + showStr := fmt.Sprintf("CREATE USER '%s'@'%s' IDENTIFIED WITH '%s' AS '%s' REQUIRE %s PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK", + e.User.Username, e.User.Hostname, authplugin, checker.GetEncodedPassword(e.User.Username, e.User.Hostname), require) e.appendRow([]interface{}{showStr}) return nil } @@ -1412,6 +1446,10 @@ func (e *ShowExec) fetchShowPrivileges() error { e.appendRow([]interface{}{"Create tablespace", "Server Admin", "To create/alter/drop tablespaces"}) e.appendRow([]interface{}{"Update", "Tables", "To update existing rows"}) e.appendRow([]interface{}{"Usage", "Server Admin", "No privileges - allow connect only"}) + + for _, priv := range privileges.GetDynamicPrivileges() { + e.appendRow([]interface{}{priv, "Server Admin", ""}) + } return nil } diff --git a/executor/show_stats.go b/executor/show_stats.go index 3a1cf0cb48adc..b6449863b0c1f 100644 --- a/executor/show_stats.go +++ b/executor/show_stats.go @@ -17,7 +17,6 @@ import ( "fmt" "sort" "strings" - "time" "github.com/pingcap/errors" "github.com/pingcap/parser/ast" @@ -203,7 +202,7 @@ func (e *ShowExec) histogramToRow(dbName, tblName, partitionName, colName string } func (e *ShowExec) versionToTime(version uint64) types.Time { - t := time.Unix(0, oracle.ExtractPhysical(version)*int64(time.Millisecond)) + t := oracle.GetTimeFromTS(version) return types.NewTime(types.FromGoTime(t), mysql.TypeDatetime, 0) } diff --git a/executor/show_stats_test.go b/executor/show_stats_test.go index e9e0ed0ebe82a..8dddc2151a7fc 100644 --- a/executor/show_stats_test.go +++ b/executor/show_stats_test.go @@ -75,10 +75,12 @@ func (s *testShowStatsSuite) TestShowStatsBuckets(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") + // Simple behavior testing. Version=1 is enough. + tk.MustExec("set @@tidb_analyze_version=1") tk.MustExec("create table t (a int, b int)") tk.MustExec("create index idx on t(a,b)") tk.MustExec("insert into t values (1,1)") - tk.MustExec("analyze table t") + tk.MustExec("analyze table t with 0 topn") result := tk.MustQuery("show stats_buckets").Sort() result.Check(testkit.Rows("test t a 0 0 1 1 1 1 0", "test t b 0 0 1 1 1 1 0", "test t idx 1 0 1 1 (1, 1) (1, 1) 0")) result = tk.MustQuery("show stats_buckets where column_name = 'idx'") @@ -87,7 +89,7 @@ func (s *testShowStatsSuite) TestShowStatsBuckets(c *C) { tk.MustExec("drop table t") tk.MustExec("create table t (`a` datetime, `b` int, key `idx`(`a`, `b`))") tk.MustExec("insert into t values (\"2020-01-01\", 1)") - tk.MustExec("analyze table t") + tk.MustExec("analyze table t with 0 topn") result = tk.MustQuery("show stats_buckets").Sort() result.Check(testkit.Rows("test t a 0 0 1 1 2020-01-01 00:00:00 2020-01-01 00:00:00 0", "test t b 0 0 1 1 1 1 0", "test t idx 1 0 1 1 (2020-01-01 00:00:00, 1) (2020-01-01 00:00:00, 1) 0")) result = tk.MustQuery("show stats_buckets where column_name = 'idx'") @@ -96,7 +98,7 @@ func (s *testShowStatsSuite) TestShowStatsBuckets(c *C) { tk.MustExec("drop table t") tk.MustExec("create table t (`a` date, `b` int, key `idx`(`a`, `b`))") tk.MustExec("insert into t values (\"2020-01-01\", 1)") - tk.MustExec("analyze table t") + tk.MustExec("analyze table t with 0 topn") result = tk.MustQuery("show stats_buckets").Sort() result.Check(testkit.Rows("test t a 0 0 1 1 2020-01-01 2020-01-01 0", "test t b 0 0 1 1 1 1 0", "test t idx 1 0 1 1 (2020-01-01, 1) (2020-01-01, 1) 0")) result = tk.MustQuery("show stats_buckets where column_name = 'idx'") @@ -105,7 +107,7 @@ func (s *testShowStatsSuite) TestShowStatsBuckets(c *C) { tk.MustExec("drop table t") tk.MustExec("create table t (`a` timestamp, `b` int, key `idx`(`a`, `b`))") tk.MustExec("insert into t values (\"2020-01-01\", 1)") - tk.MustExec("analyze table t") + tk.MustExec("analyze table t with 0 topn") result = tk.MustQuery("show stats_buckets").Sort() result.Check(testkit.Rows("test t a 0 0 1 1 2020-01-01 00:00:00 2020-01-01 00:00:00 0", "test t b 0 0 1 1 1 1 0", "test t idx 1 0 1 1 (2020-01-01 00:00:00, 1) (2020-01-01 00:00:00, 1) 0")) result = tk.MustQuery("show stats_buckets where column_name = 'idx'") @@ -118,6 +120,7 @@ func (s *testShowStatsSuite) TestShowStatsHasNullValue(c *C) { tk.MustExec("drop table if exists t") tk.MustExec("create table t (a int, index idx(a))") tk.MustExec("insert into t values(NULL)") + tk.MustExec("set @@session.tidb_analyze_version=1") tk.MustExec("analyze table t") // Null values are excluded from histogram for single-column index. tk.MustQuery("show stats_buckets").Check(testkit.Rows()) @@ -177,6 +180,8 @@ func (s *testShowStatsSuite) TestShowPartitionStats(c *C) { tk := testkit.NewTestKit(c, s.store) testkit.WithPruneMode(tk, variable.Static, func() { tk.MustExec("set @@session.tidb_enable_table_partition=1") + // Version2 is tested in TestGlobalStatsData1/2/3 and TestAnalyzeGlobalStatsWithOpts. + tk.MustExec("set @@session.tidb_analyze_version=1") tk.MustExec("use test") tk.MustExec("drop table if exists t") createTable := `CREATE TABLE t (a int, b int, primary key(a), index idx(b)) @@ -215,6 +220,7 @@ func (s *testShowStatsSuite) TestShowAnalyzeStatus(c *C) { tk.MustExec("drop table if exists t") tk.MustExec("create table t (a int, b int, primary key(a), index idx(b))") tk.MustExec(`insert into t values (1, 1), (2, 2)`) + tk.MustExec("set @@tidb_analyze_version=1") tk.MustExec("analyze table t") result := tk.MustQuery("show analyze status").Sort() @@ -237,6 +243,20 @@ func (s *testShowStatsSuite) TestShowAnalyzeStatus(c *C) { c.Assert(result.Rows()[1][5], NotNil) c.Assert(result.Rows()[1][6], NotNil) c.Assert(result.Rows()[1][7], Equals, "finished") + + statistics.ClearHistoryJobs() + tk.MustExec("set @@tidb_analyze_version=2") + tk.MustExec("analyze table t") + result = tk.MustQuery("show analyze status").Sort() + c.Assert(len(result.Rows()), Equals, 1) + c.Assert(result.Rows()[0][0], Equals, "test") + c.Assert(result.Rows()[0][1], Equals, "t") + c.Assert(result.Rows()[0][2], Equals, "") + c.Assert(result.Rows()[0][3], Equals, "analyze table") + c.Assert(result.Rows()[0][4], Equals, "2") + c.Assert(result.Rows()[0][5], NotNil) + c.Assert(result.Rows()[0][6], NotNil) + c.Assert(result.Rows()[0][7], Equals, "finished") } func (s *testShowStatsSuite) TestShowStatusSnapshot(c *C) { diff --git a/executor/show_test.go b/executor/show_test.go index a343779245a3f..4109559b30928 100644 --- a/executor/show_test.go +++ b/executor/show_test.go @@ -433,6 +433,15 @@ func (s *testSuite5) TestShowCreateUser(c *C) { // "show create user" for current user doesn't check privileges. rows = tk1.MustQuery("show create user current_user") rows.Check(testkit.Rows("CREATE USER 'check_priv'@'127.0.0.1' IDENTIFIED WITH 'mysql_native_password' AS '' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK")) + + // Creating users with `IDENTIFIED WITH 'caching_sha2_password'` is not supported yet. So manually creating an entry for now. + // later this can be changed to test the full path once 'caching_sha2_password' support is completed. + tk.MustExec("CREATE USER 'sha_test'@'%' IDENTIFIED BY 'temp_passwd'") + tk.MustExec("UPDATE mysql.user SET plugin='caching_sha2_password', authentication_string=0x24412430303524532C06366D1D1E2B2F4437681A057B6807193D1C4B6E772F667A764663534E6C3978716C3057644D73427A787747674679687632644A384F337941704A542F WHERE user='sha_test' AND host='%'") + tk.MustExec("FLUSH PRIVILEGES") + + rows = tk.MustQuery("SHOW CREATE USER 'sha_test'@'%'") + rows.Check(testkit.Rows("CREATE USER 'sha_test'@'%' IDENTIFIED WITH 'caching_sha2_password' AS '$A$005$S,\x066m\x1d\x1e+/D7h\x1a\x05{h\a\x19=\x1cKnw/fzvFcSNl9xql0WdMsBzxwGgFyhv2dJ8O3yApJT/' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK")) } func (s *testSuite5) TestUnprivilegedShow(c *C) { @@ -1102,9 +1111,10 @@ func (s *testSuite5) TestShowBuiltin(c *C) { res := tk.MustQuery("show builtins;") c.Assert(res, NotNil) rows := res.Rows() - c.Assert(268, Equals, len(rows)) + const builtinFuncNum = 269 + c.Assert(builtinFuncNum, Equals, len(rows)) c.Assert("abs", Equals, rows[0][0].(string)) - c.Assert("yearweek", Equals, rows[267][0].(string)) + c.Assert("yearweek", Equals, rows[builtinFuncNum-1][0].(string)) } func (s *testSuite5) TestShowClusterConfig(c *C) { @@ -1269,6 +1279,9 @@ func (s *testSuite5) TestShowVar(c *C) { c.Check(res.Rows(), HasLen, 0) res = tk.MustQuery("show global variables like '" + variable.TiDBPartitionPruneMode + "'") c.Check(res.Rows(), HasLen, 0) + // Test Hidden tx_read_ts + res = tk.MustQuery("show variables like '%tx_read_ts'") + c.Check(res.Rows(), HasLen, 0) } func (s *testSuite5) TestIssue19507(c *C) { @@ -1302,3 +1315,42 @@ func (s *testSuite5) TestShowPerformanceSchema(c *C) { testkit.Rows("events_statements_summary_by_digest 0 SCHEMA_NAME 1 SCHEMA_NAME A 0 YES BTREE YES NULL NO", "events_statements_summary_by_digest 0 SCHEMA_NAME 2 DIGEST A 0 YES BTREE YES NULL NO")) } + +func (s *testSuite5) TestShowTemporaryTable(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("create global temporary table t1 (id int) on commit delete rows") + tk.MustExec("create global temporary table t3 (i int primary key, j int) on commit delete rows") + // For issue https://github.com/pingcap/tidb/issues/24752 + tk.MustQuery("show create table t1").Check(testkit.Rows("t1 CREATE GLOBAL TEMPORARY TABLE `t1` (\n" + + " `id` int(11) DEFAULT NULL\n" + + ") ENGINE=memory DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ON COMMIT DELETE ROWS")) + // No panic, fix issue https://github.com/pingcap/tidb/issues/24788 + expect := "CREATE GLOBAL TEMPORARY TABLE `t3` (\n" + + " `i` int(11) NOT NULL,\n" + + " `j` int(11) DEFAULT NULL,\n" + + " PRIMARY KEY (`i`) /*T![clustered_index] CLUSTERED */\n" + + ") ENGINE=memory DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ON COMMIT DELETE ROWS" + tk.MustQuery("show create table t3").Check(testkit.Rows("t3 " + expect)) + + // Verify that the `show create table` result can be used to build the table. + createTable := strings.ReplaceAll(expect, "t3", "t4") + tk.MustExec(createTable) + + // Cover auto increment column. + tk.MustExec(`CREATE GLOBAL TEMPORARY TABLE t5 ( + id int(11) NOT NULL AUTO_INCREMENT, + b int(11) NOT NULL, + pad varbinary(255) DEFAULT NULL, + PRIMARY KEY (id), + KEY b (b)) ON COMMIT DELETE ROWS`) + expect = "CREATE GLOBAL TEMPORARY TABLE `t5` (\n" + + " `id` int(11) NOT NULL AUTO_INCREMENT,\n" + + " `b` int(11) NOT NULL,\n" + + " `pad` varbinary(255) DEFAULT NULL,\n" + + " PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */,\n" + + " KEY `b` (`b`)\n" + + ") ENGINE=memory DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ON COMMIT DELETE ROWS" + tk.MustQuery("show create table t5").Check(testkit.Rows("t5 " + expect)) +} diff --git a/executor/simple.go b/executor/simple.go index 5ed8ced28af48..95067f42621d0 100644 --- a/executor/simple.go +++ b/executor/simple.go @@ -22,7 +22,6 @@ import ( "github.com/ngaut/pools" "github.com/pingcap/errors" - "github.com/pingcap/failpoint" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/auth" "github.com/pingcap/parser/model" @@ -40,11 +39,7 @@ import ( "github.com/pingcap/tidb/privilege" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" - "github.com/pingcap/tidb/store/tikv/oracle" tikvutil "github.com/pingcap/tidb/store/tikv/util" - "github.com/pingcap/tidb/types" - driver "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/collate" @@ -76,6 +71,9 @@ type SimpleExec struct { IsFromRemote bool done bool is infoschema.InfoSchema + + // staleTxnStartTS is the StartTS that is used to execute the staleness txn during a read-only begin statement. + staleTxnStartTS uint64 } func (e *baseExecutor) getSysSession() (sessionctx.Context, error) { @@ -138,6 +136,8 @@ func (e *SimpleExec) Next(ctx context.Context, req *chunk.Chunk) (err error) { err = e.executeAlterUser(x) case *ast.DropUserStmt: err = e.executeDropUser(x) + case *ast.RenameUserStmt: + err = e.executeRenameUser(x) case *ast.SetPwdStmt: err = e.executeSetPwd(x) case *ast.KillStmt: @@ -566,18 +566,30 @@ func (e *SimpleExec) executeUse(s *ast.UseStmt) error { } func (e *SimpleExec) executeBegin(ctx context.Context, s *ast.BeginStmt) error { - // If `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND` is the first statement in TxnCtx, we should + // If `START TRANSACTION READ ONLY` is the first statement in TxnCtx, we should // always create a new Txn instead of reusing it. if s.ReadOnly { enableNoopFuncs := e.ctx.GetSessionVars().EnableNoopFuncs - if !enableNoopFuncs && s.Bound == nil { + if !enableNoopFuncs && s.AsOf == nil { return expression.ErrFunctionsNoopImpl.GenWithStackByArgs("READ ONLY") } - if s.Bound != nil { - return e.executeStartTransactionReadOnlyWithTimestampBound(ctx, s) + if s.AsOf != nil { + // start transaction read only as of failed due to we set tx_read_ts before + if e.ctx.GetSessionVars().TxnReadTS.PeakTxnReadTS() > 0 { + return errors.New("start transaction read only as of is forbidden after set transaction read only as of") + } } } - + if e.staleTxnStartTS > 0 { + if err := e.ctx.NewStaleTxnWithStartTS(ctx, e.staleTxnStartTS); err != nil { + return err + } + // With START TRANSACTION, autocommit remains disabled until you end + // the transaction with COMMIT or ROLLBACK. The autocommit mode then + // reverts to its previous state. + e.ctx.GetSessionVars().SetInTxn(true) + return nil + } // If BEGIN is the first statement in TxnCtx, we can reuse the existing transaction, without the // need to call NewTxn, which commits the existing transaction and begins a new one. // If the last un-committed/un-rollback transaction is a time-bounded read-only transaction, we should @@ -606,103 +618,14 @@ func (e *SimpleExec) executeBegin(ctx context.Context, s *ast.BeginStmt) error { return err } if e.ctx.GetSessionVars().TxnCtx.IsPessimistic { - txn.SetOption(tikvstore.Pessimistic, true) + txn.SetOption(kv.Pessimistic, true) } if s.CausalConsistencyOnly { - txn.SetOption(tikvstore.GuaranteeLinearizability, false) + txn.SetOption(kv.GuaranteeLinearizability, false) } return nil } -func (e *SimpleExec) executeStartTransactionReadOnlyWithTimestampBound(ctx context.Context, s *ast.BeginStmt) error { - opt := sessionctx.StalenessTxnOption{} - opt.Mode = s.Bound.Mode - switch s.Bound.Mode { - case ast.TimestampBoundReadTimestamp: - // TODO: support funcCallExpr in future - v, ok := s.Bound.Timestamp.(*driver.ValueExpr) - if !ok { - return errors.New("Invalid value for Bound Timestamp") - } - t, err := types.ParseTime(e.ctx.GetSessionVars().StmtCtx, v.GetString(), v.GetType().Tp, types.GetFsp(v.GetString())) - if err != nil { - return err - } - gt, err := t.GoTime(e.ctx.GetSessionVars().TimeZone) - if err != nil { - return err - } - startTS := oracle.ComposeTS(gt.Unix()*1000, 0) - opt.StartTS = startTS - case ast.TimestampBoundExactStaleness: - // TODO: support funcCallExpr in future - v, ok := s.Bound.Timestamp.(*driver.ValueExpr) - if !ok { - return errors.New("Invalid value for Bound Timestamp") - } - d, err := types.ParseDuration(e.ctx.GetSessionVars().StmtCtx, v.GetString(), types.GetFsp(v.GetString())) - if err != nil { - return err - } - opt.PrevSec = uint64(d.Seconds()) - case ast.TimestampBoundMaxStaleness: - v, ok := s.Bound.Timestamp.(*driver.ValueExpr) - if !ok { - return errors.New("Invalid value for Bound Timestamp") - } - d, err := types.ParseDuration(e.ctx.GetSessionVars().StmtCtx, v.GetString(), types.GetFsp(v.GetString())) - if err != nil { - return err - } - opt.PrevSec = uint64(d.Seconds()) - case ast.TimestampBoundMinReadTimestamp: - v, ok := s.Bound.Timestamp.(*driver.ValueExpr) - if !ok { - return errors.New("Invalid value for Bound Timestamp") - } - t, err := types.ParseTime(e.ctx.GetSessionVars().StmtCtx, v.GetString(), v.GetType().Tp, types.GetFsp(v.GetString())) - if err != nil { - return err - } - gt, err := t.GoTime(e.ctx.GetSessionVars().TimeZone) - if err != nil { - return err - } - startTS := oracle.ComposeTS(gt.Unix()*1000, 0) - opt.StartTS = startTS - } - err := e.ctx.NewTxnWithStalenessOption(ctx, opt) - if err != nil { - return err - } - dom := domain.GetDomain(e.ctx) - m, err := dom.GetSnapshotMeta(e.ctx.GetSessionVars().TxnCtx.StartTS) - if err != nil { - return err - } - staleVer, err := m.GetSchemaVersion() - if err != nil { - return err - } - failpoint.Inject("mockStalenessTxnSchemaVer", func(val failpoint.Value) { - if val.(bool) { - staleVer = e.ctx.GetSessionVars().TxnCtx.SchemaVersion - 1 - } else { - staleVer = e.ctx.GetSessionVars().TxnCtx.SchemaVersion - } - }) - // TODO: currently we directly check the schema version. In future, we can cache the stale infoschema instead. - if e.ctx.GetSessionVars().TxnCtx.SchemaVersion > staleVer { - return errors.New("schema version changed after the staleness startTS") - } - - // With START TRANSACTION, autocommit remains disabled until you end - // the transaction with COMMIT or ROLLBACK. The autocommit mode then - // reverts to its previous state. - e.ctx.GetSessionVars().SetInTxn(true) - return nil -} - func (e *SimpleExec) executeRevokeRole(s *ast.RevokeRoleStmt) error { for _, role := range s.Roles { exists, err := userExists(e.ctx, role.Username, role.Hostname) @@ -820,9 +743,9 @@ func (e *SimpleExec) executeCreateUser(ctx context.Context, s *ast.CreateUserStm sql := new(strings.Builder) if s.IsCreateRole { - sqlexec.MustFormatSQL(sql, `INSERT INTO %n.%n (Host, User, authentication_string, Account_locked) VALUES `, mysql.SystemDB, mysql.UserTable) + sqlexec.MustFormatSQL(sql, `INSERT INTO %n.%n (Host, User, authentication_string, plugin, Account_locked) VALUES `, mysql.SystemDB, mysql.UserTable) } else { - sqlexec.MustFormatSQL(sql, `INSERT INTO %n.%n (Host, User, authentication_string) VALUES `, mysql.SystemDB, mysql.UserTable) + sqlexec.MustFormatSQL(sql, `INSERT INTO %n.%n (Host, User, authentication_string, plugin) VALUES `, mysql.SystemDB, mysql.UserTable) } users := make([]*auth.UserIdentity, 0, len(s.Specs)) @@ -851,9 +774,9 @@ func (e *SimpleExec) executeCreateUser(ctx context.Context, s *ast.CreateUserStm return errors.Trace(ErrPasswordFormat) } if s.IsCreateRole { - sqlexec.MustFormatSQL(sql, `(%?, %?, %?, %?)`, spec.User.Hostname, spec.User.Username, pwd, "Y") + sqlexec.MustFormatSQL(sql, `(%?, %?, %?, %?, %?)`, spec.User.Hostname, spec.User.Username, pwd, mysql.AuthNativePassword, "Y") } else { - sqlexec.MustFormatSQL(sql, `(%?, %?, %?)`, spec.User.Hostname, spec.User.Username, pwd) + sqlexec.MustFormatSQL(sql, `(%?, %?, %?, %?)`, spec.User.Hostname, spec.User.Username, pwd, mysql.AuthNativePassword) } users = append(users, spec.User) } @@ -1048,6 +971,123 @@ func (e *SimpleExec) executeGrantRole(s *ast.GrantRoleStmt) error { return nil } +// Should cover same internal mysql.* tables as DROP USER, so this function is very similar +func (e *SimpleExec) executeRenameUser(s *ast.RenameUserStmt) error { + + var failedUser string + sysSession, err := e.getSysSession() + defer e.releaseSysSession(sysSession) + if err != nil { + return err + } + sqlExecutor := sysSession.(sqlexec.SQLExecutor) + + if _, err := sqlExecutor.ExecuteInternal(context.TODO(), "begin"); err != nil { + return err + } + + for _, userToUser := range s.UserToUsers { + oldUser, newUser := userToUser.OldUser, userToUser.NewUser + exists, err := userExistsInternal(sqlExecutor, oldUser.Username, oldUser.Hostname) + if err != nil { + return err + } + if !exists { + failedUser = oldUser.String() + " TO " + newUser.String() + " old did not exist" + break + } + + exists, err = userExistsInternal(sqlExecutor, newUser.Username, newUser.Hostname) + if err != nil { + return err + } + if exists { + // MySQL reports the old user, even when the issue is the new user. + failedUser = oldUser.String() + " TO " + newUser.String() + " new did exist" + break + } + + if err = renameUserHostInSystemTable(sqlExecutor, mysql.UserTable, "User", "Host", userToUser); err != nil { + failedUser = oldUser.String() + " TO " + newUser.String() + " " + mysql.UserTable + " error" + break + } + + // rename privileges from mysql.global_priv + if err = renameUserHostInSystemTable(sqlExecutor, mysql.GlobalPrivTable, "User", "Host", userToUser); err != nil { + failedUser = oldUser.String() + " TO " + newUser.String() + " " + mysql.GlobalPrivTable + " error" + break + } + + // rename privileges from mysql.db + if err = renameUserHostInSystemTable(sqlExecutor, mysql.DBTable, "User", "Host", userToUser); err != nil { + failedUser = oldUser.String() + " TO " + newUser.String() + " " + mysql.DBTable + " error" + break + } + + // rename privileges from mysql.tables_priv + if err = renameUserHostInSystemTable(sqlExecutor, mysql.TablePrivTable, "User", "Host", userToUser); err != nil { + failedUser = oldUser.String() + " TO " + newUser.String() + " " + mysql.TablePrivTable + " error" + break + } + + // rename relationship from mysql.role_edges + if err = renameUserHostInSystemTable(sqlExecutor, mysql.RoleEdgeTable, "TO_USER", "TO_HOST", userToUser); err != nil { + failedUser = oldUser.String() + " TO " + newUser.String() + " " + mysql.RoleEdgeTable + " (to) error" + break + } + + if err = renameUserHostInSystemTable(sqlExecutor, mysql.RoleEdgeTable, "FROM_USER", "FROM_HOST", userToUser); err != nil { + failedUser = oldUser.String() + " TO " + newUser.String() + " " + mysql.RoleEdgeTable + " (from) error" + break + } + + // rename relationship from mysql.default_roles + if err = renameUserHostInSystemTable(sqlExecutor, mysql.DefaultRoleTable, "DEFAULT_ROLE_USER", "DEFAULT_ROLE_HOST", userToUser); err != nil { + failedUser = oldUser.String() + " TO " + newUser.String() + " " + mysql.DefaultRoleTable + " (default role user) error" + break + } + + if err = renameUserHostInSystemTable(sqlExecutor, mysql.DefaultRoleTable, "USER", "HOST", userToUser); err != nil { + failedUser = oldUser.String() + " TO " + newUser.String() + " " + mysql.DefaultRoleTable + " error" + break + } + + // rename relationship from mysql.global_grants + // TODO: add global_grants into the parser + if err = renameUserHostInSystemTable(sqlExecutor, "global_grants", "User", "Host", userToUser); err != nil { + failedUser = oldUser.String() + " TO " + newUser.String() + " mysql.global_grants error" + break + } + + //TODO: need update columns_priv once we implement columns_priv functionality. + // When that is added, please refactor both executeRenameUser and executeDropUser to use an array of tables + // to loop over, so it is easier to maintain. + } + + if failedUser == "" { + if _, err := sqlExecutor.ExecuteInternal(context.TODO(), "commit"); err != nil { + return err + } + } else { + if _, err := sqlExecutor.ExecuteInternal(context.TODO(), "rollback"); err != nil { + return err + } + return ErrCannotUser.GenWithStackByArgs("RENAME USER", failedUser) + } + domain.GetDomain(e.ctx).NotifyUpdatePrivilege(e.ctx) + return nil +} + +func renameUserHostInSystemTable(sqlExecutor sqlexec.SQLExecutor, tableName, usernameColumn, hostColumn string, users *ast.UserToUser) error { + sql := new(strings.Builder) + sqlexec.MustFormatSQL(sql, `UPDATE %n.%n SET %n = %?, %n = %? WHERE %n = %? and %n = %?;`, + mysql.SystemDB, tableName, + usernameColumn, users.NewUser.Username, hostColumn, users.NewUser.Hostname, + usernameColumn, users.OldUser.Username, hostColumn, users.OldUser.Hostname) + _, err := sqlExecutor.ExecuteInternal(context.TODO(), sql.String()) + return err +} + func (e *SimpleExec) executeDropUser(s *ast.DropUserStmt) error { // Check privileges. // Check `CREATE USER` privilege. @@ -1203,6 +1243,27 @@ func userExists(ctx sessionctx.Context, name string, host string) (bool, error) return len(rows) > 0, nil } +// use the same internal executor to read within the same transaction, otherwise same as userExists +func userExistsInternal(sqlExecutor sqlexec.SQLExecutor, name string, host string) (bool, error) { + sql := new(strings.Builder) + sqlexec.MustFormatSQL(sql, `SELECT * FROM %n.%n WHERE User=%? AND Host=%?;`, mysql.SystemDB, mysql.UserTable, name, host) + recordSet, err := sqlExecutor.ExecuteInternal(context.TODO(), sql.String()) + if err != nil { + return false, err + } + req := recordSet.NewChunk() + err = recordSet.Next(context.TODO(), req) + var rows int = 0 + if err == nil { + rows = req.NumRows() + } + errClose := recordSet.Close() + if errClose != nil { + return false, errClose + } + return rows > 0, err +} + func (e *SimpleExec) executeSetPwd(s *ast.SetPwdStmt) error { var u, h string if s.User == nil { @@ -1318,6 +1379,7 @@ func killRemoteConn(ctx context.Context, sctx sessionctx.Context, connID *util.G kvReq, err := builder. SetDAGRequest(dagReq). SetFromSessionVars(sctx.GetSessionVars()). + SetFromInfoSchema(sctx.GetInfoSchema()). SetStoreType(kv.TiDB). SetTiDBServerID(connID.ServerID). Build() @@ -1406,12 +1468,12 @@ func (e *SimpleExec) executeDropStats(s *ast.DropStatsStmt) (err error) { if err := h.DeleteTableStatsFromKV(statsIDs); err != nil { return err } - return h.Update(infoschema.GetInfoSchema(e.ctx)) + return h.Update(e.ctx.GetInfoSchema().(infoschema.InfoSchema)) } func (e *SimpleExec) autoNewTxn() bool { switch e.Statement.(type) { - case *ast.CreateUserStmt, *ast.AlterUserStmt, *ast.DropUserStmt: + case *ast.CreateUserStmt, *ast.AlterUserStmt, *ast.DropUserStmt, *ast.RenameUserStmt: return true } return false diff --git a/executor/simple_test.go b/executor/simple_test.go index 8c09fc336f5d2..425e7d5204731 100644 --- a/executor/simple_test.go +++ b/executor/simple_test.go @@ -32,6 +32,7 @@ import ( "github.com/pingcap/tidb/statistics/handle" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/util" + "github.com/pingcap/tidb/util/israce" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testutil" ) @@ -604,12 +605,15 @@ func (s *testFlushSuite) TestFlushPrivilegesPanic(c *C) { } func (s *testSerialSuite) TestDropPartitionStats(c *C) { - c.Skip("unstable") + if israce.RaceEnabled { + c.Skip("unstable, skip race test") + } // Use the testSerialSuite to fix the unstable test tk := testkit.NewTestKit(c, s.store) - tk.MustExec("use test") - tk.MustExec("drop table if exists t;") - tk.MustExec(`create table t ( + tk.MustExec(`create database if not exists test_drop_gstats`) + tk.MustExec("use test_drop_gstats") + tk.MustExec("drop table if exists test_drop_gstats;") + tk.MustExec(`create table test_drop_gstats ( a int, key(a) ) @@ -620,7 +624,7 @@ partition by range (a) ( )`) tk.MustExec("set @@tidb_analyze_version = 2") tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") - tk.MustExec("insert into t values (1), (5), (11), (15), (21), (25)") + tk.MustExec("insert into test_drop_gstats values (1), (5), (11), (15), (21), (25)") c.Assert(s.domain.StatsHandle().DumpStatsDeltaToKV(handle.DumpAll), IsNil) checkPartitionStats := func(names ...string) { @@ -631,26 +635,26 @@ partition by range (a) ( } } - tk.MustExec("analyze table t") + tk.MustExec("analyze table test_drop_gstats") checkPartitionStats("global", "p0", "p1", "global") - tk.MustExec("drop stats t partition p0") + tk.MustExec("drop stats test_drop_gstats partition p0") checkPartitionStats("global", "p1", "global") - err := tk.ExecToErr("drop stats t partition abcde") + err := tk.ExecToErr("drop stats test_drop_gstats partition abcde") c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "can not found the specified partition name abcde in the table definition") - tk.MustExec("drop stats t partition global") + tk.MustExec("drop stats test_drop_gstats partition global") checkPartitionStats("global", "p1") - tk.MustExec("drop stats t global") + tk.MustExec("drop stats test_drop_gstats global") checkPartitionStats("p1") - tk.MustExec("analyze table t") + tk.MustExec("analyze table test_drop_gstats") checkPartitionStats("global", "p0", "p1", "global") - tk.MustExec("drop stats t partition p0, p1, global") + tk.MustExec("drop stats test_drop_gstats partition p0, p1, global") checkPartitionStats("global") } @@ -693,7 +697,7 @@ func (s *testSuite3) TestDropStatsFromKV(c *C) { tk.MustExec(`insert into t values("1","1"),("2","2"),("3","3"),("4","4")`) tk.MustExec("insert into t select * from t") tk.MustExec("insert into t select * from t") - tk.MustExec("analyze table t") + tk.MustExec("analyze table t with 2 topn") tblID := tk.MustQuery(`select tidb_table_id from information_schema.tables where table_name = "t" and table_schema = "test"`).Rows()[0][0].(string) tk.MustQuery("select modify_count, count from mysql.stats_meta where table_id = " + tblID).Check( testkit.Rows("0 16")) diff --git a/executor/slow_query.go b/executor/slow_query.go index 9e32cb175e4b8..ad64fa5355893 100755 --- a/executor/slow_query.go +++ b/executor/slow_query.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "os" "path/filepath" "sort" @@ -998,11 +997,11 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co } return nil } - files, err := ioutil.ReadDir(logDir) + files, err := os.ReadDir(logDir) if err != nil { return nil, err } - walkFn := func(path string, info os.FileInfo) error { + walkFn := func(path string, info os.DirEntry) error { if info.IsDir() { return nil } diff --git a/executor/stale_txn_test.go b/executor/stale_txn_test.go index c68b8a5bfa511..44def0981bed0 100644 --- a/executor/stale_txn_test.go +++ b/executor/stale_txn_test.go @@ -22,65 +22,49 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/ddl/placement" "github.com/pingcap/tidb/store/tikv/oracle" + "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/testkit" ) func (s *testStaleTxnSerialSuite) TestExactStalenessTransaction(c *C) { - c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer", "return(false)"), IsNil) - defer func() { - err := failpoint.Disable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer") - c.Assert(err, IsNil) - }() - testcases := []struct { name string preSQL string sql string IsStaleness bool expectPhysicalTS int64 - preSec int64 txnScope string zone string }{ { - name: "TimestampBoundExactStaleness", - preSQL: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND EXACT STALENESS '00:00:20';`, - sql: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND READ TIMESTAMP '2020-09-06 00:00:00';`, + name: "AsOfTimestamp", + preSQL: "begin", + sql: `START TRANSACTION READ ONLY AS OF TIMESTAMP '2020-09-06 00:00:00';`, IsStaleness: true, expectPhysicalTS: 1599321600000, txnScope: "local", zone: "sh", }, { - name: "TimestampBoundReadTimestamp", + name: "begin after AsOfTimestamp", + preSQL: `START TRANSACTION READ ONLY AS OF TIMESTAMP '2020-09-06 00:00:00';`, + sql: "begin", + IsStaleness: false, + txnScope: oracle.GlobalTxnScope, + zone: "", + }, + { + name: "AsOfTimestamp with tidb_bounded_staleness", preSQL: "begin", - sql: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND READ TIMESTAMP '2020-09-06 00:00:00';`, + sql: `START TRANSACTION READ ONLY AS OF TIMESTAMP tidb_bounded_staleness('2015-09-21 00:07:01', NOW());`, IsStaleness: true, - expectPhysicalTS: 1599321600000, + expectPhysicalTS: 1442765221000, txnScope: "local", zone: "bj", }, { - name: "TimestampBoundExactStaleness", - preSQL: "begin", - sql: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND EXACT STALENESS '00:00:20';`, - IsStaleness: true, - preSec: 20, - txnScope: "local", - zone: "sh", - }, - { - name: "TimestampBoundExactStaleness", - preSQL: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND READ TIMESTAMP '2020-09-06 00:00:00';`, - sql: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND EXACT STALENESS '00:00:20';`, - IsStaleness: true, - preSec: 20, - txnScope: "local", - zone: "sz", - }, - { - name: "begin", - preSQL: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND READ TIMESTAMP '2020-09-06 00:00:00';`, + name: "begin after AsOfTimestamp with tidb_bounded_staleness", + preSQL: `START TRANSACTION READ ONLY AS OF TIMESTAMP tidb_bounded_staleness('2015-09-21 00:07:01', NOW());`, sql: "begin", IsStaleness: false, txnScope: oracle.GlobalTxnScope, @@ -96,29 +80,160 @@ func (s *testStaleTxnSerialSuite) TestExactStalenessTransaction(c *C) { tk.MustExec(fmt.Sprintf("set @@txn_scope=%v", testcase.txnScope)) tk.MustExec(testcase.preSQL) tk.MustExec(testcase.sql) + c.Assert(tk.Se.GetSessionVars().TxnCtx.IsStaleness, Equals, testcase.IsStaleness) if testcase.expectPhysicalTS > 0 { c.Assert(oracle.ExtractPhysical(tk.Se.GetSessionVars().TxnCtx.StartTS), Equals, testcase.expectPhysicalTS) - } else if testcase.preSec > 0 { - curSec := time.Now().Unix() - startTS := oracle.ExtractPhysical(tk.Se.GetSessionVars().TxnCtx.StartTS) - // exact stale txn tolerate 2 seconds deviation for startTS - c.Assert(startTS, Greater, (curSec-testcase.preSec-2)*1000) - c.Assert(startTS, Less, (curSec-testcase.preSec+2)*1000) } else if !testcase.IsStaleness { - curSec := time.Now().Unix() + curTS := oracle.ExtractPhysical(oracle.GoTimeToTS(time.Now())) startTS := oracle.ExtractPhysical(tk.Se.GetSessionVars().TxnCtx.StartTS) - c.Assert(curSec*1000-startTS, Less, time.Second/time.Millisecond) - c.Assert(startTS-curSec*1000, Less, time.Second/time.Millisecond) + c.Assert(curTS-startTS, Less, time.Second.Milliseconds()) + c.Assert(startTS-curTS, Less, time.Second.Milliseconds()) } - c.Assert(tk.Se.GetSessionVars().TxnCtx.IsStaleness, Equals, testcase.IsStaleness) tk.MustExec("commit") - failpoint.Disable("github.com/pingcap/tidb/config/injectTxnScope") + } + failpoint.Disable("github.com/pingcap/tidb/config/injectTxnScope") +} + +func (s *testStaleTxnSerialSuite) TestSelectAsOf(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + // For mocktikv, safe point is not initialized, we manually insert it for snapshot to use. + safePointName := "tikv_gc_safe_point" + safePointValue := "20160102-15:04:05 -0700" + safePointComment := "All versions after safe point can be accessed. (DO NOT EDIT)" + updateSafePoint := fmt.Sprintf(`INSERT INTO mysql.tidb VALUES ('%[1]s', '%[2]s', '%[3]s') + ON DUPLICATE KEY + UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment) + tk.MustExec(updateSafePoint) + tk.MustExec("drop table if exists t") + tk.MustExec(`drop table if exists b`) + tk.MustExec("create table t (id int primary key);") + tk.MustExec("create table b (pid int primary key);") + defer func() { + tk.MustExec(`drop table if exists b`) + tk.MustExec(`drop table if exists t`) + }() + time.Sleep(2 * time.Second) + now := time.Now() + time.Sleep(2 * time.Second) + + testcases := []struct { + setTxnSQL string + name string + sql string + expectPhysicalTS int64 + preSec int64 + // IsStaleness is auto cleanup in select stmt. + errorStr string + }{ + { + name: "set transaction as of", + setTxnSQL: fmt.Sprintf("set transaction read only as of timestamp '%s';", now.Format("2006-1-2 15:04:05")), + sql: "select * from t;", + expectPhysicalTS: now.Unix(), + }, + { + name: "set transaction as of, expect error", + setTxnSQL: fmt.Sprintf("set transaction read only as of timestamp '%s';", now.Format("2006-1-2 15:04:05")), + sql: fmt.Sprintf("select * from t as of timestamp '%s';", now.Format("2006-1-2 15:04:05")), + errorStr: ".*can't use select as of while already set transaction as of.*", + }, + { + name: "TimestampExactRead1", + sql: fmt.Sprintf("select * from t as of timestamp '%s';", now.Format("2006-1-2 15:04:05")), + expectPhysicalTS: now.Unix(), + }, + { + name: "NormalRead", + sql: `select * from b;`, + preSec: 0, + }, + { + name: "TimestampExactRead2", + sql: fmt.Sprintf("select * from t as of timestamp TIMESTAMP('%s');", now.Format("2006-1-2 15:04:05")), + expectPhysicalTS: now.Unix(), + }, + { + name: "TimestampExactRead3", + sql: `select * from t as of timestamp NOW() - INTERVAL 2 SECOND;`, + preSec: 2, + }, + { + name: "TimestampExactRead4", + sql: `select * from t as of timestamp TIMESTAMP(NOW() - INTERVAL 2 SECOND);`, + preSec: 2, + }, + { + name: "TimestampExactRead5", + sql: `select * from t as of timestamp TIMESTAMP(NOW() - INTERVAL 1 SECOND), b as of timestamp TIMESTAMP(NOW() - INTERVAL 1 SECOND);`, + preSec: 1, + }, + { + name: "TimestampExactRead6", + sql: `select * from t as of timestamp TIMESTAMP(NOW() - INTERVAL 1 SECOND), b as of timestamp TIMESTAMP('2020-09-06 00:00:00');`, + errorStr: ".*can not set different time in the as of.*", + }, + { + name: "TimestampExactRead7", + sql: `select * from t as of timestamp TIMESTAMP(NOW() - INTERVAL 1 SECOND), b;`, + errorStr: ".*can not set different time in the as of.*", + }, + { + name: "TimestampExactRead8", + sql: `select * from t, b as of timestamp TIMESTAMP(NOW() - INTERVAL 1 SECOND);`, + errorStr: ".*can not set different time in the as of.*", + }, + { + name: "NomalRead", + sql: `select * from t, b;`, + preSec: 0, + }, + { + name: "TimestampExactRead9", + sql: `select * from (select * from t as of timestamp TIMESTAMP(NOW() - INTERVAL 1 SECOND), b as of timestamp TIMESTAMP(NOW() - INTERVAL 1 SECOND)) as c, b;`, + errorStr: ".*can not set different time in the as of.*", + }, + { + name: "TimestampExactRead10", + sql: `select * from (select * from t as of timestamp TIMESTAMP(NOW() - INTERVAL 2 SECOND), b as of timestamp TIMESTAMP(NOW() - INTERVAL 2 SECOND)) as c;`, + preSec: 2, + }, + // Cannot be supported the SubSelect + { + name: "TimestampExactRead11", + sql: `select * from (select * from t as of timestamp TIMESTAMP(NOW() - INTERVAL 20 SECOND), b as of timestamp TIMESTAMP(NOW() - INTERVAL 20 SECOND)) as c as of timestamp Now();`, + errorStr: ".*You have an error in your SQL syntax.*", + }, + } + + for _, testcase := range testcases { + c.Log(testcase.name) + if len(testcase.setTxnSQL) > 0 { + tk.MustExec(testcase.setTxnSQL) + } + if testcase.expectPhysicalTS > 0 { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/assertStaleTSO", fmt.Sprintf(`return(%d)`, testcase.expectPhysicalTS)), IsNil) + } else if testcase.preSec > 0 { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/assertStaleTSOWithTolerance", fmt.Sprintf(`return(%d)`, time.Now().Unix()-testcase.preSec)), IsNil) + } + _, err := tk.Exec(testcase.sql) + if len(testcase.errorStr) != 0 { + c.Assert(err, ErrorMatches, testcase.errorStr) + continue + } + c.Assert(err, IsNil, Commentf("sql:%s, error stack %v", testcase.sql, errors.ErrorStack(err))) + if testcase.expectPhysicalTS > 0 { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/executor/assertStaleTSO"), IsNil) + } else if testcase.preSec > 0 { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/executor/assertStaleTSOWithTolerance"), IsNil) + } + if len(testcase.setTxnSQL) > 0 { + c.Assert(tk.Se.GetSessionVars().TxnReadTS.PeakTxnReadTS(), Equals, uint64(0)) + } } } func (s *testStaleTxnSerialSuite) TestStaleReadKVRequest(c *C) { - c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer", "return(false)"), IsNil) - defer failpoint.Disable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer") tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -155,22 +270,17 @@ func (s *testStaleTxnSerialSuite) TestStaleReadKVRequest(c *C) { failpoint.Enable("github.com/pingcap/tidb/config/injectTxnScope", fmt.Sprintf(`return("%v")`, testcase.zone)) failpoint.Enable("github.com/pingcap/tidb/store/tikv/assertStoreLabels", fmt.Sprintf(`return("%v_%v")`, placement.DCLabelKey, testcase.txnScope)) failpoint.Enable("github.com/pingcap/tidb/store/tikv/assertStaleReadFlag", `return(true)`) - tk.MustExec(`START TRANSACTION READ ONLY WITH TIMESTAMP BOUND EXACT STALENESS '00:00:20';`) + // Using NOW() will cause the loss of fsp precision, so we use NOW(3) to be accurate to the millisecond. + tk.MustExec(`START TRANSACTION READ ONLY AS OF TIMESTAMP NOW(3);`) tk.MustQuery(testcase.sql) tk.MustExec(`commit`) - failpoint.Disable("github.com/pingcap/tidb/config/injectTxnScope") - failpoint.Disable("github.com/pingcap/tidb/store/tikv/assertStoreLabels") - failpoint.Disable("github.com/pingcap/tidb/store/tikv/assertStaleReadFlag") } + failpoint.Disable("github.com/pingcap/tidb/config/injectTxnScope") + failpoint.Disable("github.com/pingcap/tidb/store/tikv/assertStoreLabels") + failpoint.Disable("github.com/pingcap/tidb/store/tikv/assertStaleReadFlag") } -func (s *testStaleTxnSerialSuite) TestStalenessAndHistoryRead(c *C) { - c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer", "return(false)"), IsNil) - defer func() { - err := failpoint.Disable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer") - c.Assert(err, IsNil) - }() - +func (s *testStaleTxnSuite) TestStalenessAndHistoryRead(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") // For mocktikv, safe point is not initialized, we manually insert it for snapshot to use. @@ -182,139 +292,483 @@ func (s *testStaleTxnSerialSuite) TestStalenessAndHistoryRead(c *C) { UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment) tk.MustExec(updateSafePoint) // set @@tidb_snapshot before staleness txn - tk.MustExec(`set @@tidb_snapshot="2016-10-08 16:45:26";`) - tk.MustExec(`START TRANSACTION READ ONLY WITH TIMESTAMP BOUND READ TIMESTAMP '2020-09-06 00:00:00';`) + tk.MustExec(`START TRANSACTION READ ONLY AS OF TIMESTAMP '2020-09-06 00:00:00';`) + // 1599321600000 == 2020-09-06 00:00:00 c.Assert(oracle.ExtractPhysical(tk.Se.GetSessionVars().TxnCtx.StartTS), Equals, int64(1599321600000)) tk.MustExec("commit") // set @@tidb_snapshot during staleness txn - tk.MustExec(`START TRANSACTION READ ONLY WITH TIMESTAMP BOUND READ TIMESTAMP '2020-09-06 00:00:00';`) + tk.MustExec(`START TRANSACTION READ ONLY AS OF TIMESTAMP '2020-09-06 00:00:00';`) tk.MustExec(`set @@tidb_snapshot="2016-10-08 16:45:26";`) c.Assert(oracle.ExtractPhysical(tk.Se.GetSessionVars().TxnCtx.StartTS), Equals, int64(1599321600000)) tk.MustExec("commit") + + // test mutex + tk.MustExec(`set @@tidb_snapshot="2020-10-08 16:45:26";`) + c.Assert(tk.Se.GetSessionVars().SnapshotTS, Equals, uint64(419993151340544000)) + c.Assert(tk.Se.GetSessionVars().SnapshotInfoschema, NotNil) + tk.MustExec("SET TRANSACTION READ ONLY AS OF TIMESTAMP '2020-10-08 16:46:26'") + c.Assert(tk.Se.GetSessionVars().SnapshotTS, Equals, uint64(0)) + c.Assert(tk.Se.GetSessionVars().TxnReadTS.PeakTxnReadTS(), Equals, uint64(419993167069184000)) + + tk.MustExec("SET TRANSACTION READ ONLY AS OF TIMESTAMP '2020-10-08 16:46:26'") + c.Assert(tk.Se.GetSessionVars().TxnReadTS.PeakTxnReadTS(), Equals, uint64(419993167069184000)) + tk.MustExec(`set @@tidb_snapshot="2020-10-08 16:45:26";`) + c.Assert(tk.Se.GetSessionVars().SnapshotTS, Equals, uint64(419993151340544000)) + c.Assert(tk.Se.GetSessionVars().TxnReadTS.PeakTxnReadTS(), Equals, uint64(0)) + c.Assert(tk.Se.GetSessionVars().SnapshotInfoschema, NotNil) } -func (s *testStaleTxnSerialSuite) TestStalenessTransactionSchemaVer(c *C) { +func (s *testStaleTxnSerialSuite) TestTimeBoundedStalenessTxn(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (id int primary key);") + defer tk.MustExec(`drop table if exists t`) testcases := []struct { - name string - sql string - expectErr error + name string + sql string + injectSafeTS uint64 + // compareWithSafeTS will be 0 if StartTS==SafeTS, -1 if StartTS < SafeTS, and +1 if StartTS > SafeTS. + compareWithSafeTS int }{ { - name: "ddl change before stale txn", - sql: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND EXACT STALENESS '00:00:03'`, - expectErr: errors.New("schema version changed after the staleness startTS"), + name: "20 seconds ago to now, safeTS 10 secs ago", + sql: `START TRANSACTION READ ONLY AS OF TIMESTAMP tidb_bounded_staleness(NOW() - INTERVAL 20 SECOND, NOW())`, + injectSafeTS: oracle.GoTimeToTS(time.Now().Add(-10 * time.Second)), + compareWithSafeTS: 0, + }, + { + name: "10 seconds ago to now, safeTS 20 secs ago", + sql: `START TRANSACTION READ ONLY AS OF TIMESTAMP tidb_bounded_staleness(NOW() - INTERVAL 10 SECOND, NOW())`, + injectSafeTS: oracle.GoTimeToTS(time.Now().Add(-20 * time.Second)), + compareWithSafeTS: 1, }, { - name: "ddl change before stale txn", - sql: fmt.Sprintf("START TRANSACTION READ ONLY WITH TIMESTAMP BOUND READ TIMESTAMP '%v'", - time.Now().Truncate(3*time.Second).Format("2006-01-02 15:04:05")), - expectErr: errors.New(".*schema version changed after the staleness startTS.*"), + name: "20 seconds ago to 10 seconds ago, safeTS 5 secs ago", + sql: `START TRANSACTION READ ONLY AS OF TIMESTAMP tidb_bounded_staleness(NOW() - INTERVAL 20 SECOND, NOW() - INTERVAL 10 SECOND)`, + injectSafeTS: oracle.GoTimeToTS(time.Now().Add(-5 * time.Second)), + compareWithSafeTS: -1, }, { - name: "ddl change before stale txn", - sql: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND EXACT STALENESS '00:00:03'`, - expectErr: nil, + name: "exact timestamp 5 seconds ago, safeTS 10 secs ago", + sql: `START TRANSACTION READ ONLY AS OF TIMESTAMP NOW() - INTERVAL 5 SECOND`, + injectSafeTS: oracle.GoTimeToTS(time.Now().Add(-10 * time.Second)), + compareWithSafeTS: 1, + }, + { + name: "exact timestamp 10 seconds ago, safeTS 5 secs ago", + sql: `START TRANSACTION READ ONLY AS OF TIMESTAMP NOW() - INTERVAL 10 SECOND`, + injectSafeTS: oracle.GoTimeToTS(time.Now().Add(-5 * time.Second)), + compareWithSafeTS: -1, }, } - tk := testkit.NewTestKitWithInit(c, s.store) for _, testcase := range testcases { - check := func() { - if testcase.expectErr != nil { - c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer", "return(true)"), IsNil) - defer func() { - err := failpoint.Disable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer") - c.Assert(err, IsNil) - }() - - } else { - c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer", "return(false)"), IsNil) - defer func() { - err := failpoint.Disable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer") - c.Assert(err, IsNil) - }() - - } - _, err := tk.Exec(testcase.sql) - if testcase.expectErr != nil { - c.Assert(err, NotNil) - c.Assert(err.Error(), Matches, testcase.expectErr.Error()) - } else { - c.Assert(err, IsNil) - } + c.Log(testcase.name) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/injectSafeTS", + fmt.Sprintf("return(%v)", testcase.injectSafeTS)), IsNil) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/expression/injectSafeTS", + fmt.Sprintf("return(%v)", testcase.injectSafeTS)), IsNil) + tk.MustExec(testcase.sql) + if testcase.compareWithSafeTS == 1 { + c.Assert(tk.Se.GetSessionVars().TxnCtx.StartTS, Greater, testcase.injectSafeTS) + } else if testcase.compareWithSafeTS == 0 { + c.Assert(tk.Se.GetSessionVars().TxnCtx.StartTS, Equals, testcase.injectSafeTS) + } else { + c.Assert(tk.Se.GetSessionVars().TxnCtx.StartTS, Less, testcase.injectSafeTS) } - check() + tk.MustExec("commit") } + failpoint.Disable("github.com/pingcap/tidb/expression/injectSafeTS") + failpoint.Disable("github.com/pingcap/tidb/store/tikv/injectSafeTS") } -func (s *testStaleTxnSerialSuite) TestTimeBoundedStalenessTxn(c *C) { - c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer", "return(false)"), IsNil) - defer failpoint.Disable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer") +func (s *testStaleTxnSerialSuite) TestStalenessTransactionSchemaVer(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") + defer tk.MustExec("drop table if exists t") tk.MustExec("create table t (id int primary key);") - defer tk.MustExec(`drop table if exists t`) - tk.MustExec(`START TRANSACTION READ ONLY WITH TIMESTAMP BOUND MAX STALENESS '00:00:10'`) + + schemaVer1 := tk.Se.GetInfoSchema().SchemaMetaVersion() + time1 := time.Now() + tk.MustExec("alter table t add c int") + + // confirm schema changed + schemaVer2 := tk.Se.GetInfoSchema().SchemaMetaVersion() + c.Assert(schemaVer1, Less, schemaVer2) + + // get the specific old schema + tk.MustExec(fmt.Sprintf(`START TRANSACTION READ ONLY AS OF TIMESTAMP '%s'`, time1.Format("2006-1-2 15:04:05.000"))) + c.Assert(tk.Se.GetInfoSchema().SchemaMetaVersion(), Equals, schemaVer1) + + // schema changed back to the newest + tk.MustExec("commit") + c.Assert(tk.Se.GetInfoSchema().SchemaMetaVersion(), Equals, schemaVer2) + + // select does not affect the infoschema + tk.MustExec(fmt.Sprintf(`SELECT * from t AS OF TIMESTAMP '%s'`, time1.Format("2006-1-2 15:04:05.000"))) + c.Assert(tk.Se.GetInfoSchema().SchemaMetaVersion(), Equals, schemaVer2) +} + +func (s *testStaleTxnSerialSuite) TestSetTransactionReadOnlyAsOf(c *C) { + t1, err := time.Parse(types.TimeFormat, "2016-09-21 09:53:04") + c.Assert(err, IsNil) + tk := testkit.NewTestKit(c, s.store) + // For mocktikv, safe point is not initialized, we manually insert it for snapshot to use. + safePointName := "tikv_gc_safe_point" + safePointValue := "20160102-15:04:05 -0700" + safePointComment := "All versions after safe point can be accessed. (DO NOT EDIT)" + updateSafePoint := fmt.Sprintf(`INSERT INTO mysql.tidb VALUES ('%[1]s', '%[2]s', '%[3]s') + ON DUPLICATE KEY + UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment) + tk.MustExec(updateSafePoint) testcases := []struct { - name string - sql string - injectResolveTS uint64 - useResolveTS bool + sql string + expectedTS uint64 + injectSafeTS uint64 }{ { - name: "max 20 seconds ago, resolveTS 10 secs ago", - sql: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND MAX STALENESS '00:00:20'`, - injectResolveTS: func() uint64 { - phy := time.Now().Add(-10*time.Second).Unix() * 1000 - return oracle.ComposeTS(phy, 0) - }(), - useResolveTS: true, - }, - { - name: "max 10 seconds ago, resolveTS 20 secs ago", - sql: `START TRANSACTION READ ONLY WITH TIMESTAMP BOUND MAX STALENESS '00:00:10'`, - injectResolveTS: func() uint64 { - phy := time.Now().Add(-20*time.Second).Unix() * 1000 - return oracle.ComposeTS(phy, 0) - }(), - useResolveTS: false, - }, - { - name: "max 20 seconds ago, resolveTS 10 secs ago", - sql: func() string { - return fmt.Sprintf(`START TRANSACTION READ ONLY WITH TIMESTAMP BOUND MIN READ TIMESTAMP '%v'`, - time.Now().Add(-20*time.Second).Format("2006-01-02 15:04:05")) - }(), - injectResolveTS: func() uint64 { - phy := time.Now().Add(-10*time.Second).Unix() * 1000 - return oracle.ComposeTS(phy, 0) - }(), - useResolveTS: true, - }, - { - name: "max 10 seconds ago, resolveTS 20 secs ago", - sql: func() string { - return fmt.Sprintf(`START TRANSACTION READ ONLY WITH TIMESTAMP BOUND MIN READ TIMESTAMP '%v'`, - time.Now().Add(-10*time.Second).Format("2006-01-02 15:04:05")) - }(), - injectResolveTS: func() uint64 { - phy := time.Now().Add(-20*time.Second).Unix() * 1000 - return oracle.ComposeTS(phy, 0) - }(), - useResolveTS: false, + sql: `SET TRANSACTION READ ONLY as of timestamp '2021-04-21 00:42:12'`, + expectedTS: 424394603102208000, + injectSafeTS: 0, + }, + { + sql: `SET TRANSACTION READ ONLY as of timestamp tidb_bounded_staleness('2015-09-21 00:07:01', '2021-04-27 11:26:13')`, + expectedTS: oracle.GoTimeToTS(t1), + injectSafeTS: oracle.GoTimeToTS(t1), }, } for _, testcase := range testcases { - c.Log(testcase.name) - c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/injectResolveTS", - fmt.Sprintf("return(%v)", testcase.injectResolveTS)), IsNil) + if testcase.injectSafeTS > 0 { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/expression/injectSafeTS", + fmt.Sprintf("return(%v)", testcase.injectSafeTS)), IsNil) + } tk.MustExec(testcase.sql) - if testcase.useResolveTS { - c.Assert(tk.Se.GetSessionVars().TxnCtx.StartTS, Equals, testcase.injectResolveTS) + c.Assert(tk.Se.GetSessionVars().TxnReadTS.PeakTxnReadTS(), Equals, testcase.expectedTS) + tk.MustExec("begin") + c.Assert(tk.Se.GetSessionVars().TxnCtx.StartTS, Equals, testcase.expectedTS) + tk.MustExec("commit") + c.Assert(tk.Se.GetSessionVars().TxnReadTS.PeakTxnReadTS(), Equals, uint64(0)) + tk.MustExec("begin") + c.Assert(tk.Se.GetSessionVars().TxnCtx.StartTS, Not(Equals), testcase.expectedTS) + tk.MustExec("commit") + + failpoint.Disable("github.com/pingcap/tidb/expression/injectSafeTS") + } + + err = tk.ExecToErr(`SET TRANSACTION READ ONLY as of timestamp tidb_bounded_staleness(invalid1, invalid2')`) + c.Assert(err, NotNil) + c.Assert(tk.Se.GetSessionVars().TxnReadTS.PeakTxnReadTS(), Equals, uint64(0)) + + tk.MustExec(`SET TRANSACTION READ ONLY as of timestamp '2021-04-21 00:42:12'`) + err = tk.ExecToErr(`START TRANSACTION READ ONLY AS OF TIMESTAMP '2020-09-06 00:00:00'`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "start transaction read only as of is forbidden after set transaction read only as of") + tk.MustExec(`SET TRANSACTION READ ONLY as of timestamp '2021-04-21 00:42:12'`) +} + +func (s *testStaleTxnSerialSuite) TestValidateReadOnlyInStalenessTransaction(c *C) { + testcases := []struct { + name string + sql string + isValidate bool + }{ + { + name: "select statement", + sql: `select * from t;`, + isValidate: true, + }, + { + name: "explain statement", + sql: `explain insert into t (id) values (1);`, + isValidate: true, + }, + { + name: "explain analyze insert statement", + sql: `explain analyze insert into t (id) values (1);`, + isValidate: false, + }, + { + name: "explain analyze select statement", + sql: `explain analyze select * from t `, + isValidate: true, + }, + { + name: "execute insert statement", + sql: `EXECUTE stmt1;`, + isValidate: false, + }, + { + name: "execute select statement", + sql: `EXECUTE stmt2;`, + isValidate: true, + }, + { + name: "show statement", + sql: `show tables;`, + isValidate: true, + }, + { + name: "set union", + sql: `SELECT 1, 2 UNION SELECT 'a', 'b';`, + isValidate: true, + }, + { + name: "insert", + sql: `insert into t (id) values (1);`, + isValidate: false, + }, + { + name: "delete", + sql: `delete from t where id =1`, + isValidate: false, + }, + { + name: "update", + sql: "update t set id =2 where id =1", + isValidate: false, + }, + { + name: "point get", + sql: `select * from t where id = 1`, + isValidate: true, + }, + { + name: "batch point get", + sql: `select * from t where id in (1,2,3);`, + isValidate: true, + }, + { + name: "split table", + sql: `SPLIT TABLE t BETWEEN (0) AND (1000000000) REGIONS 16;`, + isValidate: true, + }, + { + name: "do statement", + sql: `DO SLEEP(1);`, + isValidate: true, + }, + { + name: "select for update", + sql: "select * from t where id = 1 for update", + isValidate: false, + }, + { + name: "select lock in share mode", + sql: "select * from t where id = 1 lock in share mode", + isValidate: true, + }, + { + name: "select for update union statement", + sql: "select * from t for update union select * from t;", + isValidate: false, + }, + { + name: "replace statement", + sql: "replace into t(id) values (1)", + isValidate: false, + }, + { + name: "load data statement", + sql: "LOAD DATA LOCAL INFILE '/mn/asa.csv' INTO TABLE t FIELDS TERMINATED BY x'2c' ENCLOSED BY b'100010' LINES TERMINATED BY '\r\n' IGNORE 1 LINES (id);", + isValidate: false, + }, + { + name: "update multi tables", + sql: "update t,t1 set t.id = 1,t1.id = 2 where t.1 = 2 and t1.id = 3;", + isValidate: false, + }, + { + name: "delete multi tables", + sql: "delete t from t1 where t.id = t1.id", + isValidate: false, + }, + { + name: "insert select", + sql: "insert into t select * from t1;", + isValidate: false, + }, + } + tk := testkit.NewTestKit(c, s.store) + // For mocktikv, safe point is not initialized, we manually insert it for snapshot to use. + safePointName := "tikv_gc_safe_point" + safePointValue := "20160102-15:04:05 -0700" + safePointComment := "All versions after safe point can be accessed. (DO NOT EDIT)" + updateSafePoint := fmt.Sprintf(`INSERT INTO mysql.tidb VALUES ('%[1]s', '%[2]s', '%[3]s') + ON DUPLICATE KEY + UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment) + tk.MustExec(updateSafePoint) + tk.MustExec("use test") + tk.MustExec("create table t (id int);") + tk.MustExec("create table t1 (id int);") + tk.MustExec(`PREPARE stmt1 FROM 'insert into t(id) values (5);';`) + tk.MustExec(`PREPARE stmt2 FROM 'select * from t';`) + tk.MustExec(`set @@tidb_enable_noop_functions=1;`) + for _, testcase := range testcases { + c.Log(testcase.name) + tk.MustExec(`START TRANSACTION READ ONLY AS OF TIMESTAMP NOW(3);`) + if testcase.isValidate { + _, err := tk.Exec(testcase.sql) + c.Assert(err, IsNil) + } else { + err := tk.ExecToErr(testcase.sql) + c.Assert(err, NotNil) + c.Assert(err.Error(), Matches, `.*only support read-only statement during read-only staleness transactions.*`) + } + tk.MustExec("commit") + tk.MustExec("set transaction read only as of timestamp NOW(3);") + if testcase.isValidate { + _, err := tk.Exec(testcase.sql) + c.Assert(err, IsNil) } else { - c.Assert(oracle.CompareTS(tk.Se.GetSessionVars().TxnCtx.StartTS, testcase.injectResolveTS), Equals, 1) + err := tk.ExecToErr(testcase.sql) + c.Assert(err, NotNil) + c.Assert(err.Error(), Matches, `.*only support read-only statement during read-only staleness transactions.*`) } + tk.MustExec("set transaction read only as of timestamp ''") + } +} + +func (s *testStaleTxnSuite) TestSpecialSQLInStalenessTxn(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + testcases := []struct { + name string + sql string + sameSession bool + }{ + { + name: "ddl", + sql: "create table t (id int, b int,INDEX(b));", + sameSession: false, + }, + { + name: "set global session", + sql: `SET GLOBAL sql_mode = 'STRICT_TRANS_TABLES,NO_AUTO_CREATE_USER';`, + sameSession: true, + }, + { + name: "analyze table", + sql: "analyze table t", + sameSession: true, + }, + { + name: "session binding", + sql: "CREATE SESSION BINDING FOR SELECT * FROM t WHERE b = 123 USING SELECT * FROM t IGNORE INDEX (b) WHERE b = 123;", + sameSession: true, + }, + { + name: "global binding", + sql: "CREATE GLOBAL BINDING FOR SELECT * FROM t WHERE b = 123 USING SELECT * FROM t IGNORE INDEX (b) WHERE b = 123;", + sameSession: true, + }, + { + name: "grant statements", + sql: "GRANT ALL ON test.* TO 'newuser';", + sameSession: false, + }, + { + name: "revoke statements", + sql: "REVOKE ALL ON test.* FROM 'newuser';", + sameSession: false, + }, + } + tk.MustExec("CREATE USER 'newuser' IDENTIFIED BY 'mypassword';") + for _, testcase := range testcases { + comment := Commentf(testcase.name) + tk.MustExec(`START TRANSACTION READ ONLY AS OF TIMESTAMP NOW(3);`) + c.Assert(tk.Se.GetSessionVars().TxnCtx.IsStaleness, Equals, true, comment) + tk.MustExec(testcase.sql) + c.Assert(tk.Se.GetSessionVars().TxnCtx.IsStaleness, Equals, testcase.sameSession, comment) + } +} + +func (s *testStaleTxnSuite) TestAsOfTimestampCompatibility(c *C) { + tk := testkit.NewTestKit(c, s.store) + // For mocktikv, safe point is not initialized, we manually insert it for snapshot to use. + safePointName := "tikv_gc_safe_point" + safePointValue := "20160102-15:04:05 -0700" + safePointComment := "All versions after safe point can be accessed. (DO NOT EDIT)" + updateSafePoint := fmt.Sprintf(`INSERT INTO mysql.tidb VALUES ('%[1]s', '%[2]s', '%[3]s') + ON DUPLICATE KEY + UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment) + tk.MustExec(updateSafePoint) + tk.MustExec("use test") + tk.MustExec("create table t5(id int);") + time1 := time.Now() + testcases := []struct { + beginSQL string + sql string + }{ + { + beginSQL: fmt.Sprintf("START TRANSACTION READ ONLY AS OF TIMESTAMP '%s'", time1.Format("2006-1-2 15:04:05.000")), + sql: fmt.Sprintf(`SET TRANSACTION READ ONLY AS OF TIMESTAMP '%s'`, time1.Format("2006-1-2 15:04:05.000")), + }, + { + beginSQL: "begin", + sql: fmt.Sprintf(`SET TRANSACTION READ ONLY AS OF TIMESTAMP '%s'`, time1.Format("2006-1-2 15:04:05.000")), + }, + { + beginSQL: "start transaction", + sql: fmt.Sprintf(`SET TRANSACTION READ ONLY AS OF TIMESTAMP '%s'`, time1.Format("2006-1-2 15:04:05.000")), + }, + { + beginSQL: fmt.Sprintf("START TRANSACTION READ ONLY AS OF TIMESTAMP '%s'", time1.Format("2006-1-2 15:04:05.000")), + sql: fmt.Sprintf("select * from t5 as of timestamp '%s'", time1.Format("2006-1-2 15:04:05.000")), + }, + { + beginSQL: "begin", + sql: fmt.Sprintf("select * from t5 as of timestamp '%s'", time1.Format("2006-1-2 15:04:05.000")), + }, + { + beginSQL: "start transaction", + sql: fmt.Sprintf("select * from t5 as of timestamp '%s'", time1.Format("2006-1-2 15:04:05.000")), + }, + } + for _, testcase := range testcases { + tk.MustExec(testcase.beginSQL) + err := tk.ExecToErr(testcase.sql) + c.Assert(err, NotNil) + c.Assert(err.Error(), Matches, ".*as of timestamp can't be set in transaction.*") tk.MustExec("commit") - failpoint.Disable("github.com/pingcap/tidb/store/tikv/injectResolveTS") } } + +func (s *testStaleTxnSuite) TestSetTransactionInfoSchema(c *C) { + tk := testkit.NewTestKit(c, s.store) + // For mocktikv, safe point is not initialized, we manually insert it for snapshot to use. + safePointName := "tikv_gc_safe_point" + safePointValue := "20160102-15:04:05 -0700" + safePointComment := "All versions after safe point can be accessed. (DO NOT EDIT)" + updateSafePoint := fmt.Sprintf(`INSERT INTO mysql.tidb VALUES ('%[1]s', '%[2]s', '%[3]s') + ON DUPLICATE KEY + UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment) + tk.MustExec(updateSafePoint) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + defer tk.MustExec("drop table if exists t") + tk.MustExec("create table t (id int primary key);") + + schemaVer1 := tk.Se.GetInfoSchema().SchemaMetaVersion() + time1 := time.Now() + tk.MustExec("alter table t add c int") + + // confirm schema changed + schemaVer2 := tk.Se.GetInfoSchema().SchemaMetaVersion() + time2 := time.Now() + c.Assert(schemaVer1, Less, schemaVer2) + tk.MustExec(fmt.Sprintf(`SET TRANSACTION READ ONLY AS OF TIMESTAMP '%s'`, time1.Format("2006-1-2 15:04:05.000"))) + c.Assert(tk.Se.GetInfoSchema().SchemaMetaVersion(), Equals, schemaVer1) + tk.MustExec("select * from t;") + tk.MustExec("alter table t add d int") + schemaVer3 := tk.Se.GetInfoSchema().SchemaMetaVersion() + tk.MustExec(fmt.Sprintf(`SET TRANSACTION READ ONLY AS OF TIMESTAMP '%s'`, time1.Format("2006-1-2 15:04:05.000"))) + tk.MustExec("begin;") + c.Assert(tk.Se.GetInfoSchema().SchemaMetaVersion(), Equals, schemaVer1) + tk.MustExec("commit") + tk.MustExec(fmt.Sprintf(`SET TRANSACTION READ ONLY AS OF TIMESTAMP '%s'`, time2.Format("2006-1-2 15:04:05.000"))) + tk.MustExec("begin;") + c.Assert(tk.Se.GetInfoSchema().SchemaMetaVersion(), Equals, schemaVer2) + tk.MustExec("commit") + c.Assert(tk.Se.GetInfoSchema().SchemaMetaVersion(), Equals, schemaVer3) +} diff --git a/executor/table_reader.go b/executor/table_reader.go index 767826f0c3b6c..88531e9c86363 100644 --- a/executor/table_reader.go +++ b/executor/table_reader.go @@ -21,7 +21,6 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/tidb/distsql" "github.com/pingcap/tidb/expression" - "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx" @@ -54,7 +53,8 @@ func (sr selectResultHook) SelectResult(ctx context.Context, sctx sessionctx.Con } type kvRangeBuilder interface { - buildKeyRange(pid int64) ([]kv.KeyRange, error) + buildKeyRange(pid int64, ranges []*ranger.Range) ([]kv.KeyRange, error) + buildKeyRangeSeparately(ranges []*ranger.Range) ([]int64, [][]kv.KeyRange, error) } // TableReaderExecutor sends DAG request and reads table data from kv layer. @@ -154,6 +154,25 @@ func (e *TableReaderExecutor) Open(ctx context.Context) error { } } firstPartRanges, secondPartRanges := distsql.SplitRangesAcrossInt64Boundary(e.ranges, e.keepOrder, e.desc, e.table.Meta() != nil && e.table.Meta().IsCommonHandle) + + // Treat temporary table as dummy table, avoid sending distsql request to TiKV. + // Calculate the kv ranges here, UnionScan rely on this kv ranges. + if e.table.Meta() != nil && e.table.Meta().TempTableType != model.TempTableNone { + kvReq, err := e.buildKVReq(ctx, firstPartRanges) + if err != nil { + return err + } + e.kvRanges = append(e.kvRanges, kvReq.KeyRanges...) + if len(secondPartRanges) != 0 { + kvReq, err = e.buildKVReq(ctx, secondPartRanges) + if err != nil { + return err + } + e.kvRanges = append(e.kvRanges, kvReq.KeyRanges...) + } + return nil + } + firstResult, err := e.buildResp(ctx, firstPartRanges) if err != nil { e.feedback.Invalidate() @@ -176,6 +195,12 @@ func (e *TableReaderExecutor) Open(ctx context.Context) error { // Next fills data into the chunk passed by its caller. // The task was actually done by tableReaderHandler. func (e *TableReaderExecutor) Next(ctx context.Context, req *chunk.Chunk) error { + if e.table.Meta() != nil && e.table.Meta().TempTableType != model.TempTableNone { + // Treat temporary table as dummy table, avoid sending distsql request to TiKV. + req.Reset() + return nil + } + logutil.Eventf(ctx, "table scan table: %s, range: %v", stringutil.MemoizeStr(func() string { var tableName string if meta := e.table.Meta(); meta != nil { @@ -198,6 +223,10 @@ func (e *TableReaderExecutor) Next(ctx context.Context, req *chunk.Chunk) error // Close implements the Executor Close interface. func (e *TableReaderExecutor) Close() error { + if e.table.Meta() != nil && e.table.Meta().TempTableType != model.TempTableNone { + return nil + } + var err error if e.resultHandler != nil { err = e.resultHandler.Close() @@ -210,10 +239,73 @@ func (e *TableReaderExecutor) Close() error { // buildResp first builds request and sends it to tikv using distsql.Select. It uses SelectResult returned by the callee // to fetch all results. func (e *TableReaderExecutor) buildResp(ctx context.Context, ranges []*ranger.Range) (distsql.SelectResult, error) { + if e.storeType == kv.TiFlash && e.kvRangeBuilder != nil { + // TiFlash cannot support to access multiple tables/partitions within one KVReq, so we have to build KVReq for each partition separately. + kvReqs, err := e.buildKVReqSeparately(ctx, ranges) + if err != nil { + return nil, err + } + var results []distsql.SelectResult + for _, kvReq := range kvReqs { + result, err := e.SelectResult(ctx, e.ctx, kvReq, retTypes(e), e.feedback, getPhysicalPlanIDs(e.plans), e.id) + if err != nil { + return nil, err + } + results = append(results, result) + } + return distsql.NewSerialSelectResults(results), nil + } + + kvReq, err := e.buildKVReq(ctx, ranges) + if err != nil { + return nil, err + } + e.kvRanges = append(e.kvRanges, kvReq.KeyRanges...) + + result, err := e.SelectResult(ctx, e.ctx, kvReq, retTypes(e), e.feedback, getPhysicalPlanIDs(e.plans), e.id) + if err != nil { + return nil, err + } + return result, nil +} + +func (e *TableReaderExecutor) buildKVReqSeparately(ctx context.Context, ranges []*ranger.Range) ([]*kv.Request, error) { + pids, kvRanges, err := e.kvRangeBuilder.buildKeyRangeSeparately(ranges) + if err != nil { + return nil, err + } + var kvReqs []*kv.Request + for i, kvRange := range kvRanges { + e.kvRanges = append(e.kvRanges, kvRange...) + if err := updateExecutorTableID(ctx, e.dagPB.RootExecutor, pids[i], true); err != nil { + return nil, err + } + var builder distsql.RequestBuilder + reqBuilder := builder.SetKeyRanges(kvRange) + kvReq, err := reqBuilder. + SetDAGRequest(e.dagPB). + SetStartTS(e.startTS). + SetDesc(e.desc). + SetKeepOrder(e.keepOrder). + SetStreaming(e.streaming). + SetFromSessionVars(e.ctx.GetSessionVars()). + SetFromInfoSchema(e.ctx.GetInfoSchema()). + SetMemTracker(e.memTracker). + SetStoreType(e.storeType). + SetAllowBatchCop(e.batchCop).Build() + if err != nil { + return nil, err + } + kvReqs = append(kvReqs, kvReq) + } + return kvReqs, nil +} + +func (e *TableReaderExecutor) buildKVReq(ctx context.Context, ranges []*ranger.Range) (*kv.Request, error) { var builder distsql.RequestBuilder var reqBuilder *distsql.RequestBuilder if e.kvRangeBuilder != nil { - kvRange, err := e.kvRangeBuilder.buildKeyRange(getPhysicalTableID(e.table)) + kvRange, err := e.kvRangeBuilder.buildKeyRange(getPhysicalTableID(e.table), ranges) if err != nil { return nil, err } @@ -221,28 +313,18 @@ func (e *TableReaderExecutor) buildResp(ctx context.Context, ranges []*ranger.Ra } else { reqBuilder = builder.SetHandleRanges(e.ctx.GetSessionVars().StmtCtx, getPhysicalTableID(e.table), e.table.Meta() != nil && e.table.Meta().IsCommonHandle, ranges, e.feedback) } - kvReq, err := reqBuilder. + reqBuilder. SetDAGRequest(e.dagPB). SetStartTS(e.startTS). SetDesc(e.desc). SetKeepOrder(e.keepOrder). SetStreaming(e.streaming). SetFromSessionVars(e.ctx.GetSessionVars()). + SetFromInfoSchema(e.ctx.GetInfoSchema()). SetMemTracker(e.memTracker). SetStoreType(e.storeType). - SetAllowBatchCop(e.batchCop). - SetFromInfoSchema(infoschema.GetInfoSchema(e.ctx)). - Build() - if err != nil { - return nil, err - } - e.kvRanges = append(e.kvRanges, kvReq.KeyRanges...) - - result, err := e.SelectResult(ctx, e.ctx, kvReq, retTypes(e), e.feedback, getPhysicalPlanIDs(e.plans), e.id) - if err != nil { - return nil, err - } - return result, nil + SetAllowBatchCop(e.batchCop) + return reqBuilder.Build() } func buildVirtualColumnIndex(schema *expression.Schema, columns []*model.ColumnInfo) []int { diff --git a/executor/testdata/executor_suite_in.json b/executor/testdata/executor_suite_in.json index 6abd20c740a80..cd8fa234c0117 100644 --- a/executor/testdata/executor_suite_in.json +++ b/executor/testdata/executor_suite_in.json @@ -51,5 +51,562 @@ "select count(*) from t as t1 left join t as t2 on t1.c1 = t2.c1 where t1.c1 != NULL", "select * from t as t1 left join t as t2 on t1.c1 = t2.c1 where t1.c1 is not NULL" ] + }, + { + "name": "TestRangePartitionBoundariesEq", + "cases": [ + "INSERT INTO t VALUES (999998, '999998 Filler ...'), (999999, '999999 Filler ...'), (1000000, '1000000 Filler ...'), (1000001, '1000001 Filler ...'), (1000002, '1000002 Filler ...')", + "INSERT INTO t VALUES (1999998, '1999998 Filler ...'), (1999999, '1999999 Filler ...'), (2000000, '2000000 Filler ...'), (2000001, '2000001 Filler ...'), (2000002, '2000002 Filler ...')", + "INSERT INTO t VALUES (2999998, '2999998 Filler ...'), (2999999, '2999999 Filler ...')", + "INSERT INTO t VALUES (-2147483648, 'MIN_INT filler...'), (0, '0 Filler...')", + "ANALYZE TABLE t", + "SELECT * FROM t WHERE a = -2147483648", + "SELECT * FROM t WHERE a IN (-2147483648)", + "SELECT * FROM t WHERE a = 0", + "SELECT * FROM t WHERE a IN (0)", + "SELECT * FROM t WHERE a = 999998", + "SELECT * FROM t WHERE a IN (999998)", + "SELECT * FROM t WHERE a = 999999", + "SELECT * FROM t WHERE a IN (999999)", + "SELECT * FROM t WHERE a = 1000000", + "SELECT * FROM t WHERE a IN (1000000)", + "SELECT * FROM t WHERE a = 1000001", + "SELECT * FROM t WHERE a IN (1000001)", + "SELECT * FROM t WHERE a = 1000002", + "SELECT * FROM t WHERE a IN (1000002)", + "SELECT * FROM t WHERE a = 3000000", + "SELECT * FROM t WHERE a IN (3000000)", + "SELECT * FROM t WHERE a = 3000001", + "SELECT * FROM t WHERE a IN (3000001)", + "SELECT * FROM t WHERE a IN (-2147483648, -2147483647)", + "SELECT * FROM t WHERE a IN (-2147483647, -2147483646)", + "SELECT * FROM t WHERE a IN (999997, 999998, 999999)", + "SELECT * FROM t WHERE a IN (999998, 999999, 1000000)", + "SELECT * FROM t WHERE a IN (999999, 1000000, 1000001)", + "SELECT * FROM t WHERE a IN (1000000, 1000001, 1000002)", + "SELECT * FROM t WHERE a IN (1999997, 1999998, 1999999)", + "SELECT * FROM t WHERE a IN (1999998, 1999999, 2000000)", + "SELECT * FROM t WHERE a IN (1999999, 2000000, 2000001)", + "SELECT * FROM t WHERE a IN (2000000, 2000001, 2000002)", + "SELECT * FROM t WHERE a IN (2999997, 2999998, 2999999)", + "SELECT * FROM t WHERE a IN (2999998, 2999999, 3000000)", + "SELECT * FROM t WHERE a IN (2999999, 3000000, 3000001)", + "SELECT * FROM t WHERE a IN (3000000, 3000001, 3000002)" + ] + }, + { + "name": "TestRangePartitionBoundariesNe", + "cases": [ + "INSERT INTO t VALUES (0, '0 Filler...')", + "INSERT INTO t VALUES (1, '1 Filler...')", + "INSERT INTO t VALUES (2, '2 Filler...')", + "INSERT INTO t VALUES (3, '3 Filler...')", + "INSERT INTO t VALUES (4, '4 Filler...')", + "INSERT INTO t VALUES (5, '5 Filler...')", + "INSERT INTO t VALUES (6, '6 Filler...')", + "ANALYZE TABLE t", + "SELECT * FROM t WHERE a != -1", + "SELECT * FROM t WHERE 1 = 1 AND a != -1", + "SELECT * FROM t WHERE a NOT IN (-2, -1)", + "SELECT * FROM t WHERE 1 = 0 OR a = -1", + "SELECT * FROM t WHERE a != 0", + "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0", + "SELECT * FROM t WHERE a NOT IN (-2, -1, 0)", + "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0", + "SELECT * FROM t WHERE a != 1", + "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0 AND a != 1", + "SELECT * FROM t WHERE a NOT IN (-2, -1, 0, 1)", + "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0 OR a = 1", + "SELECT * FROM t WHERE a != 2", + "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0 AND a != 1 AND a != 2", + "SELECT * FROM t WHERE a NOT IN (-2, -1, 0, 1, 2)", + "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0 OR a = 1 OR a = 2", + "SELECT * FROM t WHERE a != 3", + "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0 AND a != 1 AND a != 2 AND a != 3", + "SELECT * FROM t WHERE a NOT IN (-2, -1, 0, 1, 2, 3)", + "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0 OR a = 1 OR a = 2 OR a = 3", + "SELECT * FROM t WHERE a != 4", + "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0 AND a != 1 AND a != 2 AND a != 3 AND a != 4", + "SELECT * FROM t WHERE a NOT IN (-2, -1, 0, 1, 2, 3, 4)", + "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0 OR a = 1 OR a = 2 OR a = 3 OR a = 4", + "SELECT * FROM t WHERE a != 5", + "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0 AND a != 1 AND a != 2 AND a != 3 AND a != 4 AND a != 5", + "SELECT * FROM t WHERE a NOT IN (-2, -1, 0, 1, 2, 3, 4, 5)", + "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0 OR a = 1 OR a = 2 OR a = 3 OR a = 4 OR a = 5", + "SELECT * FROM t WHERE a != 6", + "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0 AND a != 1 AND a != 2 AND a != 3 AND a != 4 AND a != 5 AND a != 6", + "SELECT * FROM t WHERE a NOT IN (-2, -1, 0, 1, 2, 3, 4, 5, 6)", + "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0 OR a = 1 OR a = 2 OR a = 3 OR a = 4 OR a = 5 OR a = 6", + "SELECT * FROM t WHERE a != 7", + "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0 AND a != 1 AND a != 2 AND a != 3 AND a != 4 AND a != 5 AND a != 6 AND a != 7", + "SELECT * FROM t WHERE a NOT IN (-2, -1, 0, 1, 2, 3, 4, 5, 6, 7)", + "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0 OR a = 1 OR a = 2 OR a = 3 OR a = 4 OR a = 5 OR a = 6 OR a = 7" + ] + }, + { + "name": "TestRangePartitionBoundariesBetweenM", + "cases": [ + "INSERT INTO t VALUES (999998, '999998 Filler ...'), (999999, '999999 Filler ...'), (1000000, '1000000 Filler ...'), (1000001, '1000001 Filler ...'), (1000002, '1000002 Filler ...')", + "INSERT INTO t VALUES (1999998, '1999998 Filler ...'), (1999999, '1999999 Filler ...'), (2000000, '2000000 Filler ...'), (2000001, '2000001 Filler ...'), (2000002, '2000002 Filler ...')", + "INSERT INTO t VALUES (2999998, '2999998 Filler ...'), (2999999, '2999999 Filler ...')", + "INSERT INTO t VALUES (-2147483648, 'MIN_INT filler...'), (0, '0 Filler...')", + "ANALYZE TABLE t", + "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2147483649", + "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2147483648", + "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2147483647", + "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2147483646", + "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2147483638", + "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2146483650", + "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2146483649", + "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2146483648", + "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2146483647", + "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2146483646", + "SELECT * FROM t WHERE a BETWEEN 0 AND -1", + "SELECT * FROM t WHERE a BETWEEN 0 AND 0", + "SELECT * FROM t WHERE a BETWEEN 0 AND 1", + "SELECT * FROM t WHERE a BETWEEN 0 AND 2", + "SELECT * FROM t WHERE a BETWEEN 0 AND 10", + "SELECT * FROM t WHERE a BETWEEN 0 AND 999998", + "SELECT * FROM t WHERE a BETWEEN 0 AND 999999", + "SELECT * FROM t WHERE a BETWEEN 0 AND 1000000", + "SELECT * FROM t WHERE a BETWEEN 0 AND 1000001", + "SELECT * FROM t WHERE a BETWEEN 0 AND 1000002", + "SELECT * FROM t WHERE a BETWEEN 999998 AND 999997", + "SELECT * FROM t WHERE a BETWEEN 999998 AND 999998", + "SELECT * FROM t WHERE a BETWEEN 999998 AND 999999", + "SELECT * FROM t WHERE a BETWEEN 999998 AND 1000000", + "SELECT * FROM t WHERE a BETWEEN 999998 AND 1000008", + "SELECT * FROM t WHERE a BETWEEN 999998 AND 1999996", + "SELECT * FROM t WHERE a BETWEEN 999998 AND 1999997", + "SELECT * FROM t WHERE a BETWEEN 999998 AND 1999998", + "SELECT * FROM t WHERE a BETWEEN 999998 AND 1999999", + "SELECT * FROM t WHERE a BETWEEN 999998 AND 2000000", + "SELECT * FROM t WHERE a BETWEEN 999999 AND 999998", + "SELECT * FROM t WHERE a BETWEEN 999999 AND 999999", + "SELECT * FROM t WHERE a BETWEEN 999999 AND 1000000", + "SELECT * FROM t WHERE a BETWEEN 999999 AND 1000001", + "SELECT * FROM t WHERE a BETWEEN 999999 AND 1000009", + "SELECT * FROM t WHERE a BETWEEN 999999 AND 1999997", + "SELECT * FROM t WHERE a BETWEEN 999999 AND 1999998", + "SELECT * FROM t WHERE a BETWEEN 999999 AND 1999999", + "SELECT * FROM t WHERE a BETWEEN 999999 AND 2000000", + "SELECT * FROM t WHERE a BETWEEN 999999 AND 2000001", + "SELECT * FROM t WHERE a BETWEEN 1000000 AND 999999", + "SELECT * FROM t WHERE a BETWEEN 1000000 AND 1000000", + "SELECT * FROM t WHERE a BETWEEN 1000000 AND 1000001", + "SELECT * FROM t WHERE a BETWEEN 1000000 AND 1000002", + "SELECT * FROM t WHERE a BETWEEN 1000000 AND 1000010", + "SELECT * FROM t WHERE a BETWEEN 1000000 AND 1999998", + "SELECT * FROM t WHERE a BETWEEN 1000000 AND 1999999", + "SELECT * FROM t WHERE a BETWEEN 1000000 AND 2000000", + "SELECT * FROM t WHERE a BETWEEN 1000000 AND 2000001", + "SELECT * FROM t WHERE a BETWEEN 1000000 AND 2000002", + "SELECT * FROM t WHERE a BETWEEN 1000001 AND 1000000", + "SELECT * FROM t WHERE a BETWEEN 1000001 AND 1000001", + "SELECT * FROM t WHERE a BETWEEN 1000001 AND 1000002", + "SELECT * FROM t WHERE a BETWEEN 1000001 AND 1000003", + "SELECT * FROM t WHERE a BETWEEN 1000001 AND 1000011", + "SELECT * FROM t WHERE a BETWEEN 1000001 AND 1999999", + "SELECT * FROM t WHERE a BETWEEN 1000001 AND 2000000", + "SELECT * FROM t WHERE a BETWEEN 1000001 AND 2000001", + "SELECT * FROM t WHERE a BETWEEN 1000001 AND 2000002", + "SELECT * FROM t WHERE a BETWEEN 1000001 AND 2000003", + "SELECT * FROM t WHERE a BETWEEN 1000002 AND 1000001", + "SELECT * FROM t WHERE a BETWEEN 1000002 AND 1000002", + "SELECT * FROM t WHERE a BETWEEN 1000002 AND 1000003", + "SELECT * FROM t WHERE a BETWEEN 1000002 AND 1000004", + "SELECT * FROM t WHERE a BETWEEN 1000002 AND 1000012", + "SELECT * FROM t WHERE a BETWEEN 1000002 AND 2000000", + "SELECT * FROM t WHERE a BETWEEN 1000002 AND 2000001", + "SELECT * FROM t WHERE a BETWEEN 1000002 AND 2000002", + "SELECT * FROM t WHERE a BETWEEN 1000002 AND 2000003", + "SELECT * FROM t WHERE a BETWEEN 1000002 AND 2000004", + "SELECT * FROM t WHERE a BETWEEN 3000000 AND 2999999", + "SELECT * FROM t WHERE a BETWEEN 3000000 AND 3000000", + "SELECT * FROM t WHERE a BETWEEN 3000000 AND 3000001", + "SELECT * FROM t WHERE a BETWEEN 3000000 AND 3000002", + "SELECT * FROM t WHERE a BETWEEN 3000000 AND 3000010", + "SELECT * FROM t WHERE a BETWEEN 3000000 AND 3999998", + "SELECT * FROM t WHERE a BETWEEN 3000000 AND 3999999", + "SELECT * FROM t WHERE a BETWEEN 3000000 AND 4000000", + "SELECT * FROM t WHERE a BETWEEN 3000000 AND 4000001", + "SELECT * FROM t WHERE a BETWEEN 3000000 AND 4000002", + "SELECT * FROM t WHERE a BETWEEN 3000001 AND 3000000", + "SELECT * FROM t WHERE a BETWEEN 3000001 AND 3000001", + "SELECT * FROM t WHERE a BETWEEN 3000001 AND 3000002", + "SELECT * FROM t WHERE a BETWEEN 3000001 AND 3000003", + "SELECT * FROM t WHERE a BETWEEN 3000001 AND 3000011", + "SELECT * FROM t WHERE a BETWEEN 3000001 AND 3999999", + "SELECT * FROM t WHERE a BETWEEN 3000001 AND 4000000", + "SELECT * FROM t WHERE a BETWEEN 3000001 AND 4000001", + "SELECT * FROM t WHERE a BETWEEN 3000001 AND 4000002", + "SELECT * FROM t WHERE a BETWEEN 3000001 AND 4000003" + ] + }, + { + "name": "TestRangePartitionBoundariesBetweenS", + "cases": [ + "INSERT INTO t VALUES (0, '0 Filler...')", + "INSERT INTO t VALUES (1, '1 Filler...')", + "INSERT INTO t VALUES (2, '2 Filler...')", + "INSERT INTO t VALUES (3, '3 Filler...')", + "INSERT INTO t VALUES (4, '4 Filler...')", + "INSERT INTO t VALUES (5, '5 Filler...')", + "INSERT INTO t VALUES (6, '6 Filler...')", + "ANALYZE TABLE t", + "SELECT * FROM t WHERE a BETWEEN 2 AND -1", + "SELECT * FROM t WHERE a BETWEEN -1 AND 4", + "SELECT * FROM t WHERE a BETWEEN 2 AND 0", + "SELECT * FROM t WHERE a BETWEEN 0 AND 4", + "SELECT * FROM t WHERE a BETWEEN 2 AND 1", + "SELECT * FROM t WHERE a BETWEEN 1 AND 4", + "SELECT * FROM t WHERE a BETWEEN 2 AND 2", + "SELECT * FROM t WHERE a BETWEEN 2 AND 4", + "SELECT * FROM t WHERE a BETWEEN 2 AND 3", + "SELECT * FROM t WHERE a BETWEEN 3 AND 4", + "SELECT * FROM t WHERE a BETWEEN 2 AND 4", + "SELECT * FROM t WHERE a BETWEEN 4 AND 4", + "SELECT * FROM t WHERE a BETWEEN 2 AND 5", + "SELECT * FROM t WHERE a BETWEEN 5 AND 4", + "SELECT * FROM t WHERE a BETWEEN 2 AND 6", + "SELECT * FROM t WHERE a BETWEEN 6 AND 4", + "SELECT * FROM t WHERE a BETWEEN 2 AND 7", + "SELECT * FROM t WHERE a BETWEEN 7 AND 4" + ] + }, + { + "name": "TestRangePartitionBoundariesLtM", + "cases": [ + "INSERT INTO t VALUES (999998, '999998 Filler ...'), (999999, '999999 Filler ...'), (1000000, '1000000 Filler ...'), (1000001, '1000001 Filler ...'), (1000002, '1000002 Filler ...')", + "INSERT INTO t VALUES (1999998, '1999998 Filler ...'), (1999999, '1999999 Filler ...'), (2000000, '2000000 Filler ...'), (2000001, '2000001 Filler ...'), (2000002, '2000002 Filler ...')", + "INSERT INTO t VALUES (2999998, '2999998 Filler ...'), (2999999, '2999999 Filler ...')", + "INSERT INTO t VALUES (-2147483648, 'MIN_INT filler...'), (0, '0 Filler...')", + "ANALYZE TABLE t", + "SELECT * FROM t WHERE a < -2147483648", + "SELECT * FROM t WHERE a > -2147483648", + "SELECT * FROM t WHERE a <= -2147483648", + "SELECT * FROM t WHERE a >= -2147483648", + "SELECT * FROM t WHERE a < 0", + "SELECT * FROM t WHERE a > 0", + "SELECT * FROM t WHERE a <= 0", + "SELECT * FROM t WHERE a >= 0", + "SELECT * FROM t WHERE a < 999998", + "SELECT * FROM t WHERE a > 999998", + "SELECT * FROM t WHERE a <= 999998", + "SELECT * FROM t WHERE a >= 999998", + "SELECT * FROM t WHERE a < 999999", + "SELECT * FROM t WHERE a > 999999", + "SELECT * FROM t WHERE a <= 999999", + "SELECT * FROM t WHERE a >= 999999", + "SELECT * FROM t WHERE a < 1000000", + "SELECT * FROM t WHERE a > 1000000", + "SELECT * FROM t WHERE a <= 1000000", + "SELECT * FROM t WHERE a >= 1000000", + "SELECT * FROM t WHERE a < 1000001", + "SELECT * FROM t WHERE a > 1000001", + "SELECT * FROM t WHERE a <= 1000001", + "SELECT * FROM t WHERE a >= 1000001", + "SELECT * FROM t WHERE a < 1000002", + "SELECT * FROM t WHERE a > 1000002", + "SELECT * FROM t WHERE a <= 1000002", + "SELECT * FROM t WHERE a >= 1000002", + "SELECT * FROM t WHERE a < 3000000", + "SELECT * FROM t WHERE a > 3000000", + "SELECT * FROM t WHERE a <= 3000000", + "SELECT * FROM t WHERE a >= 3000000", + "SELECT * FROM t WHERE a < 3000001", + "SELECT * FROM t WHERE a > 3000001", + "SELECT * FROM t WHERE a <= 3000001", + "SELECT * FROM t WHERE a >= 3000001", + "SELECT * FROM t WHERE a < 999997", + "SELECT * FROM t WHERE a > 999997", + "SELECT * FROM t WHERE a <= 999997", + "SELECT * FROM t WHERE a >= 999997", + "SELECT * FROM t WHERE a >= 999997 AND a <= 999999", + "SELECT * FROM t WHERE a > 999997 AND a <= 999999", + "SELECT * FROM t WHERE a > 999997 AND a < 999999", + "SELECT * FROM t WHERE a > 999997 AND a <= 999999", + "SELECT * FROM t WHERE a < 999998", + "SELECT * FROM t WHERE a > 999998", + "SELECT * FROM t WHERE a <= 999998", + "SELECT * FROM t WHERE a >= 999998", + "SELECT * FROM t WHERE a >= 999998 AND a <= 1000000", + "SELECT * FROM t WHERE a > 999998 AND a <= 1000000", + "SELECT * FROM t WHERE a > 999998 AND a < 1000000", + "SELECT * FROM t WHERE a > 999998 AND a <= 1000000", + "SELECT * FROM t WHERE a < 999999", + "SELECT * FROM t WHERE a > 999999", + "SELECT * FROM t WHERE a <= 999999", + "SELECT * FROM t WHERE a >= 999999", + "SELECT * FROM t WHERE a >= 999999 AND a <= 1000001", + "SELECT * FROM t WHERE a > 999999 AND a <= 1000001", + "SELECT * FROM t WHERE a > 999999 AND a < 1000001", + "SELECT * FROM t WHERE a > 999999 AND a <= 1000001", + "SELECT * FROM t WHERE a < 1000000", + "SELECT * FROM t WHERE a > 1000000", + "SELECT * FROM t WHERE a <= 1000000", + "SELECT * FROM t WHERE a >= 1000000", + "SELECT * FROM t WHERE a >= 1000000 AND a <= 1000002", + "SELECT * FROM t WHERE a > 1000000 AND a <= 1000002", + "SELECT * FROM t WHERE a > 1000000 AND a < 1000002", + "SELECT * FROM t WHERE a > 1000000 AND a <= 1000002", + "SELECT * FROM t WHERE a < 1999997", + "SELECT * FROM t WHERE a > 1999997", + "SELECT * FROM t WHERE a <= 1999997", + "SELECT * FROM t WHERE a >= 1999997", + "SELECT * FROM t WHERE a >= 1999997 AND a <= 1999999", + "SELECT * FROM t WHERE a > 1999997 AND a <= 1999999", + "SELECT * FROM t WHERE a > 1999997 AND a < 1999999", + "SELECT * FROM t WHERE a > 1999997 AND a <= 1999999", + "SELECT * FROM t WHERE a < 1999998", + "SELECT * FROM t WHERE a > 1999998", + "SELECT * FROM t WHERE a <= 1999998", + "SELECT * FROM t WHERE a >= 1999998", + "SELECT * FROM t WHERE a >= 1999998 AND a <= 2000000", + "SELECT * FROM t WHERE a > 1999998 AND a <= 2000000", + "SELECT * FROM t WHERE a > 1999998 AND a < 2000000", + "SELECT * FROM t WHERE a > 1999998 AND a <= 2000000", + "SELECT * FROM t WHERE a < 1999999", + "SELECT * FROM t WHERE a > 1999999", + "SELECT * FROM t WHERE a <= 1999999", + "SELECT * FROM t WHERE a >= 1999999", + "SELECT * FROM t WHERE a >= 1999999 AND a <= 2000001", + "SELECT * FROM t WHERE a > 1999999 AND a <= 2000001", + "SELECT * FROM t WHERE a > 1999999 AND a < 2000001", + "SELECT * FROM t WHERE a > 1999999 AND a <= 2000001", + "SELECT * FROM t WHERE a < 2000000", + "SELECT * FROM t WHERE a > 2000000", + "SELECT * FROM t WHERE a <= 2000000", + "SELECT * FROM t WHERE a >= 2000000", + "SELECT * FROM t WHERE a >= 2000000 AND a <= 2000002", + "SELECT * FROM t WHERE a > 2000000 AND a <= 2000002", + "SELECT * FROM t WHERE a > 2000000 AND a < 2000002", + "SELECT * FROM t WHERE a > 2000000 AND a <= 2000002", + "SELECT * FROM t WHERE a < 2999997", + "SELECT * FROM t WHERE a > 2999997", + "SELECT * FROM t WHERE a <= 2999997", + "SELECT * FROM t WHERE a >= 2999997", + "SELECT * FROM t WHERE a >= 2999997 AND a <= 2999999", + "SELECT * FROM t WHERE a > 2999997 AND a <= 2999999", + "SELECT * FROM t WHERE a > 2999997 AND a < 2999999", + "SELECT * FROM t WHERE a > 2999997 AND a <= 2999999", + "SELECT * FROM t WHERE a < 2999998", + "SELECT * FROM t WHERE a > 2999998", + "SELECT * FROM t WHERE a <= 2999998", + "SELECT * FROM t WHERE a >= 2999998", + "SELECT * FROM t WHERE a >= 2999998 AND a <= 3000000", + "SELECT * FROM t WHERE a > 2999998 AND a <= 3000000", + "SELECT * FROM t WHERE a > 2999998 AND a < 3000000", + "SELECT * FROM t WHERE a > 2999998 AND a <= 3000000", + "SELECT * FROM t WHERE a < 2999999", + "SELECT * FROM t WHERE a > 2999999", + "SELECT * FROM t WHERE a <= 2999999", + "SELECT * FROM t WHERE a >= 2999999", + "SELECT * FROM t WHERE a >= 2999999 AND a <= 3000001", + "SELECT * FROM t WHERE a > 2999999 AND a <= 3000001", + "SELECT * FROM t WHERE a > 2999999 AND a < 3000001", + "SELECT * FROM t WHERE a > 2999999 AND a <= 3000001", + "SELECT * FROM t WHERE a < 3000000", + "SELECT * FROM t WHERE a > 3000000", + "SELECT * FROM t WHERE a <= 3000000", + "SELECT * FROM t WHERE a >= 3000000", + "SELECT * FROM t WHERE a >= 3000000 AND a <= 3000002", + "SELECT * FROM t WHERE a > 3000000 AND a <= 3000002", + "SELECT * FROM t WHERE a > 3000000 AND a < 3000002", + "SELECT * FROM t WHERE a > 3000000 AND a <= 3000002" + ] + }, + { + "name": "TestRangePartitionBoundariesLtS", + "cases": [ + "INSERT INTO t VALUES (0, '0 Filler...')", + "INSERT INTO t VALUES (1, '1 Filler...')", + "INSERT INTO t VALUES (2, '2 Filler...')", + "INSERT INTO t VALUES (3, '3 Filler...')", + "INSERT INTO t VALUES (4, '4 Filler...')", + "INSERT INTO t VALUES (5, '5 Filler...')", + "INSERT INTO t VALUES (6, '6 Filler...')", + "ANALYZE TABLE t", + "SELECT * FROM t WHERE a < -1", + "SELECT * FROM t WHERE a > -1", + "SELECT * FROM t WHERE a <= -1", + "SELECT * FROM t WHERE a >= -1", + "SELECT * FROM t WHERE a < 2 OR a > -1", + "SELECT * FROM t WHERE a > 2 AND a < -1", + "SELECT * FROM t WHERE NOT (a < 2 OR a > -1)", + "SELECT * FROM t WHERE NOT (a > 2 AND a < -1)", + "SELECT * FROM t WHERE a < 2 OR a >= -1", + "SELECT * FROM t WHERE a >= 2 AND a < -1", + "SELECT * FROM t WHERE NOT (a < 2 OR a >= -1)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a < -1)", + "SELECT * FROM t WHERE a <= 2 OR a > -1", + "SELECT * FROM t WHERE a > 2 AND a <= -1", + "SELECT * FROM t WHERE NOT (a <= 2 OR a > -1)", + "SELECT * FROM t WHERE NOT (a > 2 AND a <= -1)", + "SELECT * FROM t WHERE a <= 2 OR a >= -1", + "SELECT * FROM t WHERE a >= 2 AND a <= -1", + "SELECT * FROM t WHERE NOT (a <= 2 OR a >= -1)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a <= -1)", + "SELECT * FROM t WHERE a < 0", + "SELECT * FROM t WHERE a > 0", + "SELECT * FROM t WHERE a <= 0", + "SELECT * FROM t WHERE a >= 0", + "SELECT * FROM t WHERE a < 2 OR a > 0", + "SELECT * FROM t WHERE a > 2 AND a < 0", + "SELECT * FROM t WHERE NOT (a < 2 OR a > 0)", + "SELECT * FROM t WHERE NOT (a > 2 AND a < 0)", + "SELECT * FROM t WHERE a < 2 OR a >= 0", + "SELECT * FROM t WHERE a >= 2 AND a < 0", + "SELECT * FROM t WHERE NOT (a < 2 OR a >= 0)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a < 0)", + "SELECT * FROM t WHERE a <= 2 OR a > 0", + "SELECT * FROM t WHERE a > 2 AND a <= 0", + "SELECT * FROM t WHERE NOT (a <= 2 OR a > 0)", + "SELECT * FROM t WHERE NOT (a > 2 AND a <= 0)", + "SELECT * FROM t WHERE a <= 2 OR a >= 0", + "SELECT * FROM t WHERE a >= 2 AND a <= 0", + "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 0)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 0)", + "SELECT * FROM t WHERE a < 1", + "SELECT * FROM t WHERE a > 1", + "SELECT * FROM t WHERE a <= 1", + "SELECT * FROM t WHERE a >= 1", + "SELECT * FROM t WHERE a < 2 OR a > 1", + "SELECT * FROM t WHERE a > 2 AND a < 1", + "SELECT * FROM t WHERE NOT (a < 2 OR a > 1)", + "SELECT * FROM t WHERE NOT (a > 2 AND a < 1)", + "SELECT * FROM t WHERE a < 2 OR a >= 1", + "SELECT * FROM t WHERE a >= 2 AND a < 1", + "SELECT * FROM t WHERE NOT (a < 2 OR a >= 1)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a < 1)", + "SELECT * FROM t WHERE a <= 2 OR a > 1", + "SELECT * FROM t WHERE a > 2 AND a <= 1", + "SELECT * FROM t WHERE NOT (a <= 2 OR a > 1)", + "SELECT * FROM t WHERE NOT (a > 2 AND a <= 1)", + "SELECT * FROM t WHERE a <= 2 OR a >= 1", + "SELECT * FROM t WHERE a >= 2 AND a <= 1", + "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 1)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 1)", + "SELECT * FROM t WHERE a < 2", + "SELECT * FROM t WHERE a > 2", + "SELECT * FROM t WHERE a <= 2", + "SELECT * FROM t WHERE a >= 2", + "SELECT * FROM t WHERE a < 2 OR a > 2", + "SELECT * FROM t WHERE a > 2 AND a < 2", + "SELECT * FROM t WHERE NOT (a < 2 OR a > 2)", + "SELECT * FROM t WHERE NOT (a > 2 AND a < 2)", + "SELECT * FROM t WHERE a < 2 OR a >= 2", + "SELECT * FROM t WHERE a >= 2 AND a < 2", + "SELECT * FROM t WHERE NOT (a < 2 OR a >= 2)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a < 2)", + "SELECT * FROM t WHERE a <= 2 OR a > 2", + "SELECT * FROM t WHERE a > 2 AND a <= 2", + "SELECT * FROM t WHERE NOT (a <= 2 OR a > 2)", + "SELECT * FROM t WHERE NOT (a > 2 AND a <= 2)", + "SELECT * FROM t WHERE a <= 2 OR a >= 2", + "SELECT * FROM t WHERE a >= 2 AND a <= 2", + "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 2)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 2)", + "SELECT * FROM t WHERE a < 3", + "SELECT * FROM t WHERE a > 3", + "SELECT * FROM t WHERE a <= 3", + "SELECT * FROM t WHERE a >= 3", + "SELECT * FROM t WHERE a < 2 OR a > 3", + "SELECT * FROM t WHERE a > 2 AND a < 3", + "SELECT * FROM t WHERE NOT (a < 2 OR a > 3)", + "SELECT * FROM t WHERE NOT (a > 2 AND a < 3)", + "SELECT * FROM t WHERE a < 2 OR a >= 3", + "SELECT * FROM t WHERE a >= 2 AND a < 3", + "SELECT * FROM t WHERE NOT (a < 2 OR a >= 3)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a < 3)", + "SELECT * FROM t WHERE a <= 2 OR a > 3", + "SELECT * FROM t WHERE a > 2 AND a <= 3", + "SELECT * FROM t WHERE NOT (a <= 2 OR a > 3)", + "SELECT * FROM t WHERE NOT (a > 2 AND a <= 3)", + "SELECT * FROM t WHERE a <= 2 OR a >= 3", + "SELECT * FROM t WHERE a >= 2 AND a <= 3", + "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 3)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 3)", + "SELECT * FROM t WHERE a < 4", + "SELECT * FROM t WHERE a > 4", + "SELECT * FROM t WHERE a <= 4", + "SELECT * FROM t WHERE a >= 4", + "SELECT * FROM t WHERE a < 2 OR a > 4", + "SELECT * FROM t WHERE a > 2 AND a < 4", + "SELECT * FROM t WHERE NOT (a < 2 OR a > 4)", + "SELECT * FROM t WHERE NOT (a > 2 AND a < 4)", + "SELECT * FROM t WHERE a < 2 OR a >= 4", + "SELECT * FROM t WHERE a >= 2 AND a < 4", + "SELECT * FROM t WHERE NOT (a < 2 OR a >= 4)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a < 4)", + "SELECT * FROM t WHERE a <= 2 OR a > 4", + "SELECT * FROM t WHERE a > 2 AND a <= 4", + "SELECT * FROM t WHERE NOT (a <= 2 OR a > 4)", + "SELECT * FROM t WHERE NOT (a > 2 AND a <= 4)", + "SELECT * FROM t WHERE a <= 2 OR a >= 4", + "SELECT * FROM t WHERE a >= 2 AND a <= 4", + "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 4)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 4)", + "SELECT * FROM t WHERE a < 5", + "SELECT * FROM t WHERE a > 5", + "SELECT * FROM t WHERE a <= 5", + "SELECT * FROM t WHERE a >= 5", + "SELECT * FROM t WHERE a < 2 OR a > 5", + "SELECT * FROM t WHERE a > 2 AND a < 5", + "SELECT * FROM t WHERE NOT (a < 2 OR a > 5)", + "SELECT * FROM t WHERE NOT (a > 2 AND a < 5)", + "SELECT * FROM t WHERE a < 2 OR a >= 5", + "SELECT * FROM t WHERE a >= 2 AND a < 5", + "SELECT * FROM t WHERE NOT (a < 2 OR a >= 5)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a < 5)", + "SELECT * FROM t WHERE a <= 2 OR a > 5", + "SELECT * FROM t WHERE a > 2 AND a <= 5", + "SELECT * FROM t WHERE NOT (a <= 2 OR a > 5)", + "SELECT * FROM t WHERE NOT (a > 2 AND a <= 5)", + "SELECT * FROM t WHERE a <= 2 OR a >= 5", + "SELECT * FROM t WHERE a >= 2 AND a <= 5", + "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 5)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 5)", + "SELECT * FROM t WHERE a < 6", + "SELECT * FROM t WHERE a > 6", + "SELECT * FROM t WHERE a <= 6", + "SELECT * FROM t WHERE a >= 6", + "SELECT * FROM t WHERE a < 2 OR a > 6", + "SELECT * FROM t WHERE a > 2 AND a < 6", + "SELECT * FROM t WHERE NOT (a < 2 OR a > 6)", + "SELECT * FROM t WHERE NOT (a > 2 AND a < 6)", + "SELECT * FROM t WHERE a < 2 OR a >= 6", + "SELECT * FROM t WHERE a >= 2 AND a < 6", + "SELECT * FROM t WHERE NOT (a < 2 OR a >= 6)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a < 6)", + "SELECT * FROM t WHERE a <= 2 OR a > 6", + "SELECT * FROM t WHERE a > 2 AND a <= 6", + "SELECT * FROM t WHERE NOT (a <= 2 OR a > 6)", + "SELECT * FROM t WHERE NOT (a > 2 AND a <= 6)", + "SELECT * FROM t WHERE a <= 2 OR a >= 6", + "SELECT * FROM t WHERE a >= 2 AND a <= 6", + "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 6)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 6)", + "SELECT * FROM t WHERE a < 7", + "SELECT * FROM t WHERE a > 7", + "SELECT * FROM t WHERE a <= 7", + "SELECT * FROM t WHERE a >= 7", + "SELECT * FROM t WHERE a < 2 OR a > 7", + "SELECT * FROM t WHERE a > 2 AND a < 7", + "SELECT * FROM t WHERE NOT (a < 2 OR a > 7)", + "SELECT * FROM t WHERE NOT (a > 2 AND a < 7)", + "SELECT * FROM t WHERE a < 2 OR a >= 7", + "SELECT * FROM t WHERE a >= 2 AND a < 7", + "SELECT * FROM t WHERE NOT (a < 2 OR a >= 7)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a < 7)", + "SELECT * FROM t WHERE a <= 2 OR a > 7", + "SELECT * FROM t WHERE a > 2 AND a <= 7", + "SELECT * FROM t WHERE NOT (a <= 2 OR a > 7)", + "SELECT * FROM t WHERE NOT (a > 2 AND a <= 7)", + "SELECT * FROM t WHERE a <= 2 OR a >= 7", + "SELECT * FROM t WHERE a >= 2 AND a <= 7", + "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 7)", + "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 7)" + ] } ] diff --git a/executor/testdata/executor_suite_out.json b/executor/testdata/executor_suite_out.json index 2be3c8ea4894f..eab098751f2ba 100644 --- a/executor/testdata/executor_suite_out.json +++ b/executor/testdata/executor_suite_out.json @@ -598,5 +598,5776 @@ ] } ] + }, + { + "Name": "TestRangePartitionBoundariesEq", + "Cases": [ + { + "SQL": "INSERT INTO t VALUES (999998, '999998 Filler ...'), (999999, '999999 Filler ...'), (1000000, '1000000 Filler ...'), (1000001, '1000001 Filler ...'), (1000002, '1000002 Filler ...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (1999998, '1999998 Filler ...'), (1999999, '1999999 Filler ...'), (2000000, '2000000 Filler ...'), (2000001, '2000001 Filler ...'), (2000002, '2000002 Filler ...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (2999998, '2999998 Filler ...'), (2999999, '2999999 Filler ...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (-2147483648, 'MIN_INT filler...'), (0, '0 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "ANALYZE TABLE t", + "Plan": null, + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a = -2147483648", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (-2147483648)", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a = 0", + "Plan": [ + "p0" + ], + "Res": [ + "0 0 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (0)", + "Plan": [ + "p0" + ], + "Res": [ + "0 0 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a = 999998", + "Plan": [ + "p0" + ], + "Res": [ + "999998 999998 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (999998)", + "Plan": [ + "p0" + ], + "Res": [ + "999998 999998 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a = 999999", + "Plan": [ + "p0" + ], + "Res": [ + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (999999)", + "Plan": [ + "p0" + ], + "Res": [ + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a = 1000000", + "Plan": [ + "p1" + ], + "Res": [ + "1000000 1000000 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (1000000)", + "Plan": [ + "p1" + ], + "Res": [ + "1000000 1000000 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a = 1000001", + "Plan": [ + "p1" + ], + "Res": [ + "1000001 1000001 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (1000001)", + "Plan": [ + "p1" + ], + "Res": [ + "1000001 1000001 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a = 1000002", + "Plan": [ + "p1" + ], + "Res": [ + "1000002 1000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (1000002)", + "Plan": [ + "p1" + ], + "Res": [ + "1000002 1000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a = 3000000", + "Plan": null, + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a IN (3000000)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a = 3000001", + "Plan": null, + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a IN (3000001)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a IN (-2147483648, -2147483647)", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (-2147483647, -2147483646)", + "Plan": [ + "p0" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a IN (999997, 999998, 999999)", + "Plan": [ + "p0" + ], + "Res": [ + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (999998, 999999, 1000000)", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (999999, 1000000, 1000001)", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (1000000, 1000001, 1000002)", + "Plan": [ + "p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (1999997, 1999998, 1999999)", + "Plan": [ + "p1" + ], + "Res": [ + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (1999998, 1999999, 2000000)", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (1999999, 2000000, 2000001)", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (2000000, 2000001, 2000002)", + "Plan": [ + "p2" + ], + "Res": [ + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (2999997, 2999998, 2999999)", + "Plan": [ + "p2" + ], + "Res": [ + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (2999998, 2999999, 3000000)", + "Plan": [ + "p2" + ], + "Res": [ + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (2999999, 3000000, 3000001)", + "Plan": [ + "p2" + ], + "Res": [ + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a IN (3000000, 3000001, 3000002)", + "Plan": [ + "dual" + ], + "Res": null + } + ] + }, + { + "Name": "TestRangePartitionBoundariesNe", + "Cases": [ + { + "SQL": "INSERT INTO t VALUES (0, '0 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (1, '1 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (2, '2 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (3, '3 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (4, '4 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (5, '5 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (6, '6 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "ANALYZE TABLE t", + "Plan": null, + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a != -1", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 1 AND a != -1", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a NOT IN (-2, -1)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 0 OR a = -1", + "Plan": [ + "p0" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a != 0", + "Plan": [ + "all" + ], + "Res": [ + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0", + "Plan": [ + "all" + ], + "Res": [ + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a NOT IN (-2, -1, 0)", + "Plan": [ + "all" + ], + "Res": [ + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0", + "Plan": [ + "p0" + ], + "Res": [ + "0 0 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a != 1", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0 AND a != 1", + "Plan": [ + "all" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a NOT IN (-2, -1, 0, 1)", + "Plan": [ + "all" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0 OR a = 1", + "Plan": [ + "p0 p1" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a != 2", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0 AND a != 1 AND a != 2", + "Plan": [ + "all" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a NOT IN (-2, -1, 0, 1, 2)", + "Plan": [ + "all" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0 OR a = 1 OR a = 2", + "Plan": [ + "p0 p1 p2" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a != 3", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0 AND a != 1 AND a != 2 AND a != 3", + "Plan": [ + "all" + ], + "Res": [ + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a NOT IN (-2, -1, 0, 1, 2, 3)", + "Plan": [ + "all" + ], + "Res": [ + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0 OR a = 1 OR a = 2 OR a = 3", + "Plan": [ + "p0 p1 p2 p3" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a != 4", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0 AND a != 1 AND a != 2 AND a != 3 AND a != 4", + "Plan": [ + "all" + ], + "Res": [ + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a NOT IN (-2, -1, 0, 1, 2, 3, 4)", + "Plan": [ + "all" + ], + "Res": [ + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0 OR a = 1 OR a = 2 OR a = 3 OR a = 4", + "Plan": [ + "p0 p1 p2 p3 p4" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a != 5", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0 AND a != 1 AND a != 2 AND a != 3 AND a != 4 AND a != 5", + "Plan": [ + "all" + ], + "Res": [ + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a NOT IN (-2, -1, 0, 1, 2, 3, 4, 5)", + "Plan": [ + "all" + ], + "Res": [ + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0 OR a = 1 OR a = 2 OR a = 3 OR a = 4 OR a = 5", + "Plan": [ + "p0 p1 p2 p3 p4 p5" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a != 6", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0 AND a != 1 AND a != 2 AND a != 3 AND a != 4 AND a != 5 AND a != 6", + "Plan": [ + "all" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a NOT IN (-2, -1, 0, 1, 2, 3, 4, 5, 6)", + "Plan": [ + "all" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0 OR a = 1 OR a = 2 OR a = 3 OR a = 4 OR a = 5 OR a = 6", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a != 7", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 1 AND a != -1 AND a != 0 AND a != 1 AND a != 2 AND a != 3 AND a != 4 AND a != 5 AND a != 6 AND a != 7", + "Plan": [ + "all" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a NOT IN (-2, -1, 0, 1, 2, 3, 4, 5, 6, 7)", + "Plan": [ + "all" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE 1 = 0 OR a = -1 OR a = 0 OR a = 1 OR a = 2 OR a = 3 OR a = 4 OR a = 5 OR a = 6 OR a = 7", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + } + ] + }, + { + "Name": "TestRangePartitionBoundariesBetweenM", + "Cases": [ + { + "SQL": "INSERT INTO t VALUES (999998, '999998 Filler ...'), (999999, '999999 Filler ...'), (1000000, '1000000 Filler ...'), (1000001, '1000001 Filler ...'), (1000002, '1000002 Filler ...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (1999998, '1999998 Filler ...'), (1999999, '1999999 Filler ...'), (2000000, '2000000 Filler ...'), (2000001, '2000001 Filler ...'), (2000002, '2000002 Filler ...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (2999998, '2999998 Filler ...'), (2999999, '2999999 Filler ...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (-2147483648, 'MIN_INT filler...'), (0, '0 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "ANALYZE TABLE t", + "Plan": null, + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2147483649", + "Plan": [ + "p0" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2147483648", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2147483647", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2147483646", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2147483638", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2146483650", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2146483649", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2146483648", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2146483647", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN -2147483648 AND -2146483646", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 0 AND -1", + "Plan": [ + "p0" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 0 AND 0", + "Plan": [ + "p0" + ], + "Res": [ + "0 0 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 0 AND 1", + "Plan": [ + "p0" + ], + "Res": [ + "0 0 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 0 AND 2", + "Plan": [ + "p0" + ], + "Res": [ + "0 0 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 0 AND 10", + "Plan": [ + "p0" + ], + "Res": [ + "0 0 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 0 AND 999998", + "Plan": [ + "p0" + ], + "Res": [ + "0 0 Filler...", + "999998 999998 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 0 AND 999999", + "Plan": [ + "p0" + ], + "Res": [ + "0 0 Filler...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 0 AND 1000000", + "Plan": [ + "p0 p1" + ], + "Res": [ + "0 0 Filler...", + "1000000 1000000 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 0 AND 1000001", + "Plan": [ + "p0 p1" + ], + "Res": [ + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 0 AND 1000002", + "Plan": [ + "p0 p1" + ], + "Res": [ + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999998 AND 999997", + "Plan": [ + "p0" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999998 AND 999998", + "Plan": [ + "p0" + ], + "Res": [ + "999998 999998 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999998 AND 999999", + "Plan": [ + "p0" + ], + "Res": [ + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999998 AND 1000000", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999998 AND 1000008", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999998 AND 1999996", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999998 AND 1999997", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999998 AND 1999998", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999998 AND 1999999", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999998 AND 2000000", + "Plan": [ + "all" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999999 AND 999998", + "Plan": [ + "p0" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999999 AND 999999", + "Plan": [ + "p0" + ], + "Res": [ + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999999 AND 1000000", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999999 AND 1000001", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999999 AND 1000009", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999999 AND 1999997", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999999 AND 1999998", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999999 AND 1999999", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999999 AND 2000000", + "Plan": [ + "all" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 999999 AND 2000001", + "Plan": [ + "all" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000000 AND 999999", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000000 AND 1000000", + "Plan": [ + "p1" + ], + "Res": [ + "1000000 1000000 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000000 AND 1000001", + "Plan": [ + "p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000000 AND 1000002", + "Plan": [ + "p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000000 AND 1000010", + "Plan": [ + "p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000000 AND 1999998", + "Plan": [ + "p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000000 AND 1999999", + "Plan": [ + "p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000000 AND 2000000", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000000 AND 2000001", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000000 AND 2000002", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000001 AND 1000000", + "Plan": [ + "p1" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000001 AND 1000001", + "Plan": [ + "p1" + ], + "Res": [ + "1000001 1000001 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000001 AND 1000002", + "Plan": [ + "p1" + ], + "Res": [ + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000001 AND 1000003", + "Plan": [ + "p1" + ], + "Res": [ + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000001 AND 1000011", + "Plan": [ + "p1" + ], + "Res": [ + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000001 AND 1999999", + "Plan": [ + "p1" + ], + "Res": [ + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000001 AND 2000000", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000001 AND 2000001", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000001 AND 2000002", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000001 AND 2000003", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000002 AND 1000001", + "Plan": [ + "p1" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000002 AND 1000002", + "Plan": [ + "p1" + ], + "Res": [ + "1000002 1000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000002 AND 1000003", + "Plan": [ + "p1" + ], + "Res": [ + "1000002 1000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000002 AND 1000004", + "Plan": [ + "p1" + ], + "Res": [ + "1000002 1000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000002 AND 1000012", + "Plan": [ + "p1" + ], + "Res": [ + "1000002 1000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000002 AND 2000000", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000002 AND 2000001", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000002 AND 2000002", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000002 AND 2000003", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1000002 AND 2000004", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000000 AND 2999999", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000000 AND 3000000", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000000 AND 3000001", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000000 AND 3000002", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000000 AND 3000010", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000000 AND 3999998", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000000 AND 3999999", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000000 AND 4000000", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000000 AND 4000001", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000000 AND 4000002", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000001 AND 3000000", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000001 AND 3000001", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000001 AND 3000002", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000001 AND 3000003", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000001 AND 3000011", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000001 AND 3999999", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000001 AND 4000000", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000001 AND 4000001", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000001 AND 4000002", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3000001 AND 4000003", + "Plan": [ + "dual" + ], + "Res": null + } + ] + }, + { + "Name": "TestRangePartitionBoundariesBetweenS", + "Cases": [ + { + "SQL": "INSERT INTO t VALUES (0, '0 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (1, '1 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (2, '2 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (3, '3 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (4, '4 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (5, '5 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (6, '6 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "ANALYZE TABLE t", + "Plan": null, + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 2 AND -1", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN -1 AND 4", + "Plan": [ + "p0 p1 p2 p3 p4" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 2 AND 0", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 0 AND 4", + "Plan": [ + "p0 p1 p2 p3 p4" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 2 AND 1", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 1 AND 4", + "Plan": [ + "p1 p2 p3 p4" + ], + "Res": [ + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 2 AND 2", + "Plan": [ + "p2" + ], + "Res": [ + "2 2 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 2 AND 4", + "Plan": [ + "p2 p3 p4" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 2 AND 3", + "Plan": [ + "p2 p3" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 3 AND 4", + "Plan": [ + "p3 p4" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 2 AND 4", + "Plan": [ + "p2 p3 p4" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 4 AND 4", + "Plan": [ + "p4" + ], + "Res": [ + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 2 AND 5", + "Plan": [ + "p2 p3 p4 p5" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 5 AND 4", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 2 AND 6", + "Plan": [ + "p2 p3 p4 p5 p6" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 6 AND 4", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 2 AND 7", + "Plan": [ + "p2 p3 p4 p5 p6" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a BETWEEN 7 AND 4", + "Plan": [ + "dual" + ], + "Res": null + } + ] + }, + { + "Name": "TestRangePartitionBoundariesLtM", + "Cases": [ + { + "SQL": "INSERT INTO t VALUES (999998, '999998 Filler ...'), (999999, '999999 Filler ...'), (1000000, '1000000 Filler ...'), (1000001, '1000001 Filler ...'), (1000002, '1000002 Filler ...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (1999998, '1999998 Filler ...'), (1999999, '1999999 Filler ...'), (2000000, '2000000 Filler ...'), (2000001, '2000001 Filler ...'), (2000002, '2000002 Filler ...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (2999998, '2999998 Filler ...'), (2999999, '2999999 Filler ...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (-2147483648, 'MIN_INT filler...'), (0, '0 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "ANALYZE TABLE t", + "Plan": null, + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a < -2147483648", + "Plan": [ + "p0" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a > -2147483648", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= -2147483648", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= -2147483648", + "Plan": [ + "all" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 0", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 0", + "Plan": [ + "all" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 0", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 0", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 999998", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 999998", + "Plan": [ + "all" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 999998", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "999998 999998 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 999998", + "Plan": [ + "all" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 999999", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "999998 999998 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 999999", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 999999", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 999999", + "Plan": [ + "all" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 1000000", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1000000", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 1000000", + "Plan": [ + "p0 p1" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 1000000", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 1000001", + "Plan": [ + "p0 p1" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1000001", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 1000001", + "Plan": [ + "p0 p1" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 1000001", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 1000002", + "Plan": [ + "p0 p1" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1000002", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 1000002", + "Plan": [ + "p0 p1" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 1000002", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 3000000", + "Plan": [ + "all" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 3000000", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a <= 3000000", + "Plan": [ + "all" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 3000000", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a < 3000001", + "Plan": [ + "all" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 3000001", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a <= 3000001", + "Plan": [ + "all" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 3000001", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a < 999997", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 999997", + "Plan": [ + "all" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 999997", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 999997", + "Plan": [ + "all" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 999997 AND a <= 999999", + "Plan": [ + "p0" + ], + "Res": [ + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 999997 AND a <= 999999", + "Plan": [ + "p0" + ], + "Res": [ + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 999997 AND a < 999999", + "Plan": [ + "p0" + ], + "Res": [ + "999998 999998 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 999997 AND a <= 999999", + "Plan": [ + "p0" + ], + "Res": [ + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 999998", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 999998", + "Plan": [ + "all" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 999998", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "999998 999998 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 999998", + "Plan": [ + "all" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 999998 AND a <= 1000000", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 999998 AND a <= 1000000", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 999998 AND a < 1000000", + "Plan": [ + "p0" + ], + "Res": [ + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 999998 AND a <= 1000000", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 999999", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "999998 999998 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 999999", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 999999", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 999999", + "Plan": [ + "all" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 999999 AND a <= 1000001", + "Plan": [ + "p0 p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 999999 AND a <= 1000001", + "Plan": [ + "p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 999999 AND a < 1000001", + "Plan": [ + "p1" + ], + "Res": [ + "1000000 1000000 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 999999 AND a <= 1000001", + "Plan": [ + "p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 1000000", + "Plan": [ + "p0" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1000000", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 1000000", + "Plan": [ + "p0 p1" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 1000000", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 1000000 AND a <= 1000002", + "Plan": [ + "p1" + ], + "Res": [ + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1000000 AND a <= 1000002", + "Plan": [ + "p1" + ], + "Res": [ + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1000000 AND a < 1000002", + "Plan": [ + "p1" + ], + "Res": [ + "1000001 1000001 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1000000 AND a <= 1000002", + "Plan": [ + "p1" + ], + "Res": [ + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 1999997", + "Plan": [ + "p0 p1" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1999997", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 1999997", + "Plan": [ + "p0 p1" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 1999997", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 1999997 AND a <= 1999999", + "Plan": [ + "p1" + ], + "Res": [ + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1999997 AND a <= 1999999", + "Plan": [ + "p1" + ], + "Res": [ + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1999997 AND a < 1999999", + "Plan": [ + "p1" + ], + "Res": [ + "1999998 1999998 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1999997 AND a <= 1999999", + "Plan": [ + "p1" + ], + "Res": [ + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 1999998", + "Plan": [ + "p0 p1" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1999998", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 1999998", + "Plan": [ + "p0 p1" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 1999998", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 1999998 AND a <= 2000000", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1999998 AND a <= 2000000", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1999998 AND a < 2000000", + "Plan": [ + "p1" + ], + "Res": [ + "1999999 1999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1999998 AND a <= 2000000", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 1999999", + "Plan": [ + "p0 p1" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1999999", + "Plan": [ + "p2" + ], + "Res": [ + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 1999999", + "Plan": [ + "p0 p1" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 1999999", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 1999999 AND a <= 2000001", + "Plan": [ + "p1 p2" + ], + "Res": [ + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1999999 AND a <= 2000001", + "Plan": [ + "p2" + ], + "Res": [ + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1999999 AND a < 2000001", + "Plan": [ + "p2" + ], + "Res": [ + "2000000 2000000 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1999999 AND a <= 2000001", + "Plan": [ + "p2" + ], + "Res": [ + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2000000", + "Plan": [ + "p0 p1" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2000000", + "Plan": [ + "p2" + ], + "Res": [ + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2000000", + "Plan": [ + "all" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2000000", + "Plan": [ + "p2" + ], + "Res": [ + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2000000 AND a <= 2000002", + "Plan": [ + "p2" + ], + "Res": [ + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2000000 AND a <= 2000002", + "Plan": [ + "p2" + ], + "Res": [ + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2000000 AND a < 2000002", + "Plan": [ + "p2" + ], + "Res": [ + "2000001 2000001 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2000000 AND a <= 2000002", + "Plan": [ + "p2" + ], + "Res": [ + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2999997", + "Plan": [ + "all" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2999997", + "Plan": [ + "p2" + ], + "Res": [ + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2999997", + "Plan": [ + "all" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2999997", + "Plan": [ + "p2" + ], + "Res": [ + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2999997 AND a <= 2999999", + "Plan": [ + "p2" + ], + "Res": [ + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2999997 AND a <= 2999999", + "Plan": [ + "p2" + ], + "Res": [ + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2999997 AND a < 2999999", + "Plan": [ + "p2" + ], + "Res": [ + "2999998 2999998 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2999997 AND a <= 2999999", + "Plan": [ + "p2" + ], + "Res": [ + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2999998", + "Plan": [ + "all" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2999998", + "Plan": [ + "p2" + ], + "Res": [ + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2999998", + "Plan": [ + "all" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2999998", + "Plan": [ + "p2" + ], + "Res": [ + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2999998 AND a <= 3000000", + "Plan": [ + "p2" + ], + "Res": [ + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2999998 AND a <= 3000000", + "Plan": [ + "p2" + ], + "Res": [ + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2999998 AND a < 3000000", + "Plan": [ + "p2" + ], + "Res": [ + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2999998 AND a <= 3000000", + "Plan": [ + "p2" + ], + "Res": [ + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2999999", + "Plan": [ + "all" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2999999", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2999999", + "Plan": [ + "all" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2999999", + "Plan": [ + "p2" + ], + "Res": [ + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2999999 AND a <= 3000001", + "Plan": [ + "p2" + ], + "Res": [ + "2999999 2999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2999999 AND a <= 3000001", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a > 2999999 AND a < 3000001", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a > 2999999 AND a <= 3000001", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a < 3000000", + "Plan": [ + "all" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 3000000", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a <= 3000000", + "Plan": [ + "all" + ], + "Res": [ + "-2147483648 MIN_INT filler...", + "0 0 Filler...", + "1000000 1000000 Filler ...", + "1000001 1000001 Filler ...", + "1000002 1000002 Filler ...", + "1999998 1999998 Filler ...", + "1999999 1999999 Filler ...", + "2000000 2000000 Filler ...", + "2000001 2000001 Filler ...", + "2000002 2000002 Filler ...", + "2999998 2999998 Filler ...", + "2999999 2999999 Filler ...", + "999998 999998 Filler ...", + "999999 999999 Filler ..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 3000000", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a >= 3000000 AND a <= 3000002", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a > 3000000 AND a <= 3000002", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a > 3000000 AND a < 3000002", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a > 3000000 AND a <= 3000002", + "Plan": [ + "dual" + ], + "Res": null + } + ] + }, + { + "Name": "TestRangePartitionBoundariesLtS", + "Cases": [ + { + "SQL": "INSERT INTO t VALUES (0, '0 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (1, '1 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (2, '2 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (3, '3 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (4, '4 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (5, '5 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "INSERT INTO t VALUES (6, '6 Filler...')", + "Plan": null, + "Res": null + }, + { + "SQL": "ANALYZE TABLE t", + "Plan": null, + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a < -1", + "Plan": [ + "p0" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a > -1", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= -1", + "Plan": [ + "p0" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a >= -1", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a > -1", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a < -1", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a > -1)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a < -1)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a >= -1", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a < -1", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a >= -1)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a < -1)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a > -1", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a <= -1", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a > -1)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a <= -1)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a >= -1", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a <= -1", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a >= -1)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a <= -1)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 0", + "Plan": [ + "p0" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a > 0", + "Plan": [ + "p1 p2 p3 p4 p5 p6" + ], + "Res": [ + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 0", + "Plan": [ + "p0" + ], + "Res": [ + "0 0 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 0", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a > 0", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a < 0", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a > 0)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a < 0)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a >= 0", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a < 0", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a >= 0)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a < 0)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a > 0", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a <= 0", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a > 0)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a <= 0)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a >= 0", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a <= 0", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 0)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 0)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 1", + "Plan": [ + "p0" + ], + "Res": [ + "0 0 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 1", + "Plan": [ + "p2 p3 p4 p5 p6" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 1", + "Plan": [ + "p0 p1" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 1", + "Plan": [ + "p1 p2 p3 p4 p5 p6" + ], + "Res": [ + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a > 1", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a < 1", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a > 1)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a < 1)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a >= 1", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a < 1", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a >= 1)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a < 1)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a > 1", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a <= 1", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a > 1)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a <= 1)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a >= 1", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a <= 1", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 1)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 1)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2", + "Plan": [ + "p0 p1" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2", + "Plan": [ + "p3 p4 p5 p6" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2", + "Plan": [ + "p0 p1 p2" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2", + "Plan": [ + "p2 p3 p4 p5 p6" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a > 2", + "Plan": [ + "p0 p1 p3 p4 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a < 2", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a > 2)", + "Plan": [ + "p2" + ], + "Res": [ + "2 2 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a < 2)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a >= 2", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a < 2", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a >= 2)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a < 2)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a > 2", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a <= 2", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a > 2)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a <= 2)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a >= 2", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a <= 2", + "Plan": [ + "p2" + ], + "Res": [ + "2 2 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 2)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 2)", + "Plan": [ + "p0 p1 p3 p4 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 3", + "Plan": [ + "p0 p1 p2" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 3", + "Plan": [ + "p4 p5 p6" + ], + "Res": [ + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 3", + "Plan": [ + "p0 p1 p2 p3" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 3", + "Plan": [ + "p3 p4 p5 p6" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a > 3", + "Plan": [ + "p0 p1 p4 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a < 3", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a > 3)", + "Plan": [ + "p2 p3" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a < 3)", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a >= 3", + "Plan": [ + "p0 p1 p3 p4 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a < 3", + "Plan": [ + "p2" + ], + "Res": [ + "2 2 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a >= 3)", + "Plan": [ + "p2" + ], + "Res": [ + "2 2 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a < 3)", + "Plan": [ + "p0 p1 p3 p4 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a > 3", + "Plan": [ + "p0 p1 p2 p4 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a <= 3", + "Plan": [ + "p3" + ], + "Res": [ + "3 3 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a > 3)", + "Plan": [ + "p3" + ], + "Res": [ + "3 3 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a <= 3)", + "Plan": [ + "p0 p1 p2 p4 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a >= 3", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a <= 3", + "Plan": [ + "p2 p3" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 3)", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 3)", + "Plan": [ + "p0 p1 p4 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 4", + "Plan": [ + "p0 p1 p2 p3" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 4", + "Plan": [ + "p5 p6" + ], + "Res": [ + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 4", + "Plan": [ + "p0 p1 p2 p3 p4" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 4", + "Plan": [ + "p4 p5 p6" + ], + "Res": [ + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a > 4", + "Plan": [ + "p0 p1 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a < 4", + "Plan": [ + "p3" + ], + "Res": [ + "3 3 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a > 4)", + "Plan": [ + "p2 p3 p4" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a < 4)", + "Plan": [ + "p0 p1 p2 p4 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a >= 4", + "Plan": [ + "p0 p1 p4 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a < 4", + "Plan": [ + "p2 p3" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a >= 4)", + "Plan": [ + "p2 p3" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a < 4)", + "Plan": [ + "p0 p1 p4 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a > 4", + "Plan": [ + "p0 p1 p2 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a <= 4", + "Plan": [ + "p3 p4" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a > 4)", + "Plan": [ + "p3 p4" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a <= 4)", + "Plan": [ + "p0 p1 p2 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a >= 4", + "Plan": [ + "p0 p1 p2 p4 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a <= 4", + "Plan": [ + "p2 p3 p4" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 4)", + "Plan": [ + "p3" + ], + "Res": [ + "3 3 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 4)", + "Plan": [ + "p0 p1 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 5", + "Plan": [ + "p0 p1 p2 p3 p4" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 5", + "Plan": [ + "p6" + ], + "Res": [ + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 5", + "Plan": [ + "p0 p1 p2 p3 p4 p5" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 5", + "Plan": [ + "p5 p6" + ], + "Res": [ + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a > 5", + "Plan": [ + "p0 p1 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a < 5", + "Plan": [ + "p3 p4" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a > 5)", + "Plan": [ + "p2 p3 p4 p5" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a < 5)", + "Plan": [ + "p0 p1 p2 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a >= 5", + "Plan": [ + "p0 p1 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a < 5", + "Plan": [ + "p2 p3 p4" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a >= 5)", + "Plan": [ + "p2 p3 p4" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a < 5)", + "Plan": [ + "p0 p1 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a > 5", + "Plan": [ + "p0 p1 p2 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a <= 5", + "Plan": [ + "p3 p4 p5" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a > 5)", + "Plan": [ + "p3 p4 p5" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a <= 5)", + "Plan": [ + "p0 p1 p2 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a >= 5", + "Plan": [ + "p0 p1 p2 p5 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a <= 5", + "Plan": [ + "p2 p3 p4 p5" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 5)", + "Plan": [ + "p3 p4" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 5)", + "Plan": [ + "p0 p1 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 6", + "Plan": [ + "p0 p1 p2 p3 p4 p5" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 6", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a <= 6", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 6", + "Plan": [ + "p6" + ], + "Res": [ + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a > 6", + "Plan": [ + "p0 p1" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a < 6", + "Plan": [ + "p3 p4 p5" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a > 6)", + "Plan": [ + "p2 p3 p4 p5 p6" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a < 6)", + "Plan": [ + "p0 p1 p2 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a >= 6", + "Plan": [ + "p0 p1 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a < 6", + "Plan": [ + "p2 p3 p4 p5" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a >= 6)", + "Plan": [ + "p2 p3 p4 p5" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a < 6)", + "Plan": [ + "p0 p1 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a > 6", + "Plan": [ + "p0 p1 p2" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a <= 6", + "Plan": [ + "p3 p4 p5 p6" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a > 6)", + "Plan": [ + "p3 p4 p5 p6" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a <= 6)", + "Plan": [ + "p0 p1 p2" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a >= 6", + "Plan": [ + "p0 p1 p2 p6" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a <= 6", + "Plan": [ + "p2 p3 p4 p5 p6" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 6)", + "Plan": [ + "p3 p4 p5" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 6)", + "Plan": [ + "p0 p1" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 7", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 7", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a <= 7", + "Plan": [ + "all" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 7", + "Plan": [ + "dual" + ], + "Res": null + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a > 7", + "Plan": [ + "p0 p1" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a < 7", + "Plan": [ + "p3 p4 p5 p6" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a > 7)", + "Plan": [ + "p2 p3 p4 p5 p6" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a < 7)", + "Plan": [ + "p0 p1 p2" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a < 2 OR a >= 7", + "Plan": [ + "p0 p1" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a < 7", + "Plan": [ + "p2 p3 p4 p5 p6" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a < 2 OR a >= 7)", + "Plan": [ + "p2 p3 p4 p5 p6" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a < 7)", + "Plan": [ + "p0 p1" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a > 7", + "Plan": [ + "p0 p1 p2" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a > 2 AND a <= 7", + "Plan": [ + "p3 p4 p5 p6" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a > 7)", + "Plan": [ + "p3 p4 p5 p6" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a > 2 AND a <= 7)", + "Plan": [ + "p0 p1 p2" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a <= 2 OR a >= 7", + "Plan": [ + "p0 p1 p2" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler...", + "2 2 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE a >= 2 AND a <= 7", + "Plan": [ + "p2 p3 p4 p5 p6" + ], + "Res": [ + "2 2 Filler...", + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a <= 2 OR a >= 7)", + "Plan": [ + "p3 p4 p5 p6" + ], + "Res": [ + "3 3 Filler...", + "4 4 Filler...", + "5 5 Filler...", + "6 6 Filler..." + ] + }, + { + "SQL": "SELECT * FROM t WHERE NOT (a >= 2 AND a <= 7)", + "Plan": [ + "p0 p1" + ], + "Res": [ + "0 0 Filler...", + "1 1 Filler..." + ] + } + ] } ] diff --git a/executor/tiflash_test.go b/executor/tiflash_test.go index 690106a9a38e5..d4a4f873e6db5 100644 --- a/executor/tiflash_test.go +++ b/executor/tiflash_test.go @@ -15,6 +15,8 @@ package executor_test import ( "fmt" + "math/rand" + "strings" "sync" "sync/atomic" "time" @@ -85,6 +87,9 @@ func (s *tiflashTestSuite) TestReadPartitionTable(c *C) { tk.MustExec("insert into t values(2,0)") tk.MustExec("insert into t values(3,0)") tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"") + // mock executor does not support use outer table as build side for outer join, so need to + // force the inner table as build side + tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1") tk.MustQuery("select /*+ STREAM_AGG() */ count(*) from t").Check(testkit.Rows("3")) tk.MustQuery("select * from t order by a").Check(testkit.Rows("1 0", "2 0", "3 0")) @@ -129,6 +134,9 @@ func (s *tiflashTestSuite) TestReadUnsigedPK(c *C) { tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"") tk.MustExec("set @@session.tidb_allow_mpp=ON") tk.MustExec("set @@session.tidb_opt_broadcast_join=ON") + // mock executor does not support use outer table as build side for outer join, so need to + // force the inner table as build side + tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1") tk.MustQuery("select count(*) from t1 , t where t1.a = t.a").Check(testkit.Rows("5")) tk.MustQuery("select count(*) from t1 , t where t1.a = t.a and ((t1.a < 9223372036854775800 and t1.a > 2) or (t1.a <= 1 and t1.a > -1))").Check(testkit.Rows("3")) @@ -161,6 +169,9 @@ func (s *tiflashTestSuite) TestMppExecution(c *C) { tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"") tk.MustExec("set @@session.tidb_allow_mpp=ON") + // mock executor does not support use outer table as build side for outer join, so need to + // force the inner table as build side + tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1") for i := 0; i < 20; i++ { // test if it is stable. tk.MustQuery("select count(*) from t1 , t where t1.a = t.a").Check(testkit.Rows("3")) @@ -248,6 +259,148 @@ func (s *tiflashTestSuite) TestInjectExtraProj(c *C) { tk.MustQuery("select avg(a), a from t group by a").Check(testkit.Rows("9223372036854775807.0000 9223372036854775807")) } +func (s *tiflashTestSuite) TestTiFlashPartitionTableShuffledHashJoin(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKit(c, s.store) + tk.MustExec(`create database tiflash_partition_SHJ`) + tk.MustExec("use tiflash_partition_SHJ") + tk.MustExec(`create table thash (a int, b int) partition by hash(a) partitions 4`) + tk.MustExec(`create table trange (a int, b int) partition by range(a) ( + partition p0 values less than (100), partition p1 values less than (200), + partition p2 values less than (300), partition p3 values less than (400))`) + listPartitions := make([]string, 4) + for i := 0; i < 400; i++ { + idx := i % 4 + if listPartitions[idx] != "" { + listPartitions[idx] += ", " + } + listPartitions[idx] = listPartitions[idx] + fmt.Sprintf("%v", i) + } + tk.MustExec(`create table tlist (a int, b int) partition by list(a) ( + partition p0 values in (` + listPartitions[0] + `), partition p1 values in (` + listPartitions[1] + `), + partition p2 values in (` + listPartitions[2] + `), partition p3 values in (` + listPartitions[3] + `))`) + tk.MustExec(`create table tnormal (a int, b int)`) + + for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} { + tk.MustExec("alter table " + tbl + " set tiflash replica 1") + tb := testGetTableByName(c, tk.Se, "tiflash_partition_SHJ", tbl) + err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true) + c.Assert(err, IsNil) + } + + vals := make([]string, 0, 100) + for i := 0; i < 100; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(400), rand.Intn(400))) + } + for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} { + tk.MustExec(fmt.Sprintf("insert into %v values %v", tbl, strings.Join(vals, ", "))) + tk.MustExec(fmt.Sprintf("analyze table %v", tbl)) + } + + tk.MustExec("SET tidb_allow_mpp=2") + tk.MustExec("SET tidb_opt_broadcast_join=0") + tk.MustExec("SET tidb_broadcast_join_threshold_count=0") + tk.MustExec("SET tidb_broadcast_join_threshold_size=0") + tk.MustExec("set @@session.tidb_isolation_read_engines='tiflash'") + // mock executor does not support use outer table as build side for outer join, so need to + // force the inner table as build side + tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1") + + lr := func() (int, int) { + l, r := rand.Intn(400), rand.Intn(400) + if l > r { + l, r = r, l + } + return l, r + } + for i := 0; i < 2; i++ { + l1, r1 := lr() + l2, r2 := lr() + cond := fmt.Sprintf("t1.b>=%v and t1.b<=%v and t2.b>=%v and t2.b<=%v", l1, r1, l2, r2) + var res [][]interface{} + for _, mode := range []string{"static", "dynamic"} { + tk.MustExec(fmt.Sprintf("set @@tidb_partition_prune_mode = '%v'", mode)) + for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} { + q := fmt.Sprintf("select count(*) from %v t1 join %v t2 on t1.a=t2.a where %v", tbl, tbl, cond) + if res == nil { + res = tk.MustQuery(q).Sort().Rows() + } else { + tk.MustQuery(q).Check(res) + } + } + } + } +} + +func (s *tiflashTestSuite) TestTiFlashPartitionTableReader(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKit(c, s.store) + tk.MustExec(`create database tiflash_partition_tablereader`) + tk.MustExec("use tiflash_partition_tablereader") + tk.MustExec(`create table thash (a int, b int) partition by hash(a) partitions 4`) + tk.MustExec(`create table trange (a int, b int) partition by range(a) ( + partition p0 values less than (100), partition p1 values less than (200), + partition p2 values less than (300), partition p3 values less than (400))`) + listPartitions := make([]string, 4) + for i := 0; i < 400; i++ { + idx := i % 4 + if listPartitions[idx] != "" { + listPartitions[idx] += ", " + } + listPartitions[idx] = listPartitions[idx] + fmt.Sprintf("%v", i) + } + tk.MustExec(`create table tlist (a int, b int) partition by list(a) ( + partition p0 values in (` + listPartitions[0] + `), partition p1 values in (` + listPartitions[1] + `), + partition p2 values in (` + listPartitions[2] + `), partition p3 values in (` + listPartitions[3] + `))`) + tk.MustExec(`create table tnormal (a int, b int)`) + + for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} { + tk.MustExec("alter table " + tbl + " set tiflash replica 1") + tb := testGetTableByName(c, tk.Se, "tiflash_partition_tablereader", tbl) + err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true) + c.Assert(err, IsNil) + } + // mock executor does not support use outer table as build side for outer join, so need to + // force the inner table as build side + tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1") + + vals := make([]string, 0, 500) + for i := 0; i < 500; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(400), rand.Intn(400))) + } + for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} { + tk.MustExec(fmt.Sprintf("insert into %v values %v", tbl, strings.Join(vals, ", "))) + } + + tk.MustExec("SET tidb_allow_mpp=2") + tk.MustExec("set @@session.tidb_isolation_read_engines='tiflash'") + for i := 0; i < 100; i++ { + l, r := rand.Intn(400), rand.Intn(400) + if l > r { + l, r = r, l + } + cond := fmt.Sprintf("a>=%v and a<=%v", l, r) + var res [][]interface{} + for _, mode := range []string{"static", "dynamic"} { + tk.MustExec(fmt.Sprintf("set @@tidb_partition_prune_mode = '%v'", mode)) + for _, tbl := range []string{"thash", "trange", "tlist", "tnormal"} { + q := fmt.Sprintf("select * from %v where %v", tbl, cond) + if res == nil { + res = tk.MustQuery(q).Sort().Rows() + } else { + tk.MustQuery(q).Sort().Check(res) + } + } + } + } +} + func (s *tiflashTestSuite) TestPartitionTable(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -266,6 +419,9 @@ func (s *tiflashTestSuite) TestPartitionTable(c *C) { failpoint.Enable("github.com/pingcap/tidb/executor/checkUseMPP", `return(true)`) tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"") tk.MustExec("set @@session.tidb_allow_mpp=ON") + // mock executor does not support use outer table as build side for outer join, so need to + // force the inner table as build side + tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1") failpoint.Enable("github.com/pingcap/tidb/executor/checkTotalMPPTasks", `return(4)`) tk.MustQuery("select count(*) from t").Check(testkit.Rows("4")) failpoint.Disable("github.com/pingcap/tidb/executor/checkTotalMPPTasks") @@ -350,6 +506,9 @@ func (s *tiflashTestSuite) TestMppEnum(c *C) { tk.MustExec("insert into t values(3,'zca')") tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"") tk.MustExec("set @@session.tidb_allow_mpp=ON") + // mock executor does not support use outer table as build side for outer join, so need to + // force the inner table as build side + tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1") tk.MustQuery("select t1.b from t t1 join t t2 on t1.a = t2.a order by t1.b").Check(testkit.Rows("aca", "bca", "zca")) } @@ -371,6 +530,9 @@ func (s *tiflashTestSuite) TestCancelMppTasks(c *C) { c.Assert(err, IsNil) tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"") tk.MustExec("set @@session.tidb_allow_mpp=ON") + // mock executor does not support use outer table as build side for outer join, so need to + // force the inner table as build side + tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1") atomic.StoreUint32(&tk.Se.GetSessionVars().Killed, 0) c.Assert(failpoint.Enable(hang, `return(true)`), IsNil) wg := &sync.WaitGroup{} @@ -415,6 +577,9 @@ func (s *tiflashTestSuite) TestMppGoroutinesExitFromErrors(c *C) { tk.MustExec("insert into t1 values(3,0)") tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"") tk.MustExec("set @@session.tidb_allow_mpp=ON") + // mock executor does not support use outer table as build side for outer join, so need to + // force the inner table as build side + tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1") c.Assert(failpoint.Enable(mppNonRootTaskError, `return(true)`), IsNil) c.Assert(failpoint.Enable(hang, `return(true)`), IsNil) @@ -425,6 +590,51 @@ func (s *tiflashTestSuite) TestMppGoroutinesExitFromErrors(c *C) { c.Assert(failpoint.Disable(hang), IsNil) } +func (s *tiflashTestSuite) TestMppUnionAll(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists x1") + tk.MustExec("create table x1(a int , b int);") + tk.MustExec("alter table x1 set tiflash replica 1") + tk.MustExec("drop table if exists x2") + tk.MustExec("create table x2(a int , b int);") + tk.MustExec("alter table x2 set tiflash replica 1") + tb := testGetTableByName(c, tk.Se, "test", "x1") + err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true) + c.Assert(err, IsNil) + tb = testGetTableByName(c, tk.Se, "test", "x2") + err = domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true) + c.Assert(err, IsNil) + + tk.MustExec("insert into x1 values (1, 1), (2, 2), (3, 3), (4, 4)") + tk.MustExec("insert into x2 values (5, 1), (2, 2), (3, 3), (4, 4)") + // mock executor does not support use outer table as build side for outer join, so need to + // force the inner table as build side + tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1") + + // test join + union (join + select) + tk.MustQuery("select x1.a, x.a from x1 left join (select x2.b a, x1.b from x1 join x2 on x1.a = x2.b union all select * from x1 ) x on x1.a = x.a order by x1.a").Check(testkit.Rows("1 1", "1 1", "2 2", "2 2", "3 3", "3 3", "4 4", "4 4")) + tk.MustQuery("select x1.a, x.a from x1 left join (select count(*) a, sum(b) b from x1 group by a union all select * from x2 ) x on x1.a = x.a order by x1.a").Check(testkit.Rows("1 1", "1 1", "1 1", "1 1", "2 2", "3 3", "4 4")) + + tk.MustExec("drop table if exists x3") + tk.MustExec("create table x3(a int , b int);") + tk.MustExec("alter table x3 set tiflash replica 1") + tb = testGetTableByName(c, tk.Se, "test", "x3") + err = domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true) + c.Assert(err, IsNil) + + tk.MustExec("insert into x3 values (2, 2), (2, 3), (2, 4)") + // test nested union all + tk.MustQuery("select count(*) from (select a, b from x1 union all select a, b from x3 union all (select x1.a, x3.b from (select * from x3 union all select * from x2) x3 left join x1 on x3.a = x1.b))").Check(testkit.Rows("14")) + // test union all join union all + tk.MustQuery("select count(*) from (select * from x1 union all select * from x2 union all select * from x3) x join (select * from x1 union all select * from x2 union all select * from x3) y on x.a = y.b").Check(testkit.Rows("29")) + tk.MustExec("set @@session.tidb_broadcast_join_threshold_count=100000") + failpoint.Enable("github.com/pingcap/tidb/executor/checkTotalMPPTasks", `return(6)`) + tk.MustQuery("select count(*) from (select * from x1 union all select * from x2 union all select * from x3) x join (select * from x1 union all select * from x2 union all select * from x3) y on x.a = y.b").Check(testkit.Rows("29")) + failpoint.Disable("github.com/pingcap/tidb/executor/checkTotalMPPTasks") + +} + func (s *tiflashTestSuite) TestMppApply(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -446,6 +656,9 @@ func (s *tiflashTestSuite) TestMppApply(c *C) { tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"") tk.MustExec("set @@session.tidb_allow_mpp=ON") + // mock executor does not support use outer table as build side for outer join, so need to + // force the inner table as build side + tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1") // table full scan with correlated filter tk.MustQuery("select /*+ agg_to_cop(), hash_agg()*/ count(*) from x1 where a >= any (select a from x2 where x1.a = x2.a) order by 1;").Check(testkit.Rows("3")) // table range scan with correlated access conditions @@ -479,8 +692,156 @@ func (s *tiflashTestSuite) TestTiFlashVirtualColumn(c *C) { tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"") tk.MustExec("set @@session.tidb_allow_mpp=ON") + // mock executor does not support use outer table as build side for outer join, so need to + // force the inner table as build side + tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1") tk.MustQuery("select /*+ hash_agg() */ count(*) from t1 where c > b'01'").Check(testkit.Rows("2")) tk.MustQuery("select /*+ hash_agg() */ count(*) from t2 where c > 1").Check(testkit.Rows("2")) tk.MustQuery("select /*+ hash_agg() */ count(*) from t3 where c > b'01'").Check(testkit.Rows("3")) } + +func (s *tiflashTestSuite) TestTiFlashPartitionTableShuffledHashAggregation(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database tiflash_partition_AGG") + tk.MustExec("use tiflash_partition_AGG") + tk.MustExec(`create table thash (a int, b int) partition by hash(a) partitions 4`) + tk.MustExec(`create table trange (a int, b int) partition by range(a) ( + partition p0 values less than (100), partition p1 values less than (200), + partition p2 values less than (300), partition p3 values less than (400))`) + listPartitions := make([]string, 4) + for i := 0; i < 400; i++ { + idx := i % 4 + if listPartitions[idx] != "" { + listPartitions[idx] += ", " + } + listPartitions[idx] = listPartitions[idx] + fmt.Sprintf("%v", i) + } + tk.MustExec(`create table tlist (a int, b int) partition by list(a) ( + partition p0 values in (` + listPartitions[0] + `), partition p1 values in (` + listPartitions[1] + `), + partition p2 values in (` + listPartitions[2] + `), partition p3 values in (` + listPartitions[3] + `))`) + tk.MustExec(`create table tnormal (a int, b int) partition by hash(a) partitions 4`) + + for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} { + tk.MustExec("alter table " + tbl + " set tiflash replica 1") + tb := testGetTableByName(c, tk.Se, "tiflash_partition_AGG", tbl) + err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true) + c.Assert(err, IsNil) + } + + vals := make([]string, 0, 100) + for i := 0; i < 100; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(400), rand.Intn(400))) + } + for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} { + tk.MustExec(fmt.Sprintf("insert into %v values %v", tbl, strings.Join(vals, ", "))) + tk.MustExec(fmt.Sprintf("analyze table %v", tbl)) + } + tk.MustExec("set @@session.tidb_isolation_read_engines='tiflash'") + tk.MustExec("set @@session.tidb_allow_mpp=2") + // mock executor does not support use outer table as build side for outer join, so need to + // force the inner table as build side + tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1") + + lr := func() (int, int) { + l, r := rand.Intn(400), rand.Intn(400) + if l > r { + l, r = r, l + } + return l, r + } + for i := 0; i < 2; i++ { + l1, r1 := lr() + cond := fmt.Sprintf("t1.b>=%v and t1.b<=%v", l1, r1) + var res [][]interface{} + for _, mode := range []string{"static", "dynamic"} { + tk.MustExec(fmt.Sprintf("set @@tidb_partition_prune_mode = '%v'", mode)) + for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} { + q := fmt.Sprintf("select /*+ HASH_AGG() */ count(*) from %v t1 where %v", tbl, cond) + c.Assert(tk.HasPlan(q, "HashAgg"), IsTrue) + if res == nil { + res = tk.MustQuery(q).Sort().Rows() + } else { + tk.MustQuery(q).Check(res) + } + } + } + } +} + +func (s *tiflashTestSuite) TestTiFlashPartitionTableBroadcastJoin(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database tiflash_partition_BCJ") + tk.MustExec("use tiflash_partition_BCJ") + tk.MustExec(`create table thash (a int, b int) partition by hash(a) partitions 4`) + tk.MustExec(`create table trange (a int, b int) partition by range(a) ( + partition p0 values less than (100), partition p1 values less than (200), + partition p2 values less than (300), partition p3 values less than (400))`) + listPartitions := make([]string, 4) + for i := 0; i < 400; i++ { + idx := i % 4 + if listPartitions[idx] != "" { + listPartitions[idx] += ", " + } + listPartitions[idx] = listPartitions[idx] + fmt.Sprintf("%v", i) + } + tk.MustExec(`create table tlist (a int, b int) partition by list(a) ( + partition p0 values in (` + listPartitions[0] + `), partition p1 values in (` + listPartitions[1] + `), + partition p2 values in (` + listPartitions[2] + `), partition p3 values in (` + listPartitions[3] + `))`) + tk.MustExec(`create table tnormal (a int, b int) partition by hash(a) partitions 4`) + + for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} { + tk.MustExec("alter table " + tbl + " set tiflash replica 1") + tb := testGetTableByName(c, tk.Se, "tiflash_partition_BCJ", tbl) + err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true) + c.Assert(err, IsNil) + } + + vals := make([]string, 0, 100) + for i := 0; i < 100; i++ { + vals = append(vals, fmt.Sprintf("(%v, %v)", rand.Intn(400), rand.Intn(400))) + } + for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} { + tk.MustExec(fmt.Sprintf("insert into %v values %v", tbl, strings.Join(vals, ", "))) + tk.MustExec(fmt.Sprintf("analyze table %v", tbl)) + } + tk.MustExec("set @@session.tidb_isolation_read_engines='tiflash'") + tk.MustExec("set @@session.tidb_allow_mpp=2") + tk.MustExec("set @@session.tidb_opt_broadcast_join=ON") + // mock executor does not support use outer table as build side for outer join, so need to + // force the inner table as build side + tk.MustExec("set tidb_opt_mpp_outer_join_fixed_build_side=1") + + lr := func() (int, int) { + l, r := rand.Intn(400), rand.Intn(400) + if l > r { + l, r = r, l + } + return l, r + } + for i := 0; i < 2; i++ { + l1, r1 := lr() + l2, r2 := lr() + cond := fmt.Sprintf("t1.b>=%v and t1.b<=%v and t2.b>=%v and t2.b<=%v", l1, r1, l2, r2) + var res [][]interface{} + for _, mode := range []string{"static", "dynamic"} { + tk.MustExec(fmt.Sprintf("set @@tidb_partition_prune_mode = '%v'", mode)) + for _, tbl := range []string{`thash`, `trange`, `tlist`, `tnormal`} { + q := fmt.Sprintf("select count(*) from %v t1 join %v t2 on t1.a=t2.a where %v", tbl, tbl, cond) + if res == nil { + res = tk.MustQuery(q).Sort().Rows() + } else { + tk.MustQuery(q).Check(res) + } + } + } + } +} diff --git a/executor/update.go b/executor/update.go index b8c7e2a985142..cbc61f6121b88 100644 --- a/executor/update.go +++ b/executor/update.go @@ -23,8 +23,8 @@ import ( "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/kv" plannercore "github.com/pingcap/tidb/planner/core" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/tikv" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" @@ -259,9 +259,15 @@ func (e *UpdateExec) updateRows(ctx context.Context) (int, error) { memUsageOfChk = chk.MemoryUsage() e.memTracker.Consume(memUsageOfChk) if e.collectRuntimeStatsEnabled() { - txn, err := e.ctx.Txn(false) + txn, err := e.ctx.Txn(true) if err == nil && txn.GetSnapshot() != nil { - txn.GetSnapshot().SetOption(tikvstore.CollectRuntimeStats, e.stats.SnapshotRuntimeStats) + txn.GetSnapshot().SetOption(kv.CollectRuntimeStats, e.stats.SnapshotRuntimeStats) + } + } + if variable.TopSQLEnabled() { + txn, err := e.ctx.Txn(true) + if err == nil { + txn.SetOption(kv.ResourceGroupTag, e.ctx.GetSessionVars().StmtCtx.GetResourceGroupTag()) } } for rowIdx := 0; rowIdx < chk.NumRows(); rowIdx++ { @@ -408,7 +414,7 @@ func (e *UpdateExec) Close() error { if e.runtimeStats != nil && e.stats != nil { txn, err := e.ctx.Txn(false) if err == nil && txn.GetSnapshot() != nil { - txn.GetSnapshot().DelOption(tikvstore.CollectRuntimeStats) + txn.GetSnapshot().SetOption(kv.CollectRuntimeStats, nil) } } return e.children[0].Close() diff --git a/executor/window.go b/executor/window.go index 4675902318b7a..19e92687fffda 100644 --- a/executor/window.go +++ b/executor/window.go @@ -319,7 +319,9 @@ func (p *rowFrameWindowProcessor) appendResult2Chunk(ctx sessionctx.Context, row for i, windowFunc := range p.windowFuncs { slidingWindowAggFunc := slidingWindowAggFuncs[i] if slidingWindowAggFunc != nil && initializedSlidingWindow { - err = slidingWindowAggFunc.Slide(ctx, rows, lastStart, lastEnd, shiftStart, shiftEnd, p.partialResults[i]) + err = slidingWindowAggFunc.Slide(ctx, func(u uint64) chunk.Row { + return rows[u] + }, lastStart, lastEnd, shiftStart, shiftEnd, p.partialResults[i]) if err != nil { return nil, err } @@ -335,7 +337,9 @@ func (p *rowFrameWindowProcessor) appendResult2Chunk(ctx sessionctx.Context, row for i, windowFunc := range p.windowFuncs { slidingWindowAggFunc := slidingWindowAggFuncs[i] if slidingWindowAggFunc != nil && initializedSlidingWindow { - err = slidingWindowAggFunc.Slide(ctx, rows, lastStart, lastEnd, shiftStart, shiftEnd, p.partialResults[i]) + err = slidingWindowAggFunc.Slide(ctx, func(u uint64) chunk.Row { + return rows[u] + }, lastStart, lastEnd, shiftStart, shiftEnd, p.partialResults[i]) } else { // For MinMaxSlidingWindowAggFuncs, it needs the absolute value of each start of window, to compare // whether elements inside deque are out of current window. @@ -469,7 +473,9 @@ func (p *rangeFrameWindowProcessor) appendResult2Chunk(ctx sessionctx.Context, r for i, windowFunc := range p.windowFuncs { slidingWindowAggFunc := slidingWindowAggFuncs[i] if slidingWindowAggFunc != nil && initializedSlidingWindow { - err = slidingWindowAggFunc.Slide(ctx, rows, lastStart, lastEnd, shiftStart, shiftEnd, p.partialResults[i]) + err = slidingWindowAggFunc.Slide(ctx, func(u uint64) chunk.Row { + return rows[u] + }, lastStart, lastEnd, shiftStart, shiftEnd, p.partialResults[i]) if err != nil { return nil, err } @@ -485,7 +491,9 @@ func (p *rangeFrameWindowProcessor) appendResult2Chunk(ctx sessionctx.Context, r for i, windowFunc := range p.windowFuncs { slidingWindowAggFunc := slidingWindowAggFuncs[i] if slidingWindowAggFunc != nil && initializedSlidingWindow { - err = slidingWindowAggFunc.Slide(ctx, rows, lastStart, lastEnd, shiftStart, shiftEnd, p.partialResults[i]) + err = slidingWindowAggFunc.Slide(ctx, func(u uint64) chunk.Row { + return rows[u] + }, lastStart, lastEnd, shiftStart, shiftEnd, p.partialResults[i]) } else { if minMaxSlidingWindowAggFunc, ok := windowFunc.(aggfuncs.MaxMinSlidingWindowAggFunc); ok { minMaxSlidingWindowAggFunc.SetWindowStart(start) diff --git a/executor/window_test.go b/executor/window_test.go index e530aa03f0549..970d7f7354100 100644 --- a/executor/window_test.go +++ b/executor/window_test.go @@ -23,10 +23,30 @@ import ( func (s *testSuite7) TestWindowFunctions(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("set @@tidb_window_concurrency = 1") + tk.MustExec("set @@tidb_enable_pipelined_window_function = 0") + defer func() { + tk.MustExec("set @@tidb_enable_pipelined_window_function=1;") + }() doTestWindowFunctions(tk) } func (s *testSuite7) TestWindowParallelFunctions(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("set @@tidb_window_concurrency = 4") + tk.MustExec("set @@tidb_enable_pipelined_window_function = 0") + defer func() { + tk.MustExec("set @@tidb_enable_pipelined_window_function=1;") + }() + doTestWindowFunctions(tk) +} + +func (s *testSuite7) TestPipelinedWindowFunctions(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("set @@tidb_window_concurrency = 1") + doTestWindowFunctions(tk) +} + +func (s *testSuite7) TestPipelinedWindowParallelFunctions(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("set @@tidb_window_concurrency = 4") doTestWindowFunctions(tk) @@ -222,6 +242,25 @@ func (s *testSuite7) TestWindowFunctionsDataReference(c *C) { } func (s *testSuite7) TestSlidingWindowFunctions(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test;") + tk.MustExec("set @@tidb_enable_pipelined_window_function=0;") + defer func() { + tk.MustExec("set @@tidb_enable_pipelined_window_function=1;") + }() + idTypes := []string{"FLOAT", "DOUBLE"} + useHighPrecisions := []string{"ON", "OFF"} + for _, idType := range idTypes { + for _, useHighPrecision := range useHighPrecisions { + tk.MustExec("drop table if exists t;") + tk.MustExec(fmt.Sprintf("CREATE TABLE t (id %s, sex CHAR(1));", idType)) + tk.MustExec(fmt.Sprintf("SET SESSION windowing_use_high_precision = %s;", useHighPrecision)) + baseTestSlidingWindowFunctions(tk) + } + } +} + +func (s *testSuite7) TestPipelinedSlidingWindowFunctions(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test;") idTypes := []string{"FLOAT", "DOUBLE"} @@ -245,6 +284,7 @@ func baseTestSlidingWindowFunctions(tk *testkit.TestKit) { tk.MustExec("insert into t values (5,'M')") tk.MustExec("insert into t values (10,null)") tk.MustExec("insert into t values (11,null)") + tk.MustExec("PREPARE p FROM 'SELECT sex, COUNT(id) OVER (ORDER BY id ROWS BETWEEN ? PRECEDING and ? PRECEDING) FROM t';") tk.MustExec("SET @p1= 1;") tk.MustExec("SET @p2= 2;") diff --git a/executor/write_test.go b/executor/write_test.go index 27ea70ae748a5..7d39df1b71939 100644 --- a/executor/write_test.go +++ b/executor/write_test.go @@ -1554,7 +1554,7 @@ func (s *testSuite8) TestUpdate(c *C) { _, err = tk.Exec("UPDATE t SET c2=16777215 WHERE c1>= -8388608 AND c1 < -9 ORDER BY c1 LIMIT 2") c.Assert(err, IsNil) - tk.MustExec("update (select * from t) t set c1 = 1111111") + tk.MustGetErrCode("update (select * from t) t set c1 = 1111111", mysql.ErrNonUpdatableTable) // test update ignore for bad null error tk.MustExec("drop table if exists t;") @@ -1604,8 +1604,7 @@ func (s *testSuite8) TestUpdate(c *C) { tk.MustExec("drop view v") tk.MustExec("create sequence seq") - _, err = tk.Exec("update seq set minvalue=1") - c.Assert(err.Error(), Equals, "update sequence seq is not supported now.") + tk.MustGetErrCode("update seq set minvalue=1", mysql.ErrBadField) tk.MustExec("drop sequence seq") tk.MustExec("drop table if exists t1, t2") @@ -2861,6 +2860,38 @@ func (s *testSuite7) TestDeferConstraintCheckForInsert(c *C) { tk.MustExec("insert into t values (1, 3)") _, err = tk.Exec("commit") c.Assert(err, NotNil) + + // Cover the temporary table. + tk.MustExec("set tidb_enable_global_temporary_table=true") + for val := range []int{0, 1} { + tk.MustExec("set tidb_constraint_check_in_place = ?", val) + + tk.MustExec("drop table t") + tk.MustExec("create global temporary table t (a int primary key, b int) on commit delete rows") + tk.MustExec("begin") + tk.MustExec("insert into t values (1, 1)") + _, err = tk.Exec(`insert into t values (1, 3)`) + c.Assert(err, NotNil) + tk.MustExec("insert into t values (2, 2)") + _, err = tk.Exec("update t set a = a + 1 where a = 1") + c.Assert(err, NotNil) + _, err = tk.Exec("insert into t values (1, 3) on duplicated key update a = a + 1") + c.Assert(err, NotNil) + tk.MustExec("commit") + + tk.MustExec("drop table t") + tk.MustExec("create global temporary table t (a int, b int unique) on commit delete rows") + tk.MustExec("begin") + tk.MustExec("insert into t values (1, 1)") + _, err = tk.Exec(`insert into t values (3, 1)`) + c.Assert(err, NotNil) + tk.MustExec("insert into t values (2, 2)") + _, err = tk.Exec("update t set b = b + 1 where a = 1") + c.Assert(err, NotNil) + _, err = tk.Exec("insert into t values (3, 1) on duplicated key update b = b + 1") + c.Assert(err, NotNil) + tk.MustExec("commit") + } } func (s *testSuite7) TestPessimisticDeleteYourWrites(c *C) { @@ -3931,6 +3962,25 @@ func (s *testSerialSuite) TestIssue20840(c *C) { tk.MustExec("drop table t1") } +func (s *testSerialSuite) TestIssueInsertPrefixIndexForNonUTF8Collation(c *C) { + collate.SetNewCollationEnabledForTest(true) + defer collate.SetNewCollationEnabledForTest(false) + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1, t2, t3") + tk.MustExec("create table t1 ( c_int int, c_str varchar(40) character set ascii collate ascii_bin, primary key(c_int, c_str(8)) clustered , unique key(c_str))") + tk.MustExec("create table t2 ( c_int int, c_str varchar(40) character set latin1 collate latin1_bin, primary key(c_int, c_str(8)) clustered , unique key(c_str))") + tk.MustExec("insert into t1 values (3, 'fervent brattain')") + tk.MustExec("insert into t2 values (3, 'fervent brattain')") + tk.MustExec("admin check table t1") + tk.MustExec("admin check table t2") + + tk.MustExec("create table t3 (x varchar(40) CHARACTER SET ascii COLLATE ascii_bin, UNIQUE KEY uk(x(4)))") + tk.MustExec("insert into t3 select 'abc '") + tk.MustGetErrCode("insert into t3 select 'abc d'", 1062) +} + func (s *testSerialSuite) TestIssue22496(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") diff --git a/expression/aggregation/descriptor.go b/expression/aggregation/descriptor.go index af16d26b1f81a..4415b0688ce09 100644 --- a/expression/aggregation/descriptor.go +++ b/expression/aggregation/descriptor.go @@ -210,7 +210,7 @@ func (a *AggFuncDesc) GetAggFunc(ctx sessionctx.Context) Aggregation { var s string var err error var maxLen uint64 - s, err = variable.GetSessionSystemVar(ctx.GetSessionVars(), variable.GroupConcatMaxLen) + s, err = variable.GetSessionOrGlobalSystemVar(ctx.GetSessionVars(), variable.GroupConcatMaxLen) if err != nil { panic(fmt.Sprintf("Error happened when GetAggFunc: no system variable named '%s'", variable.GroupConcatMaxLen)) } diff --git a/expression/aggregation/window_func.go b/expression/aggregation/window_func.go index 8f963480dde16..64e412ad2149a 100644 --- a/expression/aggregation/window_func.go +++ b/expression/aggregation/window_func.go @@ -70,6 +70,22 @@ var noFrameWindowFuncs = map[string]struct{}{ ast.WindowFuncRowNumber: {}, } +var useDefaultFrameWindowFuncs = map[string]ast.FrameClause{ + ast.WindowFuncRowNumber: { + Type: ast.Rows, + Extent: ast.FrameExtent{ + Start: ast.FrameBound{Type: ast.CurrentRow}, + End: ast.FrameBound{Type: ast.CurrentRow}, + }, + }, +} + +// UseDefaultFrame indicates if the window function has a provided frame that will override user's designation +func UseDefaultFrame(name string) (bool, ast.FrameClause) { + frame, ok := useDefaultFrameWindowFuncs[strings.ToLower(name)] + return ok, frame +} + // NeedFrame checks if the function need frame specification. func NeedFrame(name string) bool { _, ok := noFrameWindowFuncs[strings.ToLower(name)] diff --git a/expression/bench_test.go b/expression/bench_test.go index 3a2b6559c29cb..51d001a8b07d8 100644 --- a/expression/bench_test.go +++ b/expression/bench_test.go @@ -263,6 +263,13 @@ func (g *defaultGener) gen() interface{} { case types.ETDatetime, types.ETTimestamp: gt := getRandomTime(g.randGen.Rand) t := types.NewTime(gt, convertETType(g.eType), 0) + // TiDB has DST time problem, and it causes ErrWrongValue. + // We should ignore ambiguous Time. See https://timezonedb.com/time-zones/Asia/Shanghai. + for _, err := t.GoTime(time.Local); err != nil; { + gt = getRandomTime(g.randGen.Rand) + t = types.NewTime(gt, convertETType(g.eType), 0) + _, err = t.GoTime(time.Local) + } return t case types.ETDuration: d := types.Duration{ diff --git a/expression/builtin.go b/expression/builtin.go index 9c530f92949e1..8da5528b6a975 100644 --- a/expression/builtin.go +++ b/expression/builtin.go @@ -90,7 +90,7 @@ func newBaseBuiltinFunc(ctx sessionctx.Context, funcName string, args []Expressi if ctx == nil { return baseBuiltinFunc{}, errors.New("unexpected nil session ctx") } - if err := checkIllegalMixCollation(funcName, args, retType); err != nil { + if err := CheckIllegalMixCollation(funcName, args, retType); err != nil { return baseBuiltinFunc{}, err } derivedCharset, derivedCollate := DeriveCollationFromExprs(ctx, args...) @@ -112,7 +112,8 @@ var ( coerString = []string{"EXPLICIT", "NONE", "IMPLICIT", "SYSCONST", "COERCIBLE", "NUMERIC", "IGNORABLE"} ) -func checkIllegalMixCollation(funcName string, args []Expression, evalType types.EvalType) error { +// CheckIllegalMixCollation checks illegal mix collation with expressions +func CheckIllegalMixCollation(funcName string, args []Expression, evalType types.EvalType) error { if len(args) < 2 { return nil } @@ -169,7 +170,7 @@ func newBaseBuiltinFuncWithTp(ctx sessionctx.Context, funcName string, args []Ex } } - if err = checkIllegalMixCollation(funcName, args, retType); err != nil { + if err = CheckIllegalMixCollation(funcName, args, retType); err != nil { return } @@ -687,6 +688,9 @@ var funcs = map[string]functionClass{ ast.Year: &yearFunctionClass{baseFunctionClass{ast.Year, 1, 1}}, ast.YearWeek: &yearWeekFunctionClass{baseFunctionClass{ast.YearWeek, 1, 2}}, ast.LastDay: &lastDayFunctionClass{baseFunctionClass{ast.LastDay, 1, 1}}, + // TSO functions + ast.TiDBBoundedStaleness: &tidbBoundedStalenessFunctionClass{baseFunctionClass{ast.TiDBBoundedStaleness, 2, 2}}, + ast.TiDBParseTso: &tidbParseTsoFunctionClass{baseFunctionClass{ast.TiDBParseTso, 1, 1}}, // string functions ast.ASCII: &asciiFunctionClass{baseFunctionClass{ast.ASCII, 1, 1}}, @@ -881,7 +885,6 @@ var funcs = map[string]functionClass{ // This function is used to show tidb-server version info. ast.TiDBVersion: &tidbVersionFunctionClass{baseFunctionClass{ast.TiDBVersion, 0, 0}}, ast.TiDBIsDDLOwner: &tidbIsDDLOwnerFunctionClass{baseFunctionClass{ast.TiDBIsDDLOwner, 0, 0}}, - ast.TiDBParseTso: &tidbParseTsoFunctionClass{baseFunctionClass{ast.TiDBParseTso, 1, 1}}, ast.TiDBDecodePlan: &tidbDecodePlanFunctionClass{baseFunctionClass{ast.TiDBDecodePlan, 1, 1}}, // TiDB Sequence function. diff --git a/expression/builtin_cast.go b/expression/builtin_cast.go index 0f2d5827c91d9..9d6becab44cbe 100644 --- a/expression/builtin_cast.go +++ b/expression/builtin_cast.go @@ -1804,6 +1804,7 @@ func BuildCastFunction4Union(ctx sessionctx.Context, expr Expression, tp *types. // BuildCastFunction builds a CAST ScalarFunction from the Expression. func BuildCastFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldType) (res Expression) { + expr = TryPushCastIntoControlFunctionForHybridType(ctx, expr, tp) var fc functionClass switch tp.EvalType() { case types.ETInt: @@ -1983,3 +1984,92 @@ func WrapWithCastAsJSON(ctx sessionctx.Context, expr Expression) Expression { } return BuildCastFunction(ctx, expr, tp) } + +// TryPushCastIntoControlFunctionForHybridType try to push cast into control function for Hybrid Type. +// If necessary, it will rebuild control function using changed args. +// When a hybrid type is the output of a control function, the result may be as a numeric type to subsequent calculation +// We should perform the `Cast` operation early to avoid using the wrong type for calculation +// For example, the condition `if(1, e, 'a') = 1`, `if` function will output `e` and compare with `1`. +// If the evaltype is ETString, it will get wrong result. So we can rewrite the condition to +// `IfInt(1, cast(e as int), cast('a' as int)) = 1` to get the correct result. +func TryPushCastIntoControlFunctionForHybridType(ctx sessionctx.Context, expr Expression, tp *types.FieldType) (res Expression) { + sf, ok := expr.(*ScalarFunction) + if !ok { + return expr + } + + var wrapCastFunc func(ctx sessionctx.Context, expr Expression) Expression + switch tp.EvalType() { + case types.ETInt: + wrapCastFunc = WrapWithCastAsInt + case types.ETReal: + wrapCastFunc = WrapWithCastAsReal + default: + return expr + } + + isHybrid := func(ft *types.FieldType) bool { + // todo: compatible with mysql control function using bit type. issue 24725 + return ft.Hybrid() && ft.Tp != mysql.TypeBit + } + + args := sf.GetArgs() + switch sf.FuncName.L { + case ast.If: + if isHybrid(args[1].GetType()) || isHybrid(args[2].GetType()) { + args[1] = wrapCastFunc(ctx, args[1]) + args[2] = wrapCastFunc(ctx, args[2]) + f, err := funcs[ast.If].getFunction(ctx, args) + if err != nil { + return expr + } + sf.RetType, sf.Function = f.getRetTp(), f + return sf + } + case ast.Case: + hasHybrid := false + for i := 0; i < len(args)-1; i += 2 { + hasHybrid = hasHybrid || isHybrid(args[i+1].GetType()) + } + if len(args)%2 == 1 { + hasHybrid = hasHybrid || isHybrid(args[len(args)-1].GetType()) + } + if !hasHybrid { + return expr + } + + for i := 0; i < len(args)-1; i += 2 { + args[i+1] = wrapCastFunc(ctx, args[i+1]) + } + if len(args)%2 == 1 { + args[len(args)-1] = wrapCastFunc(ctx, args[len(args)-1]) + } + f, err := funcs[ast.Case].getFunction(ctx, args) + if err != nil { + return expr + } + sf.RetType, sf.Function = f.getRetTp(), f + return sf + case ast.Elt: + hasHybrid := false + for i := 1; i < len(args); i++ { + hasHybrid = hasHybrid || isHybrid(args[i].GetType()) + } + if !hasHybrid { + return expr + } + + for i := 1; i < len(args); i++ { + args[i] = wrapCastFunc(ctx, args[i]) + } + f, err := funcs[ast.Elt].getFunction(ctx, args) + if err != nil { + return expr + } + sf.RetType, sf.Function = f.getRetTp(), f + return sf + default: + return expr + } + return expr +} diff --git a/expression/builtin_control.go b/expression/builtin_control.go index 13f539648e049..9ffc4bfe718a9 100644 --- a/expression/builtin_control.go +++ b/expression/builtin_control.go @@ -135,10 +135,16 @@ func InferType4ControlFuncs(lexp, rexp Expression) *types.FieldType { resultEvalType := resultFieldType.EvalType() if resultEvalType == types.ETInt { resultFieldType.Decimal = 0 + if resultFieldType.Tp == mysql.TypeEnum || resultFieldType.Tp == mysql.TypeSet { + resultFieldType.Tp = mysql.TypeLonglong + } } else if resultEvalType == types.ETString { if lhs.Tp != mysql.TypeNull || rhs.Tp != mysql.TypeNull { resultFieldType.Decimal = types.UnspecifiedLength } + if resultFieldType.Tp == mysql.TypeEnum || resultFieldType.Tp == mysql.TypeSet { + resultFieldType.Tp = mysql.TypeVarchar + } } return resultFieldType } diff --git a/expression/builtin_info.go b/expression/builtin_info.go index 6a41b20ef75af..7b439d6dce14d 100644 --- a/expression/builtin_info.go +++ b/expression/builtin_info.go @@ -847,7 +847,7 @@ func (b *builtinNextValSig) evalInt(row chunk.Row) (int64, bool, error) { db = b.ctx.GetSessionVars().CurrentDB } // Check the tableName valid. - sequence, err := b.ctx.GetSessionVars().TxnCtx.InfoSchema.(util.SequenceSchema).SequenceByName(model.NewCIStr(db), model.NewCIStr(seq)) + sequence, err := b.ctx.GetInfoSchema().(util.SequenceSchema).SequenceByName(model.NewCIStr(db), model.NewCIStr(seq)) if err != nil { return 0, false, err } @@ -903,7 +903,7 @@ func (b *builtinLastValSig) evalInt(row chunk.Row) (int64, bool, error) { db = b.ctx.GetSessionVars().CurrentDB } // Check the tableName valid. - sequence, err := b.ctx.GetSessionVars().TxnCtx.InfoSchema.(util.SequenceSchema).SequenceByName(model.NewCIStr(db), model.NewCIStr(seq)) + sequence, err := b.ctx.GetInfoSchema().(util.SequenceSchema).SequenceByName(model.NewCIStr(db), model.NewCIStr(seq)) if err != nil { return 0, false, err } @@ -953,7 +953,7 @@ func (b *builtinSetValSig) evalInt(row chunk.Row) (int64, bool, error) { db = b.ctx.GetSessionVars().CurrentDB } // Check the tableName valid. - sequence, err := b.ctx.GetSessionVars().TxnCtx.InfoSchema.(util.SequenceSchema).SequenceByName(model.NewCIStr(db), model.NewCIStr(seq)) + sequence, err := b.ctx.GetInfoSchema().(util.SequenceSchema).SequenceByName(model.NewCIStr(db), model.NewCIStr(seq)) if err != nil { return 0, false, err } diff --git a/expression/builtin_json.go b/expression/builtin_json.go index bb2eee747606b..287e4383bf165 100644 --- a/expression/builtin_json.go +++ b/expression/builtin_json.go @@ -14,7 +14,8 @@ package expression import ( - json2 "encoding/json" + "bytes" + goJSON "encoding/json" "strconv" "strings" @@ -841,7 +842,7 @@ func (b *builtinJSONValidStringSig) evalInt(row chunk.Row) (res int64, isNull bo } data := hack.Slice(val) - if json2.Valid(data) { + if goJSON.Valid(data) { res = 1 } else { res = 0 @@ -1072,8 +1073,45 @@ type jsonPrettyFunctionClass struct { baseFunctionClass } +type builtinJSONSPrettySig struct { + baseBuiltinFunc +} + +func (b *builtinJSONSPrettySig) Clone() builtinFunc { + newSig := &builtinJSONSPrettySig{} + newSig.cloneFrom(&b.baseBuiltinFunc) + return newSig +} + func (c *jsonPrettyFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (builtinFunc, error) { - return nil, errFunctionNotExists.GenWithStackByArgs("FUNCTION", "JSON_PRETTY") + if err := c.verifyArgs(args); err != nil { + return nil, err + } + + bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETString, types.ETJson) + if err != nil { + return nil, err + } + sig := &builtinJSONSPrettySig{bf} + sig.setPbCode(tipb.ScalarFuncSig_JsonPrettySig) + return sig, nil +} + +func (b *builtinJSONSPrettySig) evalString(row chunk.Row) (res string, isNull bool, err error) { + obj, isNull, err := b.args[0].EvalJSON(b.ctx, row) + if isNull || err != nil { + return res, isNull, err + } + + buf, err := obj.MarshalJSON() + if err != nil { + return res, isNull, err + } + var resBuf bytes.Buffer + if err = goJSON.Indent(&resBuf, buf, "", " "); err != nil { + return res, isNull, err + } + return resBuf.String(), false, nil } type jsonQuoteFunctionClass struct { diff --git a/expression/builtin_json_test.go b/expression/builtin_json_test.go index 0302e4f3c2d3d..666c1a1eb5b00 100644 --- a/expression/builtin_json_test.go +++ b/expression/builtin_json_test.go @@ -995,3 +995,76 @@ func (s *testEvaluatorSuite) TestJSONStorageSize(c *C) { } } } + +func (s *testEvaluatorSuite) TestJSONPretty(c *C) { + fc := funcs[ast.JSONPretty] + tbl := []struct { + input []interface{} + expected interface{} + success bool + }{ + // Tests scalar arguments + {[]interface{}{nil}, nil, true}, + {[]interface{}{`true`}, "true", true}, + {[]interface{}{`false`}, "false", true}, + {[]interface{}{`2223`}, "2223", true}, + // Tests simple json + {[]interface{}{`{"a":1}`}, `{ + "a": 1 +}`, true}, + {[]interface{}{`[1]`}, `[ + 1 +]`, true}, + // Test complex json + {[]interface{}{`{"a":1,"b":[{"d":1},{"e":2},{"f":3}],"c":"eee"}`}, `{ + "a": 1, + "b": [ + { + "d": 1 + }, + { + "e": 2 + }, + { + "f": 3 + } + ], + "c": "eee" +}`, true}, + {[]interface{}{`{"a":1,"b":"qwe","c":[1,2,3,"123",null],"d":{"d1":1,"d2":2}}`}, `{ + "a": 1, + "b": "qwe", + "c": [ + 1, + 2, + 3, + "123", + null + ], + "d": { + "d1": 1, + "d2": 2 + } +}`, true}, + // Tests invalid json data + {[]interface{}{`{1}`}, nil, false}, + {[]interface{}{`[1,3,4,5]]`}, nil, false}, + } + for _, t := range tbl { + args := types.MakeDatums(t.input...) + f, err := fc.getFunction(s.ctx, s.datumsToConstants(args)) + c.Assert(err, IsNil) + d, err := evalBuiltinFunc(f, chunk.Row{}) + if t.success { + c.Assert(err, IsNil) + + if t.expected == nil { + c.Assert(d.IsNull(), IsTrue) + } else { + c.Assert(d.GetString(), Equals, t.expected.(string)) + } + } else { + c.Assert(err, NotNil) + } + } +} diff --git a/expression/builtin_json_vec.go b/expression/builtin_json_vec.go index 953da67458040..86e1d64b09902 100644 --- a/expression/builtin_json_vec.go +++ b/expression/builtin_json_vec.go @@ -14,6 +14,8 @@ package expression import ( + "bytes" + goJSON "encoding/json" "strconv" "strings" @@ -1176,3 +1178,39 @@ func (b *builtinJSONUnquoteSig) vecEvalString(input *chunk.Chunk, result *chunk. } return nil } + +func (b *builtinJSONSPrettySig) vectorized() bool { + return true +} + +func (b *builtinJSONSPrettySig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { + n := input.NumRows() + buf, err := b.bufAllocator.get(types.ETJson, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf) + if err := b.args[0].VecEvalJSON(b.ctx, input, buf); err != nil { + return err + } + result.ReserveString(n) + for i := 0; i < n; i++ { + if buf.IsNull(i) { + result.AppendNull() + continue + } + + jb, err := buf.GetJSON(i).MarshalJSON() + if err != nil { + return err + } + + var resBuf bytes.Buffer + if err = goJSON.Indent(&resBuf, jb, "", " "); err != nil { + return err + } + + result.AppendString(resBuf.String()) + } + return nil +} diff --git a/expression/builtin_time.go b/expression/builtin_time.go index 1d52cf6adc2c3..9fe7a39a3d80b 100644 --- a/expression/builtin_time.go +++ b/expression/builtin_time.go @@ -27,6 +27,7 @@ import ( "github.com/cznic/mathutil" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" @@ -2107,6 +2108,7 @@ func (c *currentDateFunctionClass) getFunction(ctx sessionctx.Context, args []Ex return nil, err } bf.tp.Flen, bf.tp.Decimal = 10, 0 + bf.tp.Tp = mysql.TypeDate sig := &builtinCurrentDateSig{bf} return sig, nil } @@ -2768,20 +2770,20 @@ func (b *builtinExtractDurationSig) evalInt(row chunk.Row) (int64, bool, error) return res, err != nil, err } -// baseDateArithmitical is the base class for all "builtinAddDateXXXSig" and "builtinSubDateXXXSig", +// baseDateArithmetical is the base class for all "builtinAddDateXXXSig" and "builtinSubDateXXXSig", // which provides parameter getter and date arithmetical calculate functions. -type baseDateArithmitical struct { +type baseDateArithmetical struct { // intervalRegexp is "*Regexp" used to extract string interval for "DAY" unit. intervalRegexp *regexp.Regexp } -func newDateArighmeticalUtil() baseDateArithmitical { - return baseDateArithmitical{ +func newDateArighmeticalUtil() baseDateArithmetical { + return baseDateArithmetical{ intervalRegexp: regexp.MustCompile(`-?[\d]+`), } } -func (du *baseDateArithmitical) getDateFromString(ctx sessionctx.Context, args []Expression, row chunk.Row, unit string) (types.Time, bool, error) { +func (du *baseDateArithmetical) getDateFromString(ctx sessionctx.Context, args []Expression, row chunk.Row, unit string) (types.Time, bool, error) { dateStr, isNull, err := args[0].EvalString(ctx, row) if isNull || err != nil { return types.ZeroTime, true, err @@ -2806,7 +2808,7 @@ func (du *baseDateArithmitical) getDateFromString(ctx sessionctx.Context, args [ return date, false, handleInvalidTimeError(ctx, err) } -func (du *baseDateArithmitical) getDateFromInt(ctx sessionctx.Context, args []Expression, row chunk.Row, unit string) (types.Time, bool, error) { +func (du *baseDateArithmetical) getDateFromInt(ctx sessionctx.Context, args []Expression, row chunk.Row, unit string) (types.Time, bool, error) { dateInt, isNull, err := args[0].EvalInt(ctx, row) if isNull || err != nil { return types.ZeroTime, true, err @@ -2826,7 +2828,7 @@ func (du *baseDateArithmitical) getDateFromInt(ctx sessionctx.Context, args []Ex return date, false, nil } -func (du *baseDateArithmitical) getDateFromDatetime(ctx sessionctx.Context, args []Expression, row chunk.Row, unit string) (types.Time, bool, error) { +func (du *baseDateArithmetical) getDateFromDatetime(ctx sessionctx.Context, args []Expression, row chunk.Row, unit string) (types.Time, bool, error) { date, isNull, err := args[0].EvalTime(ctx, row) if isNull || err != nil { return types.ZeroTime, true, err @@ -2838,7 +2840,7 @@ func (du *baseDateArithmitical) getDateFromDatetime(ctx sessionctx.Context, args return date, false, nil } -func (du *baseDateArithmitical) getIntervalFromString(ctx sessionctx.Context, args []Expression, row chunk.Row, unit string) (string, bool, error) { +func (du *baseDateArithmetical) getIntervalFromString(ctx sessionctx.Context, args []Expression, row chunk.Row, unit string) (string, bool, error) { interval, isNull, err := args[1].EvalString(ctx, row) if isNull || err != nil { return "", true, err @@ -2856,7 +2858,7 @@ func (du *baseDateArithmitical) getIntervalFromString(ctx sessionctx.Context, ar return interval, false, nil } -func (du *baseDateArithmitical) getIntervalFromDecimal(ctx sessionctx.Context, args []Expression, row chunk.Row, unit string) (string, bool, error) { +func (du *baseDateArithmetical) getIntervalFromDecimal(ctx sessionctx.Context, args []Expression, row chunk.Row, unit string) (string, bool, error) { decimal, isNull, err := args[1].EvalDecimal(ctx, row) if isNull || err != nil { return "", true, err @@ -2910,7 +2912,7 @@ func (du *baseDateArithmitical) getIntervalFromDecimal(ctx sessionctx.Context, a return interval, false, nil } -func (du *baseDateArithmitical) getIntervalFromInt(ctx sessionctx.Context, args []Expression, row chunk.Row, unit string) (string, bool, error) { +func (du *baseDateArithmetical) getIntervalFromInt(ctx sessionctx.Context, args []Expression, row chunk.Row, unit string) (string, bool, error) { interval, isNull, err := args[1].EvalInt(ctx, row) if isNull || err != nil { return "", true, err @@ -2918,7 +2920,7 @@ func (du *baseDateArithmitical) getIntervalFromInt(ctx sessionctx.Context, args return strconv.FormatInt(interval, 10), false, nil } -func (du *baseDateArithmitical) getIntervalFromReal(ctx sessionctx.Context, args []Expression, row chunk.Row, unit string) (string, bool, error) { +func (du *baseDateArithmetical) getIntervalFromReal(ctx sessionctx.Context, args []Expression, row chunk.Row, unit string) (string, bool, error) { interval, isNull, err := args[1].EvalReal(ctx, row) if isNull || err != nil { return "", true, err @@ -2926,7 +2928,7 @@ func (du *baseDateArithmitical) getIntervalFromReal(ctx sessionctx.Context, args return strconv.FormatFloat(interval, 'f', args[1].GetType().Decimal, 64), false, nil } -func (du *baseDateArithmitical) add(ctx sessionctx.Context, date types.Time, interval string, unit string) (types.Time, bool, error) { +func (du *baseDateArithmetical) add(ctx sessionctx.Context, date types.Time, interval string, unit string) (types.Time, bool, error) { year, month, day, nano, err := types.ParseDurationValue(unit, interval) if err := handleInvalidTimeError(ctx, err); err != nil { return types.ZeroTime, true, err @@ -2934,7 +2936,7 @@ func (du *baseDateArithmitical) add(ctx sessionctx.Context, date types.Time, int return du.addDate(ctx, date, year, month, day, nano) } -func (du *baseDateArithmitical) addDate(ctx sessionctx.Context, date types.Time, year, month, day, nano int64) (types.Time, bool, error) { +func (du *baseDateArithmetical) addDate(ctx sessionctx.Context, date types.Time, year, month, day, nano int64) (types.Time, bool, error) { goTime, err := date.GoTime(time.UTC) if err := handleInvalidTimeError(ctx, err); err != nil { return types.ZeroTime, true, err @@ -2971,7 +2973,7 @@ func (du *baseDateArithmitical) addDate(ctx sessionctx.Context, date types.Time, return date, false, nil } -func (du *baseDateArithmitical) addDuration(ctx sessionctx.Context, d types.Duration, interval string, unit string) (types.Duration, bool, error) { +func (du *baseDateArithmetical) addDuration(ctx sessionctx.Context, d types.Duration, interval string, unit string) (types.Duration, bool, error) { dur, err := types.ExtractDurationValue(unit, interval) if err != nil { return types.ZeroDuration, true, handleInvalidTimeError(ctx, err) @@ -2983,7 +2985,7 @@ func (du *baseDateArithmitical) addDuration(ctx sessionctx.Context, d types.Dura return retDur, false, nil } -func (du *baseDateArithmitical) subDuration(ctx sessionctx.Context, d types.Duration, interval string, unit string) (types.Duration, bool, error) { +func (du *baseDateArithmetical) subDuration(ctx sessionctx.Context, d types.Duration, interval string, unit string) (types.Duration, bool, error) { dur, err := types.ExtractDurationValue(unit, interval) if err != nil { return types.ZeroDuration, true, handleInvalidTimeError(ctx, err) @@ -2995,7 +2997,7 @@ func (du *baseDateArithmitical) subDuration(ctx sessionctx.Context, d types.Dura return retDur, false, nil } -func (du *baseDateArithmitical) sub(ctx sessionctx.Context, date types.Time, interval string, unit string) (types.Time, bool, error) { +func (du *baseDateArithmetical) sub(ctx sessionctx.Context, date types.Time, interval string, unit string) (types.Time, bool, error) { year, month, day, nano, err := types.ParseDurationValue(unit, interval) if err := handleInvalidTimeError(ctx, err); err != nil { return types.ZeroTime, true, err @@ -3003,7 +3005,7 @@ func (du *baseDateArithmitical) sub(ctx sessionctx.Context, date types.Time, int return du.addDate(ctx, date, -year, -month, -day, -nano) } -func (du *baseDateArithmitical) vecGetDateFromInt(b *baseBuiltinFunc, input *chunk.Chunk, unit string, result *chunk.Column) error { +func (du *baseDateArithmetical) vecGetDateFromInt(b *baseBuiltinFunc, input *chunk.Chunk, unit string, result *chunk.Column) error { n := input.NumRows() buf, err := b.bufAllocator.get(types.ETInt, n) if err != nil { @@ -3045,7 +3047,7 @@ func (du *baseDateArithmitical) vecGetDateFromInt(b *baseBuiltinFunc, input *chu return nil } -func (du *baseDateArithmitical) vecGetDateFromString(b *baseBuiltinFunc, input *chunk.Chunk, unit string, result *chunk.Column) error { +func (du *baseDateArithmetical) vecGetDateFromString(b *baseBuiltinFunc, input *chunk.Chunk, unit string, result *chunk.Column) error { n := input.NumRows() buf, err := b.bufAllocator.get(types.ETString, n) if err != nil { @@ -3089,7 +3091,7 @@ func (du *baseDateArithmitical) vecGetDateFromString(b *baseBuiltinFunc, input * return nil } -func (du *baseDateArithmitical) vecGetDateFromDatetime(b *baseBuiltinFunc, input *chunk.Chunk, unit string, result *chunk.Column) error { +func (du *baseDateArithmetical) vecGetDateFromDatetime(b *baseBuiltinFunc, input *chunk.Chunk, unit string, result *chunk.Column) error { n := input.NumRows() result.ResizeTime(n, false) if err := b.args[0].VecEvalTime(b.ctx, input, result); err != nil { @@ -3110,7 +3112,7 @@ func (du *baseDateArithmitical) vecGetDateFromDatetime(b *baseBuiltinFunc, input return nil } -func (du *baseDateArithmitical) vecGetIntervalFromString(b *baseBuiltinFunc, input *chunk.Chunk, unit string, result *chunk.Column) error { +func (du *baseDateArithmetical) vecGetIntervalFromString(b *baseBuiltinFunc, input *chunk.Chunk, unit string, result *chunk.Column) error { n := input.NumRows() buf, err := b.bufAllocator.get(types.ETString, n) if err != nil { @@ -3147,7 +3149,7 @@ func (du *baseDateArithmitical) vecGetIntervalFromString(b *baseBuiltinFunc, inp return nil } -func (du *baseDateArithmitical) vecGetIntervalFromDecimal(b *baseBuiltinFunc, input *chunk.Chunk, unit string, result *chunk.Column) error { +func (du *baseDateArithmetical) vecGetIntervalFromDecimal(b *baseBuiltinFunc, input *chunk.Chunk, unit string, result *chunk.Column) error { n := input.NumRows() buf, err := b.bufAllocator.get(types.ETDecimal, n) if err != nil { @@ -3248,7 +3250,7 @@ func (du *baseDateArithmitical) vecGetIntervalFromDecimal(b *baseBuiltinFunc, in return nil } -func (du *baseDateArithmitical) vecGetIntervalFromInt(b *baseBuiltinFunc, input *chunk.Chunk, unit string, result *chunk.Column) error { +func (du *baseDateArithmetical) vecGetIntervalFromInt(b *baseBuiltinFunc, input *chunk.Chunk, unit string, result *chunk.Column) error { n := input.NumRows() buf, err := b.bufAllocator.get(types.ETInt, n) if err != nil { @@ -3271,7 +3273,7 @@ func (du *baseDateArithmitical) vecGetIntervalFromInt(b *baseBuiltinFunc, input return nil } -func (du *baseDateArithmitical) vecGetIntervalFromReal(b *baseBuiltinFunc, input *chunk.Chunk, unit string, result *chunk.Column) error { +func (du *baseDateArithmetical) vecGetIntervalFromReal(b *baseBuiltinFunc, input *chunk.Chunk, unit string, result *chunk.Column) error { n := input.NumRows() buf, err := b.bufAllocator.get(types.ETReal, n) if err != nil { @@ -3355,97 +3357,97 @@ func (c *addDateFunctionClass) getFunction(ctx sessionctx.Context, args []Expres case dateEvalTp == types.ETString && intervalEvalTp == types.ETString: sig = &builtinAddDateStringStringSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateStringString) case dateEvalTp == types.ETString && intervalEvalTp == types.ETInt: sig = &builtinAddDateStringIntSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateStringInt) case dateEvalTp == types.ETString && intervalEvalTp == types.ETReal: sig = &builtinAddDateStringRealSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateStringReal) case dateEvalTp == types.ETString && intervalEvalTp == types.ETDecimal: sig = &builtinAddDateStringDecimalSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateStringDecimal) case dateEvalTp == types.ETInt && intervalEvalTp == types.ETString: sig = &builtinAddDateIntStringSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateIntString) case dateEvalTp == types.ETInt && intervalEvalTp == types.ETInt: sig = &builtinAddDateIntIntSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateIntInt) case dateEvalTp == types.ETInt && intervalEvalTp == types.ETReal: sig = &builtinAddDateIntRealSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateIntReal) case dateEvalTp == types.ETInt && intervalEvalTp == types.ETDecimal: sig = &builtinAddDateIntDecimalSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateIntDecimal) case dateEvalTp == types.ETDatetime && intervalEvalTp == types.ETString: sig = &builtinAddDateDatetimeStringSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateDatetimeString) case dateEvalTp == types.ETDatetime && intervalEvalTp == types.ETInt: sig = &builtinAddDateDatetimeIntSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateDatetimeInt) case dateEvalTp == types.ETDatetime && intervalEvalTp == types.ETReal: sig = &builtinAddDateDatetimeRealSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateDatetimeReal) case dateEvalTp == types.ETDatetime && intervalEvalTp == types.ETDecimal: sig = &builtinAddDateDatetimeDecimalSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateDatetimeDecimal) case dateEvalTp == types.ETDuration && intervalEvalTp == types.ETString: sig = &builtinAddDateDurationStringSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateDurationString) case dateEvalTp == types.ETDuration && intervalEvalTp == types.ETInt: sig = &builtinAddDateDurationIntSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateDurationInt) case dateEvalTp == types.ETDuration && intervalEvalTp == types.ETReal: sig = &builtinAddDateDurationRealSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateDurationReal) case dateEvalTp == types.ETDuration && intervalEvalTp == types.ETDecimal: sig = &builtinAddDateDurationDecimalSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_AddDateDurationDecimal) } @@ -3454,11 +3456,11 @@ func (c *addDateFunctionClass) getFunction(ctx sessionctx.Context, args []Expres type builtinAddDateStringStringSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateStringStringSig) Clone() builtinFunc { - newSig := &builtinAddDateStringStringSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateStringStringSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -3487,11 +3489,11 @@ func (b *builtinAddDateStringStringSig) evalTime(row chunk.Row) (types.Time, boo type builtinAddDateStringIntSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateStringIntSig) Clone() builtinFunc { - newSig := &builtinAddDateStringIntSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateStringIntSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -3520,11 +3522,11 @@ func (b *builtinAddDateStringIntSig) evalTime(row chunk.Row) (types.Time, bool, type builtinAddDateStringRealSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateStringRealSig) Clone() builtinFunc { - newSig := &builtinAddDateStringRealSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateStringRealSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -3553,11 +3555,11 @@ func (b *builtinAddDateStringRealSig) evalTime(row chunk.Row) (types.Time, bool, type builtinAddDateStringDecimalSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateStringDecimalSig) Clone() builtinFunc { - newSig := &builtinAddDateStringDecimalSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateStringDecimalSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -3586,11 +3588,11 @@ func (b *builtinAddDateStringDecimalSig) evalTime(row chunk.Row) (types.Time, bo type builtinAddDateIntStringSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateIntStringSig) Clone() builtinFunc { - newSig := &builtinAddDateIntStringSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateIntStringSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -3619,11 +3621,11 @@ func (b *builtinAddDateIntStringSig) evalTime(row chunk.Row) (types.Time, bool, type builtinAddDateIntIntSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateIntIntSig) Clone() builtinFunc { - newSig := &builtinAddDateIntIntSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateIntIntSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -3652,11 +3654,11 @@ func (b *builtinAddDateIntIntSig) evalTime(row chunk.Row) (types.Time, bool, err type builtinAddDateIntRealSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateIntRealSig) Clone() builtinFunc { - newSig := &builtinAddDateIntRealSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateIntRealSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -3685,11 +3687,11 @@ func (b *builtinAddDateIntRealSig) evalTime(row chunk.Row) (types.Time, bool, er type builtinAddDateIntDecimalSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateIntDecimalSig) Clone() builtinFunc { - newSig := &builtinAddDateIntDecimalSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateIntDecimalSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -3718,11 +3720,11 @@ func (b *builtinAddDateIntDecimalSig) evalTime(row chunk.Row) (types.Time, bool, type builtinAddDateDatetimeStringSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateDatetimeStringSig) Clone() builtinFunc { - newSig := &builtinAddDateDatetimeStringSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateDatetimeStringSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -3751,11 +3753,11 @@ func (b *builtinAddDateDatetimeStringSig) evalTime(row chunk.Row) (types.Time, b type builtinAddDateDatetimeIntSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateDatetimeIntSig) Clone() builtinFunc { - newSig := &builtinAddDateDatetimeIntSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateDatetimeIntSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -3784,11 +3786,11 @@ func (b *builtinAddDateDatetimeIntSig) evalTime(row chunk.Row) (types.Time, bool type builtinAddDateDatetimeRealSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateDatetimeRealSig) Clone() builtinFunc { - newSig := &builtinAddDateDatetimeRealSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateDatetimeRealSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -3817,11 +3819,11 @@ func (b *builtinAddDateDatetimeRealSig) evalTime(row chunk.Row) (types.Time, boo type builtinAddDateDatetimeDecimalSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateDatetimeDecimalSig) Clone() builtinFunc { - newSig := &builtinAddDateDatetimeDecimalSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateDatetimeDecimalSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -3850,11 +3852,11 @@ func (b *builtinAddDateDatetimeDecimalSig) evalTime(row chunk.Row) (types.Time, type builtinAddDateDurationStringSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateDurationStringSig) Clone() builtinFunc { - newSig := &builtinAddDateDurationStringSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateDurationStringSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -3881,11 +3883,11 @@ func (b *builtinAddDateDurationStringSig) evalDuration(row chunk.Row) (types.Dur type builtinAddDateDurationIntSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateDurationIntSig) Clone() builtinFunc { - newSig := &builtinAddDateDurationIntSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateDurationIntSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -3911,11 +3913,11 @@ func (b *builtinAddDateDurationIntSig) evalDuration(row chunk.Row) (types.Durati type builtinAddDateDurationDecimalSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateDurationDecimalSig) Clone() builtinFunc { - newSig := &builtinAddDateDurationDecimalSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateDurationDecimalSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -3941,11 +3943,11 @@ func (b *builtinAddDateDurationDecimalSig) evalDuration(row chunk.Row) (types.Du type builtinAddDateDurationRealSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinAddDateDurationRealSig) Clone() builtinFunc { - newSig := &builtinAddDateDurationRealSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinAddDateDurationRealSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4029,97 +4031,97 @@ func (c *subDateFunctionClass) getFunction(ctx sessionctx.Context, args []Expres case dateEvalTp == types.ETString && intervalEvalTp == types.ETString: sig = &builtinSubDateStringStringSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateStringString) case dateEvalTp == types.ETString && intervalEvalTp == types.ETInt: sig = &builtinSubDateStringIntSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateStringInt) case dateEvalTp == types.ETString && intervalEvalTp == types.ETReal: sig = &builtinSubDateStringRealSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateStringReal) case dateEvalTp == types.ETString && intervalEvalTp == types.ETDecimal: sig = &builtinSubDateStringDecimalSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateStringDecimal) case dateEvalTp == types.ETInt && intervalEvalTp == types.ETString: sig = &builtinSubDateIntStringSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateIntString) case dateEvalTp == types.ETInt && intervalEvalTp == types.ETInt: sig = &builtinSubDateIntIntSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateIntInt) case dateEvalTp == types.ETInt && intervalEvalTp == types.ETReal: sig = &builtinSubDateIntRealSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateIntReal) case dateEvalTp == types.ETInt && intervalEvalTp == types.ETDecimal: sig = &builtinSubDateIntDecimalSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateIntDecimal) case dateEvalTp == types.ETDatetime && intervalEvalTp == types.ETString: sig = &builtinSubDateDatetimeStringSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateDatetimeString) case dateEvalTp == types.ETDatetime && intervalEvalTp == types.ETInt: sig = &builtinSubDateDatetimeIntSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateDatetimeInt) case dateEvalTp == types.ETDatetime && intervalEvalTp == types.ETReal: sig = &builtinSubDateDatetimeRealSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateDatetimeReal) case dateEvalTp == types.ETDatetime && intervalEvalTp == types.ETDecimal: sig = &builtinSubDateDatetimeDecimalSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateDatetimeDecimal) case dateEvalTp == types.ETDuration && intervalEvalTp == types.ETString: sig = &builtinSubDateDurationStringSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateDurationString) case dateEvalTp == types.ETDuration && intervalEvalTp == types.ETInt: sig = &builtinSubDateDurationIntSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateDurationInt) case dateEvalTp == types.ETDuration && intervalEvalTp == types.ETReal: sig = &builtinSubDateDurationRealSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateDurationReal) case dateEvalTp == types.ETDuration && intervalEvalTp == types.ETDecimal: sig = &builtinSubDateDurationDecimalSig{ baseBuiltinFunc: bf, - baseDateArithmitical: newDateArighmeticalUtil(), + baseDateArithmetical: newDateArighmeticalUtil(), } sig.setPbCode(tipb.ScalarFuncSig_SubDateDurationDecimal) } @@ -4128,11 +4130,11 @@ func (c *subDateFunctionClass) getFunction(ctx sessionctx.Context, args []Expres type builtinSubDateStringStringSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinSubDateStringStringSig) Clone() builtinFunc { - newSig := &builtinSubDateStringStringSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateStringStringSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4161,11 +4163,11 @@ func (b *builtinSubDateStringStringSig) evalTime(row chunk.Row) (types.Time, boo type builtinSubDateStringIntSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinSubDateStringIntSig) Clone() builtinFunc { - newSig := &builtinSubDateStringIntSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateStringIntSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4194,11 +4196,11 @@ func (b *builtinSubDateStringIntSig) evalTime(row chunk.Row) (types.Time, bool, type builtinSubDateStringRealSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinSubDateStringRealSig) Clone() builtinFunc { - newSig := &builtinSubDateStringRealSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateStringRealSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4227,11 +4229,11 @@ func (b *builtinSubDateStringRealSig) evalTime(row chunk.Row) (types.Time, bool, type builtinSubDateStringDecimalSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinSubDateStringDecimalSig) Clone() builtinFunc { - newSig := &builtinSubDateStringDecimalSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateStringDecimalSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4258,11 +4260,11 @@ func (b *builtinSubDateStringDecimalSig) evalTime(row chunk.Row) (types.Time, bo type builtinSubDateIntStringSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinSubDateIntStringSig) Clone() builtinFunc { - newSig := &builtinSubDateIntStringSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateIntStringSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4291,11 +4293,11 @@ func (b *builtinSubDateIntStringSig) evalTime(row chunk.Row) (types.Time, bool, type builtinSubDateIntIntSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinSubDateIntIntSig) Clone() builtinFunc { - newSig := &builtinSubDateIntIntSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateIntIntSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4324,11 +4326,11 @@ func (b *builtinSubDateIntIntSig) evalTime(row chunk.Row) (types.Time, bool, err type builtinSubDateIntRealSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinSubDateIntRealSig) Clone() builtinFunc { - newSig := &builtinSubDateIntRealSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateIntRealSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4357,16 +4359,16 @@ func (b *builtinSubDateIntRealSig) evalTime(row chunk.Row) (types.Time, bool, er type builtinSubDateDatetimeStringSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } type builtinSubDateIntDecimalSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinSubDateIntDecimalSig) Clone() builtinFunc { - newSig := &builtinSubDateIntDecimalSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateIntDecimalSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4394,7 +4396,7 @@ func (b *builtinSubDateIntDecimalSig) evalTime(row chunk.Row) (types.Time, bool, } func (b *builtinSubDateDatetimeStringSig) Clone() builtinFunc { - newSig := &builtinSubDateDatetimeStringSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateDatetimeStringSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4423,11 +4425,11 @@ func (b *builtinSubDateDatetimeStringSig) evalTime(row chunk.Row) (types.Time, b type builtinSubDateDatetimeIntSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinSubDateDatetimeIntSig) Clone() builtinFunc { - newSig := &builtinSubDateDatetimeIntSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateDatetimeIntSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4456,11 +4458,11 @@ func (b *builtinSubDateDatetimeIntSig) evalTime(row chunk.Row) (types.Time, bool type builtinSubDateDatetimeRealSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinSubDateDatetimeRealSig) Clone() builtinFunc { - newSig := &builtinSubDateDatetimeRealSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateDatetimeRealSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4489,11 +4491,11 @@ func (b *builtinSubDateDatetimeRealSig) evalTime(row chunk.Row) (types.Time, boo type builtinSubDateDatetimeDecimalSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinSubDateDatetimeDecimalSig) Clone() builtinFunc { - newSig := &builtinSubDateDatetimeDecimalSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateDatetimeDecimalSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4522,11 +4524,11 @@ func (b *builtinSubDateDatetimeDecimalSig) evalTime(row chunk.Row) (types.Time, type builtinSubDateDurationStringSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinSubDateDurationStringSig) Clone() builtinFunc { - newSig := &builtinSubDateDurationStringSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateDurationStringSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4553,11 +4555,11 @@ func (b *builtinSubDateDurationStringSig) evalDuration(row chunk.Row) (types.Dur type builtinSubDateDurationIntSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinSubDateDurationIntSig) Clone() builtinFunc { - newSig := &builtinSubDateDurationIntSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateDurationIntSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4584,11 +4586,11 @@ func (b *builtinSubDateDurationIntSig) evalDuration(row chunk.Row) (types.Durati type builtinSubDateDurationDecimalSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinSubDateDurationDecimalSig) Clone() builtinFunc { - newSig := &builtinSubDateDurationDecimalSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateDurationDecimalSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -4615,11 +4617,11 @@ func (b *builtinSubDateDurationDecimalSig) evalDuration(row chunk.Row) (types.Du type builtinSubDateDurationRealSig struct { baseBuiltinFunc - baseDateArithmitical + baseDateArithmetical } func (b *builtinSubDateDurationRealSig) Clone() builtinFunc { - newSig := &builtinSubDateDurationRealSig{baseDateArithmitical: b.baseDateArithmitical} + newSig := &builtinSubDateDurationRealSig{baseDateArithmetical: b.baseDateArithmetical} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -7113,3 +7115,97 @@ func handleInvalidZeroTime(ctx sessionctx.Context, t types.Time) (bool, error) { } return true, handleInvalidTimeError(ctx, types.ErrWrongValue.GenWithStackByArgs(types.DateTimeStr, t.String())) } + +// tidbBoundedStalenessFunctionClass reads a time window [a, b] and compares it with the latest SafeTS +// to determine which TS to use in a read only transaction. +type tidbBoundedStalenessFunctionClass struct { + baseFunctionClass +} + +func (c *tidbBoundedStalenessFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (builtinFunc, error) { + if err := c.verifyArgs(args); err != nil { + return nil, err + } + bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETDatetime, types.ETDatetime, types.ETDatetime) + if err != nil { + return nil, err + } + sig := &builtinTiDBBoundedStalenessSig{bf} + return sig, nil +} + +type builtinTiDBBoundedStalenessSig struct { + baseBuiltinFunc +} + +func (b *builtinTiDBBoundedStalenessSig) Clone() builtinFunc { + newSig := &builtinTidbParseTsoSig{} + newSig.cloneFrom(&b.baseBuiltinFunc) + return newSig +} + +func (b *builtinTiDBBoundedStalenessSig) evalTime(row chunk.Row) (types.Time, bool, error) { + leftTime, isNull, err := b.args[0].EvalTime(b.ctx, row) + if isNull || err != nil { + return types.ZeroTime, true, handleInvalidTimeError(b.ctx, err) + } + rightTime, isNull, err := b.args[1].EvalTime(b.ctx, row) + if isNull || err != nil { + return types.ZeroTime, true, handleInvalidTimeError(b.ctx, err) + } + if invalidLeftTime, invalidRightTime := leftTime.InvalidZero(), rightTime.InvalidZero(); invalidLeftTime || invalidRightTime { + if invalidLeftTime { + err = handleInvalidTimeError(b.ctx, types.ErrWrongValue.GenWithStackByArgs(types.DateTimeStr, leftTime.String())) + } + if invalidRightTime { + err = handleInvalidTimeError(b.ctx, types.ErrWrongValue.GenWithStackByArgs(types.DateTimeStr, rightTime.String())) + } + return types.ZeroTime, true, err + } + timeZone := getTimeZone(b.ctx) + minTime, err := leftTime.GoTime(timeZone) + if err != nil { + return types.ZeroTime, true, err + } + maxTime, err := rightTime.GoTime(timeZone) + if err != nil { + return types.ZeroTime, true, err + } + if minTime.After(maxTime) { + return types.ZeroTime, true, nil + } + // Because the minimum unit of a TSO is millisecond, so we only need fsp to be 3. + return types.NewTime(types.FromGoTime(calAppropriateTime(minTime, maxTime, getMinSafeTime(b.ctx, timeZone))), mysql.TypeDatetime, 3), false, nil +} + +func getMinSafeTime(sessionCtx sessionctx.Context, timeZone *time.Location) time.Time { + var minSafeTS uint64 + if store := sessionCtx.GetStore(); store != nil { + minSafeTS = store.GetMinSafeTS(sessionCtx.GetSessionVars().CheckAndGetTxnScope()) + } + // Inject mocked SafeTS for test. + failpoint.Inject("injectSafeTS", func(val failpoint.Value) { + injectTS := val.(int) + minSafeTS = uint64(injectTS) + }) + // Try to get from the stmt cache to make sure this function is deterministic. + stmtCtx := sessionCtx.GetSessionVars().StmtCtx + minSafeTS = stmtCtx.GetOrStoreStmtCache(stmtctx.StmtSafeTSCacheKey, minSafeTS).(uint64) + return oracle.GetTimeFromTS(minSafeTS).In(timeZone) +} + +// For a SafeTS t and a time range [t1, t2]: +// 1. If t < t1, we will use t1 as the result, +// and with it, a read request may fail because it's an unreached SafeTS. +// 2. If t1 <= t <= t2, we will use t as the result, and with it, +// a read request won't fail. +// 2. If t2 < t, we will use t2 as the result, +// and with it, a read request won't fail because it's bigger than the latest SafeTS. +func calAppropriateTime(minTime, maxTime, minSafeTime time.Time) time.Time { + if minSafeTime.Before(minTime) { + return minTime + } else if minSafeTime.After(maxTime) { + return maxTime + } + return minSafeTime +} diff --git a/expression/builtin_time_test.go b/expression/builtin_time_test.go index f82f6fb8f76ea..4015794377486 100644 --- a/expression/builtin_time_test.go +++ b/expression/builtin_time_test.go @@ -14,12 +14,14 @@ package expression import ( + "fmt" "math" "strings" "time" . "github.com/pingcap/check" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/charset" "github.com/pingcap/parser/mysql" @@ -27,6 +29,7 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx/variable" + "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/mock" @@ -804,7 +807,7 @@ func (s *testEvaluatorSuite) TestTime(c *C) { } func resetStmtContext(ctx sessionctx.Context) { - ctx.GetSessionVars().StmtCtx.ResetNowTs() + ctx.GetSessionVars().StmtCtx.ResetStmtCache() } func (s *testEvaluatorSuite) TestNowAndUTCTimestamp(c *C) { @@ -2854,8 +2857,105 @@ func (s *testEvaluatorSuite) TestTidbParseTso(c *C) { } } +func (s *testEvaluatorSuite) TestTiDBBoundedStaleness(c *C) { + t1, err := time.Parse(types.TimeFormat, "2015-09-21 09:53:04") + c.Assert(err, IsNil) + // time.Parse uses UTC time zone by default, we need to change it to Local manually. + t1 = t1.Local() + t1Str := t1.Format(types.TimeFormat) + t2 := time.Now() + t2Str := t2.Format(types.TimeFormat) + timeZone := time.Local + s.ctx.GetSessionVars().TimeZone = timeZone + tests := []struct { + leftTime interface{} + rightTime interface{} + injectSafeTS uint64 + isNull bool + expect time.Time + }{ + // SafeTS is in the range. + { + leftTime: t1Str, + rightTime: t2Str, + injectSafeTS: oracle.GoTimeToTS(t2.Add(-1 * time.Second)), + isNull: false, + expect: t2.Add(-1 * time.Second), + }, + // SafeTS is less than the left time. + { + leftTime: t1Str, + rightTime: t2Str, + injectSafeTS: oracle.GoTimeToTS(t1.Add(-1 * time.Second)), + isNull: false, + expect: t1, + }, + // SafeTS is bigger than the right time. + { + leftTime: t1Str, + rightTime: t2Str, + injectSafeTS: oracle.GoTimeToTS(t2.Add(time.Second)), + isNull: false, + expect: t2, + }, + // Wrong time order. + { + leftTime: t2Str, + rightTime: t1Str, + injectSafeTS: 0, + isNull: true, + expect: time.Time{}, + }, + } + + fc := funcs[ast.TiDBBoundedStaleness] + for _, test := range tests { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/expression/injectSafeTS", + fmt.Sprintf("return(%v)", test.injectSafeTS)), IsNil) + f, err := fc.getFunction(s.ctx, s.datumsToConstants([]types.Datum{types.NewDatum(test.leftTime), types.NewDatum(test.rightTime)})) + c.Assert(err, IsNil) + d, err := evalBuiltinFunc(f, chunk.Row{}) + c.Assert(err, IsNil) + if test.isNull { + c.Assert(d.IsNull(), IsTrue) + } else { + goTime, err := d.GetMysqlTime().GoTime(timeZone) + c.Assert(err, IsNil) + c.Assert(goTime.Format(types.TimeFormat), Equals, test.expect.Format(types.TimeFormat)) + } + resetStmtContext(s.ctx) + } + + // Test whether it's deterministic. + safeTime1 := t2.Add(-1 * time.Second) + safeTS1 := oracle.GoTimeToTS(safeTime1) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/expression/injectSafeTS", + fmt.Sprintf("return(%v)", safeTS1)), IsNil) + f, err := fc.getFunction(s.ctx, s.datumsToConstants([]types.Datum{types.NewDatum(t1Str), types.NewDatum(t2Str)})) + c.Assert(err, IsNil) + d, err := evalBuiltinFunc(f, chunk.Row{}) + c.Assert(err, IsNil) + goTime, err := d.GetMysqlTime().GoTime(timeZone) + c.Assert(err, IsNil) + resultTime := goTime.Format(types.TimeFormat) + c.Assert(resultTime, Equals, safeTime1.Format(types.TimeFormat)) + // SafeTS updated. + safeTime2 := t2.Add(1 * time.Second) + safeTS2 := oracle.GoTimeToTS(safeTime2) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/expression/injectSafeTS", + fmt.Sprintf("return(%v)", safeTS2)), IsNil) + f, err = fc.getFunction(s.ctx, s.datumsToConstants([]types.Datum{types.NewDatum(t1Str), types.NewDatum(t2Str)})) + c.Assert(err, IsNil) + d, err = evalBuiltinFunc(f, chunk.Row{}) + c.Assert(err, IsNil) + // Still safeTime1 + c.Assert(resultTime, Equals, safeTime1.Format(types.TimeFormat)) + resetStmtContext(s.ctx) + failpoint.Disable("github.com/pingcap/tidb/expression/injectSafeTS") +} + func (s *testEvaluatorSuite) TestGetIntervalFromDecimal(c *C) { - du := baseDateArithmitical{} + du := baseDateArithmetical{} tests := []struct { param string diff --git a/expression/builtin_time_vec.go b/expression/builtin_time_vec.go index 94c1cd8b6f0c4..6f74a8f587e50 100644 --- a/expression/builtin_time_vec.go +++ b/expression/builtin_time_vec.go @@ -854,6 +854,70 @@ func (b *builtinTidbParseTsoSig) vecEvalTime(input *chunk.Chunk, result *chunk.C return nil } +func (b *builtinTiDBBoundedStalenessSig) vectorized() bool { + return true +} + +func (b *builtinTiDBBoundedStalenessSig) vecEvalTime(input *chunk.Chunk, result *chunk.Column) error { + n := input.NumRows() + buf0, err := b.bufAllocator.get(types.ETDatetime, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf0) + if err = b.args[0].VecEvalTime(b.ctx, input, buf0); err != nil { + return err + } + buf1, err := b.bufAllocator.get(types.ETDatetime, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf1) + if err = b.args[1].VecEvalTime(b.ctx, input, buf1); err != nil { + return err + } + args0 := buf0.Times() + args1 := buf1.Times() + timeZone := getTimeZone(b.ctx) + minSafeTime := getMinSafeTime(b.ctx, timeZone) + result.ResizeTime(n, false) + result.MergeNulls(buf0, buf1) + times := result.Times() + for i := 0; i < n; i++ { + if result.IsNull(i) { + continue + } + if invalidArg0, invalidArg1 := args0[i].InvalidZero(), args1[i].InvalidZero(); invalidArg0 || invalidArg1 { + if invalidArg0 { + err = handleInvalidTimeError(b.ctx, types.ErrWrongValue.GenWithStackByArgs(types.DateTimeStr, args0[i].String())) + } + if invalidArg1 { + err = handleInvalidTimeError(b.ctx, types.ErrWrongValue.GenWithStackByArgs(types.DateTimeStr, args1[i].String())) + } + if err != nil { + return err + } + result.SetNull(i, true) + continue + } + minTime, err := args0[i].GoTime(timeZone) + if err != nil { + return err + } + maxTime, err := args1[i].GoTime(timeZone) + if err != nil { + return err + } + if minTime.After(maxTime) { + result.SetNull(i, true) + continue + } + // Because the minimum unit of a TSO is millisecond, so we only need fsp to be 3. + times[i] = types.NewTime(types.FromGoTime(calAppropriateTime(minTime, maxTime, minSafeTime)), mysql.TypeDatetime, 3) + } + return nil +} + func (b *builtinFromDaysSig) vectorized() bool { return true } diff --git a/expression/builtin_time_vec_test.go b/expression/builtin_time_vec_test.go index 593cce162d7ff..389c257460e1b 100644 --- a/expression/builtin_time_vec_test.go +++ b/expression/builtin_time_vec_test.go @@ -516,7 +516,15 @@ var vecBuiltinTimeCases = map[string][]vecExprBenchCase{ { retEvalType: types.ETDatetime, childrenTypes: []types.EvalType{types.ETInt}, - geners: []dataGenerator{newRangeInt64Gener(0, math.MaxInt64)}, + // TiDB has DST time problem. Change the random ranges to [2000-01-01 00:00:01, +inf] + geners: []dataGenerator{newRangeInt64Gener(248160190726144000, math.MaxInt64)}, + }, + }, + // Todo: how to inject the safeTS for better testing. + ast.TiDBBoundedStaleness: { + { + retEvalType: types.ETDatetime, + childrenTypes: []types.EvalType{types.ETDatetime, types.ETDatetime}, }, }, ast.LastDay: { diff --git a/expression/column.go b/expression/column.go index 006b9a3867cda..ebc0feaf93a06 100644 --- a/expression/column.go +++ b/expression/column.go @@ -38,10 +38,9 @@ type CorrelatedColumn struct { // Clone implements Expression interface. func (col *CorrelatedColumn) Clone() Expression { - var d types.Datum return &CorrelatedColumn{ Column: col.Column, - Data: &d, + Data: col.Data, } } diff --git a/expression/distsql_builtin.go b/expression/distsql_builtin.go index 0350a0b5acf4d..7fd61b30fa3fa 100644 --- a/expression/distsql_builtin.go +++ b/expression/distsql_builtin.go @@ -1268,9 +1268,13 @@ func convertEnum(val []byte, tp *tipb.FieldType) (*Constant, error) { if err != nil { return nil, errors.Errorf("invalid enum % x", val) } - e, err := types.ParseEnumValue(tp.Elems, uVal) - if err != nil { - return nil, err + // If uVal is 0, it should return Enum{} + var e = types.Enum{} + if uVal != 0 { + e, err = types.ParseEnumValue(tp.Elems, uVal) + if err != nil { + return nil, err + } } d := types.NewMysqlEnumDatum(e) return &Constant{Value: d, RetType: FieldTypeFromPB(tp)}, nil diff --git a/expression/expr_to_pb.go b/expression/expr_to_pb.go index 59ff01b73b67c..dc031145d0d95 100644 --- a/expression/expr_to_pb.go +++ b/expression/expr_to_pb.go @@ -211,6 +211,10 @@ func (pc PbConverter) columnToPBExpr(column *Column) *tipb.Expr { switch column.GetType().Tp { case mysql.TypeBit, mysql.TypeSet, mysql.TypeGeometry, mysql.TypeUnspecified: return nil + case mysql.TypeEnum: + if !IsPushDownEnabled("enum", kv.UnSpecified) { + return nil + } } if pc.client.IsRequestTypeSupported(kv.ReqTypeDAG, kv.ReqSubTypeBasic) { diff --git a/expression/expr_to_pb_test.go b/expression/expr_to_pb_test.go index d997e9cf1691f..f9b2a4d76978d 100644 --- a/expression/expr_to_pb_test.go +++ b/expression/expr_to_pb_test.go @@ -719,6 +719,11 @@ func (s *testEvaluatorSuite) TestExprPushDownToFlash(c *C) { c.Assert(err, IsNil) exprs = append(exprs, function) + // CastStringAsReal + function, err = NewFunction(mock.NewContext(), ast.Cast, types.NewFieldType(mysql.TypeDouble), stringColumn) + c.Assert(err, IsNil) + exprs = append(exprs, function) + // Substring2ArgsUTF8 function, err = NewFunction(mock.NewContext(), ast.Substr, types.NewFieldType(mysql.TypeString), stringColumn, intColumn) c.Assert(err, IsNil) @@ -729,14 +734,141 @@ func (s *testEvaluatorSuite) TestExprPushDownToFlash(c *C) { c.Assert(err, IsNil) exprs = append(exprs, function) + // sqrt + function, err = NewFunction(mock.NewContext(), ast.Sqrt, types.NewFieldType(mysql.TypeDouble), realColumn) + c.Assert(err, IsNil) + exprs = append(exprs, function) + + // ScalarFuncSig_CeilReal + function, err = NewFunction(mock.NewContext(), ast.Ceil, types.NewFieldType(mysql.TypeDouble), realColumn) + c.Assert(err, IsNil) + exprs = append(exprs, function) + + // ScalarFuncSig_CeilIntToInt + function, err = NewFunction(mock.NewContext(), ast.Ceil, types.NewFieldType(mysql.TypeLonglong), intColumn) + c.Assert(err, IsNil) + exprs = append(exprs, function) + + // ScalarFuncSig_CeilDecimalToInt + function, err = NewFunction(mock.NewContext(), ast.Ceil, types.NewFieldType(mysql.TypeLonglong), decimalColumn) + c.Assert(err, IsNil) + exprs = append(exprs, function) + + // ScalarFuncSig_CeilDecimalToDecimal + function, err = NewFunction(mock.NewContext(), ast.Ceil, types.NewFieldType(mysql.TypeNewDecimal), decimalColumn) + c.Assert(err, IsNil) + exprs = append(exprs, function) + + // ScalarFuncSig_FloorReal + function, err = NewFunction(mock.NewContext(), ast.Floor, types.NewFieldType(mysql.TypeDouble), realColumn) + c.Assert(err, IsNil) + exprs = append(exprs, function) + + // ScalarFuncSig_FloorIntToInt + function, err = NewFunction(mock.NewContext(), ast.Floor, types.NewFieldType(mysql.TypeLonglong), intColumn) + c.Assert(err, IsNil) + exprs = append(exprs, function) + + // ScalarFuncSig_FloorDecimalToInt + function, err = NewFunction(mock.NewContext(), ast.Floor, types.NewFieldType(mysql.TypeLonglong), decimalColumn) + c.Assert(err, IsNil) + exprs = append(exprs, function) + + // ScalarFuncSig_FloorDecimalToDecimal + function, err = NewFunction(mock.NewContext(), ast.Floor, types.NewFieldType(mysql.TypeNewDecimal), decimalColumn) + c.Assert(err, IsNil) + exprs = append(exprs, function) + + // Replace + function, err = NewFunction(mock.NewContext(), ast.Replace, types.NewFieldType(mysql.TypeString), stringColumn, stringColumn, stringColumn) + c.Assert(err, IsNil) + exprs = append(exprs, function) + + // ScalarFuncSig_RoundReal function, err = NewFunction(mock.NewContext(), ast.Round, types.NewFieldType(mysql.TypeDouble), realColumn) c.Assert(err, IsNil) exprs = append(exprs, function) + // ScalarFuncSig_RoundInt function, err = NewFunction(mock.NewContext(), ast.Round, types.NewFieldType(mysql.TypeLonglong), intColumn) c.Assert(err, IsNil) exprs = append(exprs, function) + // concat + function, err = NewFunction(mock.NewContext(), ast.Concat, types.NewFieldType(mysql.TypeString), stringColumn, intColumn, realColumn) + c.Assert(err, IsNil) + exprs = append(exprs, function) + + // UnixTimestampCurrent + function, err = NewFunction(mock.NewContext(), ast.UnixTimestamp, types.NewFieldType(mysql.TypeLonglong)) + c.Assert(err, IsNil) + _, ok := function.(*Constant) + c.Assert(ok, IsTrue) + + // UnixTimestampInt + datetimeColumn.RetType.Decimal = 0 + function, err = NewFunction(mock.NewContext(), ast.UnixTimestamp, types.NewFieldType(mysql.TypeLonglong), datetimeColumn) + c.Assert(err, IsNil) + c.Assert(function.(*ScalarFunction).Function.PbCode(), Equals, tipb.ScalarFuncSig_UnixTimestampInt) + exprs = append(exprs, function) + + // UnixTimestampDecimal + datetimeColumn.RetType.Decimal = types.UnspecifiedLength + function, err = NewFunction(mock.NewContext(), ast.UnixTimestamp, types.NewFieldType(mysql.TypeNewDecimal), datetimeColumn) + c.Assert(err, IsNil) + c.Assert(function.(*ScalarFunction).Function.PbCode(), Equals, tipb.ScalarFuncSig_UnixTimestampDec) + exprs = append(exprs, function) + + // Year + function, err = NewFunction(mock.NewContext(), ast.Year, types.NewFieldType(mysql.TypeLonglong), datetimeColumn) + c.Assert(err, IsNil) + c.Assert(function.(*ScalarFunction).Function.PbCode(), Equals, tipb.ScalarFuncSig_Year) + exprs = append(exprs, function) + + // Day + function, err = NewFunction(mock.NewContext(), ast.Day, types.NewFieldType(mysql.TypeLonglong), datetimeColumn) + c.Assert(err, IsNil) + c.Assert(function.(*ScalarFunction).Function.PbCode(), Equals, tipb.ScalarFuncSig_DayOfMonth) + exprs = append(exprs, function) + + // Datediff + function, err = NewFunction(mock.NewContext(), ast.DateDiff, types.NewFieldType(mysql.TypeLonglong), datetimeColumn, datetimeColumn) + c.Assert(err, IsNil) + c.Assert(function.(*ScalarFunction).Function.PbCode(), Equals, tipb.ScalarFuncSig_DateDiff) + exprs = append(exprs, function) + + // Datesub + function, err = NewFunction(mock.NewContext(), ast.DateSub, types.NewFieldType(mysql.TypeDatetime), datetimeColumn, intColumn, stringColumn) + c.Assert(err, IsNil) + c.Assert(function.(*ScalarFunction).Function.PbCode(), Equals, tipb.ScalarFuncSig_SubDateDatetimeInt) + exprs = append(exprs, function) + function, err = NewFunction(mock.NewContext(), ast.DateSub, types.NewFieldType(mysql.TypeDatetime), stringColumn, intColumn, stringColumn) + c.Assert(err, IsNil) + c.Assert(function.(*ScalarFunction).Function.PbCode(), Equals, tipb.ScalarFuncSig_SubDateStringInt) + exprs = append(exprs, function) + function, err = NewFunction(mock.NewContext(), ast.SubDate, types.NewFieldType(mysql.TypeDatetime), datetimeColumn, intColumn, stringColumn) + c.Assert(err, IsNil) + c.Assert(function.(*ScalarFunction).Function.PbCode(), Equals, tipb.ScalarFuncSig_SubDateDatetimeInt) + exprs = append(exprs, function) + + // castTimeAsString: + function, err = NewFunction(mock.NewContext(), ast.Cast, types.NewFieldType(mysql.TypeString), datetimeColumn) + c.Assert(err, IsNil) + c.Assert(function.(*ScalarFunction).Function.PbCode(), Equals, tipb.ScalarFuncSig_CastTimeAsString) + exprs = append(exprs, function) + + // concat_ws + function, err = NewFunction(mock.NewContext(), ast.ConcatWS, types.NewFieldType(mysql.TypeString), stringColumn, stringColumn, stringColumn) + c.Assert(err, IsNil) + c.Assert(function.(*ScalarFunction).Function.PbCode(), Equals, tipb.ScalarFuncSig_ConcatWS) + exprs = append(exprs, function) + + // StrToDateDateTime + function, err = NewFunction(mock.NewContext(), ast.StrToDate, types.NewFieldType(mysql.TypeDatetime), stringColumn, stringColumn) + c.Assert(err, IsNil) + c.Assert(function.(*ScalarFunction).Function.PbCode(), Equals, tipb.ScalarFuncSig_StrToDateDatetime) + exprs = append(exprs, function) + canPush := CanExprsPushDown(sc, exprs, client, kv.TiFlash) c.Assert(canPush, Equals, true) diff --git a/expression/expression.go b/expression/expression.go index de6bb1cab5e54..daa3be56a9959 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -996,36 +996,58 @@ func scalarExprSupportedByTiKV(sf *ScalarFunction) bool { func scalarExprSupportedByFlash(function *ScalarFunction) bool { switch function.FuncName.L { + case ast.Floor, ast.Ceil, ast.Ceiling: + switch function.Function.PbCode() { + case tipb.ScalarFuncSig_FloorIntToDec, tipb.ScalarFuncSig_CeilIntToDec: + return false + default: + return true + } case ast.LogicOr, ast.LogicAnd, ast.UnaryNot, ast.BitNeg, ast.Xor, ast.And, ast.Or, ast.GE, ast.LE, ast.EQ, ast.NE, ast.LT, ast.GT, ast.In, ast.IsNull, ast.Like, - ast.Plus, ast.Minus, ast.Div, ast.Mul, /*ast.Mod,*/ + ast.Plus, ast.Minus, ast.Div, ast.Mul, ast.Abs, /*ast.Mod,*/ ast.If, ast.Ifnull, ast.Case, - ast.Month, - ast.TimestampDiff, ast.DateFormat, ast.FromUnixTime, + ast.Concat, ast.ConcatWS, + ast.Year, ast.Month, ast.Day, + ast.DateDiff, ast.TimestampDiff, ast.DateFormat, ast.FromUnixTime, + ast.Sqrt, ast.JSONLength: return true - case ast.Substr, ast.Substring: + case ast.Substr, ast.Substring, ast.Left, ast.Right, ast.CharLength: switch function.Function.PbCode() { case + tipb.ScalarFuncSig_LeftUTF8, + tipb.ScalarFuncSig_RightUTF8, + tipb.ScalarFuncSig_CharLengthUTF8, tipb.ScalarFuncSig_Substring2ArgsUTF8, tipb.ScalarFuncSig_Substring3ArgsUTF8: return true } case ast.Cast: switch function.Function.PbCode() { - case tipb.ScalarFuncSig_CastIntAsInt, tipb.ScalarFuncSig_CastIntAsDecimal, tipb.ScalarFuncSig_CastIntAsString, tipb.ScalarFuncSig_CastIntAsTime, - tipb.ScalarFuncSig_CastRealAsInt, tipb.ScalarFuncSig_CastRealAsDecimal, tipb.ScalarFuncSig_CastRealAsString, tipb.ScalarFuncSig_CastRealAsTime, - tipb.ScalarFuncSig_CastStringAsInt, tipb.ScalarFuncSig_CastStringAsDecimal, tipb.ScalarFuncSig_CastStringAsString, tipb.ScalarFuncSig_CastStringAsTime, - tipb.ScalarFuncSig_CastDecimalAsInt, tipb.ScalarFuncSig_CastDecimalAsDecimal, tipb.ScalarFuncSig_CastDecimalAsString, tipb.ScalarFuncSig_CastDecimalAsTime, - tipb.ScalarFuncSig_CastTimeAsInt, tipb.ScalarFuncSig_CastTimeAsDecimal, tipb.ScalarFuncSig_CastTimeAsTime: + case tipb.ScalarFuncSig_CastIntAsInt, tipb.ScalarFuncSig_CastIntAsReal, tipb.ScalarFuncSig_CastIntAsDecimal, tipb.ScalarFuncSig_CastIntAsString, tipb.ScalarFuncSig_CastIntAsTime, + tipb.ScalarFuncSig_CastRealAsInt, tipb.ScalarFuncSig_CastRealAsReal, tipb.ScalarFuncSig_CastRealAsDecimal, tipb.ScalarFuncSig_CastRealAsString, tipb.ScalarFuncSig_CastRealAsTime, + tipb.ScalarFuncSig_CastStringAsInt, tipb.ScalarFuncSig_CastStringAsReal, tipb.ScalarFuncSig_CastStringAsDecimal, tipb.ScalarFuncSig_CastStringAsString, tipb.ScalarFuncSig_CastStringAsTime, + tipb.ScalarFuncSig_CastDecimalAsInt /*, tipb.ScalarFuncSig_CastDecimalAsReal*/, tipb.ScalarFuncSig_CastDecimalAsDecimal, tipb.ScalarFuncSig_CastDecimalAsString, tipb.ScalarFuncSig_CastDecimalAsTime, + tipb.ScalarFuncSig_CastTimeAsInt /*, tipb.ScalarFuncSig_CastTimeAsReal*/, tipb.ScalarFuncSig_CastTimeAsDecimal, tipb.ScalarFuncSig_CastTimeAsTime, tipb.ScalarFuncSig_CastTimeAsString: return true } - case ast.DateAdd: + case ast.DateAdd, ast.AddDate: switch function.Function.PbCode() { case tipb.ScalarFuncSig_AddDateDatetimeInt, tipb.ScalarFuncSig_AddDateStringInt: return true } + case ast.DateSub, ast.SubDate: + switch function.Function.PbCode() { + case tipb.ScalarFuncSig_SubDateDatetimeInt, tipb.ScalarFuncSig_SubDateStringInt: + return true + } + case ast.UnixTimestamp: + switch function.Function.PbCode() { + case tipb.ScalarFuncSig_UnixTimestampInt, tipb.ScalarFuncSig_UnixTimestampDec: + return true + } case ast.Round: switch function.Function.PbCode() { case tipb.ScalarFuncSig_RoundInt, tipb.ScalarFuncSig_RoundReal: @@ -1036,6 +1058,20 @@ func scalarExprSupportedByFlash(function *ScalarFunction) bool { case tipb.ScalarFuncSig_ExtractDatetime: return true } + case ast.Replace: + switch function.Function.PbCode() { + case tipb.ScalarFuncSig_Replace: + return true + } + case ast.StrToDate: + switch function.Function.PbCode() { + case + tipb.ScalarFuncSig_StrToDateDate, + tipb.ScalarFuncSig_StrToDateDatetime: + return true + default: + return false + } } return false } diff --git a/expression/generator/compare_vec.go b/expression/generator/compare_vec.go index 5864e61c02c3c..4908b63af477b 100644 --- a/expression/generator/compare_vec.go +++ b/expression/generator/compare_vec.go @@ -19,8 +19,8 @@ import ( "bytes" "flag" "go/format" - "io/ioutil" "log" + "os" "path/filepath" "text/template" @@ -414,7 +414,7 @@ func generateDotGo(fileName string, compares []CompareContext, types []TypeConte log.Println("[Warn]", fileName+": gofmt failed", err) data = w.Bytes() // write original data for debugging } - return ioutil.WriteFile(fileName, data, 0644) + return os.WriteFile(fileName, data, 0644) } func generateTestDotGo(fileName string, compares []CompareContext, types []TypeContext) error { @@ -459,7 +459,7 @@ func generateTestDotGo(fileName string, compares []CompareContext, types []TypeC log.Println("[Warn]", fileName+": gofmt failed", err) data = w.Bytes() // write original data for debugging } - return ioutil.WriteFile(fileName, data, 0644) + return os.WriteFile(fileName, data, 0644) } // generateOneFile generate one xxx.go file and the associated xxx_test.go file. diff --git a/expression/generator/control_vec.go b/expression/generator/control_vec.go index 0b77d127d6f9a..99e77fd4052b8 100644 --- a/expression/generator/control_vec.go +++ b/expression/generator/control_vec.go @@ -18,8 +18,8 @@ package main import ( "bytes" "go/format" - "io/ioutil" "log" + "os" "path/filepath" "text/template" @@ -636,7 +636,7 @@ func generateDotGo(fileName string) error { log.Println("[Warn]", fileName+": gofmt failed", err) data = w.Bytes() // write original data for debugging } - return ioutil.WriteFile(fileName, data, 0644) + return os.WriteFile(fileName, data, 0644) } func generateTestDotGo(fileName string) error { @@ -650,7 +650,7 @@ func generateTestDotGo(fileName string) error { log.Println("[Warn]", fileName+": gofmt failed", err) data = w.Bytes() // write original data for debugging } - return ioutil.WriteFile(fileName, data, 0644) + return os.WriteFile(fileName, data, 0644) } // generateOneFile generate one xxx.go file and the associated xxx_test.go file. diff --git a/expression/generator/other_vec.go b/expression/generator/other_vec.go index 8016ba7619766..9c57d179f123d 100644 --- a/expression/generator/other_vec.go +++ b/expression/generator/other_vec.go @@ -18,8 +18,8 @@ package main import ( "bytes" "go/format" - "io/ioutil" "log" + "os" "path/filepath" "text/template" @@ -459,7 +459,7 @@ func generateDotGo(fileName string) error { log.Println("[Warn]", fileName+": gofmt failed", err) data = w.Bytes() // write original data for debugging } - return ioutil.WriteFile(fileName, data, 0644) + return os.WriteFile(fileName, data, 0644) } func generateTestDotGo(fileName string) error { @@ -473,7 +473,7 @@ func generateTestDotGo(fileName string) error { log.Println("[Warn]", fileName+": gofmt failed", err) data = w.Bytes() // write original data for debugging } - return ioutil.WriteFile(fileName, data, 0644) + return os.WriteFile(fileName, data, 0644) } // generateOneFile generate one xxx.go file and the associated xxx_test.go file. diff --git a/expression/generator/string_vec.go b/expression/generator/string_vec.go index 8eaeaee32a387..72785b98b7df4 100644 --- a/expression/generator/string_vec.go +++ b/expression/generator/string_vec.go @@ -19,8 +19,8 @@ import ( "bytes" "flag" "go/format" - "io/ioutil" "log" + "os" "path/filepath" "text/template" @@ -168,7 +168,7 @@ func generateDotGo(fileName string, types []TypeContext) (err error) { log.Println("[Warn]", fileName+": gofmt failed", err) data = w.Bytes() // write original data for debugging } - return ioutil.WriteFile(fileName, data, 0644) + return os.WriteFile(fileName, data, 0644) } func generateTestDotGo(fileName string, types []TypeContext) error { @@ -186,7 +186,7 @@ func generateTestDotGo(fileName string, types []TypeContext) error { log.Println("[Warn]", fileName+": gofmt failed", err) data = w.Bytes() // write original data for debugging } - return ioutil.WriteFile(fileName, data, 0644) + return os.WriteFile(fileName, data, 0644) } // generateOneFile generate one xxx.go file and the associated xxx_test.go file. diff --git a/expression/generator/time_vec.go b/expression/generator/time_vec.go index b6c13dc1af760..72b35ec17f8b0 100644 --- a/expression/generator/time_vec.go +++ b/expression/generator/time_vec.go @@ -18,8 +18,8 @@ package main import ( "bytes" "go/format" - "io/ioutil" "log" + "os" "path/filepath" "text/template" @@ -944,7 +944,7 @@ func generateDotGo(fileName string) error { log.Println("[Warn]", fileName+": gofmt failed", err) data = w.Bytes() // write original data for debugging } - return ioutil.WriteFile(fileName, data, 0644) + return os.WriteFile(fileName, data, 0644) } func generateTestDotGo(fileName string) error { @@ -958,7 +958,7 @@ func generateTestDotGo(fileName string) error { log.Println("[Warn]", fileName+": gofmt failed", err) data = w.Bytes() // write original data for debugging } - return ioutil.WriteFile(fileName, data, 0644) + return os.WriteFile(fileName, data, 0644) } // generateOneFile generate one xxx.go file and the associated xxx_test.go file. diff --git a/expression/helper.go b/expression/helper.go index 1e89c86e705f2..d9f1e22610b62 100644 --- a/expression/helper.go +++ b/expression/helper.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" driver "github.com/pingcap/tidb/types/parser_driver" @@ -139,7 +140,7 @@ func getStmtTimestamp(ctx sessionctx.Context) (time.Time, error) { } sessionVars := ctx.GetSessionVars() - timestampStr, err := variable.GetSessionSystemVar(sessionVars, "timestamp") + timestampStr, err := variable.GetSessionOrGlobalSystemVar(sessionVars, "timestamp") if err != nil { return now, err } @@ -155,5 +156,5 @@ func getStmtTimestamp(ctx sessionctx.Context) (time.Time, error) { return time.Unix(timestamp, 0), nil } stmtCtx := ctx.GetSessionVars().StmtCtx - return stmtCtx.GetNowTsCached(), nil + return stmtCtx.GetOrStoreStmtCache(stmtctx.StmtNowTsCacheKey, time.Now()).(time.Time), nil } diff --git a/expression/integration_test.go b/expression/integration_test.go index 732af641ff5e7..a1dc733e16cce 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -34,6 +34,7 @@ import ( "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/ddl/placement" "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/kv" plannercore "github.com/pingcap/tidb/planner/core" @@ -41,6 +42,7 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/mockstore" + "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/tablecodec" @@ -212,6 +214,74 @@ func (s *testIntegrationSuite) TestFuncLpadAndRpad(c *C) { result.Check(testkit.Rows(" ")) } +func (s *testIntegrationSuite) TestBuiltinFuncJsonPretty(c *C) { + ctx := context.Background() + tk := testkit.NewTestKit(c, s.store) + defer s.cleanEnv(c) + + tk.MustExec(`use test;`) + tk.MustExec(`drop table if exists t;`) + tk.MustExec("CREATE TABLE t (`id` int NOT NULL AUTO_INCREMENT, `j` json,vc VARCHAR(500) , PRIMARY KEY (`id`));") + tk.MustExec(`INSERT INTO t ( id, j, vc ) VALUES + ( 1, '{"a":1,"b":"qwe","c":[1,2,3,"123",null],"d":{"d1":1,"d2":2}}', '{"a":1,"b":"qwe","c":[1,2,3,"123",null],"d":{"d1":1,"d2":2}}' ), + ( 2, '[1,2,34]', '{' );`) + + // valid json format in json and varchar + checkResult := []string{ + `{ + "a": 1, + "b": "qwe", + "c": [ + 1, + 2, + 3, + "123", + null + ], + "d": { + "d1": 1, + "d2": 2 + } +}`, + `{ + "a": 1, + "b": "qwe", + "c": [ + 1, + 2, + 3, + "123", + null + ], + "d": { + "d1": 1, + "d2": 2 + } +}`, + } + tk. + MustQuery("select JSON_PRETTY(t.j),JSON_PRETTY(vc) from t where id = 1;"). + Check(testkit.Rows(strings.Join(checkResult, " "))) + + // invalid json format in varchar + rs, _ := tk.Exec("select JSON_PRETTY(t.j),JSON_PRETTY(vc) from t where id = 2;") + _, err := session.GetRows4Test(ctx, tk.Se, rs) + terr := errors.Cause(err).(*terror.Error) + c.Assert(terr.Code(), Equals, errors.ErrCode(mysql.ErrInvalidJSONText)) + + // invalid json format in one row + rs, _ = tk.Exec("select JSON_PRETTY(t.j),JSON_PRETTY(vc) from t where id in (1,2);") + _, err = session.GetRows4Test(ctx, tk.Se, rs) + terr = errors.Cause(err).(*terror.Error) + c.Assert(terr.Code(), Equals, errors.ErrCode(mysql.ErrInvalidJSONText)) + + // invalid json string + rs, _ = tk.Exec(`select JSON_PRETTY("[1,2,3]}");`) + _, err = session.GetRows4Test(ctx, tk.Se, rs) + terr = errors.Cause(err).(*terror.Error) + c.Assert(terr.Code(), Equals, errors.ErrCode(mysql.ErrInvalidJSONText)) +} + func (s *testIntegrationSuite) TestMiscellaneousBuiltin(c *C) { ctx := context.Background() defer s.cleanEnv(c) @@ -327,6 +397,15 @@ func (s *testIntegrationSuite) TestConvertToBit(c *C) { "Warning 1690 constant 599999999 overflows tinyint", "Warning 1406 Data Too Long, field len 63")) tk.MustQuery("select * from t;").Check(testkit.Rows("127 \u007f\xff\xff\xff\xff\xff\xff\xff")) + + // For issue 24900 + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(b bit(16));") + tk.MustExec("insert ignore into t values(0x3635313836),(0x333830);") + tk.MustQuery("show warnings;").Check(testkit.Rows( + "Warning 1406 Data Too Long, field len 16", + "Warning 1406 Data Too Long, field len 16")) + tk.MustQuery("select * from t;").Check(testkit.Rows("\xff\xff", "\xff\xff")) } func (s *testIntegrationSuite2) TestMathBuiltin(c *C) { @@ -2263,6 +2342,79 @@ func (s *testIntegrationSuite2) TestTimeBuiltin(c *C) { result = tk.MustQuery(`select tidb_parse_tso(-1)`) result.Check(testkit.Rows("")) + // for tidb_bounded_staleness + tk.MustExec("SET time_zone = '+00:00';") + t := time.Now().UTC() + ts := oracle.GoTimeToTS(t) + tidbBoundedStalenessTests := []struct { + sql string + injectSafeTS uint64 + expect string + }{ + { + sql: `select tidb_bounded_staleness(DATE_SUB(NOW(), INTERVAL 600 SECOND), DATE_ADD(NOW(), INTERVAL 600 SECOND))`, + injectSafeTS: ts, + expect: t.Format(types.TimeFSPFormat[:len(types.TimeFSPFormat)-3]), + }, + { + sql: `select tidb_bounded_staleness("2021-04-27 12:00:00.000", "2021-04-27 13:00:00.000")`, + injectSafeTS: func() uint64 { + t, err := time.Parse("2006-01-02 15:04:05.000", "2021-04-27 13:30:04.877") + c.Assert(err, IsNil) + return oracle.GoTimeToTS(t) + }(), + expect: "2021-04-27 13:00:00.000", + }, + { + sql: `select tidb_bounded_staleness("2021-04-27 12:00:00.000", "2021-04-27 13:00:00.000")`, + injectSafeTS: func() uint64 { + t, err := time.Parse("2006-01-02 15:04:05.000", "2021-04-27 11:30:04.877") + c.Assert(err, IsNil) + return oracle.GoTimeToTS(t) + }(), + expect: "2021-04-27 12:00:00.000", + }, + { + sql: `select tidb_bounded_staleness("2021-04-27 12:00:00.000", "2021-04-27 11:00:00.000")`, + injectSafeTS: 0, + expect: "", + }, + // Time is too small. + { + sql: `select tidb_bounded_staleness("0020-04-27 12:00:00.000", "2021-04-27 11:00:00.000")`, + injectSafeTS: 0, + expect: "1970-01-01 00:00:00.000", + }, + // Wrong value. + { + sql: `select tidb_bounded_staleness(1, 2)`, + injectSafeTS: 0, + expect: "", + }, + { + sql: `select tidb_bounded_staleness("invalid_time_1", "invalid_time_2")`, + injectSafeTS: 0, + expect: "", + }, + } + for _, test := range tidbBoundedStalenessTests { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/expression/injectSafeTS", + fmt.Sprintf("return(%v)", test.injectSafeTS)), IsNil) + result = tk.MustQuery(test.sql) + result.Check(testkit.Rows(test.expect)) + } + failpoint.Disable("github.com/pingcap/tidb/expression/injectSafeTS") + // test whether tidb_bounded_staleness is deterministic + result = tk.MustQuery(`select tidb_bounded_staleness(NOW(), DATE_ADD(NOW(), INTERVAL 600 SECOND)), tidb_bounded_staleness(NOW(), DATE_ADD(NOW(), INTERVAL 600 SECOND))`) + c.Assert(result.Rows()[0], HasLen, 2) + c.Assert(result.Rows()[0][0], Equals, result.Rows()[0][1]) + preResult := result.Rows()[0][0] + time.Sleep(time.Second) + result = tk.MustQuery(`select tidb_bounded_staleness(NOW(), DATE_ADD(NOW(), INTERVAL 600 SECOND)), tidb_bounded_staleness(NOW(), DATE_ADD(NOW(), INTERVAL 600 SECOND))`) + c.Assert(result.Rows()[0], HasLen, 2) + c.Assert(result.Rows()[0][0], Equals, result.Rows()[0][1]) + c.Assert(result.Rows()[0][0], Not(Equals), preResult) + // fix issue 10308 result = tk.MustQuery("select time(\"- -\");") result.Check(testkit.Rows("00:00:00")) @@ -4708,10 +4860,10 @@ func (s *testIntegrationSuite) TestFilterExtractFromDNF(c *C) { stmts, err := session.Parse(sctx, sql) c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, tt.exprStr)) c.Assert(stmts, HasLen, 1) - is := domain.GetDomain(sctx).InfoSchema() - err = plannercore.Preprocess(sctx, stmts[0], is) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) c.Assert(err, IsNil, Commentf("error %v, for resolve name, expr %s", err, tt.exprStr)) - p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is) + p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], ret.InfoSchema) c.Assert(err, IsNil, Commentf("error %v, for build plan, expr %s", err, tt.exprStr)) selection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) conds := make([]expression.Expression, len(selection.Conditions)) @@ -5474,16 +5626,14 @@ func (s *testIntegrationSuite) TestExprPushdownBlacklist(c *C) { // > pushed to both TiKV and TiFlash rows := tk.MustQuery("explain format = 'brief' select * from test.t where b > date'1988-01-01' and b < date'1994-01-01' " + "and cast(a as decimal(10,2)) > 10.10 and date_format(b,'%m') = '11'").Rows() - c.Assert(fmt.Sprintf("%v", rows[0][4]), Equals, "lt(test.t.b, 1994-01-01)") - c.Assert(fmt.Sprintf("%v", rows[1][4]), Equals, "gt(cast(test.t.a, decimal(10,2) BINARY), 10.10)") - c.Assert(fmt.Sprintf("%v", rows[3][4]), Equals, "eq(date_format(test.t.b, \"%m\"), \"11\"), gt(test.t.b, 1988-01-01)") + c.Assert(fmt.Sprintf("%v", rows[0][4]), Equals, "gt(cast(test.t.a, decimal(10,2) BINARY), 10.10), lt(test.t.b, 1994-01-01)") + c.Assert(fmt.Sprintf("%v", rows[2][4]), Equals, "eq(date_format(test.t.b, \"%m\"), \"11\"), gt(test.t.b, 1988-01-01)") tk.MustExec("set @@session.tidb_isolation_read_engines = 'tikv'") rows = tk.MustQuery("explain format = 'brief' select * from test.t where b > date'1988-01-01' and b < date'1994-01-01' " + "and cast(a as decimal(10,2)) > 10.10 and date_format(b,'%m') = '11'").Rows() - c.Assert(fmt.Sprintf("%v", rows[0][4]), Equals, "lt(test.t.b, 1994-01-01)") - c.Assert(fmt.Sprintf("%v", rows[1][4]), Equals, "eq(date_format(test.t.b, \"%m\"), \"11\")") - c.Assert(fmt.Sprintf("%v", rows[3][4]), Equals, "gt(cast(test.t.a, decimal(10,2) BINARY), 10.10), gt(test.t.b, 1988-01-01)") + c.Assert(fmt.Sprintf("%v", rows[0][4]), Equals, "eq(date_format(test.t.b, \"%m\"), \"11\"), lt(test.t.b, 1994-01-01)") + c.Assert(fmt.Sprintf("%v", rows[2][4]), Equals, "gt(cast(test.t.a, decimal(10,2) BINARY), 10.10), gt(test.t.b, 1988-01-01)") tk.MustExec("delete from mysql.expr_pushdown_blacklist where name = '<' and store_type = 'tikv,tiflash,tidb' and reason = 'for test'") tk.MustExec("delete from mysql.expr_pushdown_blacklist where name = 'date_format' and store_type = 'tikv' and reason = 'for test'") @@ -5835,7 +5985,7 @@ func (s *testIntegrationSuite) TestDecodetoChunkReuse(c *C) { tk.MustExec("create table chk (a int,b varchar(20))") for i := 0; i < 200; i++ { if i%5 == 0 { - tk.MustExec(fmt.Sprintf("insert chk values (NULL,NULL)")) + tk.MustExec("insert chk values (NULL,NULL)") continue } tk.MustExec(fmt.Sprintf("insert chk values (%d,'%s')", i, strconv.Itoa(i))) @@ -7221,6 +7371,32 @@ func (s *testIntegrationSuite) TestIssue15992(c *C) { tk.MustExec("drop table t0;") } +func (s *testIntegrationSuite) TestCTEWithDML(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("use test;") + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(a int);") + tk.MustExec("insert into t1 values(2),(3);") + tk.MustQuery("with t1 as (select 36 as col from t1 where a=3) select * from t1;").Check(testkit.Rows("36")) + tk.MustExec("insert into t1 with t1 as (select 36 as col from t1) select * from t1;") + tk.MustQuery("select * from t1").Check(testkit.Rows("2", "3", "36", "36")) + tk.MustExec("with cte1(a) as (select 36) update t1 set a = 1 where a in (select a from cte1);") + tk.MustQuery("select * from t1").Check(testkit.Rows("2", "3", "1", "1")) + tk.MustExec("with recursive cte(a) as (select 1 union select a + 1 from cte where a < 10) update cte, t1 set t1.a=1") + tk.MustQuery("select * from t1").Check(testkit.Rows("1", "1", "1", "1")) + + tk.MustGetErrCode("with recursive cte(a) as (select 1 union select a + 1 from cte where a < 10) update cte set a=1", mysql.ErrNonUpdatableTable) + tk.MustGetErrCode("with recursive cte(a) as (select 1 union select a + 1 from cte where a < 10) delete from cte", mysql.ErrNonUpdatableTable) + tk.MustGetErrCode("with cte(a) as (select a from t1) delete from cte", mysql.ErrNonUpdatableTable) + tk.MustGetErrCode("with cte(a) as (select a from t1) update cte set a=1", mysql.ErrNonUpdatableTable) + + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(a int, b int, primary key(a));") + tk.MustExec("insert into t1 values (1, 1),(2,1),(3,1);") + tk.MustExec("replace into t1 with recursive cte(a,b) as (select 1, 1 union select a + 1,b+1 from cte where a < 5) select * from cte;") + tk.MustQuery("select * from t1").Check(testkit.Rows("1 1", "2 2", "3 3", "4 4", "5 5")) +} + func (s *testIntegrationSuite) TestIssue16419(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("use test;") @@ -7901,7 +8077,7 @@ func (s *testIntegrationSerialSuite) TestIssue19804(c *C) { tk.MustExec(`create table t(a set('a', 'b', 'c'));`) tk.MustExec(`alter table t change a a set('a', 'b', 'c', 'd');`) tk.MustExec(`insert into t values('d');`) - tk.MustGetErrMsg(`alter table t change a a set('a', 'b', 'c', 'e', 'f');`, "[ddl:8200]Unsupported modify column: cannot modify set column value d to e, and tidb_enable_change_column_type is false") + tk.MustGetErrMsg(`alter table t change a a set('a', 'b', 'c', 'e', 'f');`, "[types:1265]Data truncated for column 'a', value is 'KindMysqlSet d'") } func (s *testIntegrationSerialSuite) TestIssue20209(c *C) { @@ -8757,7 +8933,7 @@ PARTITION BY RANGE (c) ( GroupID: groupID, Role: placement.Leader, Count: 1, - LabelConstraints: []placement.Constraint{ + Constraints: []placement.Constraint{ { Key: placement.DCLabelKey, Op: placement.In, @@ -9088,9 +9264,15 @@ func (s *testIntegrationSuite) TestEnumPushDown(c *C) { func (s *testIntegrationSuite) TestJiraSetInnoDBDefaultRowFormat(c *C) { // For issue #23541 // JIRA needs to be able to set this to be happy. + // See: https://nova.moe/run-jira-on-tidb/ tk := testkit.NewTestKit(c, s.store) tk.MustExec("set global innodb_default_row_format = dynamic") tk.MustExec("set global innodb_default_row_format = 'dynamic'") + tk.MustQuery("SHOW VARIABLES LIKE 'innodb_default_row_format'").Check(testkit.Rows("innodb_default_row_format dynamic")) + tk.MustQuery("SHOW VARIABLES LIKE 'character_set_server'").Check(testkit.Rows("character_set_server utf8mb4")) + tk.MustQuery("SHOW VARIABLES LIKE 'innodb_file_format'").Check(testkit.Rows("innodb_file_format Barracuda")) + tk.MustQuery("SHOW VARIABLES LIKE 'innodb_large_prefix'").Check(testkit.Rows("innodb_large_prefix ON")) + } func (s *testIntegrationSerialSuite) TestCollationForBinaryLiteral(c *C) { @@ -9158,6 +9340,17 @@ func (s *testIntegrationSerialSuite) TestIssue23805(c *C) { tk.MustExec("insert ignore into tbl_5 set col_28 = 'ZmZIdSnq' , col_25 = '18:50:52.00' on duplicate key update col_26 = 'y';\n") } +func (s *testIntegrationSuite) TestIssue24429(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("set @@sql_mode = ANSI_QUOTES;") + tk.MustExec("use test") + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int);") + tk.MustQuery(`select t."a"=10 from t;`).Check(testkit.Rows()) + tk.MustExec("drop table if exists t;") +} + func (s *testIntegrationSuite) TestVitessHash(c *C) { defer s.cleanEnv(c) tk := testkit.NewTestKit(c, s.store) @@ -9261,6 +9454,47 @@ func (s *testIntegrationSuite) TestIssue23925(c *C) { tk.MustQuery("select max(b) + 0 from t group by a;").Check(testkit.Rows("2")) } +func (s *testIntegrationSuite) TestCTEInvalidUsage(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("use test;") + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(a int);") + // A CTE can refer to CTEs defined earlier in the same WITH clause, but not those defined later. + tk.MustGetErrCode("with cte1 as (select * from cte2), cte2 as (select 1) select * from cte1;", errno.ErrNoSuchTable) + // A CTE in a given query block can refer to CTEs defined in query blocks at a more outer level, but not CTEs defined in query blocks at a more inner level. + // MySQL allows this statement, and it should be a bug of MySQL. PostgreSQL also reports an error. + tk.MustGetErrCode("with cte1 as (select * from cte2) select * from (with cte2 as (select 2) select * from cte1 ) q;", errno.ErrNoSuchTable) + // Aggregation function is not allowed in the recursive part. + tk.MustGetErrCode("with recursive cte(n) as (select 1 union select sum(n) from cte group by n) select * from cte;", errno.ErrCTERecursiveForbidsAggregation) + // Window function is not allowed in the recursive part. + tk.MustGetErrCode("with recursive cte(n) as (select 1 union select row_number() over(partition by n) from cte ) select * from cte;", errno.ErrCTERecursiveForbidsAggregation) + // Group by is not allowed in the recursive part. + tk.MustGetErrCode("with recursive cte(n) as (select 1 union (select * from cte order by n)) select * from cte;", errno.ErrNotSupportedYet) + tk.MustGetErrCode("with recursive cte(n) as (select 1 union (select * from cte order by n)) select * from cte;", errno.ErrNotSupportedYet) + // Distinct is not allowed in the recursive part. + tk.MustGetErrCode("with recursive cte(n) as (select 1 union select distinct * from cte) select * from cte;", errno.ErrNotSupportedYet) + // Limit is not allowed in the recursive part. + tk.MustGetErrCode("with recursive cte(n) as (select 1 union (select * from cte limit 2)) select * from cte;", errno.ErrNotSupportedYet) + // The recursive SELECT part must reference the CTE only once and only in its FROM clause, not in any subquery. + tk.MustGetErrCode("with recursive cte(n) as (select 1 union select * from cte, cte c1) select * from cte;", errno.ErrInvalidRequiresSingleReference) + tk.MustGetErrCode("with recursive cte(n) as (select 1 union select * from (select * from cte) c1) select * from cte;", errno.ErrInvalidRequiresSingleReference) + tk.MustGetErrCode("with recursive cte(n) as (select 1 union select * from cte where 1 in (select * from cte)) select * from cte;", errno.ErrInvalidRequiresSingleReference) + tk.MustGetErrCode("with recursive cte(n) as (select 1 union select * from cte where exists (select * from cte)) select * from cte;", errno.ErrInvalidRequiresSingleReference) + tk.MustGetErrCode("with recursive cte(n) as (select 1 union select * from cte where 1 > (select * from cte)) select * from cte;", errno.ErrInvalidRequiresSingleReference) + tk.MustGetErrCode("with recursive cte(n) as (select 1 union select (select * from cte) c1) select * from cte;", errno.ErrInvalidRequiresSingleReference) + // The recursive part can reference tables other than the CTE and join them with the CTE. If used in a join like this, the CTE must not be on the right side of a LEFT JOIN. + tk.MustGetErrCode("with recursive cte(n) as (select 1 union select * from t left join cte on t.a=cte.n) select * from cte;", errno.ErrCTERecursiveForbiddenJoinOrder) + // Recursive part containing non-recursive query is not allowed. + tk.MustGetErrCode("with recursive cte(n) as (select 1 intersect select 2 union select * from cte union select 1) select * from cte;", errno.ErrCTERecursiveRequiresNonRecursiveFirst) + tk.MustGetErrCode("with recursive cte(n) as (select * from cte union select * from cte) select * from cte;", errno.ErrCTERecursiveRequiresNonRecursiveFirst) + // Invalid use of intersect/except. + tk.MustGetErrCode("with recursive cte(n) as (select 1 intersect select * from cte) select * from cte;", errno.ErrNotSupportedYet) + tk.MustGetErrCode("with recursive cte(n) as (select 1 union select 1 intersect select * from cte) select * from cte;", errno.ErrNotSupportedYet) + tk.MustGetErrCode("with recursive cte(n) as (select 1 except select * from cte) select * from cte;", errno.ErrNotSupportedYet) + tk.MustGetErrCode("with recursive cte(n) as (select 1 union select 1 except select * from cte) select * from cte;", errno.ErrNotSupportedYet) +} + func (s *testIntegrationSuite) TestIssue23889(c *C) { defer s.cleanEnv(c) tk := testkit.NewTestKit(c, s.store) @@ -9335,6 +9569,165 @@ func (s *testIntegrationSuite) TestEnumIndex(c *C) { testkit.Rows( "OFJHCEKCQGT:MXI7P3[YO4N0DF=2XJWJ4Z9Z;HQ8TMUTZV8YLQAHWJ4BDZHR3A -30 ", "ZOHBSCRMZPOI`IVTSEZAIDAF7DS@1TT20AP9 -30 ")) + + // issue 24576 + tk.MustExec("use test") + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(col1 enum('a','b','c'), col2 enum('a','b','c'), col3 int, index idx(col1,col2));") + tk.MustExec("insert into t values(1,1,1),(2,2,2),(3,3,3);") + tk.MustQuery("select /*+ use_index(t,idx) */ col3 from t where col2 between 'b' and 'b' and col1 is not null;").Check( + testkit.Rows("2")) + tk.MustQuery("select /*+ use_index(t,idx) */ col3 from t where col2 = 'b' and col1 is not null;").Check( + testkit.Rows("2")) + + // issue25099 + tk.MustExec("use test") + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(e enum(\"a\",\"b\",\"c\"), index idx(e));") + tk.MustExec("insert ignore into t values(0),(1),(2),(3);") + tk.MustQuery("select * from t where e = '';").Check( + testkit.Rows("")) + tk.MustQuery("select * from t where e != 'a';").Sort().Check( + testkit.Rows("", "b", "c")) + tk.MustExec("alter table t drop index idx;") + tk.MustQuery("select * from t where e = '';").Check( + testkit.Rows("")) + tk.MustQuery("select * from t where e != 'a';").Sort().Check( + testkit.Rows("", "b", "c")) + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(e enum(\"\"), index idx(e));") + tk.MustExec("insert ignore into t values(0),(1);") + tk.MustQuery("select * from t where e = '';").Check( + testkit.Rows("", "")) + tk.MustExec("alter table t drop index idx;") + tk.MustQuery("select * from t where e = '';").Check( + testkit.Rows("", "")) + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(e enum(\"a\",\"b\",\"c\"), index idx(e));") + tk.MustExec("insert ignore into t values(0);") + tk.MustExec("select * from t t1 join t t2 on t1.e=t2.e;") + tk.MustQuery("select /*+ inl_join(t1,t2) */ * from t t1 join t t2 on t1.e=t2.e;").Check( + testkit.Rows(" ")) + tk.MustQuery("select /*+ hash_join(t1,t2) */ * from t t1 join t t2 on t1.e=t2.e;").Check( + testkit.Rows(" ")) + tk.MustQuery("select /*+ inl_hash_join(t1,t2) */ * from t t1 join t t2 on t1.e=t2.e;").Check( + testkit.Rows(" ")) +} + +// Previously global values were cached. This is incorrect. +// See: https://github.com/pingcap/tidb/issues/24368 +func (s *testIntegrationSuite) TestGlobalCacheCorrectness(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustQuery("SHOW VARIABLES LIKE 'max_connections'").Check(testkit.Rows("max_connections 151")) + tk.MustExec("SET GLOBAL max_connections=1234") + tk.MustQuery("SHOW VARIABLES LIKE 'max_connections'").Check(testkit.Rows("max_connections 1234")) + // restore + tk.MustExec("SET GLOBAL max_connections=151") +} + +func (s *testIntegrationSuite) TestControlFunctionWithEnumOrSet(c *C) { + defer s.cleanEnv(c) + + // issue 23114 + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists e;") + tk.MustExec("create table e(e enum('c', 'b', 'a'));") + tk.MustExec("insert into e values ('a'),('b'),('a'),('b');") + tk.MustQuery("select e from e where if(e>1, e, e);").Sort().Check( + testkit.Rows("a", "a", "b", "b")) + tk.MustQuery("select e from e where case e when 1 then e else e end;").Sort().Check( + testkit.Rows("a", "a", "b", "b")) + tk.MustQuery("select e from e where case 1 when e then e end;").Check(testkit.Rows()) + + tk.MustQuery("select if(e>1,e,e)='a' from e").Sort().Check( + testkit.Rows("0", "0", "1", "1")) + tk.MustQuery("select if(e>1,e,e)=1 from e").Sort().Check( + testkit.Rows("0", "0", "0", "0")) + // if and if + tk.MustQuery("select if(e>2,e,e) and if(e<=2,e,e) from e;").Sort().Check( + testkit.Rows("1", "1", "1", "1")) + tk.MustQuery("select if(e>2,e,e) and (if(e<3,0,e) or if(e>=2,0,e)) from e;").Sort().Check( + testkit.Rows("0", "0", "1", "1")) + tk.MustQuery("select * from e where if(e>2,e,e) and if(e<=2,e,e);").Sort().Check( + testkit.Rows("a", "a", "b", "b")) + tk.MustQuery("select * from e where if(e>2,e,e) and (if(e<3,0,e) or if(e>=2,0,e));").Sort().Check( + testkit.Rows("a", "a")) + + // issue 24494 + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(a int,b enum(\"b\",\"y\",\"1\"));") + tk.MustExec("insert into t values(0,\"y\"),(1,\"b\"),(null,null),(2,\"1\");") + tk.MustQuery("SELECT count(*) FROM t where if(a,b ,null);").Check(testkit.Rows("2")) + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(a int,b enum(\"b\"),c enum(\"c\"));") + tk.MustExec("insert into t values(1,1,1),(2,1,1),(1,1,1),(2,1,1);") + tk.MustQuery("select a from t where if(a=1,b,c)=\"b\";").Check(testkit.Rows("1", "1")) + tk.MustQuery("select a from t where if(a=1,b,c)=\"c\";").Check(testkit.Rows("2", "2")) + tk.MustQuery("select a from t where if(a=1,b,c)=1;").Sort().Check(testkit.Rows("1", "1", "2", "2")) + tk.MustQuery("select a from t where if(a=1,b,c);").Sort().Check(testkit.Rows("1", "1", "2", "2")) + + tk.MustExec("drop table if exists e;") + tk.MustExec("create table e(e enum('c', 'b', 'a'));") + tk.MustExec("insert into e values(3)") + tk.MustQuery("select elt(1,e) = 'a' from e").Check(testkit.Rows("1")) + tk.MustQuery("select elt(1,e) = 3 from e").Check(testkit.Rows("1")) + tk.MustQuery("select e from e where elt(1,e)").Check(testkit.Rows("a")) + + // test set type + tk.MustExec("drop table if exists s;") + tk.MustExec("create table s(s set('c', 'b', 'a'));") + tk.MustExec("insert into s values ('a'),('b'),('a'),('b');") + tk.MustQuery("select s from s where if(s>1, s, s);").Sort().Check( + testkit.Rows("a", "a", "b", "b")) + tk.MustQuery("select s from s where case s when 1 then s else s end;").Sort().Check( + testkit.Rows("a", "a", "b", "b")) + tk.MustQuery("select s from s where case 1 when s then s end;").Check(testkit.Rows()) + + tk.MustQuery("select if(s>1,s,s)='a' from s").Sort().Check( + testkit.Rows("0", "0", "1", "1")) + tk.MustQuery("select if(s>1,s,s)=4 from s").Sort().Check( + testkit.Rows("0", "0", "1", "1")) + + tk.MustExec("drop table if exists s;") + tk.MustExec("create table s(s set('c', 'b', 'a'));") + tk.MustExec("insert into s values('a')") + tk.MustQuery("select elt(1,s) = 'a' from s").Check(testkit.Rows("1")) + tk.MustQuery("select elt(1,s) = 4 from s").Check(testkit.Rows("1")) + tk.MustQuery("select s from s where elt(1,s)").Check(testkit.Rows("a")) + + // issue 24543 + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(a int,b enum(\"b\"),c enum(\"c\"));") + tk.MustExec("insert into t values(1,1,1),(2,1,1),(1,1,1),(2,1,1);") + tk.MustQuery("select if(A, null,b)=1 from t;").Check(testkit.Rows("", "", "", "")) + tk.MustQuery("select if(A, null,b)='a' from t;").Check(testkit.Rows("", "", "", "")) + + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(a int,b set(\"b\"),c set(\"c\"));") + tk.MustExec("insert into t values(1,1,1),(2,1,1),(1,1,1),(2,1,1);") + tk.MustQuery("select if(A, null,b)=1 from t;").Check(testkit.Rows("", "", "", "")) + tk.MustQuery("select if(A, null,b)='a' from t;").Check(testkit.Rows("", "", "", "")) +} + +func (s *testIntegrationSuite) TestComplexShowVariables(c *C) { + // This is an example SHOW VARIABLES from mysql-connector-java-5.1.34 + // It returns 19 rows in MySQL 5.7 (the language sysvar no longer exists in 5.6+) + // and 16 rows in MySQL 8.0 (the aliases for tx_isolation is removed, along with query cache) + // In the event that we hide noop sysvars in future, we must keep these variables. + tk := testkit.NewTestKit(c, s.store) + c.Assert(tk.MustQuery(`SHOW VARIABLES WHERE Variable_name ='language' OR Variable_name = 'net_write_timeout' OR Variable_name = 'interactive_timeout' +OR Variable_name = 'wait_timeout' OR Variable_name = 'character_set_client' OR Variable_name = 'character_set_connection' +OR Variable_name = 'character_set' OR Variable_name = 'character_set_server' OR Variable_name = 'tx_isolation' +OR Variable_name = 'transaction_isolation' OR Variable_name = 'character_set_results' OR Variable_name = 'timezone' +OR Variable_name = 'time_zone' OR Variable_name = 'system_time_zone' +OR Variable_name = 'lower_case_table_names' OR Variable_name = 'max_allowed_packet' OR Variable_name = 'net_buffer_length' +OR Variable_name = 'sql_mode' OR Variable_name = 'query_cache_type' OR Variable_name = 'query_cache_size' +OR Variable_name = 'license' OR Variable_name = 'init_connect'`).Rows(), HasLen, 19) + } func (s *testIntegrationSuite) TestFloat64Inf(c *C) { diff --git a/expression/testdata/expression_suite_out.json b/expression/testdata/expression_suite_out.json index ea36fc2923764..8b89ee1e4dc81 100644 --- a/expression/testdata/expression_suite_out.json +++ b/expression/testdata/expression_suite_out.json @@ -186,7 +186,7 @@ { "SQL": "explain format = 'brief' select * from t1 left join t2 on true where t1.a = 1 and t1.a = 1", "Result": [ - "HashJoin 80000.00 root CARTESIAN left outer join", + "HashJoin 100000.00 root CARTESIAN left outer join", "├─TableReader(Build) 10.00 root data:Selection", "│ └─Selection 10.00 cop[tikv] eq(test.t1.a, 1)", "│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo", @@ -253,7 +253,7 @@ { "SQL": "explain format = 'brief' select * from t1 left join t2 on true where t1.a = 1 or (t1.a = 2 and t1.a = 3)", "Result": [ - "HashJoin 80000.00 root CARTESIAN left outer join", + "HashJoin 100000.00 root CARTESIAN left outer join", "├─TableReader(Build) 10.00 root data:Selection", "│ └─Selection 10.00 cop[tikv] or(eq(test.t1.a, 1), 0)", "│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo", diff --git a/expression/typeinfer_test.go b/expression/typeinfer_test.go index bce929c933852..e0324123bbaae 100644 --- a/expression/typeinfer_test.go +++ b/expression/typeinfer_test.go @@ -20,7 +20,6 @@ import ( "github.com/pingcap/parser" "github.com/pingcap/parser/charset" "github.com/pingcap/parser/mysql" - "github.com/pingcap/tidb/domain" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx" @@ -139,10 +138,10 @@ func (s *testInferTypeSuite) TestInferType(c *C) { err = se.NewTxn(context.Background()) c.Assert(err, IsNil) - is := domain.GetDomain(sctx).InfoSchema() - err = plannercore.Preprocess(sctx, stmt, is) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(sctx, stmt, plannercore.WithPreprocessorReturn(ret)) c.Assert(err, IsNil, comment) - p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmt, is) + p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmt, ret.InfoSchema) c.Assert(err, IsNil, comment) tp := p.Schema().Columns[0].RetType @@ -1698,9 +1697,10 @@ func (s *testInferTypeSuite) createTestCase4TimeFuncs() []typeInferTestCase { {"utc_time(6) ", mysql.TypeDuration, charset.CharsetBin, mysql.BinaryFlag | mysql.NotNullFlag, 15, 6}, {"utc_time(7) ", mysql.TypeDuration, charset.CharsetBin, mysql.BinaryFlag, 15, 6}, - {"utc_date() ", mysql.TypeDatetime, charset.CharsetBin, mysql.BinaryFlag | mysql.NotNullFlag, 10, 0}, - {"curdate()", mysql.TypeDatetime, charset.CharsetBin, mysql.BinaryFlag | mysql.NotNullFlag, 10, 0}, - {"sysdate(4)", mysql.TypeDatetime, charset.CharsetBin, mysql.BinaryFlag, 19, 0}, + {"curdate() ", mysql.TypeDate, charset.CharsetBin, mysql.BinaryFlag | mysql.NotNullFlag, 10, 0}, + {"current_date() ", mysql.TypeDate, charset.CharsetBin, mysql.BinaryFlag | mysql.NotNullFlag, 10, 0}, + {"utc_date() ", mysql.TypeDatetime, charset.CharsetBin, mysql.BinaryFlag | mysql.NotNullFlag, 10, 0}, + {"sysdate(4) ", mysql.TypeDatetime, charset.CharsetBin, mysql.BinaryFlag, 19, 0}, {"date(c_int_d )", mysql.TypeDate, charset.CharsetBin, mysql.BinaryFlag, 10, 0}, {"date(c_bigint_d )", mysql.TypeDate, charset.CharsetBin, mysql.BinaryFlag, 10, 0}, diff --git a/expression/util.go b/expression/util.go index 9819dbb447ac1..3a9ecfe27b53a 100644 --- a/expression/util.go +++ b/expression/util.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/types" - "github.com/pingcap/tidb/types/parser_driver" + driver "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/collate" "github.com/pingcap/tidb/util/logutil" diff --git a/go.mod b/go.mod index bf927f9cc55ce..daf00d34eb2fc 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d github.com/carlmjohnson/flagext v0.21.0 // indirect github.com/cheggaaa/pb/v3 v3.0.4 // indirect - github.com/codahale/hdrhistogram v0.9.0 // indirect github.com/coocood/freecache v1.1.1 github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 @@ -23,14 +22,13 @@ require ( github.com/golang/protobuf v1.3.4 github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf github.com/google/btree v1.0.0 - github.com/google/go-cmp v0.5.2 // indirect + github.com/google/go-cmp v0.5.5 // indirect github.com/google/pprof v0.0.0-20200407044318-7d83b28da2e9 github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 github.com/joho/sqltocsv v0.0.0-20210208114054-cb2c3a95fb99 // indirect - github.com/klauspost/cpuid v1.2.1 github.com/kr/text v0.2.0 // indirect github.com/mattn/go-runewidth v0.0.10 // indirect github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 @@ -39,24 +37,24 @@ require ( github.com/opentracing/opentracing-go v1.1.0 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/pingcap/badger v1.5.1-0.20200908111422-2e78ee155d19 - github.com/pingcap/br v5.0.0-nightly.0.20210419090151-03762465b589+incompatible + github.com/pingcap/br v5.1.0-alpha.0.20210604015827-db22c6d284a0+incompatible github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 - github.com/pingcap/kvproto v0.0.0-20210429093846-65f54a202d7e + github.com/pingcap/kvproto v0.0.0-20210602120243-804ac0a6ce21 github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 - github.com/pingcap/parser v0.0.0-20210427084954-8e8ed7927bde + github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307 github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3 github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible - github.com/pingcap/tipb v0.0.0-20210422074242-57dd881b81b1 + github.com/pingcap/tipb v0.0.0-20210603161937-cfb5a9225f95 github.com/prometheus/client_golang v1.5.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.9.1 github.com/rivo/uniseg v0.2.0 // indirect github.com/shirou/gopsutil v3.21.2+incompatible - github.com/sirupsen/logrus v1.6.0 + github.com/sirupsen/logrus v1.6.0 // indirect github.com/soheilhy/cmux v0.1.4 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d @@ -64,11 +62,12 @@ require ( github.com/uber-go/atomic v1.4.0 github.com/uber/jaeger-client-go v2.22.1+incompatible github.com/uber/jaeger-lib v2.4.0+incompatible // indirect + github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457 // indirect go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b go.uber.org/atomic v1.7.0 - go.uber.org/automaxprocs v1.2.0 - go.uber.org/multierr v1.6.0 // indirect + go.uber.org/automaxprocs v1.4.0 + go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.16.0 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect golang.org/x/mod v0.4.2 // indirect @@ -79,13 +78,14 @@ require ( golang.org/x/tools v0.1.0 google.golang.org/grpc v1.27.1 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect - honnef.co/go/tools v0.1.3 // indirect + honnef.co/go/tools v0.2.0 // indirect modernc.org/mathutil v1.2.2 // indirect sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) -go 1.13 +go 1.16 + +// Fix panic in unit test with go >= 1.14, ref: etcd-io/bbolt#201 https://github.com/etcd-io/bbolt/pull/201 +replace go.etcd.io/bbolt => go.etcd.io/bbolt v1.3.5 diff --git a/go.sum b/go.sum index 74b4f623789b8..bab7be1bc60be 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,6 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.9.0 h1:9GjrtRI+mLEFPtTfR/AZhcxp+Ii8NZYWq5104FbZQY0= -github.com/codahale/hdrhistogram v0.9.0/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 h1:W1SHiII3e0jVwvaQFglwu3kS9NLxOeTpvik7MbKCyuQ= github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64/go.mod h1:F86k/6c7aDUdwSUevnLpHS/3Q9hzYCE99jGk2xsHnt0= @@ -151,7 +149,6 @@ github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsouza/fake-gcs-server v1.17.0 h1:OeH75kBZcZa3ZE+zz/mFdJ2btt9FgqfjI7gIh9+5fvk= github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw= @@ -240,8 +237,8 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -410,8 +407,8 @@ github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5e github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= github.com/pingcap/badger v1.5.1-0.20200908111422-2e78ee155d19 h1:IXpGy7y9HyoShAFmzW2OPF0xCA5EOoSTyZHwsgYk9Ro= github.com/pingcap/badger v1.5.1-0.20200908111422-2e78ee155d19/go.mod h1:LyrqUOHZrUDf9oGi1yoz1+qw9ckSIhQb5eMa1acOLNQ= -github.com/pingcap/br v5.0.0-nightly.0.20210419090151-03762465b589+incompatible h1:VF2oZgvBqSIMmplEWXGGmktuQGdGGIGWwptmjJFhQbU= -github.com/pingcap/br v5.0.0-nightly.0.20210419090151-03762465b589+incompatible/go.mod h1:ymVmo50lQydxib0tmK5hHk4oteB7hZ0IMCArunwy3UQ= +github.com/pingcap/br v5.1.0-alpha.0.20210604015827-db22c6d284a0+incompatible h1:+kzI8gD8buDo3y/IQiifiMHD5uIBB0GQJf3Myoj14uY= +github.com/pingcap/br v5.1.0-alpha.0.20210604015827-db22c6d284a0+incompatible/go.mod h1:ymVmo50lQydxib0tmK5hHk4oteB7hZ0IMCArunwy3UQ= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= @@ -436,23 +433,23 @@ github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17Xtb github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20210219064844-c1844a4775d6/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20210429093846-65f54a202d7e h1:oUMZ6X/Kpaoxfejh9/jQ+4UZ5xk9MRYcouWJ0oXRKNE= -github.com/pingcap/kvproto v0.0.0-20210429093846-65f54a202d7e/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20210602120243-804ac0a6ce21 h1:WLViPiCazVwK0wULKpmwLVP/aA8NvyyfOUqQy0bPEPk= +github.com/pingcap/kvproto v0.0.0-20210602120243-804ac0a6ce21/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 h1:ERrF0fTuIOnwfGbt71Ji3DKbOEaP189tjym50u8gpC8= github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/parser v0.0.0-20210427084954-8e8ed7927bde h1:CcGOCE3kr8aYBy6rRcWWldidL1X5smQxV79nlnzOk+o= -github.com/pingcap/parser v0.0.0-20210427084954-8e8ed7927bde/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= +github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307 h1:v7SipssMu4X1tVQOe3PIVE73keJNHCFXe4Cza5uNDZ8= +github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3 h1:A9KL9R+lWSVPH8IqUuH1QSTRJ5FGoY1bT2IcfPKsWD8= github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3/go.mod h1:tckvA041UWP+NqYzrJ3fMgC/Hw9wnmQ/tUkp/JaHly8= github.com/pingcap/tidb-dashboard v0.0.0-20210312062513-eef5d6404638/go.mod h1:OzFN8H0EDMMqeulPhPMw2i2JaiZWOKFQ7zdRPhENNgo= github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible h1:ceznmu/lLseGHP/jKyOa/3u/5H3wtLLLqkH2V3ssSjg= github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20210422074242-57dd881b81b1 h1:Kcp3jIcQrqG+pT1JQ0oWyRncVKQtDgnMFzRt3zJBaBo= -github.com/pingcap/tipb v0.0.0-20210422074242-57dd881b81b1/go.mod h1:nsEhnMokcn7MRqd2J60yxpn/ac3ZH8A6GOJ9NslabUo= +github.com/pingcap/tipb v0.0.0-20210603161937-cfb5a9225f95 h1:Cj7FhGvYn8hrXDNcaHi0aTl0KdV67KTL+P5gBp3vqT4= +github.com/pingcap/tipb v0.0.0-20210603161937-cfb5a9225f95/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -541,8 +538,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E= github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI= github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba/go.mod h1:O1lAbCgAAX/KZ80LM/OXwtWFI/5TvZlwxSg8Cq08PV0= @@ -588,21 +586,21 @@ github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKn github.com/vmihailenco/msgpack/v4 v4.3.11/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1/go.mod h1:xlngVLeyQ/Qi05oQxhQ+oTuqa03RjMwMfk/7/TCs+QI= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f h1:9DDCDwOyEy/gId+IEMrFHLuQ5R/WV0KNxWLler8X2OY= +github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f/go.mod h1:8sdOQnirw1PrcnTJYkmW1iOHtUmblMmGdUOHyWYycLI= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457 h1:tBbuFCtyJNKT+BFAv6qjvTFpVdy97IYNaBwGUXifIUs= github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457/go.mod h1:pheqtXeHQFzxJk45lRQ0UIGIivKnLXvialZSFWs81A8= github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA= -github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 h1:a742S4V5A15F93smuVxA60LQWsrCnN8bKeWDBARU1/k= github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b h1:3kC4J3eQF6p1UEfQTkC67eEeb3rTk+shQqdX6tFyq9Q= @@ -618,8 +616,8 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/automaxprocs v1.2.0 h1:+RUihKM+nmYUoB9w0D0Ov5TJ2PpFO2FgenTxMJiZBZA= -go.uber.org/automaxprocs v1.2.0/go.mod h1:YfO3fm683kQpzETxlTGZhGIVmXAhaw3gxeBADbpZtnU= +go.uber.org/automaxprocs v1.4.0 h1:CpDZl6aOlLhReez+8S3eEotD7Jx0Os++lemPlMULQP0= +go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q= go.uber.org/dig v1.8.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= go.uber.org/fx v1.10.0/go.mod h1:vLRicqpG/qQEzno4SYU86iCwfT95EZza+Eba0ItuxqY= go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= @@ -628,8 +626,8 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -924,16 +922,16 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.1.3 h1:qTakTkI6ni6LFD5sBwwsdSO+AQqbSIxOauHTTQKZ/7o= -honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +honnef.co/go/tools v0.2.0 h1:ws8AfbgTX3oIczLPNPCu5166oBg9ST2vNs0rcht+mDE= +honnef.co/go/tools v0.2.0/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= modernc.org/mathutil v1.2.2 h1:+yFk8hBprV+4c0U9GjFtL+dV3N8hOJ8JCituQcMShFY= modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= diff --git a/infoschema/builder.go b/infoschema/builder.go index 28591d8679baf..88e8b71add319 100644 --- a/infoschema/builder.go +++ b/infoschema/builder.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl/placement" "github.com/pingcap/tidb/domain/infosync" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/table" @@ -35,8 +36,10 @@ import ( // Builder builds a new InfoSchema. type Builder struct { - is *infoSchema - handle *Handle + is *infoSchema + // TODO: store is only used by autoid allocators + // detach allocators from storage, use passed transaction in the feature + store kv.Storage } // ApplyDiff applies SchemaDiff to the new InfoSchema. @@ -352,14 +355,14 @@ func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID i ConvertOldVersionUTF8ToUTF8MB4IfNeed(tblInfo) if len(allocs) == 0 { - allocs = autoid.NewAllocatorsFromTblInfo(b.handle.store, dbInfo.ID, tblInfo) + allocs = autoid.NewAllocatorsFromTblInfo(b.store, dbInfo.ID, tblInfo) } else { switch tp { case model.ActionRebaseAutoID, model.ActionModifyTableAutoIdCache: - newAlloc := autoid.NewAllocator(b.handle.store, dbInfo.ID, tblInfo.IsAutoIncColUnsigned(), autoid.RowIDAllocType) + newAlloc := autoid.NewAllocator(b.store, dbInfo.ID, tblInfo.IsAutoIncColUnsigned(), autoid.RowIDAllocType) allocs = append(allocs, newAlloc) case model.ActionRebaseAutoRandomBase: - newAlloc := autoid.NewAllocator(b.handle.store, dbInfo.ID, tblInfo.IsAutoRandomBitColUnsigned(), autoid.AutoRandomType) + newAlloc := autoid.NewAllocator(b.store, dbInfo.ID, tblInfo.IsAutoRandomBitColUnsigned(), autoid.AutoRandomType) allocs = append(allocs, newAlloc) case model.ActionModifyColumn: // Change column attribute from auto_increment to auto_random. @@ -368,7 +371,7 @@ func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID i allocs = allocs.Filter(func(a autoid.Allocator) bool { return a.GetType() != autoid.AutoIncrementType && a.GetType() != autoid.RowIDAllocType }) - newAlloc := autoid.NewAllocator(b.handle.store, dbInfo.ID, tblInfo.IsAutoRandomBitColUnsigned(), autoid.AutoRandomType) + newAlloc := autoid.NewAllocator(b.store, dbInfo.ID, tblInfo.IsAutoRandomBitColUnsigned(), autoid.AutoRandomType) allocs = append(allocs, newAlloc) } } @@ -470,9 +473,14 @@ func (b *Builder) applyPlacementUpdate(id string) error { return nil } +// Build builds and returns the built infoschema. +func (b *Builder) Build() InfoSchema { + return b.is +} + // InitWithOldInfoSchema initializes an empty new InfoSchema by copies all the data from old InfoSchema. -func (b *Builder) InitWithOldInfoSchema() *Builder { - oldIS := b.handle.Get().(*infoSchema) +func (b *Builder) InitWithOldInfoSchema(oldSchema InfoSchema) *Builder { + oldIS := oldSchema.(*infoSchema) b.is.schemaMetaVersion = oldIS.schemaMetaVersion b.copySchemasMap(oldIS) b.copyBundlesMap(oldIS) @@ -549,7 +557,7 @@ func (b *Builder) createSchemaTablesForDB(di *model.DBInfo, tableFromMeta tableF b.is.schemaMap[di.Name.L] = schTbls for _, t := range di.Tables { - allocs := autoid.NewAllocatorsFromTblInfo(b.handle.store, di.ID, t) + allocs := autoid.NewAllocatorsFromTblInfo(b.store, di.ID, t) var tbl table.Table tbl, err := tableFromMeta(allocs, t) if err != nil { @@ -574,21 +582,16 @@ func RegisterVirtualTable(dbInfo *model.DBInfo, tableFromMeta tableFromMetaFunc) drivers = append(drivers, &virtualTableDriver{dbInfo, tableFromMeta}) } -// Build sets new InfoSchema to the handle in the Builder. -func (b *Builder) Build() { - b.handle.value.Store(b.is) -} - // NewBuilder creates a new Builder with a Handle. -func NewBuilder(handle *Handle) *Builder { - b := new(Builder) - b.handle = handle - b.is = &infoSchema{ - schemaMap: map[string]*schemaTables{}, - ruleBundleMap: map[string]*placement.Bundle{}, - sortedTablesBuckets: make([]sortedTables, bucketCount), +func NewBuilder(store kv.Storage) *Builder { + return &Builder{ + store: store, + is: &infoSchema{ + schemaMap: map[string]*schemaTables{}, + ruleBundleMap: map[string]*placement.Bundle{}, + sortedTablesBuckets: make([]sortedTables, bucketCount), + }, } - return b } func tableBucketIdx(tableID int64) int { diff --git a/infoschema/cache.go b/infoschema/cache.go new file mode 100644 index 0000000000000..4c3371b1bc354 --- /dev/null +++ b/infoschema/cache.go @@ -0,0 +1,95 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package infoschema + +import ( + "sort" + "sync" + + "github.com/pingcap/tidb/metrics" +) + +// InfoCache handles information schema, including getting and setting. +// The cache behavior, however, is transparent and under automatic management. +// It only promised to cache the infoschema, if it is newer than all the cached. +type InfoCache struct { + mu sync.RWMutex + // cache is sorted by SchemaVersion in descending order + cache []InfoSchema +} + +// NewCache creates a new InfoCache. +func NewCache(capcity int) *InfoCache { + return &InfoCache{cache: make([]InfoSchema, 0, capcity)} +} + +// GetLatest gets the newest information schema. +func (h *InfoCache) GetLatest() InfoSchema { + h.mu.RLock() + defer h.mu.RUnlock() + metrics.InfoCacheCounters.WithLabelValues("get").Inc() + if len(h.cache) > 0 { + metrics.InfoCacheCounters.WithLabelValues("hit").Inc() + return h.cache[0] + } + return nil +} + +// GetByVersion gets the information schema based on schemaVersion. Returns nil if it is not loaded. +func (h *InfoCache) GetByVersion(version int64) InfoSchema { + h.mu.RLock() + defer h.mu.RUnlock() + metrics.InfoCacheCounters.WithLabelValues("get").Inc() + i := sort.Search(len(h.cache), func(i int) bool { + return h.cache[i].SchemaMetaVersion() <= version + }) + if i < len(h.cache) && h.cache[i].SchemaMetaVersion() == version { + metrics.InfoCacheCounters.WithLabelValues("hit").Inc() + return h.cache[i] + } + return nil +} + +// Insert will **TRY** to insert the infoschema into the cache. +// It only promised to cache the newest infoschema. +// It returns 'true' if it is cached, 'false' otherwise. +func (h *InfoCache) Insert(is InfoSchema) bool { + h.mu.Lock() + defer h.mu.Unlock() + + version := is.SchemaMetaVersion() + i := sort.Search(len(h.cache), func(i int) bool { + return h.cache[i].SchemaMetaVersion() <= version + }) + + // cached entry + if i < len(h.cache) && h.cache[i].SchemaMetaVersion() == version { + return true + } + + if len(h.cache) < cap(h.cache) { + // has free space, grown the slice + h.cache = h.cache[:len(h.cache)+1] + copy(h.cache[i+1:], h.cache[i:]) + h.cache[i] = is + return true + } else if i < len(h.cache) { + // drop older schema + copy(h.cache[i+1:], h.cache[i:]) + h.cache[i] = is + return true + } + // older than all cached schemas, refuse to cache it + return false +} diff --git a/infoschema/cache_test.go b/infoschema/cache_test.go new file mode 100644 index 0000000000000..a8e9ddcc0df5a --- /dev/null +++ b/infoschema/cache_test.go @@ -0,0 +1,119 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package infoschema_test + +import ( + . "github.com/pingcap/check" + "github.com/pingcap/tidb/infoschema" +) + +var _ = Suite(&testInfoCacheSuite{}) + +type testInfoCacheSuite struct { +} + +func (s *testInfoCacheSuite) TestNewCache(c *C) { + ic := infoschema.NewCache(16) + c.Assert(ic, NotNil) +} + +func (s *testInfoCacheSuite) TestInsert(c *C) { + ic := infoschema.NewCache(3) + c.Assert(ic, NotNil) + + is2 := infoschema.MockInfoSchemaWithSchemaVer(nil, 2) + ic.Insert(is2) + c.Assert(ic.GetByVersion(2), NotNil) + + // newer + is5 := infoschema.MockInfoSchemaWithSchemaVer(nil, 5) + ic.Insert(is5) + c.Assert(ic.GetByVersion(5), NotNil) + c.Assert(ic.GetByVersion(2), NotNil) + + // older + is0 := infoschema.MockInfoSchemaWithSchemaVer(nil, 0) + ic.Insert(is0) + c.Assert(ic.GetByVersion(5), NotNil) + c.Assert(ic.GetByVersion(2), NotNil) + c.Assert(ic.GetByVersion(0), NotNil) + + // replace 5, drop 0 + is6 := infoschema.MockInfoSchemaWithSchemaVer(nil, 6) + ic.Insert(is6) + c.Assert(ic.GetByVersion(6), NotNil) + c.Assert(ic.GetByVersion(5), NotNil) + c.Assert(ic.GetByVersion(2), NotNil) + c.Assert(ic.GetByVersion(0), IsNil) + + // replace 2, drop 2 + is3 := infoschema.MockInfoSchemaWithSchemaVer(nil, 3) + ic.Insert(is3) + c.Assert(ic.GetByVersion(6), NotNil) + c.Assert(ic.GetByVersion(5), NotNil) + c.Assert(ic.GetByVersion(3), NotNil) + c.Assert(ic.GetByVersion(2), IsNil) + c.Assert(ic.GetByVersion(0), IsNil) + + // insert 2, but failed silently + ic.Insert(is2) + c.Assert(ic.GetByVersion(6), NotNil) + c.Assert(ic.GetByVersion(5), NotNil) + c.Assert(ic.GetByVersion(3), NotNil) + c.Assert(ic.GetByVersion(2), IsNil) + c.Assert(ic.GetByVersion(0), IsNil) + + // insert 5, but it is already in + ic.Insert(is5) + c.Assert(ic.GetByVersion(6), NotNil) + c.Assert(ic.GetByVersion(5), NotNil) + c.Assert(ic.GetByVersion(3), NotNil) + c.Assert(ic.GetByVersion(2), IsNil) + c.Assert(ic.GetByVersion(0), IsNil) +} + +func (s *testInfoCacheSuite) TestGetByVersion(c *C) { + ic := infoschema.NewCache(2) + c.Assert(ic, NotNil) + is1 := infoschema.MockInfoSchemaWithSchemaVer(nil, 1) + ic.Insert(is1) + is3 := infoschema.MockInfoSchemaWithSchemaVer(nil, 3) + ic.Insert(is3) + + c.Assert(ic.GetByVersion(1), Equals, is1) + c.Assert(ic.GetByVersion(3), Equals, is3) + c.Assert(ic.GetByVersion(0), IsNil, Commentf("index == 0, but not found")) + c.Assert(ic.GetByVersion(2), IsNil, Commentf("index in the middle, but not found")) + c.Assert(ic.GetByVersion(4), IsNil, Commentf("index == length, but not found")) +} + +func (s *testInfoCacheSuite) TestGetLatest(c *C) { + ic := infoschema.NewCache(16) + c.Assert(ic, NotNil) + c.Assert(ic.GetLatest(), IsNil) + + is1 := infoschema.MockInfoSchemaWithSchemaVer(nil, 1) + ic.Insert(is1) + c.Assert(ic.GetLatest(), Equals, is1) + + // newer change the newest + is2 := infoschema.MockInfoSchemaWithSchemaVer(nil, 2) + ic.Insert(is2) + c.Assert(ic.GetLatest(), Equals, is2) + + // older schema doesn't change the newest + is0 := infoschema.MockInfoSchemaWithSchemaVer(nil, 0) + ic.Insert(is0) + c.Assert(ic.GetLatest(), Equals, is2) +} diff --git a/infoschema/cluster.go b/infoschema/cluster.go index f113e90a0f587..20589ad7a0c67 100644 --- a/infoschema/cluster.go +++ b/infoschema/cluster.go @@ -37,6 +37,10 @@ const ( ClusterTableStatementsSummary = "CLUSTER_STATEMENTS_SUMMARY" // ClusterTableStatementsSummaryHistory is the string constant of cluster statement summary history table. ClusterTableStatementsSummaryHistory = "CLUSTER_STATEMENTS_SUMMARY_HISTORY" + // ClusterTableTiDBTrx is the string constant of cluster transaction running table. + ClusterTableTiDBTrx = "CLUSTER_TIDB_TRX" + // ClusterTableDeadlocks is the string constant of cluster dead lock table. + ClusterTableDeadlocks = "CLUSTER_DEADLOCKS" ) // memTableToClusterTables means add memory table to cluster table. @@ -45,6 +49,8 @@ var memTableToClusterTables = map[string]string{ TableProcesslist: ClusterTableProcesslist, TableStatementsSummary: ClusterTableStatementsSummary, TableStatementsSummaryHistory: ClusterTableStatementsSummaryHistory, + TableTiDBTrx: ClusterTableTiDBTrx, + TableDeadlocks: ClusterTableDeadlocks, } func init() { diff --git a/infoschema/error.go b/infoschema/error.go index a0ef7ab9c8760..cb49e48419dec 100644 --- a/infoschema/error.go +++ b/infoschema/error.go @@ -69,4 +69,6 @@ var ( ErrTableLocked = dbterror.ClassSchema.NewStd(mysql.ErrTableLocked) // ErrWrongObject returns when the table/view/sequence is not the expected object. ErrWrongObject = dbterror.ClassSchema.NewStd(mysql.ErrWrongObject) + // ErrAdminCheckTable returns when the check table in temporary mode. + ErrAdminCheckTable = dbterror.ClassSchema.NewStd(mysql.ErrAdminCheckTable) ) diff --git a/infoschema/infoschema.go b/infoschema/infoschema.go index 4fcbdc042de85..2494e89b4d57f 100644 --- a/infoschema/infoschema.go +++ b/infoschema/infoschema.go @@ -17,19 +17,13 @@ import ( "fmt" "sort" "sync" - "sync/atomic" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/ddl/placement" - "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta/autoid" - "github.com/pingcap/tidb/sessionctx" - "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/util" - "github.com/pingcap/tidb/util/logutil" - "go.uber.org/zap" ) // InfoSchema is the interface used to retrieve the schema information. @@ -316,40 +310,6 @@ func (is *infoSchema) SequenceByName(schema, sequence model.CIStr) (util.Sequenc return tbl.(util.SequenceTable), nil } -// Handle handles information schema, including getting and setting. -type Handle struct { - value atomic.Value - store kv.Storage -} - -// NewHandle creates a new Handle. -func NewHandle(store kv.Storage) *Handle { - h := &Handle{ - store: store, - } - return h -} - -// Get gets information schema from Handle. -func (h *Handle) Get() InfoSchema { - v := h.value.Load() - schema, _ := v.(InfoSchema) - return schema -} - -// IsValid uses to check whether handle value is valid. -func (h *Handle) IsValid() bool { - return h.value.Load() != nil -} - -// EmptyClone creates a new Handle with the same store and memSchema, but the value is not set. -func (h *Handle) EmptyClone() *Handle { - newHandle := &Handle{ - store: h.store, - } - return newHandle -} - func init() { // Initialize the information shema database and register the driver to `drivers` dbID := autoid.InformationSchemaDBID @@ -386,28 +346,6 @@ func HasAutoIncrementColumn(tbInfo *model.TableInfo) (bool, string) { return false, "" } -// GetInfoSchema gets TxnCtx InfoSchema if snapshot schema is not set, -// Otherwise, snapshot schema is returned. -func GetInfoSchema(ctx sessionctx.Context) InfoSchema { - return GetInfoSchemaBySessionVars(ctx.GetSessionVars()) -} - -// GetInfoSchemaBySessionVars gets TxnCtx InfoSchema if snapshot schema is not set, -// Otherwise, snapshot schema is returned. -func GetInfoSchemaBySessionVars(sessVar *variable.SessionVars) InfoSchema { - var is InfoSchema - if snap := sessVar.SnapshotInfoschema; snap != nil { - is = snap.(InfoSchema) - logutil.BgLogger().Info("use snapshot schema", zap.Uint64("conn", sessVar.ConnectionID), zap.Int64("schemaVersion", is.SchemaMetaVersion())) - } else { - if sessVar.TxnCtx == nil || sessVar.TxnCtx.InfoSchema == nil { - return nil - } - is = sessVar.TxnCtx.InfoSchema.(InfoSchema) - } - return is -} - func (is *infoSchema) BundleByName(name string) (*placement.Bundle, bool) { is.ruleBundleMutex.RLock() defer is.ruleBundleMutex.RUnlock() diff --git a/infoschema/infoschema_test.go b/infoschema/infoschema_test.go index c3892e6527962..61f34032942fe 100644 --- a/infoschema/infoschema_test.go +++ b/infoschema/infoschema_test.go @@ -15,7 +15,6 @@ package infoschema_test import ( "context" - "sync" "testing" . "github.com/pingcap/check" @@ -57,7 +56,6 @@ func (*testSuite) TestT(c *C) { c.Assert(err, IsNil) defer dom.Close() - handle := infoschema.NewHandle(store) dbName := model.NewCIStr("Test") tbName := model.NewCIStr("T") colName := model.NewCIStr("A") @@ -116,7 +114,7 @@ func (*testSuite) TestT(c *C) { }) c.Assert(err, IsNil) - builder, err := infoschema.NewBuilder(handle).InitWithDBInfos(dbInfos, nil, 1) + builder, err := infoschema.NewBuilder(dom.Store()).InitWithDBInfos(dbInfos, nil, 1) c.Assert(err, IsNil) txn, err := store.Begin() @@ -126,8 +124,7 @@ func (*testSuite) TestT(c *C) { err = txn.Rollback() c.Assert(err, IsNil) - builder.Build() - is := handle.Get() + is := builder.Build() schemaNames := is.AllSchemaNames() c.Assert(schemaNames, HasLen, 4) @@ -213,14 +210,10 @@ func (*testSuite) TestT(c *C) { c.Assert(err, IsNil) err = txn.Rollback() c.Assert(err, IsNil) - builder.Build() - is = handle.Get() + is = builder.Build() schema, ok = is.SchemaByID(dbID) c.Assert(ok, IsTrue) c.Assert(len(schema.Tables), Equals, 1) - - emptyHandle := handle.EmptyClone() - c.Assert(emptyHandle.Get(), IsNil) } func (testSuite) TestMockInfoSchema(c *C) { @@ -258,32 +251,6 @@ func checkApplyCreateNonExistsTableDoesNotPanic(c *C, txn kv.Transaction, builde c.Assert(infoschema.ErrTableNotExists.Equal(err), IsTrue) } -// TestConcurrent makes sure it is safe to concurrently create handle on multiple stores. -func (testSuite) TestConcurrent(c *C) { - defer testleak.AfterTest(c)() - storeCount := 5 - stores := make([]kv.Storage, storeCount) - for i := 0; i < storeCount; i++ { - store, err := mockstore.NewMockStore() - c.Assert(err, IsNil) - stores[i] = store - } - defer func() { - for _, store := range stores { - store.Close() - } - }() - var wg sync.WaitGroup - wg.Add(storeCount) - for _, store := range stores { - go func(s kv.Storage) { - defer wg.Done() - _ = infoschema.NewHandle(s) - }(store) - } - wg.Wait() -} - // TestInfoTables makes sure that all tables of information_schema could be found in infoschema handle. func (*testSuite) TestInfoTables(c *C) { defer testleak.AfterTest(c)() @@ -293,12 +260,10 @@ func (*testSuite) TestInfoTables(c *C) { err := store.Close() c.Assert(err, IsNil) }() - handle := infoschema.NewHandle(store) - builder, err := infoschema.NewBuilder(handle).InitWithDBInfos(nil, nil, 0) + + builder, err := infoschema.NewBuilder(store).InitWithDBInfos(nil, nil, 0) c.Assert(err, IsNil) - builder.Build() - is := handle.Get() - c.Assert(is, NotNil) + is := builder.Build() infoTables := []string{ "SCHEMATA", @@ -332,6 +297,8 @@ func (*testSuite) TestInfoTables(c *C) { "TABLESPACES", "COLLATION_CHARACTER_SET_APPLICABILITY", "PROCESSLIST", + "TIDB_TRX", + "DEADLOCKS", } for _, t := range infoTables { tb, err1 := is.TableByName(util.InformationSchemaName, model.NewCIStr(t)) @@ -359,12 +326,9 @@ func (*testSuite) TestGetBundle(c *C) { c.Assert(err, IsNil) }() - handle := infoschema.NewHandle(store) - builder, err := infoschema.NewBuilder(handle).InitWithDBInfos(nil, nil, 0) + builder, err := infoschema.NewBuilder(store).InitWithDBInfos(nil, nil, 0) c.Assert(err, IsNil) - builder.Build() - - is := handle.Get() + is := builder.Build() bundle := &placement.Bundle{ ID: placement.PDBundleID, diff --git a/infoschema/metrics_schema.go b/infoschema/metrics_schema.go index 49a57e4ac9eeb..3b4654f90f7f2 100644 --- a/infoschema/metrics_schema.go +++ b/infoschema/metrics_schema.go @@ -100,17 +100,9 @@ func (def *MetricTableDef) genColumnInfos() []columnInfo { // GenPromQL generates the promQL. func (def *MetricTableDef) GenPromQL(sctx sessionctx.Context, labels map[string]set.StringSet, quantile float64) string { promQL := def.PromQL - if strings.Contains(promQL, promQLQuantileKey) { - promQL = strings.Replace(promQL, promQLQuantileKey, strconv.FormatFloat(quantile, 'f', -1, 64), -1) - } - - if strings.Contains(promQL, promQLLabelConditionKey) { - promQL = strings.Replace(promQL, promQLLabelConditionKey, def.genLabelCondition(labels), -1) - } - - if strings.Contains(promQL, promQRangeDurationKey) { - promQL = strings.Replace(promQL, promQRangeDurationKey, strconv.FormatInt(sctx.GetSessionVars().MetricSchemaRangeDuration, 10)+"s", -1) - } + promQL = strings.Replace(promQL, promQLQuantileKey, strconv.FormatFloat(quantile, 'f', -1, 64), -1) + promQL = strings.Replace(promQL, promQLLabelConditionKey, def.genLabelCondition(labels), -1) + promQL = strings.Replace(promQL, promQRangeDurationKey, strconv.FormatInt(sctx.GetSessionVars().MetricSchemaRangeDuration, 10)+"s", -1) return promQL } diff --git a/infoschema/tables.go b/infoschema/tables.go index 085bc6a96a77d..df6c926b6354d 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -17,7 +17,7 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" + "io" "net" "net/http" "sort" @@ -31,11 +31,13 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl/placement" "github.com/pingcap/tidb/domain/infosync" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta/autoid" + "github.com/pingcap/tidb/session/txninfo" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/tikv" @@ -147,6 +149,8 @@ const ( TableStatementsSummary = "STATEMENTS_SUMMARY" // TableStatementsSummaryHistory is the string constant of statements summary history table. TableStatementsSummaryHistory = "STATEMENTS_SUMMARY_HISTORY" + // TableStatementsSummaryEvicted is the string constant of statements summary evicted table. + TableStatementsSummaryEvicted = "STATEMENTS_SUMMARY_EVICTED" // TableStorageStats is a table that contains all tables disk usage TableStorageStats = "TABLE_STORAGE_STATS" // TableTiFlashTables is the string constant of tiflash tables table. @@ -161,6 +165,12 @@ const ( TableClientErrorsSummaryByUser = "CLIENT_ERRORS_SUMMARY_BY_USER" // TableClientErrorsSummaryByHost is the string constant of client errors table. TableClientErrorsSummaryByHost = "CLIENT_ERRORS_SUMMARY_BY_HOST" + // TableTiDBTrx is current running transaction status table. + TableTiDBTrx = "TIDB_TRX" + // TableDeadlocks is the string constatnt of deadlock table. + TableDeadlocks = "DEADLOCKS" + // TableDataLockWaits is current lock waiting status table. + TableDataLockWaits = "DATA_LOCK_WAITS" ) var tableIDMap = map[string]int64{ @@ -233,22 +243,29 @@ var tableIDMap = map[string]int64{ TableClientErrorsSummaryGlobal: autoid.InformationSchemaDBID + 67, TableClientErrorsSummaryByUser: autoid.InformationSchemaDBID + 68, TableClientErrorsSummaryByHost: autoid.InformationSchemaDBID + 69, + TableTiDBTrx: autoid.InformationSchemaDBID + 70, + ClusterTableTiDBTrx: autoid.InformationSchemaDBID + 71, + TableDeadlocks: autoid.InformationSchemaDBID + 72, + ClusterTableDeadlocks: autoid.InformationSchemaDBID + 73, + TableDataLockWaits: autoid.InformationSchemaDBID + 74, + TableStatementsSummaryEvicted: autoid.InformationSchemaDBID + 75, } type columnInfo struct { - name string - tp byte - size int - decimal int - flag uint - deflt interface{} - comment string + name string + tp byte + size int + decimal int + flag uint + deflt interface{} + comment string + enumElems []string } func buildColumnInfo(col columnInfo) *model.ColumnInfo { mCharset := charset.CharsetBin mCollation := charset.CharsetBin - if col.tp == mysql.TypeVarchar || col.tp == mysql.TypeBlob || col.tp == mysql.TypeLongBlob { + if col.tp == mysql.TypeVarchar || col.tp == mysql.TypeBlob || col.tp == mysql.TypeLongBlob || col.tp == mysql.TypeEnum { mCharset = charset.CharsetUTF8MB4 mCollation = charset.CollationUTF8MB4 } @@ -259,6 +276,7 @@ func buildColumnInfo(col columnInfo) *model.ColumnInfo { Flen: col.size, Decimal: col.decimal, Flag: col.flag, + Elems: col.enumElems, } return &model.ColumnInfo{ Name: model.NewCIStr(col.name), @@ -1332,6 +1350,45 @@ var tableClientErrorsSummaryByHostCols = []columnInfo{ {name: "LAST_SEEN", tp: mysql.TypeTimestamp, size: 26}, } +var tableTiDBTrxCols = []columnInfo{ + {name: "ID", tp: mysql.TypeLonglong, size: 21, flag: mysql.PriKeyFlag | mysql.NotNullFlag | mysql.UnsignedFlag}, + {name: "START_TIME", tp: mysql.TypeTimestamp, decimal: 6, size: 26, comment: "Start time of the transaction"}, + {name: "CURRENT_SQL_DIGEST", tp: mysql.TypeVarchar, size: 64, comment: "Digest of the sql the transaction are currently running"}, + {name: "STATE", tp: mysql.TypeEnum, enumElems: txninfo.TxnRunningStateStrs, comment: "Current running state of the transaction"}, + {name: "WAITING_START_TIME", tp: mysql.TypeTimestamp, decimal: 6, size: 26, comment: "Current lock waiting's start time"}, + {name: "MEM_BUFFER_KEYS", tp: mysql.TypeLonglong, size: 64, comment: "How many entries are in MemDB"}, + {name: "MEM_BUFFER_BYTES", tp: mysql.TypeLonglong, size: 64, comment: "MemDB used memory"}, + {name: "SESSION_ID", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag, comment: "Which session this transaction belongs to"}, + {name: "USER", tp: mysql.TypeVarchar, size: 16, comment: "The user who open this session"}, + {name: "DB", tp: mysql.TypeVarchar, size: 64, comment: "The schema this transaction works on"}, + {name: "ALL_SQL_DIGESTS", tp: mysql.TypeBlob, size: types.UnspecifiedLength, comment: "A list of the digests of SQL statements that the transaction has executed"}, +} + +var tableDeadlocksCols = []columnInfo{ + {name: "DEADLOCK_ID", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag, comment: "The ID to distinguish different deadlock events"}, + {name: "OCCUR_TIME", tp: mysql.TypeTimestamp, decimal: 6, size: 26, comment: "The physical time when the deadlock occurs"}, + {name: "RETRYABLE", tp: mysql.TypeTiny, size: 1, flag: mysql.NotNullFlag, comment: "Whether the deadlock is retryable. Retryable deadlocks are usually not reported to the client"}, + {name: "TRY_LOCK_TRX_ID", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "The transaction ID (start ts) of the transaction that's trying to acquire the lock"}, + {name: "CURRENT_SQL_DIGEST", tp: mysql.TypeVarchar, size: 64, comment: "The digest of the SQL that's being blocked"}, + {name: "KEY", tp: mysql.TypeBlob, size: types.UnspecifiedLength, comment: "The key on which a transaction is waiting for another"}, + {name: "TRX_HOLDING_LOCK", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "The transaction ID (start ts) of the transaction that's currently holding the lock"}, + // TODO: Implement the ALL_SQL_DIGESTS column + // {name: "ALL_SQL_DIGESTS", tp: mysql.TypeBlob, size: types.UnspecifiedLength, comment: "A list of the digests of SQL statements that the transaction has executed"}, +} + +var tableDataLockWaitsCols = []columnInfo{ + {name: "KEY", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "The key that's being waiting on"}, + {name: "TRX_ID", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Current transaction that's waiting for the lock"}, + {name: "CURRENT_HOLDING_TRX_ID", tp: mysql.TypeLonglong, size: 21, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "The transaction that's holding the lock and blocks the current transaction"}, + {name: "SQL_DIGEST", tp: mysql.TypeVarchar, size: 64, comment: "Digest of the SQL that's trying to acquire the lock"}, +} + +var tableStatementsSummaryEvictedCols = []columnInfo{ + {name: "BEGIN_TIME", tp: mysql.TypeTimestamp, size: 26}, + {name: "END_TIME", tp: mysql.TypeTimestamp, size: 26}, + {name: "EVICTED_COUNT", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, +} + // GetShardingInfo returns a nil or description string for the sharding information of given TableInfo. // The returned description string may be: // - "NOT_SHARDED": for tables that SHARD_ROW_ID_BITS is not specified. @@ -1521,7 +1578,7 @@ func GetPDServerInfo(ctx sessionctx.Context) ([]ServerInfo, error) { if err != nil { return nil, errors.Trace(err) } - pdVersion, err := ioutil.ReadAll(resp.Body) + pdVersion, err := io.ReadAll(resp.Body) terror.Log(resp.Body.Close()) if err != nil { return nil, errors.Trace(err) @@ -1694,6 +1751,7 @@ var tableNameToColumns = map[string][]columnInfo{ TableSequences: tableSequencesCols, TableStatementsSummary: tableStatementsSummaryCols, TableStatementsSummaryHistory: tableStatementsSummaryCols, + TableStatementsSummaryEvicted: tableStatementsSummaryEvictedCols, TableStorageStats: tableStorageStatsCols, TableTiFlashTables: tableTableTiFlashTablesCols, TableTiFlashSegments: tableTableTiFlashSegmentsCols, @@ -1701,6 +1759,9 @@ var tableNameToColumns = map[string][]columnInfo{ TableClientErrorsSummaryGlobal: tableClientErrorsSummaryGlobalCols, TableClientErrorsSummaryByUser: tableClientErrorsSummaryByUserCols, TableClientErrorsSummaryByHost: tableClientErrorsSummaryByHostCols, + TableTiDBTrx: tableTiDBTrxCols, + TableDeadlocks: tableDeadlocksCols, + TableDataLockWaits: tableDataLockWaitsCols, } func createInfoSchemaTable(_ autoid.Allocators, meta *model.TableInfo) (table.Table, error) { @@ -1737,7 +1798,7 @@ func (s SchemasSorter) Less(i, j int) bool { } func (it *infoschemaTable) getRows(ctx sessionctx.Context, cols []*table.Column) (fullRows [][]types.Datum, err error) { - is := GetInfoSchema(ctx) + is := ctx.GetInfoSchema().(InfoSchema) dbs := is.AllSchemas() sort.Sort(SchemasSorter(dbs)) switch it.meta.Name.O { diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index f30f25ba6abfa..dbba2dbcdba38 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -15,6 +15,7 @@ package infoschema_test import ( "crypto/tls" + "encoding/hex" "fmt" "math" "net" @@ -23,11 +24,14 @@ import ( "runtime" "strings" "time" + "unsafe" "github.com/gorilla/mux" . "github.com/pingcap/check" "github.com/pingcap/failpoint" "github.com/pingcap/fn" + "github.com/pingcap/kvproto/pkg/deadlock" + "github.com/pingcap/parser" "github.com/pingcap/parser/auth" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" @@ -42,11 +46,16 @@ import ( plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/server" "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/session/txninfo" "github.com/pingcap/tidb/store/helper" "github.com/pingcap/tidb/store/mockstore" + "github.com/pingcap/tidb/store/mockstore/mockstorage" + "github.com/pingcap/tidb/store/mockstore/unistore" + "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/kvcache" "github.com/pingcap/tidb/util/pdapi" + "github.com/pingcap/tidb/util/resourcegrouptag" "github.com/pingcap/tidb/util/set" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testleak" @@ -55,12 +64,17 @@ import ( ) var _ = Suite(&testTableSuite{&testTableSuiteBase{}}) +var _ = Suite(&testDataLockWaitSuite{&testTableSuiteBase{}}) var _ = SerialSuites(&testClusterTableSuite{testTableSuiteBase: &testTableSuiteBase{}}) type testTableSuite struct { *testTableSuiteBase } +type testDataLockWaitSuite struct { + *testTableSuiteBase +} + type testTableSuiteBase struct { store kv.Storage dom *domain.Domain @@ -121,7 +135,7 @@ func (s *testClusterTableSuite) setUpRPCService(c *C, addr string) (*grpc.Server lis, err := net.Listen("tcp", addr) c.Assert(err, IsNil) // Fix issue 9836 - sm := &mockSessionManager{make(map[uint64]*util.ProcessInfo, 1)} + sm := &mockSessionManager{make(map[uint64]*util.ProcessInfo, 1), nil} sm.processInfoMap[1] = &util.ProcessInfo{ ID: 1, User: "root", @@ -276,7 +290,7 @@ func (s *testTableSuite) TestInfoschemaFieldValue(c *C) { tk1.MustQuery("select distinct(table_schema) from information_schema.tables").Check(testkit.Rows("INFORMATION_SCHEMA")) // Fix issue 9836 - sm := &mockSessionManager{make(map[uint64]*util.ProcessInfo, 1)} + sm := &mockSessionManager{make(map[uint64]*util.ProcessInfo, 1), nil} sm.processInfoMap[1] = &util.ProcessInfo{ ID: 1, User: "root", @@ -433,6 +447,11 @@ func (s *testTableSuite) TestCurrentTimestampAsDefault(c *C) { type mockSessionManager struct { processInfoMap map[uint64]*util.ProcessInfo + txnInfo []*txninfo.TxnInfo +} + +func (sm *mockSessionManager) ShowTxnList() []*txninfo.TxnInfo { + return sm.txnInfo } func (sm *mockSessionManager) ShowProcessList() map[uint64]*util.ProcessInfo { @@ -459,7 +478,7 @@ func (s *testTableSuite) TestSomeTables(c *C) { c.Assert(err, IsNil) tk := testkit.NewTestKit(c, s.store) tk.Se = se - sm := &mockSessionManager{make(map[uint64]*util.ProcessInfo, 2)} + sm := &mockSessionManager{make(map[uint64]*util.ProcessInfo, 2), nil} sm.processInfoMap[1] = &util.ProcessInfo{ ID: 1, User: "user-1", @@ -516,7 +535,7 @@ func (s *testTableSuite) TestSomeTables(c *C) { fmt.Sprintf("3 user-3 127.0.0.1:12345 test Init DB 9223372036 %s %s", "in transaction", "check port"), )) - sm = &mockSessionManager{make(map[uint64]*util.ProcessInfo, 2)} + sm = &mockSessionManager{make(map[uint64]*util.ProcessInfo, 2), nil} sm.processInfoMap[1] = &util.ProcessInfo{ ID: 1, User: "user-1", @@ -958,8 +977,6 @@ func (s *testTableSuite) TestStmtSummaryTable(c *C) { tk.MustExec("set global tidb_enable_stmt_summary = 1") tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("1")) - // Invalidate the cache manually so that tidb_enable_stmt_summary works immediately. - s.dom.GetGlobalVarsCache().Disable() // Disable refreshing summary. tk.MustExec("set global tidb_stmt_summary_refresh_interval = 999999999") tk.MustQuery("select @@global.tidb_stmt_summary_refresh_interval").Check(testkit.Rows("999999999")) @@ -1202,8 +1219,6 @@ func (s *testClusterTableSuite) TestStmtSummaryHistoryTable(c *C) { tk.MustExec("set global tidb_enable_stmt_summary = 1") tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("1")) - // Invalidate the cache manually so that tidb_enable_stmt_summary works immediately. - s.dom.GetGlobalVarsCache().Disable() // Disable refreshing summary. tk.MustExec("set global tidb_stmt_summary_refresh_interval = 999999999") tk.MustQuery("select @@global.tidb_stmt_summary_refresh_interval").Check(testkit.Rows("999999999")) @@ -1259,8 +1274,6 @@ func (s *testTableSuite) TestStmtSummaryInternalQuery(c *C) { tk.MustExec("create global binding for select * from t where t.a = 1 using select * from t ignore index(k) where t.a = 1") tk.MustExec("set global tidb_enable_stmt_summary = 1") tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("1")) - // Invalidate the cache manually so that tidb_enable_stmt_summary works immediately. - s.dom.GetGlobalVarsCache().Disable() // Disable refreshing summary. tk.MustExec("set global tidb_stmt_summary_refresh_interval = 999999999") tk.MustQuery("select @@global.tidb_stmt_summary_refresh_interval").Check(testkit.Rows("999999999")) @@ -1361,6 +1374,33 @@ func (s *testTableSuite) TestStmtSummarySensitiveQuery(c *C) { )) } +func (s *testTableSuite) TestSimpleStmtSummaryEvictedCount(c *C) { + now := time.Now().Unix() + interval := int64(1800) + beginTimeForCurInterval := now - now%interval + tk := s.newTestKitWithPlanCache(c) + tk.MustExec(fmt.Sprintf("set global tidb_stmt_summary_refresh_interval = %v", interval)) + tk.MustExec("set global tidb_enable_stmt_summary = 0") + tk.MustExec("set global tidb_enable_stmt_summary = 1") + // first sql + tk.MustExec("set global tidb_stmt_summary_max_stmt_count = 1") + // second sql + tk.MustQuery("show databases;") + // query `evicted table` is also a SQL, passing it leads to the eviction of the previous SQLs. + tk.MustQuery("select * from `information_schema`.`STATEMENTS_SUMMARY_EVICTED`;"). + Check(testkit.Rows( + fmt.Sprintf("%s %s %v", + time.Unix(beginTimeForCurInterval, 0).Format("2006-01-02 15:04:05"), + time.Unix(beginTimeForCurInterval+interval, 0).Format("2006-01-02 15:04:05"), + int64(2)), + )) + // TODO: Add more tests. + + // clean up side effects + tk.MustExec("set global tidb_stmt_summary_max_stmt_count = 100") + tk.MustExec("set global tidb_stmt_summary_refresh_interval = 1800") +} + func (s *testTableSuite) TestPerformanceSchemaforPlanCache(c *C) { orgEnable := plannercore.PreparedPlanCacheEnabled() defer func() { @@ -1442,7 +1482,7 @@ func (s *testTableSuite) TestPlacementPolicy(c *C) { ID: "0", Role: "voter", Count: 3, - LabelConstraints: []placement.Constraint{ + Constraints: []placement.Constraint{ { Key: "zone", Op: "in", @@ -1509,3 +1549,107 @@ func (s *testTableSuite) TestInfoschemaClientErrors(c *C) { err = tk.ExecToErr("FLUSH CLIENT_ERRORS_SUMMARY") c.Assert(err.Error(), Equals, "[planner:1227]Access denied; you need (at least one of) the RELOAD privilege(s) for this operation") } + +func (s *testTableSuite) TestTrx(c *C) { + tk := s.newTestKitWithRoot(c) + _, digest := parser.NormalizeDigest("select * from trx for update;") + sm := &mockSessionManager{nil, make([]*txninfo.TxnInfo, 2)} + sm.txnInfo[0] = &txninfo.TxnInfo{ + StartTS: 424768545227014155, + CurrentSQLDigest: digest.String(), + State: txninfo.TxnRunningNormal, + BlockStartTime: nil, + EntriesCount: 1, + EntriesSize: 19, + ConnectionID: 2, + Username: "root", + CurrentDB: "test", + } + blockTime2 := time.Date(2021, 05, 20, 13, 18, 30, 123456000, time.Local) + sm.txnInfo[1] = &txninfo.TxnInfo{ + StartTS: 425070846483628033, + CurrentSQLDigest: "", + AllSQLDigests: []string{"sql1", "sql2"}, + State: txninfo.TxnLockWaiting, + BlockStartTime: unsafe.Pointer(&blockTime2), + ConnectionID: 10, + Username: "user1", + CurrentDB: "db1", + } + tk.Se.SetSessionManager(sm) + tk.MustQuery("select * from information_schema.TIDB_TRX;").Check(testkit.Rows( + "424768545227014155 2021-05-07 12:56:48.001000 "+digest.String()+" Normal 1 19 2 root test []", + "425070846483628033 2021-05-20 21:16:35.778000 LockWaiting 2021-05-20 13:18:30.123456 0 0 10 user1 db1 [sql1, sql2]")) +} + +func (s *testTableSuite) TestInfoschemaDeadlockPrivilege(c *C) { + tk := s.newTestKitWithRoot(c) + tk.MustExec("create user 'testuser'@'localhost'") + c.Assert(tk.Se.Auth(&auth.UserIdentity{ + Username: "testuser", + Hostname: "localhost", + }, nil, nil), IsTrue) + err := tk.QueryToErr("select * from information_schema.deadlocks") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[planner:1227]Access denied; you need (at least one of) the PROCESS privilege(s) for this operation") + + tk = s.newTestKitWithRoot(c) + tk.MustExec("create user 'testuser2'@'localhost'") + tk.MustExec("grant process on *.* to 'testuser2'@'localhost'") + c.Assert(tk.Se.Auth(&auth.UserIdentity{ + Username: "testuser2", + Hostname: "localhost", + }, nil, nil), IsTrue) + _ = tk.MustQuery("select * from information_schema.deadlocks") +} + +func (s *testDataLockWaitSuite) SetUpSuite(c *C) { + testleak.BeforeTest() + + client, pdClient, cluster, err := unistore.New("") + c.Assert(err, IsNil) + unistore.BootstrapWithSingleStore(cluster) + kvstore, err := tikv.NewTestTiKVStore(client, pdClient, nil, nil, 0) + c.Assert(err, IsNil) + _, digest1 := parser.NormalizeDigest("select * from t1 for update;") + _, digest2 := parser.NormalizeDigest("update t1 set f1=1 where id=2;") + s.store, err = mockstorage.NewMockStorageWithLockWaits(kvstore, []*deadlock.WaitForEntry{ + {Txn: 1, WaitForTxn: 2, KeyHash: 3, Key: []byte("a"), ResourceGroupTag: resourcegrouptag.EncodeResourceGroupTag(digest1, nil)}, + {Txn: 4, WaitForTxn: 5, KeyHash: 6, Key: []byte("b"), ResourceGroupTag: resourcegrouptag.EncodeResourceGroupTag(digest2, nil)}, + }) + c.Assert(err, IsNil) + session.DisableStats4Test() + s.dom, err = session.BootstrapSession(s.store) + c.Assert(err, IsNil) +} + +func (s *testDataLockWaitSuite) TestDataLockWait(c *C) { + _, digest1 := parser.NormalizeDigest("select * from t1 for update;") + _, digest2 := parser.NormalizeDigest("update t1 set f1=1 where id=2;") + keyHex1 := hex.EncodeToString([]byte("a")) + keyHex2 := hex.EncodeToString([]byte("b")) + tk := s.newTestKitWithRoot(c) + tk.MustQuery("select * from information_schema.DATA_LOCK_WAITS;"). + Check(testkit.Rows(keyHex1+" 1 2 "+digest1.String(), keyHex2+" 4 5 "+digest2.String())) +} + +func (s *testDataLockWaitSuite) TestDataLockPrivilege(c *C) { + tk := s.newTestKitWithRoot(c) + tk.MustExec("create user 'testuser'@'localhost'") + c.Assert(tk.Se.Auth(&auth.UserIdentity{ + Username: "testuser", + Hostname: "localhost", + }, nil, nil), IsTrue) + err := tk.QueryToErr("select * from information_schema.DATA_LOCK_WAITS") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[planner:1227]Access denied; you need (at least one of) the PROCESS privilege(s) for this operation") + + tk = s.newTestKitWithRoot(c) + tk.MustExec("create user 'testuser2'@'localhost'") + tk.MustExec("grant process on *.* to 'testuser2'@'localhost'") + c.Assert(tk.Se.Auth(&auth.UserIdentity{ + Username: "testuser2", + Hostname: "localhost", + }, nil, nil), IsTrue) + _ = tk.MustQuery("select * from information_schema.DATA_LOCK_WAITS") +} diff --git a/kv/fault_injection.go b/kv/fault_injection.go index 218ca9cbd6966..d61685a7f8a71 100644 --- a/kv/fault_injection.go +++ b/kv/fault_injection.go @@ -16,6 +16,8 @@ package kv import ( "context" "sync" + + "github.com/pingcap/tidb/store/tikv" ) // InjectionConfig is used for fault injections for KV components. @@ -64,7 +66,7 @@ func (s *InjectedStore) Begin() (Transaction, error) { } // BeginWithOption creates an injected Transaction with given option. -func (s *InjectedStore) BeginWithOption(option TransactionOption) (Transaction, error) { +func (s *InjectedStore) BeginWithOption(option tikv.StartTSOption) (Transaction, error) { txn, err := s.Storage.BeginWithOption(option) return &InjectedTransaction{ Transaction: txn, diff --git a/kv/fault_injection_test.go b/kv/fault_injection_test.go index 33b6535214b2c..09ee3a14dae3c 100644 --- a/kv/fault_injection_test.go +++ b/kv/fault_injection_test.go @@ -19,7 +19,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/errors" "github.com/pingcap/parser/terror" - "github.com/pingcap/tidb/store/tikv/oracle" + "github.com/pingcap/tidb/store/tikv" ) type testFaultInjectionSuite struct{} @@ -35,7 +35,7 @@ func (s testFaultInjectionSuite) TestFaultInjectionBasic(c *C) { storage := NewInjectedStore(newMockStorage(), &cfg) txn, err := storage.Begin() c.Assert(err, IsNil) - _, err = storage.BeginWithOption(TransactionOption{}.SetTxnScope(oracle.GlobalTxnScope).SetStartTs(0)) + _, err = storage.BeginWithOption(tikv.DefaultStartTSOption().SetTxnScope(GlobalTxnScope).SetStartTS(0)) c.Assert(err, IsNil) ver := Version{Ver: 1} snap := storage.GetSnapshot(ver) diff --git a/kv/interface_mock_test.go b/kv/interface_mock_test.go index 2388c4f48b9f3..2c01cbbb80deb 100644 --- a/kv/interface_mock_test.go +++ b/kv/interface_mock_test.go @@ -16,7 +16,9 @@ package kv import ( "context" + deadlockpb "github.com/pingcap/kvproto/pkg/deadlock" "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/oracle" ) @@ -48,10 +50,6 @@ func (t *mockTxn) SetOption(opt int, val interface{}) { t.opts[opt] = val } -func (t *mockTxn) DelOption(opt int) { - delete(t.opts, opt) -} - func (t *mockTxn) GetOption(opt int) interface{} { return t.opts[opt] } @@ -106,10 +104,6 @@ func (t *mockTxn) GetSnapshot() Snapshot { return nil } -func (t *mockTxn) GetUnionStore() UnionStore { - return nil -} - func (t *mockTxn) NewStagingBuffer() MemBuffer { return nil } @@ -158,7 +152,7 @@ func (s *mockStorage) Begin() (Transaction, error) { return newMockTxn(), nil } -func (s *mockStorage) BeginWithOption(option TransactionOption) (Transaction, error) { +func (s *mockStorage) BeginWithOption(option tikv.StartTSOption) (Transaction, error) { return newMockTxn(), nil } @@ -217,6 +211,14 @@ func (s *mockStorage) GetMemCache() MemManager { return nil } +func (s *mockStorage) GetLockWaits() ([]*deadlockpb.WaitForEntry, error) { + return nil, nil +} + +func (s *mockStorage) GetMinSafeTS(txnScope string) uint64 { + return 0 +} + // newMockStorage creates a new mockStorage. func newMockStorage() Storage { return &mockStorage{} @@ -258,4 +260,7 @@ func (s *mockSnapshot) IterReverse(k Key) (Iterator, error) { } func (s *mockSnapshot) SetOption(opt int, val interface{}) {} -func (s *mockSnapshot) DelOption(opt int) {} + +func (s *mockSnapshot) GetLockWaits() []deadlockpb.WaitForEntry { + return nil +} diff --git a/kv/keyflags.go b/kv/keyflags.go new file mode 100644 index 0000000000000..8e721061409d1 --- /dev/null +++ b/kv/keyflags.go @@ -0,0 +1,55 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kv + +// KeyFlags are metadata associated with key +type KeyFlags uint8 + +const ( + flagPresumeKNE KeyFlags = 1 << iota + flagNeedLocked +) + +// HasPresumeKeyNotExists returns whether the associated key use lazy check. +func (f KeyFlags) HasPresumeKeyNotExists() bool { + return f&flagPresumeKNE != 0 +} + +// HasNeedLocked returns whether the key needed to be locked +func (f KeyFlags) HasNeedLocked() bool { + return f&flagNeedLocked != 0 +} + +// FlagsOp describes KeyFlags modify operation. +type FlagsOp uint16 + +const ( + // SetPresumeKeyNotExists marks the existence of the associated key is checked lazily. + SetPresumeKeyNotExists FlagsOp = iota + // SetNeedLocked marks the associated key need to be acquired lock. + SetNeedLocked +) + +// ApplyFlagsOps applys flagspos to origin. +func ApplyFlagsOps(origin KeyFlags, ops ...FlagsOp) KeyFlags { + for _, op := range ops { + switch op { + case SetPresumeKeyNotExists: + origin |= flagPresumeKNE + case SetNeedLocked: + origin |= flagNeedLocked + } + } + return origin +} diff --git a/kv/kv.go b/kv/kv.go index 1b1e1a5f46a4a..283bc40078247 100644 --- a/kv/kv.go +++ b/kv/kv.go @@ -18,9 +18,11 @@ import ( "crypto/tls" "time" + deadlockpb "github.com/pingcap/kvproto/pkg/deadlock" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/store/tikv" tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/util/memory" @@ -43,9 +45,6 @@ var ( TxnTotalSizeLimit uint64 = config.DefTxnTotalSizeLimit ) -// FlagsOp describes KeyFlags modify operation. TODO:remove it when br is ready -type FlagsOp = tikvstore.FlagsOp - // Getter is the interface for the Get method. type Getter interface { // Get gets the value for key k from kv store. @@ -107,11 +106,11 @@ type MemBuffer interface { RUnlock() // GetFlags returns the latest flags associated with key. - GetFlags(Key) (tikvstore.KeyFlags, error) + GetFlags(Key) (KeyFlags, error) // SetWithFlags put key-value into the last active staging buffer with the given KeyFlags. - SetWithFlags(Key, []byte, ...tikvstore.FlagsOp) error + SetWithFlags(Key, []byte, ...FlagsOp) error // DeleteWithFlags delete key with the given KeyFlags - DeleteWithFlags(Key, ...tikvstore.FlagsOp) error + DeleteWithFlags(Key, ...FlagsOp) error // Staging create a new staging buffer inside the MemBuffer. // Subsequent writes will be temporarily stored in this new staging buffer. @@ -123,7 +122,7 @@ type MemBuffer interface { // If the changes are not published by `Release`, they will be discarded. Cleanup(StagingHandle) // InspectStage used to inspect the value updates in the given stage. - InspectStage(StagingHandle, func(Key, tikvstore.KeyFlags, []byte)) + InspectStage(StagingHandle, func(Key, KeyFlags, []byte)) // SnapshotGetter returns a Getter for a snapshot of MemBuffer. SnapshotGetter() Getter @@ -154,14 +153,13 @@ type Transaction interface { // String implements fmt.Stringer interface. String() string // LockKeys tries to lock the entries with the keys in KV store. + // Will block until all keys are locked successfully or an error occurs. LockKeys(ctx context.Context, lockCtx *LockCtx, keys ...Key) error // SetOption sets an option with a value, when val is nil, uses the default // value of this option. SetOption(opt int, val interface{}) // GetOption returns the option GetOption(opt int) interface{} - // DelOption deletes an option. - DelOption(opt int) // IsReadOnly checks if the transaction has only performed read operations. IsReadOnly() bool // StartTS returns the transaction start timestamp. @@ -173,8 +171,6 @@ type Transaction interface { GetMemBuffer() MemBuffer // GetSnapshot returns the Snapshot binding to this transaction. GetSnapshot() Snapshot - // GetUnionStore returns the UnionStore binding to this transaction. - GetUnionStore() UnionStore // SetVars sets variables to the transaction. SetVars(vars interface{}) // GetVars gets variables from the transaction. @@ -274,7 +270,7 @@ type Request struct { // call would not corresponds to a whole region result. Streaming bool // ReplicaRead is used for reading data from replicas, only follower is supported at this time. - ReplicaRead tikvstore.ReplicaReadType + ReplicaRead ReplicaReadType // StoreType represents this request is sent to the which type of store. StoreType StoreType // Cacheable is true if the request can be cached. Currently only deterministic DAG requests can be cached. @@ -287,10 +283,14 @@ type Request struct { TaskID uint64 // TiDBServerID is the specified TiDB serverID to execute request. `0` means all TiDB instances. TiDBServerID uint64 + // TxnScope is the scope of the current txn. + TxnScope string // IsStaleness indicates whether the request read staleness data IsStaleness bool // MatchStoreLabels indicates the labels the store should be matched MatchStoreLabels []*metapb.StoreLabel + // ResourceGroupTag indicates the kv request task group. + ResourceGroupTag []byte } // ResultSubset represents a result subset from a single storage unit. @@ -323,8 +323,6 @@ type Snapshot interface { // SetOption sets an option with a value, when val is nil, uses the default // value of this option. Only ReplicaRead is supported for snapshot SetOption(opt int, val interface{}) - // DelOption deletes an option. - DelOption(opt int) } // BatchGetter is the interface for BatchGet. @@ -340,52 +338,13 @@ type Driver interface { Open(path string) (Storage, error) } -// TransactionOption indicates the option when beginning a transaction -type TransactionOption struct { - TxnScope string - StartTS *uint64 - PrevSec *uint64 - MinStartTS *uint64 - MaxPrevSec *uint64 -} - -// SetMaxPrevSec set maxPrevSec -func (to TransactionOption) SetMaxPrevSec(maxPrevSec uint64) TransactionOption { - to.MaxPrevSec = &maxPrevSec - return to -} - -// SetMinStartTS set minStartTS -func (to TransactionOption) SetMinStartTS(minStartTS uint64) TransactionOption { - to.MinStartTS = &minStartTS - return to -} - -// SetStartTs set startTS -func (to TransactionOption) SetStartTs(startTS uint64) TransactionOption { - to.StartTS = &startTS - return to -} - -// SetPrevSec set prevSec -func (to TransactionOption) SetPrevSec(prevSec uint64) TransactionOption { - to.PrevSec = &prevSec - return to -} - -// SetTxnScope set txnScope -func (to TransactionOption) SetTxnScope(txnScope string) TransactionOption { - to.TxnScope = txnScope - return to -} - // Storage defines the interface for storage. // Isolation should be at least SI(SNAPSHOT ISOLATION) type Storage interface { // Begin a global transaction Begin() (Transaction, error) - // Begin a transaction with given option - BeginWithOption(option TransactionOption) (Transaction, error) + // BeginWithOption begins a transaction with given option + BeginWithOption(option tikv.StartTSOption) (Transaction, error) // GetSnapshot gets a snapshot that is able to read any data which data is <= ver. // if ver is MaxVersion or > current max committed version, we will use current version for this snapshot. GetSnapshot(ver Version) Snapshot @@ -411,6 +370,10 @@ type Storage interface { ShowStatus(ctx context.Context, key string) (interface{}, error) // GetMemCache return memory manager of the storage. GetMemCache() MemManager + // GetMinSafeTS return the minimal SafeTS of the storage with given txnScope. + GetMinSafeTS(txnScope string) uint64 + // GetLockWaits return all lock wait information + GetLockWaits() ([]*deadlockpb.WaitForEntry, error) } // EtcdBackend is used for judging a storage is a real TiKV. diff --git a/kv/mock_test.go b/kv/mock_test.go index 45e45d5941251..5efaa146920df 100644 --- a/kv/mock_test.go +++ b/kv/mock_test.go @@ -17,8 +17,6 @@ import ( "context" . "github.com/pingcap/check" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" - "github.com/pingcap/tidb/store/tikv/oracle" ) var _ = Suite(testMockSuite{}) @@ -30,12 +28,12 @@ func (s testMockSuite) TestInterface(c *C) { storage := newMockStorage() storage.GetClient() storage.UUID() - version, err := storage.CurrentVersion(oracle.GlobalTxnScope) + version, err := storage.CurrentVersion(GlobalTxnScope) c.Check(err, IsNil) snapshot := storage.GetSnapshot(version) _, err = snapshot.BatchGet(context.Background(), []Key{Key("abc"), Key("def")}) c.Check(err, IsNil) - snapshot.SetOption(tikvstore.Priority, PriorityNormal) + snapshot.SetOption(Priority, PriorityNormal) transaction, err := storage.Begin() c.Check(err, IsNil) @@ -46,7 +44,6 @@ func (s testMockSuite) TestInterface(c *C) { mock.GetOption(23) } transaction.StartTS() - transaction.DelOption(23) if transaction.IsReadOnly() { _, err = transaction.Get(context.TODO(), Key("lock")) c.Check(err, IsNil) diff --git a/store/tikv/kv/option.go b/kv/option.go similarity index 79% rename from store/tikv/kv/option.go rename to kv/option.go index bac9316d41773..de5a1d8834c40 100644 --- a/store/tikv/kv/option.go +++ b/kv/option.go @@ -59,12 +59,23 @@ const ( IsStalenessReadOnly // MatchStoreLabels indicates the labels the store should be matched MatchStoreLabels + // ResourceGroupTag indicates the resource group of the kv request. + ResourceGroupTag ) -// Priority value for transaction priority. -// TODO: remove after BR update. +// ReplicaReadType is the type of replica to read data from +type ReplicaReadType byte + const ( - PriorityNormal = iota - PriorityLow - PriorityHigh + // ReplicaReadLeader stands for 'read from leader'. + ReplicaReadLeader ReplicaReadType = iota + // ReplicaReadFollower stands for 'read from follower'. + ReplicaReadFollower + // ReplicaReadMixed stands for 'read from leader and follower and learner'. + ReplicaReadMixed ) + +// IsFollowerRead checks if follower is going to be used to read data. +func (r ReplicaReadType) IsFollowerRead() bool { + return r != ReplicaReadLeader +} diff --git a/kv/txn_scope_var.go b/kv/txn_scope_var.go new file mode 100644 index 0000000000000..941fdaff5f26f --- /dev/null +++ b/kv/txn_scope_var.go @@ -0,0 +1,73 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kv + +import ( + "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/store/tikv/oracle" +) + +// TxnScopeVar indicates the used txnScope for oracle +type TxnScopeVar struct { + // varValue indicates the value of @@txn_scope, which can only be `global` or `local` + varValue string + // txnScope indicates the value which the tidb-server holds to request tso to pd + txnScope string +} + +// GetTxnScopeVar gets TxnScopeVar from config +func GetTxnScopeVar() TxnScopeVar { + isGlobal, location := config.GetTxnScopeFromConfig() + if isGlobal { + return NewGlobalTxnScopeVar() + } + return NewLocalTxnScopeVar(location) +} + +// NewGlobalTxnScopeVar creates a Global TxnScopeVar +func NewGlobalTxnScopeVar() TxnScopeVar { + return newTxnScopeVar(GlobalTxnScope, GlobalTxnScope) +} + +// NewLocalTxnScopeVar creates a Local TxnScopeVar with given real txnScope value. +func NewLocalTxnScopeVar(txnScope string) TxnScopeVar { + return newTxnScopeVar(LocalTxnScope, txnScope) +} + +// GetVarValue returns the value of @@txn_scope which can only be `global` or `local` +func (t TxnScopeVar) GetVarValue() string { + return t.varValue +} + +// GetTxnScope returns the value of the tidb-server holds to request tso to pd. +func (t TxnScopeVar) GetTxnScope() string { + return t.txnScope +} + +func newTxnScopeVar(varValue string, txnScope string) TxnScopeVar { + return TxnScopeVar{ + varValue: varValue, + txnScope: txnScope, + } +} + +// Transaction scopes constants. +const ( + // GlobalTxnScope is synced with PD's define of global scope. + // If we want to remove the dependency on store/tikv here, we need to map + // the two GlobalTxnScopes in the driver layer. + GlobalTxnScope = oracle.GlobalTxnScope + // LocalTxnScope indicates the transaction should use local ts. + LocalTxnScope = "local" +) diff --git a/kv/union_store.go b/kv/union_store.go deleted file mode 100644 index 0e9a6768c5ebc..0000000000000 --- a/kv/union_store.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2015 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kv - -// UnionStore is a store that wraps a snapshot for read and a MemBuffer for buffered write. -// Also, it provides some transaction related utilities. -type UnionStore interface { - Retriever - - // HasPresumeKeyNotExists returns whether the key presumed key not exists error for the lazy check. - HasPresumeKeyNotExists(k Key) bool - // UnmarkPresumeKeyNotExists deletes the key presume key not exists error flag for the lazy check. - UnmarkPresumeKeyNotExists(k Key) - - // SetOption sets an option with a value, when val is nil, uses the default - // value of this option. - SetOption(opt int, val interface{}) - // DelOption deletes an option. - DelOption(opt int) - // GetOption gets an option. - GetOption(opt int) interface{} - // GetMemBuffer return the MemBuffer binding to this unionStore. - GetMemBuffer() MemBuffer -} diff --git a/meta/autoid/memid.go b/meta/autoid/memid.go index 703ffe1db4bf8..d9a6be9f6058a 100644 --- a/meta/autoid/memid.go +++ b/meta/autoid/memid.go @@ -64,7 +64,7 @@ func (alloc *inMemoryAllocator) GetType() AllocatorType { // NextGlobalAutoID implements autoid.Allocator NextGlobalAutoID interface. func (alloc *inMemoryAllocator) NextGlobalAutoID(tableID int64) (int64, error) { - return 0, errNotImplemented.GenWithStackByArgs() + return alloc.base, nil } func (alloc *inMemoryAllocator) Alloc(ctx context.Context, tableID int64, n uint64, increment, offset int64) (int64, int64, error) { diff --git a/meta/meta.go b/meta/meta.go index 2682ed5b47d1e..3f76d2948e9b1 100644 --- a/meta/meta.go +++ b/meta/meta.go @@ -30,7 +30,6 @@ import ( "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/metrics" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/structure" "github.com/pingcap/tidb/util/dbterror" "github.com/pingcap/tidb/util/logutil" @@ -94,8 +93,8 @@ type Meta struct { // NewMeta creates a Meta in transaction txn. // If the current Meta needs to handle a job, jobListKey is the type of the job's list. func NewMeta(txn kv.Transaction, jobListKeys ...JobListKeyType) *Meta { - txn.SetOption(tikvstore.Priority, kv.PriorityHigh) - txn.SetOption(tikvstore.SyncLog, struct{}{}) + txn.SetOption(kv.Priority, kv.PriorityHigh) + txn.SetOption(kv.SyncLog, struct{}{}) t := structure.NewStructure(txn, txn, mMetaPrefix) listKey := DefaultJobListKey if len(jobListKeys) != 0 { diff --git a/meta/meta_test.go b/meta/meta_test.go index 590e85fc2a21e..4ba54f1935a3a 100644 --- a/meta/meta_test.go +++ b/meta/meta_test.go @@ -27,7 +27,6 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/store/mockstore" - "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/util/testleak" . "github.com/pingcap/tidb/util/testutil" ) @@ -291,7 +290,7 @@ func (s *testSuite) TestSnapshot(c *C) { err = txn.Commit(context.Background()) c.Assert(err, IsNil) - ver1, _ := store.CurrentVersion(oracle.GlobalTxnScope) + ver1, _ := store.CurrentVersion(kv.GlobalTxnScope) time.Sleep(time.Millisecond) txn, _ = store.Begin() m = meta.NewMeta(txn) diff --git a/metrics/domain.go b/metrics/domain.go index dd3912555d59c..a05b25dd6a46a 100644 --- a/metrics/domain.go +++ b/metrics/domain.go @@ -38,6 +38,19 @@ var ( Buckets: prometheus.ExponentialBuckets(0.001, 2, 20), // 1ms ~ 524s }) + // InfoCacheCounters are the counters of get/hit. + InfoCacheCounters = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "domain", + Name: "infocache_counters", + Help: "Counters of infoCache: get/hit.", + }, []string{LblType}) + // InfoCacheCounterGet is the total number of getting entry. + InfoCacheCounterGet = "get" + // InfoCacheCounterHit is the cache hit numbers for get. + InfoCacheCounterHit = "hit" + // LoadPrivilegeCounter records the counter of load privilege. LoadPrivilegeCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -47,6 +60,15 @@ var ( Help: "Counter of load privilege", }, []string{LblType}) + // LoadSysVarCacheCounter records the counter of loading sysvars + LoadSysVarCacheCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "domain", + Name: "load_sysvarcache_total", + Help: "Counter of load sysvar cache", + }, []string{LblType}) + SchemaValidatorStop = "stop" SchemaValidatorRestart = "restart" SchemaValidatorReset = "reset" diff --git a/metrics/grafana/tidb.json b/metrics/grafana/tidb.json index eb3434a541bc0..5c78605bb62c1 100644 --- a/metrics/grafana/tidb.json +++ b/metrics/grafana/tidb.json @@ -3923,7 +3923,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", + "expr": "sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", type=\"commit\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -5063,7 +5063,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "This metric shows OPS of async commit transactions.", + "description": "This metric shows the OPS of different types of transactions.", "editable": true, "error": false, "fill": 1, @@ -5079,13 +5079,13 @@ "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -5103,6 +5103,13 @@ "stack": false, "steppedLine": false, "targets": [ + { + "expr": "sum(rate(tidb_tikvclient_commit_txn_counter{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "2PC-{{type}}", + "refId": "C" + }, { "expr": "sum(rate(tidb_tikvclient_async_commit_txn_counter{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", @@ -5111,18 +5118,18 @@ "refId": "A" }, { - "refId": "B", "expr": "sum(rate(tidb_tikvclient_one_pc_txn_counter{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", - "intervalFactor": 1, "format": "time_series", - "legendFormat": "1PC-{{type}}" + "intervalFactor": 1, + "legendFormat": "1PC-{{type}}", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Async Commit Transaction Counter", + "title": "Transaction Types Per Second", "tooltip": { "msResolution": false, "shared": true, @@ -5159,6 +5166,123 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "99th percentile of backoff count and duration in a transaction commit", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 46 + }, + "id": 224, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/count.*/", + "yaxis": 1 + }, + { + "alias": "/duration.*/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_count_bucket{tidb_cluster=\"$tidb_cluster\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "count - {{instance}}", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "duration - {{instance}}", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Transaction Commit .99 Backoff", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": "duration", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "repeat": null, @@ -7003,6 +7127,102 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "This metric shows the reasons of replica selector failure (which needs a backoff).", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 223, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tidb_tikvclient_replica_selector_failure_counter{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Replica Selector Failure Per Second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "repeat": null, diff --git a/metrics/metrics.go b/metrics/metrics.go index ff2ac3b1aa08d..8cbca511c078e 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -98,6 +98,7 @@ func RegisterMetrics() { prometheus.MustRegister(JobsGauge) prometheus.MustRegister(KeepAliveCounter) prometheus.MustRegister(LoadPrivilegeCounter) + prometheus.MustRegister(InfoCacheCounters) prometheus.MustRegister(LoadSchemaCounter) prometheus.MustRegister(LoadSchemaDuration) prometheus.MustRegister(MetaHistogram) @@ -150,6 +151,8 @@ func RegisterMetrics() { prometheus.MustRegister(TiFlashQueryTotalCounter) prometheus.MustRegister(SmallTxnWriteDuration) prometheus.MustRegister(TxnWriteThroughput) + prometheus.MustRegister(TiKVSmallReadDuration) + prometheus.MustRegister(LoadSysVarCacheCounter) tikvmetrics.InitMetrics(TiDB, TiKVClient) tikvmetrics.RegisterMetrics() diff --git a/metrics/session.go b/metrics/session.go index f2d682a6dd22b..775ffc0f5c630 100644 --- a/metrics/session.go +++ b/metrics/session.go @@ -146,4 +146,5 @@ const ( LblInTxn = "in_txn" LblVersion = "version" LblHash = "hash" + LblCTEType = "cte_type" ) diff --git a/metrics/sli.go b/metrics/sli.go index 2e926de099997..c6d0239550851 100644 --- a/metrics/sli.go +++ b/metrics/sli.go @@ -37,4 +37,14 @@ var ( Help: "Bucketed histogram of transaction write throughput (bytes/second).", Buckets: prometheus.ExponentialBuckets(64, 1.3, 40), // 64 bytes/s ~ 2.3MB/s }) + + // TiKVSmallReadDuration uses to collect small request read duration. + TiKVSmallReadDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "tidb", + Subsystem: "sli", + Name: "tikv_small_read_duration", + Help: "Read time of TiKV small read.", + Buckets: prometheus.ExponentialBuckets(0.0005, 2, 28), // 0.5ms ~ 74h + }) ) diff --git a/metrics/telemetry.go b/metrics/telemetry.go new file mode 100644 index 0000000000000..054001f1a7c48 --- /dev/null +++ b/metrics/telemetry.go @@ -0,0 +1,68 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" +) + +// Metrics +var ( + TelemetrySQLCTECnt = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "telemetry", + Name: "non_recursive_cte_usage", + Help: "Counter of usage of CTE", + }, []string{LblCTEType}) +) + +// readCounter reads the value of a prometheus.Counter. +// Returns -1 when failing to read the value. +func readCounter(m prometheus.Counter) int64 { + // Actually, it's not recommended to read the value of prometheus metric types directly: + // https://github.com/prometheus/client_golang/issues/486#issuecomment-433345239 + pb := &dto.Metric{} + // It's impossible to return an error though. + if err := m.Write(pb); err != nil { + return -1 + } + return int64(pb.GetCounter().GetValue()) +} + +// CTEUsageCounter records the usages of CTE. +type CTEUsageCounter struct { + NonRecursiveCTEUsed int64 `json:"nonRecursiveCTEUsed"` + RecursiveUsed int64 `json:"recursiveUsed"` + NonCTEUsed int64 `json:"nonCTEUsed"` +} + +// Sub returns the difference of two counters. +func (c CTEUsageCounter) Sub(rhs CTEUsageCounter) CTEUsageCounter { + new := CTEUsageCounter{} + new.NonRecursiveCTEUsed = c.NonRecursiveCTEUsed - rhs.NonRecursiveCTEUsed + new.RecursiveUsed = c.RecursiveUsed - rhs.RecursiveUsed + new.NonCTEUsed = c.NonCTEUsed - rhs.NonCTEUsed + return new +} + +// GetCTECounter gets the TxnCommitCounter. +func GetCTECounter() CTEUsageCounter { + return CTEUsageCounter{ + NonRecursiveCTEUsed: readCounter(TelemetrySQLCTECnt.With(prometheus.Labels{LblCTEType: "nonRecurCTE"})), + RecursiveUsed: readCounter(TelemetrySQLCTECnt.With(prometheus.Labels{LblCTEType: "recurCTE"})), + NonCTEUsed: readCounter(TelemetrySQLCTECnt.With(prometheus.Labels{LblCTEType: "notCTE"})), + } +} diff --git a/owner/manager_test.go b/owner/manager_test.go index e25b204e6bbb4..e239419057291 100644 --- a/owner/manager_test.go +++ b/owner/manager_test.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/parser/terror" . "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/owner" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/util/logutil" @@ -72,11 +73,14 @@ func TestSingle(t *testing.T) { defer clus.Terminate(t) cli := clus.RandClient() ctx := goctx.Background() + ic := infoschema.NewCache(2) + ic.Insert(infoschema.MockInfoSchemaWithSchemaVer(nil, 0)) d := NewDDL( ctx, WithEtcdClient(cli), WithStore(store), WithLease(testLease), + WithInfoCache(ic), ) err = d.Start(nil) if err != nil { @@ -142,11 +146,14 @@ func TestCluster(t *testing.T) { defer clus.Terminate(t) cli := clus.Client(0) + ic := infoschema.NewCache(2) + ic.Insert(infoschema.MockInfoSchemaWithSchemaVer(nil, 0)) d := NewDDL( goctx.Background(), WithEtcdClient(cli), WithStore(store), WithLease(testLease), + WithInfoCache(ic), ) err = d.Start(nil) if err != nil { @@ -157,11 +164,14 @@ func TestCluster(t *testing.T) { t.Fatalf("expect true, got isOwner:%v", isOwner) } cli1 := clus.Client(1) + ic2 := infoschema.NewCache(2) + ic2.Insert(infoschema.MockInfoSchemaWithSchemaVer(nil, 0)) d1 := NewDDL( goctx.Background(), WithEtcdClient(cli1), WithStore(store), WithLease(testLease), + WithInfoCache(ic2), ) err = d1.Start(nil) if err != nil { @@ -189,11 +199,14 @@ func TestCluster(t *testing.T) { // d3 (not owner) stop cli3 := clus.Client(3) + ic3 := infoschema.NewCache(2) + ic3.Insert(infoschema.MockInfoSchemaWithSchemaVer(nil, 0)) d3 := NewDDL( goctx.Background(), WithEtcdClient(cli3), WithStore(store), WithLease(testLease), + WithInfoCache(ic3), ) err = d3.Start(nil) if err != nil { diff --git a/planner/cascades/implementation_rules.go b/planner/cascades/implementation_rules.go index d7a08b4fabaab..56c2141cfb213 100644 --- a/planner/cascades/implementation_rules.go +++ b/planner/cascades/implementation_rules.go @@ -95,10 +95,7 @@ type ImplTableDual struct { // Match implements ImplementationRule Match interface. func (r *ImplTableDual) Match(expr *memo.GroupExpr, prop *property.PhysicalProperty) (matched bool) { - if !prop.IsEmpty() { - return false - } - return true + return prop.IsEmpty() } // OnImplement implements ImplementationRule OnImplement interface. @@ -116,10 +113,7 @@ type ImplMemTableScan struct { // Match implements ImplementationRule Match interface. func (r *ImplMemTableScan) Match(expr *memo.GroupExpr, prop *property.PhysicalProperty) (matched bool) { - if !prop.IsEmpty() { - return false - } - return true + return prop.IsEmpty() } // OnImplement implements ImplementationRule OnImplement interface. diff --git a/planner/cascades/transformation_rules.go b/planner/cascades/transformation_rules.go index 6d23e063f5877..9961509299a52 100644 --- a/planner/cascades/transformation_rules.go +++ b/planner/cascades/transformation_rules.go @@ -1503,10 +1503,7 @@ func NewRuleMergeAggregationProjection() Transformation { // Match implements Transformation interface. func (r *MergeAggregationProjection) Match(old *memo.ExprIter) bool { proj := old.Children[0].GetExpr().ExprNode.(*plannercore.LogicalProjection) - if plannercore.ExprsHasSideEffects(proj.Exprs) { - return false - } - return true + return !plannercore.ExprsHasSideEffects(proj.Exprs) } // OnTransform implements Transformation interface. diff --git a/planner/core/cache.go b/planner/core/cache.go index f97c207d189de..0e5a624b3d635 100644 --- a/planner/core/cache.go +++ b/planner/core/cache.go @@ -18,6 +18,7 @@ import ( "sync/atomic" "time" + "github.com/pingcap/parser" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" @@ -199,7 +200,7 @@ type CachedPrepareStmt struct { Executor interface{} NormalizedSQL string NormalizedPlan string - SQLDigest string - PlanDigest string + SQLDigest *parser.Digest + PlanDigest *parser.Digest ForUpdateRead bool } diff --git a/planner/core/cacheable_checker.go b/planner/core/cacheable_checker.go index 9e2593f380d3b..b78e02fa6a1fb 100644 --- a/planner/core/cacheable_checker.go +++ b/planner/core/cacheable_checker.go @@ -17,7 +17,7 @@ import ( "github.com/pingcap/parser/ast" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/infoschema" - "github.com/pingcap/tidb/types/parser_driver" + driver "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util/logutil" "go.uber.org/zap" ) diff --git a/planner/core/cacheable_checker_test.go b/planner/core/cacheable_checker_test.go index eb33790dfd74f..7d6db5dd611d5 100644 --- a/planner/core/cacheable_checker_test.go +++ b/planner/core/cacheable_checker_test.go @@ -20,7 +20,7 @@ import ( "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/planner/core" - "github.com/pingcap/tidb/types/parser_driver" + driver "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util/testkit" ) @@ -42,7 +42,7 @@ func (s *testCacheableSuite) TestCacheable(c *C) { tk.MustExec("create table t2(a int, b int) partition by hash(a) partitions 11") tk.MustExec("create table t3(a int, b int)") tbl := &ast.TableName{Schema: model.NewCIStr("test"), Name: model.NewCIStr("t3")} - is := infoschema.GetInfoSchema(tk.Se) + is := tk.Se.GetInfoSchema().(infoschema.InfoSchema) // test non-SelectStmt/-InsertStmt/-DeleteStmt/-UpdateStmt/-SetOprStmt var stmt ast.Node = &ast.ShowStmt{} c.Assert(core.Cacheable(stmt, is), IsFalse) diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index 822e52a0a0254..84a42702e945e 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -17,7 +17,7 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" + "os" "path/filepath" "strings" "testing" @@ -60,7 +60,7 @@ func (s *testAnalyzeSuite) TearDownSuite(c *C) { func (s *testAnalyzeSuite) loadTableStats(fileName string, dom *domain.Domain) error { statsPath := filepath.Join("testdata", fileName) - bytes, err := ioutil.ReadFile(statsPath) + bytes, err := os.ReadFile(statsPath) if err != nil { return err } @@ -295,10 +295,10 @@ func (s *testAnalyzeSuite) TestIndexRead(c *C) { c.Assert(err, IsNil) c.Assert(stmts, HasLen, 1) stmt := stmts[0] - is := domain.GetDomain(ctx).InfoSchema() - err = core.Preprocess(ctx, stmt, is) + ret := &core.PreprocessorReturn{} + err = core.Preprocess(ctx, stmt, core.WithPreprocessorReturn(ret)) c.Assert(err, IsNil) - p, _, err := planner.Optimize(context.TODO(), ctx, stmt, is) + p, _, err := planner.Optimize(context.TODO(), ctx, stmt, ret.InfoSchema) c.Assert(err, IsNil) planString := core.ToString(p) s.testData.OnRecord(func() { @@ -330,10 +330,10 @@ func (s *testAnalyzeSuite) TestEmptyTable(c *C) { c.Assert(err, IsNil) c.Assert(stmts, HasLen, 1) stmt := stmts[0] - is := domain.GetDomain(ctx).InfoSchema() - err = core.Preprocess(ctx, stmt, is) + ret := &core.PreprocessorReturn{} + err = core.Preprocess(ctx, stmt, core.WithPreprocessorReturn(ret)) c.Assert(err, IsNil) - p, _, err := planner.Optimize(context.TODO(), ctx, stmt, is) + p, _, err := planner.Optimize(context.TODO(), ctx, stmt, ret.InfoSchema) c.Assert(err, IsNil) planString := core.ToString(p) s.testData.OnRecord(func() { @@ -402,10 +402,10 @@ func (s *testAnalyzeSuite) TestAnalyze(c *C) { stmt := stmts[0] err = executor.ResetContextOfStmt(ctx, stmt) c.Assert(err, IsNil) - is := domain.GetDomain(ctx).InfoSchema() - err = core.Preprocess(ctx, stmt, is) + ret := &core.PreprocessorReturn{} + err = core.Preprocess(ctx, stmt, core.WithPreprocessorReturn(ret)) c.Assert(err, IsNil) - p, _, err := planner.Optimize(context.TODO(), ctx, stmt, is) + p, _, err := planner.Optimize(context.TODO(), ctx, stmt, ret.InfoSchema) c.Assert(err, IsNil) planString := core.ToString(p) s.testData.OnRecord(func() { @@ -491,10 +491,10 @@ func (s *testAnalyzeSuite) TestPreparedNullParam(c *C) { c.Assert(err, IsNil) stmt := stmts[0] - is := domain.GetDomain(ctx).InfoSchema() - err = core.Preprocess(ctx, stmt, is, core.InPrepare) + ret := &core.PreprocessorReturn{} + err = core.Preprocess(ctx, stmt, core.InPrepare, core.WithPreprocessorReturn(ret)) c.Assert(err, IsNil) - p, _, err := planner.Optimize(context.TODO(), ctx, stmt, is) + p, _, err := planner.Optimize(context.TODO(), ctx, stmt, ret.InfoSchema) c.Assert(err, IsNil) c.Assert(core.ToString(p), Equals, best, Commentf("for %s", sql)) @@ -578,6 +578,7 @@ func (s *testAnalyzeSuite) TestInconsistentEstimation(c *C) { for i := 0; i < 10; i++ { tk.MustExec("insert into t values (5,5,5), (10,10,10)") } + tk.MustExec("set @@tidb_analyze_version=1") tk.MustExec("analyze table t with 2 buckets") // Force using the histogram to estimate. tk.MustExec("update mysql.stats_histograms set stats_ver = 0") @@ -726,14 +727,14 @@ func BenchmarkOptimize(b *testing.B) { c.Assert(err, IsNil) c.Assert(stmts, HasLen, 1) stmt := stmts[0] - is := domain.GetDomain(ctx).InfoSchema() - err = core.Preprocess(ctx, stmt, is) + ret := &core.PreprocessorReturn{} + err = core.Preprocess(ctx, stmt, core.WithPreprocessorReturn(ret)) c.Assert(err, IsNil) b.Run(tt.sql, func(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - _, _, err := planner.Optimize(context.TODO(), ctx, stmt, is) + _, _, err := planner.Optimize(context.TODO(), ctx, stmt, ret.InfoSchema) c.Assert(err, IsNil) } b.ReportAllocs() diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 3818486955646..8e3baf262101c 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -265,7 +265,9 @@ func (e *Execute) OptimizePreparedPlan(ctx context.Context, sctx sessionctx.Cont preparedObj.Executor = nil // If the schema version has changed we need to preprocess it again, // if this time it failed, the real reason for the error is schema changed. - err := Preprocess(sctx, prepared.Stmt, is, InPrepare) + // FIXME: compatible with prepare https://github.com/pingcap/tidb/issues/24932 + ret := &PreprocessorReturn{InfoSchema: is} + err := Preprocess(sctx, prepared.Stmt, InPrepare, WithPreprocessorReturn(ret)) if err != nil { return ErrSchemaChanged.GenWithStack("Schema change caused error: %s", err.Error()) } @@ -448,18 +450,6 @@ func (e *Execute) tryCachePointPlan(ctx context.Context, sctx sessionctx.Context if err != nil { return err } - case *Update: - // Temporarily turn off the cache for UPDATE to solve #21884. - - //ok, err = IsPointUpdateByAutoCommit(sctx, p) - //if err != nil { - // return err - //} - //if ok { - // // make constant expression store paramMarker - // sctx.GetSessionVars().StmtCtx.PointExec = true - // p, names, err = OptimizeAstNode(ctx, sctx, prepared.Stmt, is) - //} } if ok { // just cache point plan now @@ -555,6 +545,7 @@ func (e *Execute) rebuildRange(p Plan) error { // The code should never run here as long as we're not using point get for partition table. // And if we change the logic one day, here work as defensive programming to cache the error. if x.PartitionInfo != nil { + // TODO: relocate the partition after rebuilding range to make PlanCache support PointGet return errors.New("point get for partition table can not use plan cache") } if x.HandleParam != nil { @@ -723,6 +714,9 @@ type Simple struct { // and executing in co-processor. // Used for `global kill`. See https://github.com/pingcap/tidb/blob/master/docs/design/2020-06-01-global-kill.md. IsFromRemote bool + + // StaleTxnStartTS is the StartTS that is used to build a staleness transaction by 'START TRANSACTION READ ONLY' statement. + StaleTxnStartTS uint64 } // PhysicalSimpleWrapper is a wrapper of `Simple` to implement physical plan interface. @@ -840,8 +834,8 @@ func (h *AnalyzeTableID) Equals(t *AnalyzeTableID) bool { return h.TableID == t.TableID && h.PartitionID == t.PartitionID } -// analyzeInfo is used to store the database name, table name and partition name of analyze task. -type analyzeInfo struct { +// AnalyzeInfo is used to store the database name, table name and partition name of analyze task. +type AnalyzeInfo struct { DBName string TableName string PartitionName string @@ -857,14 +851,14 @@ type AnalyzeColumnsTask struct { ColsInfo []*model.ColumnInfo TblInfo *model.TableInfo Indexes []*model.IndexInfo - analyzeInfo + AnalyzeInfo } // AnalyzeIndexTask is used for analyze index. type AnalyzeIndexTask struct { IndexInfo *model.IndexInfo TblInfo *model.TableInfo - analyzeInfo + AnalyzeInfo } // Analyze represents an analyze plan @@ -962,6 +956,8 @@ type Explain struct { Rows [][]string ExplainRows [][]string explainedPlans map[int]bool + + ctes []*PhysicalCTE } // GetExplainRowsForPlan get explain rows for plan. @@ -1023,6 +1019,10 @@ func (e *Explain) RenderResult() error { if err != nil { return err } + err = e.explainPlanInRowFormatCTE() + if err != nil { + return err + } } case ast.ExplainFormatDOT: if physicalPlan, ok := e.TargetPlan.(PhysicalPlan); ok { @@ -1038,6 +1038,26 @@ func (e *Explain) RenderResult() error { return nil } +func (e *Explain) explainPlanInRowFormatCTE() (err error) { + explainedCTEPlan := make(map[int]struct{}) + for i := 0; i < len(e.ctes); i++ { + x := (*CTEDefinition)(e.ctes[i]) + // skip if the CTE has been explained, the same CTE has same IDForStorage + if _, ok := explainedCTEPlan[x.CTE.IDForStorage]; ok { + continue + } + e.prepareOperatorInfo(x, "root", "", "", true) + childIndent := texttree.Indent4Child("", true) + err = e.explainPlanInRowFormat(x.SeedPlan, "root", "(Seed Part)", childIndent, x.RecurPlan == nil) + if x.RecurPlan != nil { + err = e.explainPlanInRowFormat(x.RecurPlan, "root", "(Recursive Part)", childIndent, true) + } + explainedCTEPlan[x.CTE.IDForStorage] = struct{}{} + } + + return +} + // explainPlanInRowFormat generates explain information for root-tasks. func (e *Explain) explainPlanInRowFormat(p Plan, taskType, driverSide, indent string, isLastChild bool) (err error) { e.prepareOperatorInfo(p, taskType, driverSide, indent, isLastChild) @@ -1141,6 +1161,8 @@ func (e *Explain) explainPlanInRowFormat(p Plan, taskType, driverSide, indent st if x.Plan != nil { err = e.explainPlanInRowFormat(x.Plan, "root", "", indent, true) } + case *PhysicalCTE: + e.ctes = append(e.ctes, x) } return } @@ -1355,21 +1377,3 @@ func IsPointGetWithPKOrUniqueKeyByAutoCommit(ctx sessionctx.Context, p Plan) (bo func IsAutoCommitTxn(ctx sessionctx.Context) bool { return ctx.GetSessionVars().IsAutocommit() && !ctx.GetSessionVars().InTxn() } - -// IsPointUpdateByAutoCommit checks if plan p is point update and is in autocommit context -func IsPointUpdateByAutoCommit(ctx sessionctx.Context, p Plan) (bool, error) { - if !IsAutoCommitTxn(ctx) { - return false, nil - } - - // check plan - updPlan, ok := p.(*Update) - if !ok { - return false, nil - } - if _, isFastSel := updPlan.SelectPlan.(*PointGetPlan); isFastSel { - return true, nil - } - - return false, nil -} diff --git a/planner/core/encode.go b/planner/core/encode.go index d1cad479d52f8..ba4fc643821b5 100644 --- a/planner/core/encode.go +++ b/planner/core/encode.go @@ -16,11 +16,11 @@ package core import ( "bytes" "crypto/sha256" - "fmt" "hash" "sync" "github.com/pingcap/failpoint" + "github.com/pingcap/parser" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/util/plancodec" ) @@ -34,6 +34,8 @@ var encoderPool = sync.Pool{ type planEncoder struct { buf bytes.Buffer encodedPlans map[int]bool + + ctes []*PhysicalCTE } // EncodePlan is used to encodePlan the plan to the plan tree with compressing. @@ -58,10 +60,35 @@ func EncodePlan(p Plan) string { func (pn *planEncoder) encodePlanTree(p Plan) string { pn.encodedPlans = make(map[int]bool) pn.buf.Reset() + pn.ctes = pn.ctes[:0] pn.encodePlan(p, true, kv.TiKV, 0) + pn.encodeCTEPlan() return plancodec.Compress(pn.buf.Bytes()) } +func (pn *planEncoder) encodeCTEPlan() { + explainedCTEPlan := make(map[int]struct{}) + for i := 0; i < len(pn.ctes); i++ { + x := (*CTEDefinition)(pn.ctes[i]) + // skip if the CTE has been explained, the same CTE has same IDForStorage + if _, ok := explainedCTEPlan[x.CTE.IDForStorage]; ok { + continue + } + taskTypeInfo := plancodec.EncodeTaskType(true, kv.TiKV) + actRows, analyzeInfo, memoryInfo, diskInfo := getRuntimeInfo(x.SCtx(), x, nil) + rowCount := 0.0 + if statsInfo := x.statsInfo(); statsInfo != nil { + rowCount = x.statsInfo().RowCount + } + plancodec.EncodePlanNode(0, x.CTE.IDForStorage, plancodec.TypeCTEDefinition, rowCount, taskTypeInfo, x.ExplainInfo(), actRows, analyzeInfo, memoryInfo, diskInfo, &pn.buf) + pn.encodePlan(x.SeedPlan, true, kv.TiKV, 1) + if x.RecurPlan != nil { + pn.encodePlan(x.RecurPlan, true, kv.TiKV, 1) + } + explainedCTEPlan[x.CTE.IDForStorage] = struct{}{} + } +} + func (pn *planEncoder) encodePlan(p Plan, isRoot bool, store kv.StoreType, depth int) { taskTypeInfo := plancodec.EncodeTaskType(isRoot, store) actRows, analyzeInfo, memoryInfo, diskInfo := getRuntimeInfo(p.SCtx(), p, nil) @@ -102,6 +129,8 @@ func (pn *planEncoder) encodePlan(p Plan, isRoot bool, store kv.StoreType, depth if copPlan.tablePlan != nil { pn.encodePlan(copPlan.tablePlan, false, store, depth) } + case *PhysicalCTE: + pn.ctes = append(pn.ctes, copPlan) } } @@ -120,10 +149,10 @@ type planDigester struct { } // NormalizePlan is used to normalize the plan and generate plan digest. -func NormalizePlan(p Plan) (normalized, digest string) { +func NormalizePlan(p Plan) (normalized string, digest *parser.Digest) { selectPlan := getSelectPlan(p) if selectPlan == nil { - return "", "" + return "", parser.NewDigest(nil) } d := digesterPool.Get().(*planDigester) defer digesterPool.Put(d) @@ -134,7 +163,7 @@ func NormalizePlan(p Plan) (normalized, digest string) { panic(err) } d.buf.Reset() - digest = fmt.Sprintf("%x", d.hasher.Sum(nil)) + digest = parser.NewDigest(d.hasher.Sum(nil)) d.hasher.Reset() return } diff --git a/planner/core/errors.go b/planner/core/errors.go index c713aab6367c1..dd457e0ca4efe 100644 --- a/planner/core/errors.go +++ b/planner/core/errors.go @@ -20,76 +20,84 @@ import ( // error definitions. var ( - ErrUnsupportedType = dbterror.ClassOptimizer.NewStd(mysql.ErrUnsupportedType) - ErrAnalyzeMissIndex = dbterror.ClassOptimizer.NewStd(mysql.ErrAnalyzeMissIndex) - ErrWrongParamCount = dbterror.ClassOptimizer.NewStd(mysql.ErrWrongParamCount) - ErrSchemaChanged = dbterror.ClassOptimizer.NewStd(mysql.ErrSchemaChanged) - ErrTablenameNotAllowedHere = dbterror.ClassOptimizer.NewStd(mysql.ErrTablenameNotAllowedHere) - ErrNotSupportedYet = dbterror.ClassOptimizer.NewStd(mysql.ErrNotSupportedYet) - ErrWrongUsage = dbterror.ClassOptimizer.NewStd(mysql.ErrWrongUsage) - ErrUnknown = dbterror.ClassOptimizer.NewStd(mysql.ErrUnknown) - ErrUnknownTable = dbterror.ClassOptimizer.NewStd(mysql.ErrUnknownTable) - ErrNoSuchTable = dbterror.ClassOptimizer.NewStd(mysql.ErrNoSuchTable) - ErrViewRecursive = dbterror.ClassOptimizer.NewStd(mysql.ErrViewRecursive) - ErrWrongArguments = dbterror.ClassOptimizer.NewStd(mysql.ErrWrongArguments) - ErrWrongNumberOfColumnsInSelect = dbterror.ClassOptimizer.NewStd(mysql.ErrWrongNumberOfColumnsInSelect) - ErrBadGeneratedColumn = dbterror.ClassOptimizer.NewStd(mysql.ErrBadGeneratedColumn) - ErrFieldNotInGroupBy = dbterror.ClassOptimizer.NewStd(mysql.ErrFieldNotInGroupBy) - ErrAggregateOrderNonAggQuery = dbterror.ClassOptimizer.NewStd(mysql.ErrAggregateOrderNonAggQuery) - ErrFieldInOrderNotSelect = dbterror.ClassOptimizer.NewStd(mysql.ErrFieldInOrderNotSelect) - ErrAggregateInOrderNotSelect = dbterror.ClassOptimizer.NewStd(mysql.ErrAggregateInOrderNotSelect) - ErrBadTable = dbterror.ClassOptimizer.NewStd(mysql.ErrBadTable) - ErrKeyDoesNotExist = dbterror.ClassOptimizer.NewStd(mysql.ErrKeyDoesNotExist) - ErrOperandColumns = dbterror.ClassOptimizer.NewStd(mysql.ErrOperandColumns) - ErrInvalidGroupFuncUse = dbterror.ClassOptimizer.NewStd(mysql.ErrInvalidGroupFuncUse) - ErrIllegalReference = dbterror.ClassOptimizer.NewStd(mysql.ErrIllegalReference) - ErrNoDB = dbterror.ClassOptimizer.NewStd(mysql.ErrNoDB) - ErrUnknownExplainFormat = dbterror.ClassOptimizer.NewStd(mysql.ErrUnknownExplainFormat) - ErrWrongGroupField = dbterror.ClassOptimizer.NewStd(mysql.ErrWrongGroupField) - ErrDupFieldName = dbterror.ClassOptimizer.NewStd(mysql.ErrDupFieldName) - ErrNonUpdatableTable = dbterror.ClassOptimizer.NewStd(mysql.ErrNonUpdatableTable) - ErrMultiUpdateKeyConflict = dbterror.ClassOptimizer.NewStd(mysql.ErrMultiUpdateKeyConflict) - ErrInternal = dbterror.ClassOptimizer.NewStd(mysql.ErrInternal) - ErrNonUniqTable = dbterror.ClassOptimizer.NewStd(mysql.ErrNonuniqTable) - ErrWindowInvalidWindowFuncUse = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowInvalidWindowFuncUse) - ErrWindowInvalidWindowFuncAliasUse = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowInvalidWindowFuncAliasUse) - ErrWindowNoSuchWindow = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowNoSuchWindow) - ErrWindowCircularityInWindowGraph = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowCircularityInWindowGraph) - ErrWindowNoChildPartitioning = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowNoChildPartitioning) - ErrWindowNoInherentFrame = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowNoInherentFrame) - ErrWindowNoRedefineOrderBy = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowNoRedefineOrderBy) - ErrWindowDuplicateName = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowDuplicateName) - ErrPartitionClauseOnNonpartitioned = dbterror.ClassOptimizer.NewStd(mysql.ErrPartitionClauseOnNonpartitioned) - ErrWindowFrameStartIllegal = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowFrameStartIllegal) - ErrWindowFrameEndIllegal = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowFrameEndIllegal) - ErrWindowFrameIllegal = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowFrameIllegal) - ErrWindowRangeFrameOrderType = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowRangeFrameOrderType) - ErrWindowRangeFrameTemporalType = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowRangeFrameTemporalType) - ErrWindowRangeFrameNumericType = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowRangeFrameNumericType) - ErrWindowRangeBoundNotConstant = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowRangeBoundNotConstant) - ErrWindowRowsIntervalUse = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowRowsIntervalUse) - ErrWindowFunctionIgnoresFrame = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowFunctionIgnoresFrame) - ErrUnsupportedOnGeneratedColumn = dbterror.ClassOptimizer.NewStd(mysql.ErrUnsupportedOnGeneratedColumn) - ErrPrivilegeCheckFail = dbterror.ClassOptimizer.NewStd(mysql.ErrPrivilegeCheckFail) - ErrInvalidWildCard = dbterror.ClassOptimizer.NewStd(mysql.ErrInvalidWildCard) - ErrMixOfGroupFuncAndFields = dbterror.ClassOptimizer.NewStd(mysql.ErrMixOfGroupFuncAndFieldsIncompatible) - errTooBigPrecision = dbterror.ClassExpression.NewStd(mysql.ErrTooBigPrecision) - ErrDBaccessDenied = dbterror.ClassOptimizer.NewStd(mysql.ErrDBaccessDenied) - ErrTableaccessDenied = dbterror.ClassOptimizer.NewStd(mysql.ErrTableaccessDenied) - ErrSpecificAccessDenied = dbterror.ClassOptimizer.NewStd(mysql.ErrSpecificAccessDenied) - ErrViewNoExplain = dbterror.ClassOptimizer.NewStd(mysql.ErrViewNoExplain) - ErrWrongValueCountOnRow = dbterror.ClassOptimizer.NewStd(mysql.ErrWrongValueCountOnRow) - ErrViewInvalid = dbterror.ClassOptimizer.NewStd(mysql.ErrViewInvalid) - ErrNoSuchThread = dbterror.ClassOptimizer.NewStd(mysql.ErrNoSuchThread) - ErrUnknownColumn = dbterror.ClassOptimizer.NewStd(mysql.ErrBadField) - ErrCartesianProductUnsupported = dbterror.ClassOptimizer.NewStd(mysql.ErrCartesianProductUnsupported) - ErrStmtNotFound = dbterror.ClassOptimizer.NewStd(mysql.ErrPreparedStmtNotFound) - ErrAmbiguous = dbterror.ClassOptimizer.NewStd(mysql.ErrNonUniq) - ErrUnresolvedHintName = dbterror.ClassOptimizer.NewStd(mysql.ErrUnresolvedHintName) - ErrNotHintUpdatable = dbterror.ClassOptimizer.NewStd(mysql.ErrNotHintUpdatable) - ErrWarnConflictingHint = dbterror.ClassOptimizer.NewStd(mysql.ErrWarnConflictingHint) + ErrUnsupportedType = dbterror.ClassOptimizer.NewStd(mysql.ErrUnsupportedType) + ErrAnalyzeMissIndex = dbterror.ClassOptimizer.NewStd(mysql.ErrAnalyzeMissIndex) + ErrWrongParamCount = dbterror.ClassOptimizer.NewStd(mysql.ErrWrongParamCount) + ErrSchemaChanged = dbterror.ClassOptimizer.NewStd(mysql.ErrSchemaChanged) + ErrTablenameNotAllowedHere = dbterror.ClassOptimizer.NewStd(mysql.ErrTablenameNotAllowedHere) + ErrNotSupportedYet = dbterror.ClassOptimizer.NewStd(mysql.ErrNotSupportedYet) + ErrWrongUsage = dbterror.ClassOptimizer.NewStd(mysql.ErrWrongUsage) + ErrUnknown = dbterror.ClassOptimizer.NewStd(mysql.ErrUnknown) + ErrUnknownTable = dbterror.ClassOptimizer.NewStd(mysql.ErrUnknownTable) + ErrNoSuchTable = dbterror.ClassOptimizer.NewStd(mysql.ErrNoSuchTable) + ErrViewRecursive = dbterror.ClassOptimizer.NewStd(mysql.ErrViewRecursive) + ErrWrongArguments = dbterror.ClassOptimizer.NewStd(mysql.ErrWrongArguments) + ErrWrongNumberOfColumnsInSelect = dbterror.ClassOptimizer.NewStd(mysql.ErrWrongNumberOfColumnsInSelect) + ErrBadGeneratedColumn = dbterror.ClassOptimizer.NewStd(mysql.ErrBadGeneratedColumn) + ErrFieldNotInGroupBy = dbterror.ClassOptimizer.NewStd(mysql.ErrFieldNotInGroupBy) + ErrAggregateOrderNonAggQuery = dbterror.ClassOptimizer.NewStd(mysql.ErrAggregateOrderNonAggQuery) + ErrFieldInOrderNotSelect = dbterror.ClassOptimizer.NewStd(mysql.ErrFieldInOrderNotSelect) + ErrAggregateInOrderNotSelect = dbterror.ClassOptimizer.NewStd(mysql.ErrAggregateInOrderNotSelect) + ErrBadTable = dbterror.ClassOptimizer.NewStd(mysql.ErrBadTable) + ErrKeyDoesNotExist = dbterror.ClassOptimizer.NewStd(mysql.ErrKeyDoesNotExist) + ErrOperandColumns = dbterror.ClassOptimizer.NewStd(mysql.ErrOperandColumns) + ErrInvalidGroupFuncUse = dbterror.ClassOptimizer.NewStd(mysql.ErrInvalidGroupFuncUse) + ErrIllegalReference = dbterror.ClassOptimizer.NewStd(mysql.ErrIllegalReference) + ErrNoDB = dbterror.ClassOptimizer.NewStd(mysql.ErrNoDB) + ErrUnknownExplainFormat = dbterror.ClassOptimizer.NewStd(mysql.ErrUnknownExplainFormat) + ErrWrongGroupField = dbterror.ClassOptimizer.NewStd(mysql.ErrWrongGroupField) + ErrDupFieldName = dbterror.ClassOptimizer.NewStd(mysql.ErrDupFieldName) + ErrNonUpdatableTable = dbterror.ClassOptimizer.NewStd(mysql.ErrNonUpdatableTable) + ErrMultiUpdateKeyConflict = dbterror.ClassOptimizer.NewStd(mysql.ErrMultiUpdateKeyConflict) + ErrInternal = dbterror.ClassOptimizer.NewStd(mysql.ErrInternal) + ErrNonUniqTable = dbterror.ClassOptimizer.NewStd(mysql.ErrNonuniqTable) + ErrWindowInvalidWindowFuncUse = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowInvalidWindowFuncUse) + ErrWindowInvalidWindowFuncAliasUse = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowInvalidWindowFuncAliasUse) + ErrWindowNoSuchWindow = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowNoSuchWindow) + ErrWindowCircularityInWindowGraph = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowCircularityInWindowGraph) + ErrWindowNoChildPartitioning = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowNoChildPartitioning) + ErrWindowNoInherentFrame = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowNoInherentFrame) + ErrWindowNoRedefineOrderBy = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowNoRedefineOrderBy) + ErrWindowDuplicateName = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowDuplicateName) + ErrPartitionClauseOnNonpartitioned = dbterror.ClassOptimizer.NewStd(mysql.ErrPartitionClauseOnNonpartitioned) + ErrWindowFrameStartIllegal = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowFrameStartIllegal) + ErrWindowFrameEndIllegal = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowFrameEndIllegal) + ErrWindowFrameIllegal = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowFrameIllegal) + ErrWindowRangeFrameOrderType = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowRangeFrameOrderType) + ErrWindowRangeFrameTemporalType = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowRangeFrameTemporalType) + ErrWindowRangeFrameNumericType = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowRangeFrameNumericType) + ErrWindowRangeBoundNotConstant = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowRangeBoundNotConstant) + ErrWindowRowsIntervalUse = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowRowsIntervalUse) + ErrWindowFunctionIgnoresFrame = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowFunctionIgnoresFrame) + ErrUnsupportedOnGeneratedColumn = dbterror.ClassOptimizer.NewStd(mysql.ErrUnsupportedOnGeneratedColumn) + ErrPrivilegeCheckFail = dbterror.ClassOptimizer.NewStd(mysql.ErrPrivilegeCheckFail) + ErrInvalidWildCard = dbterror.ClassOptimizer.NewStd(mysql.ErrInvalidWildCard) + ErrMixOfGroupFuncAndFields = dbterror.ClassOptimizer.NewStd(mysql.ErrMixOfGroupFuncAndFieldsIncompatible) + errTooBigPrecision = dbterror.ClassExpression.NewStd(mysql.ErrTooBigPrecision) + ErrDBaccessDenied = dbterror.ClassOptimizer.NewStd(mysql.ErrDBaccessDenied) + ErrTableaccessDenied = dbterror.ClassOptimizer.NewStd(mysql.ErrTableaccessDenied) + ErrSpecificAccessDenied = dbterror.ClassOptimizer.NewStd(mysql.ErrSpecificAccessDenied) + ErrViewNoExplain = dbterror.ClassOptimizer.NewStd(mysql.ErrViewNoExplain) + ErrWrongValueCountOnRow = dbterror.ClassOptimizer.NewStd(mysql.ErrWrongValueCountOnRow) + ErrViewInvalid = dbterror.ClassOptimizer.NewStd(mysql.ErrViewInvalid) + ErrNoSuchThread = dbterror.ClassOptimizer.NewStd(mysql.ErrNoSuchThread) + ErrUnknownColumn = dbterror.ClassOptimizer.NewStd(mysql.ErrBadField) + ErrCartesianProductUnsupported = dbterror.ClassOptimizer.NewStd(mysql.ErrCartesianProductUnsupported) + ErrStmtNotFound = dbterror.ClassOptimizer.NewStd(mysql.ErrPreparedStmtNotFound) + ErrAmbiguous = dbterror.ClassOptimizer.NewStd(mysql.ErrNonUniq) + ErrUnresolvedHintName = dbterror.ClassOptimizer.NewStd(mysql.ErrUnresolvedHintName) + ErrNotHintUpdatable = dbterror.ClassOptimizer.NewStd(mysql.ErrNotHintUpdatable) + ErrWarnConflictingHint = dbterror.ClassOptimizer.NewStd(mysql.ErrWarnConflictingHint) + ErrCTERecursiveRequiresUnion = dbterror.ClassOptimizer.NewStd(mysql.ErrCTERecursiveRequiresUnion) + ErrCTERecursiveRequiresNonRecursiveFirst = dbterror.ClassOptimizer.NewStd(mysql.ErrCTERecursiveRequiresNonRecursiveFirst) + ErrCTERecursiveForbidsAggregation = dbterror.ClassOptimizer.NewStd(mysql.ErrCTERecursiveForbidsAggregation) + ErrCTERecursiveForbiddenJoinOrder = dbterror.ClassOptimizer.NewStd(mysql.ErrCTERecursiveForbiddenJoinOrder) + ErrInvalidRequiresSingleReference = dbterror.ClassOptimizer.NewStd(mysql.ErrInvalidRequiresSingleReference) // Since we cannot know if user logged in with a password, use message of ErrAccessDeniedNoPassword instead ErrAccessDenied = dbterror.ClassOptimizer.NewStdErr(mysql.ErrAccessDenied, mysql.MySQLErrName[mysql.ErrAccessDeniedNoPassword]) ErrBadNull = dbterror.ClassOptimizer.NewStd(mysql.ErrBadNull) ErrNotSupportedWithSem = dbterror.ClassOptimizer.NewStd(mysql.ErrNotSupportedWithSem) + ErrDifferentAsOf = dbterror.ClassOptimizer.NewStd(mysql.ErrUnknown) + ErrAsOf = dbterror.ClassOptimizer.NewStd(mysql.ErrUnknown) + ErrOptOnTemporaryTable = dbterror.ClassOptimizer.NewStd(mysql.ErrOptOnTemporaryTable) ) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index d4f2923b6220a..d7a17d3017221 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -39,16 +39,16 @@ import ( "go.uber.org/zap" ) -func (p *LogicalUnionScan) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { +func (p *LogicalUnionScan) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { if prop.IsFlashProp() { - return nil, true + return nil, true, nil } childProp := prop.CloneEssentialFields() us := PhysicalUnionScan{ Conditions: p.conditions, HandleCols: p.handleCols, }.Init(p.ctx, p.stats, p.blockOffset, childProp) - return []PhysicalPlan{us}, true + return []PhysicalPlan{us}, true, nil } func getMaxSortPrefix(sortCols, allCols []*expression.Column) []int { @@ -147,6 +147,19 @@ func (p *LogicalJoin) GetMergeJoin(prop *property.PhysicalProperty, schema *expr joins := make([]PhysicalPlan, 0, len(p.leftProperties)+1) // The leftProperties caches all the possible properties that are provided by its children. leftJoinKeys, rightJoinKeys, isNullEQ, hasNullEQ := p.GetJoinKeys() + + // EnumType Unsupported: merge join conflicts with index order. ref: https://github.com/pingcap/tidb/issues/24473 + for _, leftKey := range leftJoinKeys { + if leftKey.RetType.Tp == mysql.TypeEnum { + return nil + } + } + for _, rightKey := range rightJoinKeys { + if rightKey.RetType.Tp == mysql.TypeEnum { + return nil + } + } + // TODO: support null equal join keys for merge join if hasNullEQ { return nil @@ -513,6 +526,19 @@ func (p *LogicalJoin) constructIndexMergeJoin( if len(join.InnerHashKeys) > len(join.InnerJoinKeys) { return nil } + + // EnumType Unsupported: merge join conflicts with index order. ref: https://github.com/pingcap/tidb/issues/24473 + for _, innerKey := range join.InnerJoinKeys { + if innerKey.RetType.Tp == mysql.TypeEnum { + return nil + } + } + for _, outerKey := range join.OuterJoinKeys { + if outerKey.RetType.Tp == mysql.TypeEnum { + return nil + } + } + hasPrefixCol := false for _, l := range join.IdxColLens { if l != types.UnspecifiedLength { @@ -930,7 +956,7 @@ func (p *LogicalJoin) constructInnerTableScanTask( copTask := &copTask{ tablePlan: ts, indexPlanFinished: true, - cst: sessVars.ScanFactor * rowSize * ts.stats.RowCount, + cst: sessVars.GetScanFactor(ts.Table) * rowSize * ts.stats.RowCount, tblColHists: ds.TblColHists, keepOrder: ts.KeepOrder, } @@ -1088,7 +1114,7 @@ func (p *LogicalJoin) constructInnerIndexScanTask( is.stats = ds.tableStats.ScaleByExpectCnt(tmpPath.CountAfterAccess) rowSize := is.indexScanRowSize(path.Index, ds, true) sessVars := ds.ctx.GetSessionVars() - cop.cst = tmpPath.CountAfterAccess * rowSize * sessVars.ScanFactor + cop.cst = tmpPath.CountAfterAccess * rowSize * sessVars.GetScanFactor(ds.tableInfo) finalStats := ds.tableStats.ScaleByExpectCnt(rowCount) is.addPushedDownSelection(cop, ds, tmpPath, finalStats) t := cop.convertToRootTask(ds.ctx) @@ -1416,6 +1442,7 @@ func (ijHelper *indexJoinBuildHelper) updateBestChoice(ranges []*ranger.Range, p func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAndInFuncs []expression.Expression, nextColRange []*ranger.Range, haveExtraCol bool) (ranges []*ranger.Range, emptyRange bool, err error) { pointLength := matchedKeyCnt + len(eqAndInFuncs) + //nolint:gosimple // false positive unnecessary nil check if nextColRange != nil { for _, colRan := range nextColRange { // The range's exclude status is the same with last col's. @@ -1634,6 +1661,9 @@ func (p *LogicalJoin) shouldUseMPPBCJ() bool { if p.ctx.GetSessionVars().BroadcastJoinThresholdSize == 0 || p.ctx.GetSessionVars().BroadcastJoinThresholdCount == 0 { return p.ctx.GetSessionVars().AllowBCJ } + if len(p.EqualConditions) == 0 && p.ctx.GetSessionVars().AllowCartesianBCJ == 2 { + return true + } if p.JoinType == LeftOuterJoin || p.JoinType == SemiJoin || p.JoinType == AntiSemiJoin { return checkChildFitBC(p.children[1]) } else if p.JoinType == RightOuterJoin { @@ -1646,27 +1676,27 @@ func (p *LogicalJoin) shouldUseMPPBCJ() bool { // Firstly we check the hint, if hint is figured by user, we force to choose the corresponding physical plan. // If the hint is not matched, it will get other candidates. // If the hint is not figured, we will pick all candidates. -func (p *LogicalJoin) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { +func (p *LogicalJoin) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { failpoint.Inject("MockOnlyEnableIndexHashJoin", func(val failpoint.Value) { if val.(bool) { indexJoins, _ := p.tryToGetIndexJoin(prop) - failpoint.Return(indexJoins, true) + failpoint.Return(indexJoins, true, nil) } }) if prop.IsFlashProp() && ((p.preferJoinType&preferBCJoin) == 0 && p.preferJoinType > 0) { - return nil, false + return nil, false, nil } if prop.PartitionTp == property.BroadcastType { - return nil, false + return nil, false, nil } joins := make([]PhysicalPlan, 0, 8) canPushToTiFlash := p.canPushToCop(kv.TiFlash) - if p.ctx.GetSessionVars().AllowMPPExecution && canPushToTiFlash { + if p.ctx.GetSessionVars().IsMPPAllowed() && canPushToTiFlash { if p.shouldUseMPPBCJ() { mppJoins := p.tryToGetMppHashJoin(prop, true) if (p.preferJoinType & preferBCJoin) > 0 { - return mppJoins, true + return mppJoins, true, nil } joins = append(joins, mppJoins...) } else { @@ -1676,29 +1706,29 @@ func (p *LogicalJoin) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]P } else if p.ctx.GetSessionVars().AllowBCJ && canPushToTiFlash { broadCastJoins := p.tryToGetBroadCastJoin(prop) if (p.preferJoinType & preferBCJoin) > 0 { - return broadCastJoins, true + return broadCastJoins, true, nil } joins = append(joins, broadCastJoins...) } if prop.IsFlashProp() { - return joins, true + return joins, true, nil } mergeJoins := p.GetMergeJoin(prop, p.schema, p.Stats(), p.children[0].statsInfo(), p.children[1].statsInfo()) if (p.preferJoinType&preferMergeJoin) > 0 && len(mergeJoins) > 0 { - return mergeJoins, true + return mergeJoins, true, nil } joins = append(joins, mergeJoins...) indexJoins, forced := p.tryToGetIndexJoin(prop) if forced { - return indexJoins, true + return indexJoins, true, nil } joins = append(joins, indexJoins...) hashJoins := p.getHashJoins(prop) if (p.preferJoinType&preferHashJoin) > 0 && len(hashJoins) > 0 { - return hashJoins, true + return hashJoins, true, nil } joins = append(joins, hashJoins...) @@ -1706,9 +1736,9 @@ func (p *LogicalJoin) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]P // If we reach here, it means we have a hint that doesn't work. // It might be affected by the required property, so we enforce // this property and try the hint again. - return joins, false + return joins, false, nil } - return joins, true + return joins, true, nil } func canExprsInJoinPushdown(p *LogicalJoin, storeType kv.StoreType) bool { @@ -1742,9 +1772,19 @@ func (p *LogicalJoin) tryToGetMppHashJoin(prop *property.PhysicalProperty, useBC return nil } - if (p.JoinType != InnerJoin && p.JoinType != LeftOuterJoin && p.JoinType != RightOuterJoin && p.JoinType != SemiJoin && p.JoinType != AntiSemiJoin) || len(p.EqualConditions) == 0 { + if p.JoinType != InnerJoin && p.JoinType != LeftOuterJoin && p.JoinType != RightOuterJoin && p.JoinType != SemiJoin && p.JoinType != AntiSemiJoin { + return nil + } + + if len(p.EqualConditions) == 0 { + if p.ctx.GetSessionVars().AllowCartesianBCJ == 0 || !useBCJ { + return nil + } + } + if (len(p.LeftConditions) != 0 && p.JoinType != LeftOuterJoin) || (len(p.RightConditions) != 0 && p.JoinType != RightOuterJoin) { return nil } + if prop.PartitionTp == property.BroadcastType { return nil } @@ -1768,9 +1808,23 @@ func (p *LogicalJoin) tryToGetMppHashJoin(prop *property.PhysicalProperty, useBC if p.children[0].statsInfo().Count() > p.children[1].statsInfo().Count() { preferredBuildIndex = 1 } - } else if p.JoinType == SemiJoin || p.JoinType == AntiSemiJoin || p.JoinType == LeftOuterJoin { + } else if p.JoinType == SemiJoin || p.JoinType == AntiSemiJoin { preferredBuildIndex = 1 } + if p.JoinType == LeftOuterJoin || p.JoinType == RightOuterJoin { + // TiFlash does not requires that the build side must be the inner table for outer join + // so we can choose the build side based on the row count, except that + // 1. it is a broadcast join(for broadcast join, it make sense to use the broadcast side as the build side) + // 2. or session variable MPPOuterJoinFixedBuildSide is set to true + // 3. or there are otherConditions for this join + if useBCJ || p.ctx.GetSessionVars().MPPOuterJoinFixedBuildSide || len(p.OtherConditions) > 0 { + if p.JoinType == LeftOuterJoin { + preferredBuildIndex = 1 + } + } else if p.children[0].statsInfo().Count() > p.children[1].statsInfo().Count() { + preferredBuildIndex = 1 + } + } baseJoin.InnerChildIdx = preferredBuildIndex childrenProps := make([]*property.PhysicalProperty, 2) if useBCJ { @@ -1814,7 +1868,8 @@ func (p *LogicalJoin) tryToGetMppHashJoin(prop *property.PhysicalProperty, useBC EqualConditions: p.EqualConditions, storeTp: kv.TiFlash, mppShuffleJoin: !useBCJ, - }.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), p.blockOffset, childrenProps...) + // Mpp Join has quite heavy cost. Even limit might not suspend it in time, so we dont scale the count. + }.Init(p.ctx, p.stats, p.blockOffset, childrenProps...) return []PhysicalPlan{join} } @@ -1938,10 +1993,10 @@ func (p *LogicalProjection) TryToGetChildProp(prop *property.PhysicalProperty) ( return newProp, true } -func (p *LogicalProjection) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { +func (p *LogicalProjection) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { newProp, ok := p.TryToGetChildProp(prop) if !ok { - return nil, true + return nil, true, nil } proj := PhysicalProjection{ Exprs: p.Exprs, @@ -1949,7 +2004,7 @@ func (p *LogicalProjection) exhaustPhysicalPlans(prop *property.PhysicalProperty AvoidColumnEvaluator: p.AvoidColumnEvaluator, }.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), p.blockOffset, newProp) proj.SetSchema(p.schema) - return []PhysicalPlan{proj}, true + return []PhysicalPlan{proj}, true, nil } func (lt *LogicalTopN) getPhysTopN(prop *property.PhysicalProperty) []PhysicalPlan { @@ -1965,7 +2020,7 @@ func (lt *LogicalTopN) getPhysTopN(prop *property.PhysicalProperty) []PhysicalPl if !lt.limitHints.preferLimitToCop { allTaskTypes = append(allTaskTypes, property.RootTaskType) } - if lt.ctx.GetSessionVars().AllowMPPExecution { + if lt.ctx.GetSessionVars().IsMPPAllowed() { allTaskTypes = append(allTaskTypes, property.MppTaskType) } ret := make([]PhysicalPlan, 0, len(allTaskTypes)) @@ -2007,6 +2062,7 @@ func (lt *LogicalTopN) getPhysLimits(prop *property.PhysicalProperty) []Physical Count: lt.Count, Offset: lt.Offset, }.Init(lt.ctx, lt.stats, lt.blockOffset, resultProp) + limit.SetSchema(lt.Schema()) ret = append(ret, limit) } return ret @@ -2026,11 +2082,11 @@ func MatchItems(p *property.PhysicalProperty, items []*util.ByItems) bool { return true } -func (lt *LogicalTopN) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { +func (lt *LogicalTopN) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { if MatchItems(prop, lt.ByItems) { - return append(lt.getPhysTopN(prop), lt.getPhysLimits(prop)...), true + return append(lt.getPhysTopN(prop), lt.getPhysLimits(prop)...), true, nil } - return nil, true + return nil, true, nil } // GetHashJoin is public for cascades planner. @@ -2038,9 +2094,9 @@ func (la *LogicalApply) GetHashJoin(prop *property.PhysicalProperty) *PhysicalHa return la.LogicalJoin.getHashJoin(prop, 1, false) } -func (la *LogicalApply) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { +func (la *LogicalApply) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { if !prop.AllColsFromSchema(la.children[0].Schema()) || prop.IsFlashProp() { // for convenient, we don't pass through any prop - return nil, true + return nil, true, nil } disableAggPushDownToCop(la.children[0]) join := la.GetHashJoin(prop) @@ -2073,7 +2129,7 @@ func (la *LogicalApply) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([ &property.PhysicalProperty{ExpectedCnt: math.MaxFloat64, SortItems: prop.SortItems}, &property.PhysicalProperty{ExpectedCnt: math.MaxFloat64}) apply.SetSchema(la.schema) - return []PhysicalPlan{apply}, true + return []PhysicalPlan{apply}, true, nil } func disableAggPushDownToCop(p LogicalPlan) { @@ -2085,16 +2141,16 @@ func disableAggPushDownToCop(p LogicalPlan) { } } -func (p *LogicalWindow) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { +func (p *LogicalWindow) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { if prop.IsFlashProp() { - return nil, true + return nil, true, nil } var byItems []property.SortItem byItems = append(byItems, p.PartitionBy...) byItems = append(byItems, p.OrderBy...) childProperty := &property.PhysicalProperty{ExpectedCnt: math.MaxFloat64, SortItems: byItems, CanAddEnforcer: true} if !prop.IsPrefix(childProperty) { - return nil, true + return nil, true, nil } window := PhysicalWindow{ WindowFuncDescs: p.WindowFuncDescs, @@ -2103,17 +2159,21 @@ func (p *LogicalWindow) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([ Frame: p.Frame, }.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), p.blockOffset, childProperty) window.SetSchema(p.Schema()) - return []PhysicalPlan{window}, true + return []PhysicalPlan{window}, true, nil } // exhaustPhysicalPlans is only for implementing interface. DataSource and Dual generate task in `findBestTask` directly. -func (p *baseLogicalPlan) exhaustPhysicalPlans(_ *property.PhysicalProperty) ([]PhysicalPlan, bool) { +func (p *baseLogicalPlan) exhaustPhysicalPlans(_ *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { panic("baseLogicalPlan.exhaustPhysicalPlans() should never be called.") } // canPushToCop checks if it can be pushed to some stores. For TiKV, it only checks datasource. // For TiFlash, it will check whether the operator is supported, but note that the check might be inaccrute. func (p *baseLogicalPlan) canPushToCop(storeTp kv.StoreType) bool { + return p.canPushToCopImpl(storeTp, false) +} + +func (p *baseLogicalPlan) canPushToCopImpl(storeTp kv.StoreType, considerDual bool) bool { ret := true for _, ch := range p.children { switch c := ch.(type) { @@ -2125,7 +2185,21 @@ func (p *baseLogicalPlan) canPushToCop(storeTp kv.StoreType) bool { } } ret = ret && validDs - case *LogicalAggregation, *LogicalProjection, *LogicalSelection, *LogicalJoin: + case *LogicalUnionAll: + if storeTp == kv.TiFlash { + ret = ret && c.canPushToCopImpl(storeTp, true) + } else { + return false + } + case *LogicalProjection: + if storeTp == kv.TiFlash { + ret = ret && c.canPushToCopImpl(storeTp, considerDual) + } else { + return false + } + case *LogicalTableDual: + return storeTp == kv.TiFlash && considerDual + case *LogicalAggregation, *LogicalSelection, *LogicalJoin: if storeTp == kv.TiFlash { ret = ret && c.canPushToCop(storeTp) } else { @@ -2355,7 +2429,7 @@ func (la *LogicalAggregation) getHashAggs(prop *property.PhysicalProperty) []Phy taskTypes = append(taskTypes, property.CopTiFlashLocalReadTaskType) } canPushDownToTiFlash := la.canPushToCop(kv.TiFlash) - canPushDownToMPP := la.ctx.GetSessionVars().AllowMPPExecution && la.checkCanPushDownToMPP() && canPushDownToTiFlash + canPushDownToMPP := la.ctx.GetSessionVars().IsMPPAllowed() && la.checkCanPushDownToMPP() && canPushDownToTiFlash if la.HasDistinct() { // TODO: remove after the cost estimation of distinct pushdown is implemented. if !la.ctx.GetSessionVars().AllowDistinctAggPushDown { @@ -2406,7 +2480,7 @@ func (la *LogicalAggregation) ResetHintIfConflicted() (preferHash bool, preferSt return } -func (la *LogicalAggregation) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { +func (la *LogicalAggregation) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { if la.aggHints.preferAggToCop { if !la.canPushToCop(kv.TiKV) { errMsg := "Optimizer Hint AGG_TO_COP is inapplicable" @@ -2420,12 +2494,12 @@ func (la *LogicalAggregation) exhaustPhysicalPlans(prop *property.PhysicalProper hashAggs := la.getHashAggs(prop) if hashAggs != nil && preferHash { - return hashAggs, true + return hashAggs, true, nil } streamAggs := la.getStreamAggs(prop) if streamAggs != nil && preferStream { - return streamAggs, true + return streamAggs, true, nil } aggs := append(hashAggs, streamAggs...) @@ -2436,20 +2510,20 @@ func (la *LogicalAggregation) exhaustPhysicalPlans(prop *property.PhysicalProper la.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) } - return aggs, !(preferStream || preferHash) + return aggs, !(preferStream || preferHash), nil } -func (p *LogicalSelection) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { +func (p *LogicalSelection) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { childProp := prop.CloneEssentialFields() sel := PhysicalSelection{ Conditions: p.Conditions, }.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), p.blockOffset, childProp) - return []PhysicalPlan{sel}, true + return []PhysicalPlan{sel}, true, nil } -func (p *LogicalLimit) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { +func (p *LogicalLimit) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { if !prop.IsEmpty() { - return nil, true + return nil, true, nil } if p.limitHints.preferLimitToCop { @@ -2465,6 +2539,9 @@ func (p *LogicalLimit) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([] if !p.limitHints.preferLimitToCop { allTaskTypes = append(allTaskTypes, property.RootTaskType) } + if p.canPushToCop(kv.TiFlash) && p.ctx.GetSessionVars().IsMPPAllowed() { + allTaskTypes = append(allTaskTypes, property.MppTaskType) + } ret := make([]PhysicalPlan, 0, len(allTaskTypes)) for _, tp := range allTaskTypes { resultProp := &property.PhysicalProperty{TaskTp: tp, ExpectedCnt: float64(p.Count + p.Offset)} @@ -2475,12 +2552,12 @@ func (p *LogicalLimit) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([] limit.SetSchema(p.Schema()) ret = append(ret, limit) } - return ret, true + return ret, true, nil } -func (p *LogicalLock) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { +func (p *LogicalLock) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { if prop.IsFlashProp() { - return nil, true + return nil, true, nil } childProp := prop.CloneEssentialFields() lock := PhysicalLock{ @@ -2488,29 +2565,58 @@ func (p *LogicalLock) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]P TblID2Handle: p.tblID2Handle, PartitionedTable: p.partitionedTable, }.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), childProp) - return []PhysicalPlan{lock}, true + return []PhysicalPlan{lock}, true, nil } -func (p *LogicalUnionAll) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { +func (p *LogicalUnionAll) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { // TODO: UnionAll can not pass any order, but we can change it to sort merge to keep order. - if !prop.IsEmpty() || prop.IsFlashProp() { - return nil, true + if !prop.IsEmpty() || (prop.IsFlashProp() && prop.TaskTp != property.MppTaskType) { + return nil, true, nil + } + // TODO: UnionAll can pass partition info, but for briefness, we prevent it from pushing down. + if prop.TaskTp == property.MppTaskType && prop.PartitionTp != property.AnyType { + return nil, true, nil } + canUseMpp := p.ctx.GetSessionVars().IsMPPAllowed() && p.canPushToCopImpl(kv.TiFlash, true) chReqProps := make([]*property.PhysicalProperty, 0, len(p.children)) for range p.children { - chReqProps = append(chReqProps, &property.PhysicalProperty{ExpectedCnt: prop.ExpectedCnt}) + if canUseMpp && prop.TaskTp == property.MppTaskType { + chReqProps = append(chReqProps, &property.PhysicalProperty{ + ExpectedCnt: prop.ExpectedCnt, + TaskTp: property.MppTaskType, + }) + } else { + chReqProps = append(chReqProps, &property.PhysicalProperty{ExpectedCnt: prop.ExpectedCnt}) + } } - ua := PhysicalUnionAll{}.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), p.blockOffset, chReqProps...) + ua := PhysicalUnionAll{ + mpp: canUseMpp && prop.TaskTp == property.MppTaskType, + }.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), p.blockOffset, chReqProps...) ua.SetSchema(p.Schema()) - return []PhysicalPlan{ua}, true + if canUseMpp && prop.TaskTp == property.RootTaskType { + chReqProps = make([]*property.PhysicalProperty, 0, len(p.children)) + for range p.children { + chReqProps = append(chReqProps, &property.PhysicalProperty{ + ExpectedCnt: prop.ExpectedCnt, + TaskTp: property.MppTaskType, + }) + } + mppUA := PhysicalUnionAll{mpp: true}.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), p.blockOffset, chReqProps...) + mppUA.SetSchema(p.Schema()) + return []PhysicalPlan{ua, mppUA}, true, nil + } + return []PhysicalPlan{ua}, true, nil } -func (p *LogicalPartitionUnionAll) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { - uas, flagHint := p.LogicalUnionAll.exhaustPhysicalPlans(prop) +func (p *LogicalPartitionUnionAll) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { + uas, flagHint, err := p.LogicalUnionAll.exhaustPhysicalPlans(prop) + if err != nil { + return nil, false, err + } for _, ua := range uas { ua.(*PhysicalUnionAll).tp = plancodec.TypePartitionUnion } - return uas, flagHint + return uas, flagHint, nil } func (ls *LogicalSort) getPhysicalSort(prop *property.PhysicalProperty) *PhysicalSort { @@ -2529,7 +2635,7 @@ func (ls *LogicalSort) getNominalSort(reqProp *property.PhysicalProperty) *Nomin return ps } -func (ls *LogicalSort) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { +func (ls *LogicalSort) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { if MatchItems(prop, ls.ByItems) { ret := make([]PhysicalPlan, 0, 2) ret = append(ret, ls.getPhysicalSort(prop)) @@ -2537,15 +2643,15 @@ func (ls *LogicalSort) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([] if ns != nil { ret = append(ret, ns) } - return ret, true + return ret, true, nil } - return nil, true + return nil, true, nil } -func (p *LogicalMaxOneRow) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { +func (p *LogicalMaxOneRow) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { if !prop.IsEmpty() || prop.IsFlashProp() { - return nil, true + return nil, true, nil } mor := PhysicalMaxOneRow{}.Init(p.ctx, p.stats, p.blockOffset, &property.PhysicalProperty{ExpectedCnt: 2}) - return []PhysicalPlan{mor}, true + return []PhysicalPlan{mor}, true, nil } diff --git a/planner/core/explain.go b/planner/core/explain.go index e3e7e4e06d0b3..a8f5609b43e9a 100644 --- a/planner/core/explain.go +++ b/planner/core/explain.go @@ -306,7 +306,7 @@ func (p *PhysicalTableReader) accessObject(sctx sessionctx.Context) string { return "" } - is := infoschema.GetInfoSchema(sctx) + is := sctx.GetInfoSchema().(infoschema.InfoSchema) tmp, ok := is.TableByID(ts.Table.ID) if !ok { return "partition table not found" + strconv.FormatInt(ts.Table.ID, 10) @@ -366,7 +366,7 @@ func (p *PhysicalIndexReader) accessObject(sctx sessionctx.Context) string { } var buffer bytes.Buffer - is := infoschema.GetInfoSchema(sctx) + is := sctx.GetInfoSchema().(infoschema.InfoSchema) tmp, ok := is.TableByID(ts.Table.ID) if !ok { fmt.Fprintf(&buffer, "partition table not found: %d", ts.Table.ID) @@ -394,7 +394,7 @@ func (p *PhysicalIndexLookUpReader) accessObject(sctx sessionctx.Context) string } var buffer bytes.Buffer - is := infoschema.GetInfoSchema(sctx) + is := sctx.GetInfoSchema().(infoschema.InfoSchema) tmp, ok := is.TableByID(ts.Table.ID) if !ok { fmt.Fprintf(&buffer, "partition table not found: %d", ts.Table.ID) @@ -417,7 +417,7 @@ func (p *PhysicalIndexMergeReader) accessObject(sctx sessionctx.Context) string return "" } - is := infoschema.GetInfoSchema(sctx) + is := sctx.GetInfoSchema().(infoschema.InfoSchema) tmp, ok := is.TableByID(ts.Table.ID) if !ok { return "partition table not found" + strconv.FormatInt(ts.Table.ID, 10) @@ -842,16 +842,16 @@ func (p *LogicalTableDual) ExplainInfo() string { } // ExplainInfo implements Plan interface. -func (p *DataSource) ExplainInfo() string { +func (ds *DataSource) ExplainInfo() string { buffer := bytes.NewBufferString("") - tblName := p.tableInfo.Name.O - if p.TableAsName != nil && p.TableAsName.O != "" { - tblName = p.TableAsName.O + tblName := ds.tableInfo.Name.O + if ds.TableAsName != nil && ds.TableAsName.O != "" { + tblName = ds.TableAsName.O } fmt.Fprintf(buffer, "table:%s", tblName) - if p.isPartition { - if pi := p.tableInfo.GetPartitionInfo(); pi != nil { - partitionName := pi.GetNameByID(p.physicalTableID) + if ds.isPartition { + if pi := ds.tableInfo.GetPartitionInfo(); pi != nil { + partitionName := pi.GetNameByID(ds.physicalTableID) fmt.Fprintf(buffer, ", partition:%s", partitionName) } } diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index 5d6a11ad982a9..4259f690c6985 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -42,6 +42,7 @@ import ( "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/collate" "github.com/pingcap/tidb/util/hint" + "github.com/pingcap/tidb/util/sem" "github.com/pingcap/tidb/util/stringutil" ) @@ -63,8 +64,9 @@ func evalAstExpr(sctx sessionctx.Context, expr ast.ExprNode) (types.Datum, error // rewriteAstExpr rewrites ast expression directly. func rewriteAstExpr(sctx sessionctx.Context, expr ast.ExprNode, schema *expression.Schema, names types.NameSlice) (expression.Expression, error) { var is infoschema.InfoSchema - if sctx.GetSessionVars().TxnCtx.InfoSchema != nil { - is = sctx.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema) + // in tests, it may be null + if s, ok := sctx.GetInfoSchema().(infoschema.InfoSchema); ok { + is = s } b, savedBlockNames := NewPlanBuilder(sctx, is, &hint.BlockHintProcessor{}) fakePlan := LogicalTableDual{}.Init(sctx, 0) @@ -168,6 +170,7 @@ func (b *PlanBuilder) getExpressionRewriter(ctx context.Context, p LogicalPlan) rewriter.ctxStack = rewriter.ctxStack[:0] rewriter.ctxNameStk = rewriter.ctxNameStk[:0] rewriter.ctx = ctx + rewriter.err = nil return } @@ -495,6 +498,8 @@ func (er *expressionRewriter) buildSemiApplyFromEqualSubq(np LogicalPlan, l, r e } func (er *expressionRewriter) handleCompareSubquery(ctx context.Context, v *ast.CompareSubqueryExpr) (ast.Node, bool) { + ci := er.b.prepareCTECheckForSubQuery() + defer resetCTECheckForSubQuery(ci) v.L.Accept(er) if er.err != nil { return v, true @@ -534,6 +539,15 @@ func (er *expressionRewriter) handleCompareSubquery(ctx context.Context, v *ast. return v, true } } + + // Lexpr cannot compare with rexpr by different collate + opString := new(strings.Builder) + v.Op.Format(opString) + er.err = expression.CheckIllegalMixCollation(opString.String(), []expression.Expression{lexpr, rexpr}, 0) + if er.err != nil { + return v, true + } + switch v.Op { // Only EQ, NE and NullEQ can be composed with and. case opcode.EQ, opcode.NE, opcode.NullEQ: @@ -769,6 +783,8 @@ func (er *expressionRewriter) handleEQAll(lexpr, rexpr expression.Expression, np } func (er *expressionRewriter) handleExistSubquery(ctx context.Context, v *ast.ExistsSubqueryExpr) (ast.Node, bool) { + ci := er.b.prepareCTECheckForSubQuery() + defer resetCTECheckForSubQuery(ci) subq, ok := v.Sel.(*ast.SubqueryExpr) if !ok { er.err = errors.Errorf("Unknown exists type %T.", v.Sel) @@ -834,6 +850,8 @@ out: } func (er *expressionRewriter) handleInSubquery(ctx context.Context, v *ast.PatternInExpr) (ast.Node, bool) { + ci := er.b.prepareCTECheckForSubQuery() + defer resetCTECheckForSubQuery(ci) asScalar := er.asScalar er.asScalar = true v.Expr.Accept(er) @@ -943,6 +961,8 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, v *ast.Patte } func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, v *ast.SubqueryExpr) (ast.Node, bool) { + ci := er.b.prepareCTECheckForSubQuery() + defer resetCTECheckForSubQuery(ci) np, err := er.buildSubquery(ctx, v) if err != nil { er.err = err @@ -1214,25 +1234,33 @@ func (er *expressionRewriter) rewriteVariable(v *ast.VariableExpr) { er.ctxStackAppend(f, types.EmptyName) return } - var val string - var err error - if v.ExplicitScope { - err = variable.ValidateGetSystemVar(name, v.IsGlobal) - if err != nil { - er.err = err - return - } - } sysVar := variable.GetSysVar(name) if sysVar == nil { er.err = variable.ErrUnknownSystemVar.GenWithStackByArgs(name) return } - // Variable is @@gobal.variable_name or variable is only global scope variable. - if v.IsGlobal || sysVar.Scope == variable.ScopeGlobal { + if sem.IsEnabled() && sem.IsInvisibleSysVar(sysVar.Name) { + err := ErrSpecificAccessDenied.GenWithStackByArgs("RESTRICTED_VARIABLES_ADMIN") + er.b.visitInfo = appendDynamicVisitInfo(er.b.visitInfo, "RESTRICTED_VARIABLES_ADMIN", false, err) + } + if v.ExplicitScope && !sysVar.HasNoneScope() { + if v.IsGlobal && !sysVar.HasGlobalScope() { + er.err = variable.ErrIncorrectScope.GenWithStackByArgs(name, "GLOBAL") + return + } + if !v.IsGlobal && !sysVar.HasSessionScope() { + er.err = variable.ErrIncorrectScope.GenWithStackByArgs(name, "SESSION") + return + } + } + var val string + var err error + if sysVar.HasNoneScope() { + val = sysVar.Value + } else if v.IsGlobal { val, err = variable.GetGlobalSystemVar(sessionVars, name) } else { - val, err = variable.GetSessionSystemVar(sessionVars, name) + val, err = variable.GetSessionOrGlobalSystemVar(sessionVars, name) } if err != nil { er.err = err @@ -1240,8 +1268,8 @@ func (er *expressionRewriter) rewriteVariable(v *ast.VariableExpr) { } nativeVal, nativeType, nativeFlag := sysVar.GetNativeValType(val) e := expression.DatumToConstant(nativeVal, nativeType, nativeFlag) - e.GetType().Charset, _ = er.sctx.GetSessionVars().GetSystemVar(variable.CharacterSetConnection) - e.GetType().Collate, _ = er.sctx.GetSessionVars().GetSystemVar(variable.CollationConnection) + e.GetType().Charset, _ = sessionVars.GetSystemVar(variable.CharacterSetConnection) + e.GetType().Collate, _ = sessionVars.GetSystemVar(variable.CollationConnection) er.ctxStackAppend(e, types.EmptyName) } @@ -1637,9 +1665,13 @@ func (er *expressionRewriter) rewriteFuncCall(v *ast.FuncCallExpr) bool { stackLen := len(er.ctxStack) arg1 := er.ctxStack[stackLen-2] col, isColumn := arg1.(*expression.Column) + var isEnumSet bool + if arg1.GetType().Tp == mysql.TypeEnum || arg1.GetType().Tp == mysql.TypeSet { + isEnumSet = true + } // if expr1 is a column and column has not null flag, then we can eliminate ifnull on // this column. - if isColumn && mysql.HasNotNullFlag(col.RetType.Flag) { + if isColumn && !isEnumSet && mysql.HasNotNullFlag(col.RetType.Flag) { name := er.ctxNameStk[stackLen-2] newCol := col.Clone().(*expression.Column) er.ctxStackPop(len(v.Args)) diff --git a/planner/core/expression_rewriter_test.go b/planner/core/expression_rewriter_test.go index 66bb860a52f0b..fd23efe8f3dd3 100644 --- a/planner/core/expression_rewriter_test.go +++ b/planner/core/expression_rewriter_test.go @@ -43,6 +43,16 @@ func (s *testExpressionRewriterSuite) TestIfNullEliminateColName(c *C) { c.Assert(err, IsNil) fields := rs.Fields() c.Assert(fields[0].Column.Name.L, Equals, "ifnull(a,b)") + + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(e int not null, b int)") + tk.MustExec("insert into t values(1, 1)") + tk.MustExec("create table t1(e int not null, b int)") + tk.MustExec("insert into t1 values(1, 1)") + rows := tk.MustQuery("select b from t where ifnull(e, b)") + rows.Check(testkit.Rows("1")) + rows = tk.MustQuery("select b from t1 where ifnull(e, b)") + rows.Check(testkit.Rows("1")) } func (s *testExpressionRewriterSuite) TestBinaryOpFunction(c *C) { @@ -395,3 +405,21 @@ func (s *testExpressionRewriterSuite) TestIssue22818(c *C) { tk.MustQuery("select * from t where a between \"23:22:22\" and \"23:22:22\"").Check( testkit.Rows("23:22:22")) } + +func (s *testExpressionRewriterSuite) TestIssue24705(c *C) { + defer testleak.AfterTest(c)() + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + tk := testkit.NewTestKit(c, store) + defer func() { + dom.Close() + store.Close() + }() + + tk.MustExec("use test;") + tk.MustExec("drop table if exists t1,t2;") + tk.MustExec("create table t1 (c_int int, c_str varchar(40) character set utf8 collate utf8_general_ci);") + tk.MustExec("create table t2 (c_int int, c_str varchar(40) character set utf8 collate utf8_unicode_ci);") + err = tk.ExecToErr("select * from t1 where c_str < any (select c_str from t2 where c_int between 6 and 9);") + c.Assert(err.Error(), Equals, "[expression:1267]Illegal mix of collations (utf8_general_ci,IMPLICIT) and (utf8_unicode_ci,IMPLICIT) for operation '<'") +} diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 8e8086b8061de..71d85a7d8b503 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -14,6 +14,7 @@ package core import ( + "context" "math" "github.com/pingcap/errors" @@ -145,7 +146,7 @@ func (p *LogicalTableDual) findBestTask(prop *property.PhysicalProperty, planCou }.Init(p.ctx, p.stats, p.blockOffset) dual.SetSchema(p.schema) planCounter.Dec(1) - return &rootTask{p: dual}, 1, nil + return &rootTask{p: dual, isEmpty: p.RowCount == 0}, 1, nil } func (p *LogicalShow) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (task, int64, error) { @@ -211,7 +212,7 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPl childTasks = childTasks[:0] // The curCntPlan records the number of possible plans for pp curCntPlan = 1 - TimeStampNow := p.GetlogicalTS4TaskMap() + TimeStampNow := p.GetLogicalTS4TaskMap() savedPlanID := p.ctx.GetSessionVars().PlanID for j, child := range p.children { childTask, cnt, err := child.findBestTask(pp.GetChildReqProps(j), &PlanCounterDisabled) @@ -311,7 +312,10 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun var hintWorksWithProp bool // Maybe the plan can satisfy the required property, // so we try to get the task without the enforced sort first. - plansFitsProp, hintWorksWithProp = p.self.exhaustPhysicalPlans(newProp) + plansFitsProp, hintWorksWithProp, err = p.self.exhaustPhysicalPlans(newProp) + if err != nil { + return nil, 0, err + } if !hintWorksWithProp && !newProp.IsEmpty() { // If there is a hint in the plan and the hint cannot satisfy the property, // we enforce this property and try to generate the PhysicalPlan again to @@ -327,7 +331,10 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun newProp.PartitionCols = nil newProp.PartitionTp = property.AnyType var hintCanWork bool - plansNeedEnforce, hintCanWork = p.self.exhaustPhysicalPlans(newProp) + plansNeedEnforce, hintCanWork, err = p.self.exhaustPhysicalPlans(newProp) + if err != nil { + return nil, 0, err + } if hintCanWork && !hintWorksWithProp { // If the hint can work with the empty property, but cannot work with // the required property, we give up `plansFitProp` to make sure the hint @@ -873,10 +880,10 @@ func (ds *DataSource) convertToPartialIndexScan(prop *property.PhysicalProperty, } indexPlan := PhysicalSelection{Conditions: indexConds}.Init(is.ctx, stats, ds.blockOffset) indexPlan.SetChildren(is) - partialCost += rowCount * rowSize * sessVars.NetworkFactor + partialCost += rowCount * rowSize * sessVars.GetNetworkFactor(ds.tableInfo) return indexPlan, partialCost } - partialCost += rowCount * rowSize * sessVars.NetworkFactor + partialCost += rowCount * rowSize * sessVars.GetNetworkFactor(ds.tableInfo) indexPlan = is return indexPlan, partialCost } @@ -896,10 +903,10 @@ func (ds *DataSource) convertToPartialTableScan(prop *property.PhysicalProperty, tablePlan = PhysicalSelection{Conditions: ts.filterCondition}.Init(ts.ctx, ts.stats.ScaleByExpectCnt(selectivity*rowCount), ds.blockOffset) tablePlan.SetChildren(ts) partialCost += rowCount * sessVars.CopCPUFactor - partialCost += selectivity * rowCount * rowSize * sessVars.NetworkFactor + partialCost += selectivity * rowCount * rowSize * sessVars.GetNetworkFactor(ds.tableInfo) return tablePlan, partialCost } - partialCost += rowCount * rowSize * sessVars.NetworkFactor + partialCost += rowCount * rowSize * sessVars.GetNetworkFactor(ds.tableInfo) tablePlan = ts return tablePlan, partialCost } @@ -964,7 +971,7 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, } } rowSize := ds.TblColHists.GetTableAvgRowSize(ds.ctx, ds.TblCols, ts.StoreType, true) - partialCost += totalRowCount * rowSize * sessVars.ScanFactor + partialCost += totalRowCount * rowSize * sessVars.GetScanFactor(ds.tableInfo) ts.stats = ds.tableStats.ScaleByExpectCnt(totalRowCount) if ds.statisticTable.Pseudo { ts.stats.StatsVersion = statistics.PseudoVersion @@ -1298,7 +1305,7 @@ func getMostCorrCol4Handle(exprs []expression.Expression, histColl *statistics.T } // getColumnRangeCounts estimates row count for each range respectively. -func getColumnRangeCounts(sc *stmtctx.StatementContext, colID int64, ranges []*ranger.Range, histColl *statistics.Table, idxID int64) ([]float64, bool) { +func getColumnRangeCounts(sc *stmtctx.StatementContext, colID int64, ranges []*ranger.Range, histColl *statistics.HistColl, idxID int64) ([]float64, bool) { var err error var count float64 rangeCounts := make([]float64, len(ranges)) @@ -1376,7 +1383,7 @@ func (ds *DataSource) crossEstimateRowCount(path *util.AccessPath, conds []expre if col == nil || len(path.AccessConds) > 0 { return 0, false, corr } - colInfoID, colID := col.ID, col.UniqueID + colID := col.UniqueID if corr < 0 { desc = !desc } @@ -1393,7 +1400,7 @@ func (ds *DataSource) crossEstimateRowCount(path *util.AccessPath, conds []expre if !idxExists { idxID = -1 } - rangeCounts, ok := getColumnRangeCounts(sc, colInfoID, ranges, ds.statisticTable, idxID) + rangeCounts, ok := getColumnRangeCounts(sc, colID, ranges, ds.tableStats.HistColl, idxID) if !ok { return 0, false, corr } @@ -1403,9 +1410,9 @@ func (ds *DataSource) crossEstimateRowCount(path *util.AccessPath, conds []expre } var rangeCount float64 if idxExists { - rangeCount, err = ds.statisticTable.GetRowCountByIndexRanges(sc, idxID, convertedRanges) + rangeCount, err = ds.tableStats.HistColl.GetRowCountByIndexRanges(sc, idxID, convertedRanges) } else { - rangeCount, err = ds.statisticTable.GetRowCountByColumnRanges(sc, colInfoID, convertedRanges) + rangeCount, err = ds.tableStats.HistColl.GetRowCountByColumnRanges(sc, colID, convertedRanges) } if err != nil { return 0, false, corr @@ -1716,6 +1723,7 @@ func (ds *DataSource) convertToBatchPointGet(prop *property.PhysicalProperty, ca Columns: ds.Columns, SinglePart: ds.isPartition, PartTblID: ds.physicalTableID, + PartitionExpr: getPartitionExpr(ds.ctx, ds.TableInfo()), }.Init(ds.ctx, ds.tableStats.ScaleByExpectCnt(accessCnt), ds.schema.Clone(), ds.names, ds.blockOffset) if batchPointGetPlan.KeepOrder { batchPointGetPlan.Desc = prop.SortItems[0].Desc @@ -1741,7 +1749,7 @@ func (ds *DataSource) convertToBatchPointGet(prop *property.PhysicalProperty, ca batchPointGetPlan.IndexInfo = candidate.path.Index batchPointGetPlan.IdxCols = candidate.path.IdxCols batchPointGetPlan.IdxColLens = candidate.path.IdxColLens - batchPointGetPlan.PartitionColPos = getPartitionColumnPos(candidate.path.Index, hashPartColName) + batchPointGetPlan.PartitionColPos = getHashPartitionColumnPos(candidate.path.Index, hashPartColName) for _, ran := range candidate.path.Ranges { batchPointGetPlan.IndexValues = append(batchPointGetPlan.IndexValues, ran.LowVal) } @@ -1865,22 +1873,22 @@ func (ds *DataSource) getOriginalPhysicalTableScan(prop *property.PhysicalProper rowSize = ds.TblColHists.GetTableAvgRowSize(ds.ctx, ts.Schema().Columns, ts.StoreType, ds.handleCols != nil) } sessVars := ds.ctx.GetSessionVars() - cost := rowCount * rowSize * sessVars.ScanFactor + cost := rowCount * rowSize * sessVars.GetScanFactor(ds.tableInfo) if ts.IsGlobalRead { - cost += rowCount * sessVars.NetworkFactor * rowSize + cost += rowCount * sessVars.GetNetworkFactor(ds.tableInfo) * rowSize } if isMatchProp { ts.Desc = prop.SortItems[0].Desc if prop.SortItems[0].Desc && prop.ExpectedCnt >= smallScanThreshold { - cost = rowCount * rowSize * sessVars.DescScanFactor + cost = rowCount * rowSize * sessVars.GetDescScanFactor(ds.tableInfo) } ts.KeepOrder = true } switch ts.StoreType { case kv.TiKV: - cost += float64(len(ts.Ranges)) * sessVars.SeekFactor + cost += float64(len(ts.Ranges)) * sessVars.GetSeekFactor(ds.tableInfo) case kv.TiFlash: - cost += float64(len(ts.Ranges)) * float64(len(ts.Columns)) * sessVars.SeekFactor + cost += float64(len(ts.Ranges)) * float64(len(ts.Columns)) * sessVars.GetSeekFactor(ds.tableInfo) } return ts, cost, rowCount } @@ -1920,15 +1928,54 @@ func (ds *DataSource) getOriginalPhysicalIndexScan(prop *property.PhysicalProper is.stats = ds.tableStats.ScaleByExpectCnt(rowCount) rowSize := is.indexScanRowSize(idx, ds, true) sessVars := ds.ctx.GetSessionVars() - cost := rowCount * rowSize * sessVars.ScanFactor + cost := rowCount * rowSize * sessVars.GetScanFactor(ds.tableInfo) if isMatchProp { is.Desc = prop.SortItems[0].Desc if prop.SortItems[0].Desc && prop.ExpectedCnt >= smallScanThreshold { - cost = rowCount * rowSize * sessVars.DescScanFactor + cost = rowCount * rowSize * sessVars.GetDescScanFactor(ds.tableInfo) } is.KeepOrder = true } - cost += float64(len(is.Ranges)) * sessVars.SeekFactor + cost += float64(len(is.Ranges)) * sessVars.GetSeekFactor(ds.tableInfo) is.cost = cost return is, cost, rowCount } + +func (p *LogicalCTE) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (t task, cntPlan int64, err error) { + if !prop.IsEmpty() { + return invalidTask, 1, nil + } + if p.cte.cteTask != nil { + // Already built it. + return p.cte.cteTask, 1, nil + } + sp, _, err := DoOptimize(context.TODO(), p.ctx, p.cte.optFlag, p.cte.seedPartLogicalPlan) + if err != nil { + return nil, 1, err + } + + var rp PhysicalPlan + if p.cte.recursivePartLogicalPlan != nil { + rp, _, err = DoOptimize(context.TODO(), p.ctx, p.cte.optFlag, p.cte.recursivePartLogicalPlan) + if err != nil { + return nil, 1, err + } + } + + pcte := PhysicalCTE{SeedPlan: sp, RecurPlan: rp, CTE: p.cte, cteAsName: p.cteAsName}.Init(p.ctx, p.stats) + pcte.SetSchema(p.schema) + t = &rootTask{pcte, sp.statsInfo().RowCount, false} + p.cte.cteTask = t + return t, 1, nil +} + +func (p *LogicalCTETable) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (t task, cntPlan int64, err error) { + if !prop.IsEmpty() { + return nil, 1, nil + } + + pcteTable := PhysicalCTETable{IDForStorage: p.idForStorage}.Init(p.ctx, p.stats) + pcteTable.SetSchema(p.schema) + t = &rootTask{p: pcteTable} + return t, 1, nil +} diff --git a/planner/core/find_best_task_test.go b/planner/core/find_best_task_test.go index dc83476d1cfac..c744282db0c45 100644 --- a/planner/core/find_best_task_test.go +++ b/planner/core/find_best_task_test.go @@ -96,14 +96,14 @@ func (p *mockLogicalPlan4Test) getPhysicalPlan2(prop *property.PhysicalProperty) return physicalPlan2 } -func (p *mockLogicalPlan4Test) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { +func (p *mockLogicalPlan4Test) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool, error) { plan1 := make([]PhysicalPlan, 0, 1) plan2 := make([]PhysicalPlan, 0, 1) if prop.IsEmpty() && p.canGeneratePlan2 { // Generate PhysicalPlan2 when the property is empty. plan2 = append(plan2, p.getPhysicalPlan2(prop)) if p.hasHintForPlan2 { - return plan2, true + return plan2, true, nil } } if all, _ := prop.AllSameOrder(); all { @@ -115,9 +115,9 @@ func (p *mockLogicalPlan4Test) exhaustPhysicalPlans(prop *property.PhysicalPrope if prop.IsEmpty() { p.ctx.GetSessionVars().StmtCtx.AppendWarning(fmt.Errorf("the hint is inapplicable for plan2")) } - return plan1, false + return plan1, false, nil } - return append(plan1, plan2...), true + return append(plan1, plan2...), true, nil } type mockPhysicalPlan4Test struct { diff --git a/planner/core/fragment.go b/planner/core/fragment.go index d0ab808c742e7..7315da176e6b8 100644 --- a/planner/core/fragment.go +++ b/planner/core/fragment.go @@ -38,32 +38,49 @@ type Fragment struct { // following fields are filled after scheduling. ExchangeSender *PhysicalExchangeSender // data exporter + + IsRoot bool +} + +type tasksAndFrags struct { + tasks []*kv.MPPTask + frags []*Fragment } type mppTaskGenerator struct { ctx sessionctx.Context startTS uint64 is infoschema.InfoSchema + frags []*Fragment + cache map[int]tasksAndFrags } // GenerateRootMPPTasks generate all mpp tasks and return root ones. -func GenerateRootMPPTasks(ctx sessionctx.Context, startTs uint64, sender *PhysicalExchangeSender, is infoschema.InfoSchema) ([]*kv.MPPTask, error) { - g := &mppTaskGenerator{ctx: ctx, startTS: startTs, is: is} +func GenerateRootMPPTasks(ctx sessionctx.Context, startTs uint64, sender *PhysicalExchangeSender, is infoschema.InfoSchema) ([]*Fragment, error) { + g := &mppTaskGenerator{ + ctx: ctx, + startTS: startTs, + is: is, + cache: make(map[int]tasksAndFrags), + } return g.generateMPPTasks(sender) } -func (e *mppTaskGenerator) generateMPPTasks(s *PhysicalExchangeSender) ([]*kv.MPPTask, error) { +func (e *mppTaskGenerator) generateMPPTasks(s *PhysicalExchangeSender) ([]*Fragment, error) { logutil.BgLogger().Info("Mpp will generate tasks", zap.String("plan", ToString(s))) tidbTask := &kv.MPPTask{ StartTs: e.startTS, ID: -1, } - rootTasks, err := e.generateMPPTasksForFragment(s) + _, frags, err := e.generateMPPTasksForExchangeSender(s) if err != nil { return nil, errors.Trace(err) } - s.TargetTasks = []*kv.MPPTask{tidbTask} - return rootTasks, nil + for _, frag := range frags { + frag.ExchangeSender.TargetTasks = []*kv.MPPTask{tidbTask} + frag.IsRoot = true + } + return e.frags, nil } type mppAddr struct { @@ -105,6 +122,8 @@ func (f *Fragment) init(p PhysicalPlan) error { f.TableScan = x case *PhysicalExchangeReceiver: f.ExchangeReceivers = append(f.ExchangeReceivers, x) + case *PhysicalUnionAll: + return errors.New("unexpected union all detected") default: for _, ch := range p.Children() { if err := f.init(ch); err != nil { @@ -115,20 +134,107 @@ func (f *Fragment) init(p PhysicalPlan) error { return nil } -func newFragment(s *PhysicalExchangeSender) (*Fragment, error) { - f := &Fragment{ExchangeSender: s} - s.Fragment = f - err := f.init(s) - return f, errors.Trace(err) +// We would remove all the union-all operators by 'untwist'ing and copying the plans above union-all. +// This will make every route from root (ExchangeSender) to leaf nodes (ExchangeReceiver and TableScan) +// a new ioslated tree (and also a fragment) without union all. These trees (fragments then tasks) will +// finally be gathered to TiDB or be exchanged to upper tasks again. +// For instance, given a plan "select c1 from t union all select c1 from s" +// after untwist, there will be two plans in `forest` slice: +// - ExchangeSender -> Projection (c1) -> TableScan(t) +// - ExchangeSender -> Projection (c2) -> TableScan(s) +func untwistPlanAndRemoveUnionAll(stack []PhysicalPlan, forest *[]*PhysicalExchangeSender) error { + cur := stack[len(stack)-1] + switch x := cur.(type) { + case *PhysicalTableScan, *PhysicalExchangeReceiver: // This should be the leave node. + p, err := stack[0].Clone() + if err != nil { + return errors.Trace(err) + } + *forest = append(*forest, p.(*PhysicalExchangeSender)) + for i := 1; i < len(stack); i++ { + if _, ok := stack[i].(*PhysicalUnionAll); ok { + continue + } + ch, err := stack[i].Clone() + if err != nil { + return errors.Trace(err) + } + if join, ok := p.(*PhysicalHashJoin); ok { + join.SetChild(1-join.InnerChildIdx, ch) + } else { + p.SetChildren(ch) + } + p = ch + } + case *PhysicalHashJoin: + stack = append(stack, x.children[1-x.InnerChildIdx]) + err := untwistPlanAndRemoveUnionAll(stack, forest) + stack = stack[:len(stack)-1] + return errors.Trace(err) + case *PhysicalUnionAll: + for _, ch := range x.children { + stack = append(stack, ch) + err := untwistPlanAndRemoveUnionAll(stack, forest) + stack = stack[:len(stack)-1] + if err != nil { + return errors.Trace(err) + } + } + default: + if len(cur.Children()) != 1 { + return errors.Trace(errors.New("unexpected plan " + cur.ExplainID().String())) + } + ch := cur.Children()[0] + stack = append(stack, ch) + err := untwistPlanAndRemoveUnionAll(stack, forest) + stack = stack[:len(stack)-1] + return errors.Trace(err) + } + return nil } -func (e *mppTaskGenerator) generateMPPTasksForFragment(s *PhysicalExchangeSender) (tasks []*kv.MPPTask, err error) { - f, err := newFragment(s) +func buildFragments(s *PhysicalExchangeSender) ([]*Fragment, error) { + forest := make([]*PhysicalExchangeSender, 0, 1) + err := untwistPlanAndRemoveUnionAll([]PhysicalPlan{s}, &forest) if err != nil { return nil, errors.Trace(err) } + fragments := make([]*Fragment, 0, len(forest)) + for _, s := range forest { + f := &Fragment{ExchangeSender: s} + err = f.init(s) + if err != nil { + return nil, errors.Trace(err) + } + fragments = append(fragments, f) + } + return fragments, nil +} + +func (e *mppTaskGenerator) generateMPPTasksForExchangeSender(s *PhysicalExchangeSender) ([]*kv.MPPTask, []*Fragment, error) { + if cached, ok := e.cache[s.ID()]; ok { + return cached.tasks, cached.frags, nil + } + frags, err := buildFragments(s) + if err != nil { + return nil, nil, errors.Trace(err) + } + results := make([]*kv.MPPTask, 0, len(frags)) + for _, f := range frags { + tasks, err := e.generateMPPTasksForFragment(f) + if err != nil { + return nil, nil, errors.Trace(err) + } + results = append(results, tasks...) + } + e.frags = append(e.frags, frags...) + e.cache[s.ID()] = tasksAndFrags{results, frags} + return results, frags, nil +} + +func (e *mppTaskGenerator) generateMPPTasksForFragment(f *Fragment) (tasks []*kv.MPPTask, err error) { for _, r := range f.ExchangeReceivers { - r.Tasks, err = e.generateMPPTasksForFragment(r.GetExchangeSender()) + r.Tasks, r.frags, err = e.generateMPPTasksForExchangeSender(r.GetExchangeSender()) if err != nil { return nil, errors.Trace(err) } @@ -149,8 +255,9 @@ func (e *mppTaskGenerator) generateMPPTasksForFragment(s *PhysicalExchangeSender return nil, errors.New("cannot find mpp task") } for _, r := range f.ExchangeReceivers { - s := r.GetExchangeSender() - s.TargetTasks = tasks + for _, frag := range r.frags { + frag.ExchangeSender.TargetTasks = append(frag.ExchangeSender.TargetTasks, tasks...) + } } f.ExchangeSender.Tasks = tasks return tasks, nil diff --git a/planner/core/indexmerge_test.go b/planner/core/indexmerge_test.go index a78b91b54f889..f1ebf4ee8144b 100644 --- a/planner/core/indexmerge_test.go +++ b/planner/core/indexmerge_test.go @@ -90,7 +90,7 @@ func (s *testIndexMergeSuite) TestIndexMergePathGeneration(c *C) { comment := Commentf("case:%v sql:%s", i, tc) stmt, err := s.ParseOneStmt(tc, "", "") c.Assert(err, IsNil, comment) - err = Preprocess(s.ctx, stmt, s.is) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil) builder, _ := NewPlanBuilder(MockContext(), s.is, &hint.BlockHintProcessor{}) p, err := builder.Build(ctx, stmt) diff --git a/planner/core/initialize.go b/planner/core/initialize.go index c63d4efa7ba31..cbf099baf4113 100644 --- a/planner/core/initialize.go +++ b/planner/core/initialize.go @@ -419,7 +419,7 @@ func (p PhysicalTableReader) Init(ctx sessionctx.Context, offset int) *PhysicalT if p.tablePlan != nil { p.TablePlans = flattenPushDownPlan(p.tablePlan) p.schema = p.tablePlan.Schema() - if p.StoreType == kv.TiFlash && !p.GetTableScan().KeepOrder { + if p.StoreType == kv.TiFlash && p.GetTableScan() != nil && !p.GetTableScan().KeepOrder { // When allow batch cop is 1, only agg / topN uses batch cop. // When allow batch cop is 2, every query uses batch cop. switch ctx.GetSessionVars().AllowBatchCop { @@ -528,3 +528,29 @@ func flattenPushDownPlan(p PhysicalPlan) []PhysicalPlan { } return plans } + +// Init only assigns type and context. +func (p LogicalCTE) Init(ctx sessionctx.Context, offset int) *LogicalCTE { + p.baseLogicalPlan = newBaseLogicalPlan(ctx, plancodec.TypeCTE, &p, offset) + return &p +} + +// Init only assigns type and context. +func (p PhysicalCTE) Init(ctx sessionctx.Context, stats *property.StatsInfo) *PhysicalCTE { + p.basePlan = newBasePlan(ctx, plancodec.TypeCTE, 0) + p.stats = stats + return &p +} + +// Init only assigns type and context. +func (p LogicalCTETable) Init(ctx sessionctx.Context, offset int) *LogicalCTETable { + p.baseLogicalPlan = newBaseLogicalPlan(ctx, plancodec.TypeCTETable, &p, offset) + return &p +} + +// Init only assigns type and context. +func (p PhysicalCTETable) Init(ctx sessionctx.Context, stats *property.StatsInfo) *PhysicalCTETable { + p.basePlan = newBasePlan(ctx, plancodec.TypeCTETable, 0) + p.stats = stats + return &p +} diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 816c94fa32a66..9c95d5026f79b 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -180,6 +180,29 @@ func (s *testIntegrationSuite) TestPushLimitDownIndexLookUpReader(c *C) { } } +func (s *testIntegrationSuite) TestAggColumnPrune(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int)") + tk.MustExec("insert into t values(1),(2)") + + var input []string + var output []struct { + SQL string + Res []string + } + s.testData.GetTestCases(c, &input, &output) + for i, tt := range input { + s.testData.OnRecord(func() { + output[i].SQL = tt + output[i].Res = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + }) + tk.MustQuery(tt).Check(testkit.Rows(output[i].Res...)) + } +} + func (s *testIntegrationSuite) TestIsFromUnixtimeNullRejective(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -196,6 +219,20 @@ func (s *testIntegrationSuite) TestIssue22298(c *C) { tk.MustGetErrMsg(`select * from t where 0 and c = 10;`, "[planner:1054]Unknown column 'c' in 'where clause'") } +func (s *testIntegrationSuite) TestIssue24571(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec(`create view v as select 1 as b;`) + tk.MustExec(`create table t (a int);`) + tk.MustExec(`update v, t set a=2;`) + tk.MustGetErrCode(`update v, t set b=2;`, mysql.ErrNonUpdatableTable) + tk.MustExec("create database db1") + tk.MustExec("use db1") + tk.MustExec("update test.t, (select 1 as a) as t set test.t.a=1;") + // bug in MySQL: ERROR 1288 (HY000): The target table t of the UPDATE is not updatable + tk.MustExec("update (select 1 as a) as t, test.t set test.t.a=1;") +} + func (s *testIntegrationSuite) TestIssue22828(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -517,50 +554,122 @@ func (s *testIntegrationSerialSuite) TestMPPJoin(c *C) { } } -func (s *testIntegrationSerialSuite) TestMPPShuffledJoin(c *C) { +func (s *testIntegrationSerialSuite) TestMPPOuterJoinBuildSideForBroadcastJoin(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - tk.MustExec("drop table if exists d1_t") - tk.MustExec("create table d1_t(d1_k int, value int)") - tk.MustExec("insert into d1_t values(1,2),(2,3)") - tk.MustExec("insert into d1_t values(1,2),(2,3)") - tk.MustExec("analyze table d1_t") - tk.MustExec("drop table if exists d2_t") - tk.MustExec("create table d2_t(d2_k decimal(10,2), value int)") - tk.MustExec("insert into d2_t values(10.11,2),(10.12,3)") - tk.MustExec("insert into d2_t values(10.11,2),(10.12,3)") - tk.MustExec("analyze table d2_t") - tk.MustExec("drop table if exists d3_t") - tk.MustExec("create table d3_t(d3_k date, value int)") - tk.MustExec("insert into d3_t values(date'2010-01-01',2),(date'2010-01-02',3)") - tk.MustExec("insert into d3_t values(date'2010-01-01',2),(date'2010-01-02',3)") - tk.MustExec("analyze table d3_t") - tk.MustExec("drop table if exists fact_t") - tk.MustExec("create table fact_t(d1_k int, d2_k decimal(10,2), d3_k date, col1 int, col2 int, col3 int)") - tk.MustExec("insert into fact_t values(1,10.11,date'2010-01-01',1,2,3),(1,10.11,date'2010-01-02',1,2,3),(1,10.12,date'2010-01-01',1,2,3),(1,10.12,date'2010-01-02',1,2,3)") - tk.MustExec("insert into fact_t values(2,10.11,date'2010-01-01',1,2,3),(2,10.11,date'2010-01-02',1,2,3),(2,10.12,date'2010-01-01',1,2,3),(2,10.12,date'2010-01-02',1,2,3)") - tk.MustExec("insert into fact_t values(2,10.11,date'2010-01-01',1,2,3),(2,10.11,date'2010-01-02',1,2,3),(2,10.12,date'2010-01-01',1,2,3),(2,10.12,date'2010-01-02',1,2,3)") - tk.MustExec("insert into fact_t values(2,10.11,date'2010-01-01',1,2,3),(2,10.11,date'2010-01-02',1,2,3),(2,10.12,date'2010-01-01',1,2,3),(2,10.12,date'2010-01-02',1,2,3)") - tk.MustExec("analyze table fact_t") + tk.MustExec("drop table if exists a") + tk.MustExec("create table a(id int, value int)") + tk.MustExec("insert into a values(1,2),(2,3)") + tk.MustExec("analyze table a") + tk.MustExec("drop table if exists b") + tk.MustExec("create table b(id int, value int)") + tk.MustExec("insert into b values(1,2),(2,3),(3,4)") + tk.MustExec("analyze table b") + // Create virtual tiflash replica info. + dom := domain.GetDomain(tk.Se) + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test")) + c.Assert(exists, IsTrue) + for _, tblInfo := range db.Tables { + if tblInfo.Name.L == "a" || tblInfo.Name.L == "b" { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } + tk.MustExec("set @@session.tidb_isolation_read_engines = 'tiflash'") + tk.MustExec("set @@session.tidb_opt_mpp_outer_join_fixed_build_side = 0") + tk.MustExec("set @@session.tidb_broadcast_join_threshold_size = 10000") + tk.MustExec("set @@session.tidb_broadcast_join_threshold_count = 10000") + var input []string + var output []struct { + SQL string + Plan []string + } + s.testData.GetTestCases(c, &input, &output) + for i, tt := range input { + s.testData.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + }) + res := tk.MustQuery(tt) + res.Check(testkit.Rows(output[i].Plan...)) + } +} +func (s *testIntegrationSerialSuite) TestMPPOuterJoinBuildSideForShuffleJoinWithFixedBuildSide(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists a") + tk.MustExec("create table a(id int, value int)") + tk.MustExec("insert into a values(1,2),(2,3)") + tk.MustExec("analyze table a") + tk.MustExec("drop table if exists b") + tk.MustExec("create table b(id int, value int)") + tk.MustExec("insert into b values(1,2),(2,3),(3,4)") + tk.MustExec("analyze table b") // Create virtual tiflash replica info. dom := domain.GetDomain(tk.Se) is := dom.InfoSchema() db, exists := is.SchemaByName(model.NewCIStr("test")) c.Assert(exists, IsTrue) for _, tblInfo := range db.Tables { - if tblInfo.Name.L == "fact_t" || tblInfo.Name.L == "d1_t" || tblInfo.Name.L == "d2_t" || tblInfo.Name.L == "d3_t" { + if tblInfo.Name.L == "a" || tblInfo.Name.L == "b" { tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ Count: 1, Available: true, } } } + tk.MustExec("set @@session.tidb_isolation_read_engines = 'tiflash'") + tk.MustExec("set @@session.tidb_opt_mpp_outer_join_fixed_build_side = 1") + tk.MustExec("set @@session.tidb_broadcast_join_threshold_size = 0") + tk.MustExec("set @@session.tidb_broadcast_join_threshold_count = 0") + var input []string + var output []struct { + SQL string + Plan []string + } + s.testData.GetTestCases(c, &input, &output) + for i, tt := range input { + s.testData.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + }) + res := tk.MustQuery(tt) + res.Check(testkit.Rows(output[i].Plan...)) + } +} +func (s *testIntegrationSerialSuite) TestMPPOuterJoinBuildSideForShuffleJoin(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists a") + tk.MustExec("create table a(id int, value int)") + tk.MustExec("insert into a values(1,2),(2,3)") + tk.MustExec("analyze table a") + tk.MustExec("drop table if exists b") + tk.MustExec("create table b(id int, value int)") + tk.MustExec("insert into b values(1,2),(2,3),(3,4)") + tk.MustExec("analyze table b") + // Create virtual tiflash replica info. + dom := domain.GetDomain(tk.Se) + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test")) + c.Assert(exists, IsTrue) + for _, tblInfo := range db.Tables { + if tblInfo.Name.L == "a" || tblInfo.Name.L == "b" { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } tk.MustExec("set @@session.tidb_isolation_read_engines = 'tiflash'") - tk.MustExec("set @@session.tidb_allow_mpp = 1") - tk.MustExec("set @@session.tidb_broadcast_join_threshold_size = 1") - tk.MustExec("set @@session.tidb_broadcast_join_threshold_count = 1") + tk.MustExec("set @@session.tidb_opt_mpp_outer_join_fixed_build_side = 0") + tk.MustExec("set @@session.tidb_broadcast_join_threshold_size = 0") + tk.MustExec("set @@session.tidb_broadcast_join_threshold_count = 0") var input []string var output []struct { SQL string @@ -577,26 +686,30 @@ func (s *testIntegrationSerialSuite) TestMPPShuffledJoin(c *C) { } } -func (s *testIntegrationSerialSuite) TestBroadcastJoin(c *C) { +func (s *testIntegrationSerialSuite) TestMPPShuffledJoin(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - tk.MustExec("set session tidb_allow_mpp = OFF") tk.MustExec("drop table if exists d1_t") tk.MustExec("create table d1_t(d1_k int, value int)") tk.MustExec("insert into d1_t values(1,2),(2,3)") + tk.MustExec("insert into d1_t values(1,2),(2,3)") tk.MustExec("analyze table d1_t") tk.MustExec("drop table if exists d2_t") tk.MustExec("create table d2_t(d2_k decimal(10,2), value int)") tk.MustExec("insert into d2_t values(10.11,2),(10.12,3)") + tk.MustExec("insert into d2_t values(10.11,2),(10.12,3)") tk.MustExec("analyze table d2_t") tk.MustExec("drop table if exists d3_t") tk.MustExec("create table d3_t(d3_k date, value int)") tk.MustExec("insert into d3_t values(date'2010-01-01',2),(date'2010-01-02',3)") + tk.MustExec("insert into d3_t values(date'2010-01-01',2),(date'2010-01-02',3)") tk.MustExec("analyze table d3_t") tk.MustExec("drop table if exists fact_t") tk.MustExec("create table fact_t(d1_k int, d2_k decimal(10,2), d3_k date, col1 int, col2 int, col3 int)") tk.MustExec("insert into fact_t values(1,10.11,date'2010-01-01',1,2,3),(1,10.11,date'2010-01-02',1,2,3),(1,10.12,date'2010-01-01',1,2,3),(1,10.12,date'2010-01-02',1,2,3)") tk.MustExec("insert into fact_t values(2,10.11,date'2010-01-01',1,2,3),(2,10.11,date'2010-01-02',1,2,3),(2,10.12,date'2010-01-01',1,2,3),(2,10.12,date'2010-01-02',1,2,3)") + tk.MustExec("insert into fact_t values(2,10.11,date'2010-01-01',1,2,3),(2,10.11,date'2010-01-02',1,2,3),(2,10.12,date'2010-01-01',1,2,3),(2,10.12,date'2010-01-02',1,2,3)") + tk.MustExec("insert into fact_t values(2,10.11,date'2010-01-01',1,2,3),(2,10.11,date'2010-01-02',1,2,3),(2,10.12,date'2010-01-01',1,2,3),(2,10.12,date'2010-01-02',1,2,3)") tk.MustExec("analyze table fact_t") // Create virtual tiflash replica info. @@ -614,10 +727,9 @@ func (s *testIntegrationSerialSuite) TestBroadcastJoin(c *C) { } tk.MustExec("set @@session.tidb_isolation_read_engines = 'tiflash'") - tk.MustExec("set @@session.tidb_allow_batch_cop = 1") - tk.MustExec("set @@session.tidb_opt_broadcast_join = 1") - // make cbo force choose broadcast join since sql hint does not work for semi/anti-semi join - tk.MustExec("set @@session.tidb_opt_cpu_factor=10000000;") + tk.MustExec("set @@session.tidb_allow_mpp = 1") + tk.MustExec("set @@session.tidb_broadcast_join_threshold_size = 1") + tk.MustExec("set @@session.tidb_broadcast_join_threshold_count = 1") var input []string var output []struct { SQL string @@ -632,23 +744,6 @@ func (s *testIntegrationSerialSuite) TestBroadcastJoin(c *C) { res := tk.MustQuery(tt) res.Check(testkit.Rows(output[i].Plan...)) } - - // out table of out join should not be global - _, err := tk.Exec("explain format = 'brief' select /*+ broadcast_join(fact_t, d1_t), broadcast_join_local(d1_t) */ count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k") - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "[planner:1815]Internal : Can't find a proper physical plan for this query") - // nullEQ not supported - _, err = tk.Exec("explain format = 'brief' select /*+ broadcast_join(fact_t, d1_t) */ count(*) from fact_t join d1_t on fact_t.d1_k <=> d1_t.d1_k") - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "[planner:1815]Internal : Can't find a proper physical plan for this query") - // not supported if join condition has unsupported expr - _, err = tk.Exec("explain format = 'brief' select /*+ broadcast_join(fact_t, d1_t) */ count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k and sqrt(fact_t.col1) > 2") - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "[planner:1815]Internal : Can't find a proper physical plan for this query") - // cartsian join not supported - _, err = tk.Exec("explain format = 'brief' select /*+ broadcast_join(fact_t, d1_t) */ count(*) from fact_t join d1_t") - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "[planner:1815]Internal : Can't find a proper physical plan for this query") } func (s *testIntegrationSerialSuite) TestJoinNotSupportedByTiFlash(c *C) { @@ -806,6 +901,46 @@ func (s *testIntegrationSerialSuite) TestMPPWithBroadcastExchangeUnderNewCollati } } +func (s *testIntegrationSerialSuite) TestPartitionTableDynamicModeUnderNewCollation(c *C) { + collate.SetNewCollationEnabledForTest(true) + defer collate.SetNewCollationEnabledForTest(false) + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_new_collation") + tk.MustExec("use test_new_collation") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + // hash + range partition + tk.MustExec(`CREATE TABLE thash (a int, c varchar(20) charset utf8mb4 collate utf8mb4_general_ci, key(a)) partition by hash(a) partitions 4`) + tk.MustExec(`CREATE TABLE trange (a int, c varchar(20) charset utf8mb4 collate utf8mb4_general_ci, key(a)) partition by range(a) ( + partition p0 values less than (10), + partition p1 values less than (20), + partition p2 values less than (30), + partition p3 values less than (40))`) + tk.MustExec(`insert into thash values (1, 'a'), (1, 'A'), (11, 'a'), (11, 'A'), (21, 'a'), (21, 'A'), (31, 'a'), (31, 'A')`) + tk.MustExec(`insert into trange values (1, 'a'), (1, 'A'), (11, 'a'), (11, 'A'), (21, 'a'), (21, 'A'), (31, 'a'), (31, 'A')`) + tk.MustQuery(`select * from thash use index(a) where a in (1, 11, 31) and c='a'`).Sort().Check(testkit.Rows("1 A", "1 a", "11 A", "11 a", "31 A", "31 a")) + tk.MustQuery(`select * from thash ignore index(a) where a in (1, 11, 31) and c='a'`).Sort().Check(testkit.Rows("1 A", "1 a", "11 A", "11 a", "31 A", "31 a")) + tk.MustQuery(`select * from trange use index(a) where a in (1, 11, 31) and c='a'`).Sort().Check(testkit.Rows("1 A", "1 a", "11 A", "11 a", "31 A", "31 a")) + tk.MustQuery(`select * from trange ignore index(a) where a in (1, 11, 31) and c='a'`).Sort().Check(testkit.Rows("1 A", "1 a", "11 A", "11 a", "31 A", "31 a")) + + // range partition and partitioned by utf8mb4_general_ci + tk.MustExec(`create table strrange(a varchar(10) charset utf8mb4 collate utf8mb4_general_ci, b int) partition by range columns(a) ( + partition p0 values less than ('a'), + partition p1 values less than ('k'), + partition p2 values less than ('z'))`) + tk.MustExec("insert into strrange values ('a', 1), ('A', 1), ('y', 1), ('Y', 1), ('q', 1)") + tk.MustQuery("select * from strrange where a in ('a', 'y')").Sort().Check(testkit.Rows("A 1", "Y 1", "a 1", "y 1")) + + // list partition and partitioned by utf8mb4_general_ci + tk.MustExec(`create table strlist(a varchar(10) charset utf8mb4 collate utf8mb4_general_ci, b int) partition by list(a) ( + partition p0 values in ('a', 'b'), + partition p1 values in ('c', 'd'), + partition p2 values in ('e', 'f'))`) + tk.MustExec("insert into strlist values ('a', 1), ('A', 1), ('d', 1), ('D', 1), ('e', 1)") + tk.MustQuery(`select * from strlist where a='a'`).Sort().Check(testkit.Rows("A 1", "a 1")) + tk.MustQuery(`select * from strlist where a in ('D', 'e')`).Sort().Check(testkit.Rows("D 1", "d 1", "e 1")) +} + func (s *testIntegrationSerialSuite) TestMPPAvgRewrite(c *C) { defer collate.SetNewCollationEnabledForTest(false) tk := testkit.NewTestKit(c, s.store) @@ -1165,7 +1300,7 @@ func (s *testIntegrationSuite) TestPartitionPruningForEQ(c *C) { tk.MustExec("drop table if exists t") tk.MustExec("create table t(a datetime, b int) partition by range(weekday(a)) (partition p0 values less than(10), partition p1 values less than (100))") - is := infoschema.GetInfoSchema(tk.Se) + is := tk.Se.GetInfoSchema().(infoschema.InfoSchema) tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) pt := tbl.(table.PartitionedTable) @@ -3212,6 +3347,45 @@ func (s *testIntegrationSerialSuite) TestPushDownAggForMPP(c *C) { } } +func (s *testIntegrationSerialSuite) TestMppUnionAll(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t (a int not null, b int, c varchar(20))") + tk.MustExec("create table t1 (a int, b int not null, c double)") + + // Create virtual tiflash replica info. + dom := domain.GetDomain(tk.Se) + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test")) + c.Assert(exists, IsTrue) + for _, tblInfo := range db.Tables { + if tblInfo.Name.L == "t" || tblInfo.Name.L == "t1" { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } + + var input []string + var output []struct { + SQL string + Plan []string + } + s.testData.GetTestCases(c, &input, &output) + for i, tt := range input { + s.testData.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + }) + res := tk.MustQuery(tt) + res.Check(testkit.Rows(output[i].Plan...)) + } + +} + func (s *testIntegrationSerialSuite) TestMppJoinDecimal(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -3375,6 +3549,15 @@ func (s *testIntegrationSuite) TestIssue22850(c *C) { tk.MustQuery("SELECT @v:=(SELECT 1 FROM t1 t2 LEFT JOIN t1 ON t1.a GROUP BY t1.a) FROM t1").Check(testkit.Rows()) // work fine } +func (s *testIntegrationSuite) TestJoinSchemaChange(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1, t2") + tk.MustExec("create table t1(a int(11))") + tk.MustExec("create table t2(a decimal(40,20) unsigned, b decimal(40,20))") + tk.MustQuery("select count(*) as x from t1 group by a having x not in (select a from t2 where x = t2.b)").Check(testkit.Rows()) +} + // #22949: test HexLiteral Used in GetVar expr func (s *testIntegrationSuite) TestGetVarExprWithHexLiteral(c *C) { tk := testkit.NewTestKit(c, s.store) @@ -3505,6 +3688,21 @@ func (s *testIntegrationSuite) TestIssue23736(c *C) { c.Assert(tk.MustUseIndex("select /*+ stream_agg() */ count(1) from t0 where c > 10 and b < 2", "c"), IsFalse) } +// https://github.com/pingcap/tidb/issues/23802 +func (s *testIntegrationSuite) TestPanicWhileQueryTableWithIsNull(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + tk.MustExec("drop table if exists NT_HP27193") + tk.MustExec("CREATE TABLE `NT_HP27193` ( `COL1` int(20) DEFAULT NULL, `COL2` varchar(20) DEFAULT NULL, `COL4` datetime DEFAULT NULL, `COL3` bigint(20) DEFAULT NULL, `COL5` float DEFAULT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin PARTITION BY HASH ( `COL1`%`COL3` ) PARTITIONS 10;") + _, err := tk.Exec("select col1 from NT_HP27193 where col1 is null;") + c.Assert(err, IsNil) + tk.MustExec("INSERT INTO NT_HP27193 (COL2, COL4, COL3, COL5) VALUES ('m', '2020-05-04 13:15:27', 8, 2602)") + _, err = tk.Exec("select col1 from NT_HP27193 where col1 is null;") + c.Assert(err, IsNil) + tk.MustExec("drop table if exists NT_HP27193") +} + func (s *testIntegrationSuite) TestIssue23846(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -3557,6 +3755,48 @@ func (s *testIntegrationSuite) TestIssue24095(c *C) { } } +func (s *testIntegrationSuite) TestIssue24281(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists member, agent, deposit, view_member_agents") + tk.MustExec("create table member(login varchar(50) NOT NULL, agent_login varchar(100) DEFAULT NULL, PRIMARY KEY(login))") + tk.MustExec("create table agent(login varchar(50) NOT NULL, data varchar(100) DEFAULT NULL, share_login varchar(50) NOT NULL, PRIMARY KEY(login))") + tk.MustExec("create table deposit(id varchar(50) NOT NULL, member_login varchar(50) NOT NULL, transfer_amount int NOT NULL, PRIMARY KEY(id), KEY midx(member_login, transfer_amount))") + tk.MustExec("create definer='root'@'localhost' view view_member_agents (member, share_login) as select m.login as member, a.share_login AS share_login from member as m join agent as a on m.agent_login = a.login") + + tk.MustExec(" select s.member_login as v1, SUM(s.transfer_amount) AS v2 " + + "FROM deposit AS s " + + "JOIN view_member_agents AS v ON s.member_login = v.member " + + "WHERE 1 = 1 AND v.share_login = 'somevalue' " + + "GROUP BY s.member_login " + + "UNION select 1 as v1, 2 as v2") +} + +func (s *testIntegrationSuite) TestIncrementalAnalyzeStatsVer2(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int primary key, b int, index idx_b(b))") + tk.MustExec("insert into t values(1,1),(2,2),(3,3)") + tk.MustExec("set @@session.tidb_analyze_version = 2") + tk.MustExec("analyze table t") + is := tk.Se.GetInfoSchema().(infoschema.InfoSchema) + tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + c.Assert(err, IsNil) + tblID := tbl.Meta().ID + rows := tk.MustQuery(fmt.Sprintf("select distinct_count from mysql.stats_histograms where table_id = %d and is_index = 1", tblID)).Rows() + c.Assert(len(rows), Equals, 1) + c.Assert(rows[0][0], Equals, "3") + tk.MustExec("insert into t values(4,4),(5,5),(6,6)") + tk.MustExec("analyze incremental table t index idx_b") + c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings(), HasLen, 2) + c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings()[0].Err.Error(), Equals, "The version 2 would collect all statistics not only the selected indexes") + c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings()[1].Err.Error(), Equals, "The version 2 stats would ignore the INCREMENTAL keyword and do full sampling") + rows = tk.MustQuery(fmt.Sprintf("select distinct_count from mysql.stats_histograms where table_id = %d and is_index = 1", tblID)).Rows() + c.Assert(len(rows), Equals, 1) + c.Assert(rows[0][0], Equals, "6") +} + func (s *testIntegrationSuite) TestConflictReadFromStorage(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -3611,3 +3851,167 @@ func (s *testIntegrationSuite) TestSequenceAsDataSource(c *C) { tk.MustQuery("explain format = 'brief' " + tt).Check(testkit.Rows(output[i].Plan...)) } } + +func (s *testIntegrationSerialSuite) TestMergeContinuousSelections(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists ts") + tk.MustExec("create table ts (col_char_64 char(64), col_varchar_64_not_null varchar(64) not null, col_varchar_key varchar(1), id int primary key, col_varchar_64 varchar(64),col_char_64_not_null char(64) not null);") + + // Create virtual tiflash replica info. + dom := domain.GetDomain(tk.Se) + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test")) + c.Assert(exists, IsTrue) + for _, tblInfo := range db.Tables { + if tblInfo.Name.L == "ts" { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } + + tk.MustExec(" set @@tidb_allow_mpp=1;") + + var input []string + var output []struct { + SQL string + Plan []string + } + s.testData.GetTestCases(c, &input, &output) + for i, tt := range input { + s.testData.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + }) + res := tk.MustQuery(tt) + res.Check(testkit.Rows(output[i].Plan...)) + } +} + +func (s *testIntegrationSerialSuite) TestEnforceMPP(c *C) { + tk := testkit.NewTestKit(c, s.store) + + // test value limit of tidb_opt_tiflash_concurrency_factor + err := tk.ExecToErr("set @@tidb_opt_tiflash_concurrency_factor = 0") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, `[variable:1231]Variable 'tidb_opt_tiflash_concurrency_factor' can't be set to the value of '0'`) + + tk.MustExec("set @@tidb_opt_tiflash_concurrency_factor = 1") + tk.MustQuery("select @@tidb_opt_tiflash_concurrency_factor").Check(testkit.Rows("1")) + tk.MustExec("set @@tidb_opt_tiflash_concurrency_factor = 24") + tk.MustQuery("select @@tidb_opt_tiflash_concurrency_factor").Check(testkit.Rows("24")) + + // test set tidb_allow_mpp + tk.MustExec("set @@session.tidb_allow_mpp = 0") + tk.MustQuery("select @@session.tidb_allow_mpp").Check(testkit.Rows("OFF")) + tk.MustExec("set @@session.tidb_allow_mpp = 1") + tk.MustQuery("select @@session.tidb_allow_mpp").Check(testkit.Rows("ON")) + tk.MustExec("set @@session.tidb_allow_mpp = 2") + tk.MustQuery("select @@session.tidb_allow_mpp").Check(testkit.Rows("ENFORCE")) + + tk.MustExec("set @@session.tidb_allow_mpp = off") + tk.MustQuery("select @@session.tidb_allow_mpp").Check(testkit.Rows("OFF")) + tk.MustExec("set @@session.tidb_allow_mpp = oN") + tk.MustQuery("select @@session.tidb_allow_mpp").Check(testkit.Rows("ON")) + tk.MustExec("set @@session.tidb_allow_mpp = enForcE") + tk.MustQuery("select @@session.tidb_allow_mpp").Check(testkit.Rows("ENFORCE")) + + tk.MustExec("set @@global.tidb_allow_mpp = faLsE") + tk.MustQuery("select @@global.tidb_allow_mpp").Check(testkit.Rows("OFF")) + tk.MustExec("set @@global.tidb_allow_mpp = True") + tk.MustQuery("select @@global.tidb_allow_mpp").Check(testkit.Rows("ON")) + + err = tk.ExecToErr("set @@global.tidb_allow_mpp = enforceWithTypo") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, `[variable:1231]Variable 'tidb_allow_mpp' can't be set to the value of 'enforceWithTypo'`) + + // test query + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int)") + tk.MustExec("create index idx on t(a)") + + // Create virtual tiflash replica info. + dom := domain.GetDomain(tk.Se) + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test")) + c.Assert(exists, IsTrue) + for _, tblInfo := range db.Tables { + if tblInfo.Name.L == "t" { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } + + // ban mpp + tk.MustExec("set @@session.tidb_allow_mpp = 0") + tk.MustQuery("select @@session.tidb_allow_mpp").Check(testkit.Rows("OFF")) + + // read from tiflash, batch cop. + tk.MustQuery("explain format='verbose' select /*+ read_from_storage(tiflash[t]) */ count(*) from t where a=1").Check(testkit.Rows( + "StreamAgg_20 1.00 285050.00 root funcs:count(Column#5)->Column#3", + "└─TableReader_21 1.00 19003.88 root data:StreamAgg_9", + " └─StreamAgg_9 1.00 19006.88 batchCop[tiflash] funcs:count(1)->Column#5", + " └─Selection_19 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", + " └─TableFullScan_18 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo")) + + // open mpp + tk.MustExec("set @@session.tidb_allow_mpp = 1") + tk.MustQuery("select @@session.tidb_allow_mpp").Check(testkit.Rows("ON")) + + // should use tikv to index read + tk.MustQuery("explain format='verbose' select count(*) from t where a=1;").Check(testkit.Rows( + "StreamAgg_30 1.00 485.00 root funcs:count(Column#6)->Column#3", + "└─IndexReader_31 1.00 32.88 root index:StreamAgg_10", + " └─StreamAgg_10 1.00 35.88 cop[tikv] funcs:count(1)->Column#6", + " └─IndexRangeScan_29 10.00 455.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo")) + + // read from tikv, indexRead + tk.MustQuery("explain format='verbose' select /*+ read_from_storage(tikv[t]) */ count(*) from t where a=1;").Check(testkit.Rows( + "StreamAgg_18 1.00 485.00 root funcs:count(Column#5)->Column#3", + "└─IndexReader_19 1.00 32.88 root index:StreamAgg_10", + " └─StreamAgg_10 1.00 35.88 cop[tikv] funcs:count(1)->Column#5", + " └─IndexRangeScan_17 10.00 455.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo")) + + // read from tiflash, mpp with large cost + tk.MustQuery("explain format='verbose' select /*+ read_from_storage(tiflash[t]) */ count(*) from t where a=1").Check(testkit.Rows( + "HashAgg_21 1.00 11910.73 root funcs:count(Column#5)->Column#3", + "└─TableReader_23 1.00 11877.13 root data:ExchangeSender_22", + " └─ExchangeSender_22 1.00 285050.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_9 1.00 285050.00 batchCop[tiflash] funcs:count(1)->Column#5", + " └─Selection_20 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", + " └─TableFullScan_19 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo")) + + // enforce mpp + tk.MustExec("set @@session.tidb_allow_mpp = 2") + tk.MustQuery("select @@session.tidb_allow_mpp").Check(testkit.Rows("ENFORCE")) + + // should use mpp + tk.MustQuery("explain format='verbose' select count(*) from t where a=1;").Check(testkit.Rows( + "HashAgg_24 1.00 33.60 root funcs:count(Column#5)->Column#3", + "└─TableReader_26 1.00 0.00 root data:ExchangeSender_25", + " └─ExchangeSender_25 1.00 285050.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_9 1.00 285050.00 batchCop[tiflash] funcs:count(1)->Column#5", + " └─Selection_23 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", + " └─TableFullScan_22 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo")) + + // read from tikv, indexRead + tk.MustQuery("explain format='verbose' select /*+ read_from_storage(tikv[t]) */ count(*) from t where a=1;").Check(testkit.Rows( + "StreamAgg_18 1.00 485.00 root funcs:count(Column#5)->Column#3", + "└─IndexReader_19 1.00 32.88 root index:StreamAgg_10", + " └─StreamAgg_10 1.00 35.88 cop[tikv] funcs:count(1)->Column#5", + " └─IndexRangeScan_17 10.00 455.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo")) + + // read from tiflash, mpp with little cost + tk.MustQuery("explain format='verbose' select /*+ read_from_storage(tiflash[t]) */ count(*) from t where a=1").Check(testkit.Rows( + "HashAgg_21 1.00 33.60 root funcs:count(Column#5)->Column#3", + "└─TableReader_23 1.00 0.00 root data:ExchangeSender_22", + " └─ExchangeSender_22 1.00 285050.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_9 1.00 285050.00 batchCop[tiflash] funcs:count(1)->Column#5", + " └─Selection_20 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", + " └─TableFullScan_19 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo")) +} diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 7ff0e2ac6c6aa..e575bb79e7135 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -33,6 +33,7 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/opcode" "github.com/pingcap/parser/terror" + "github.com/pingcap/tidb/ddl" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/expression/aggregation" @@ -43,6 +44,7 @@ import ( "github.com/pingcap/tidb/planner/util" "github.com/pingcap/tidb/privilege" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" @@ -305,6 +307,12 @@ func (b *PlanBuilder) buildTableRefs(ctx context.Context, from *ast.TableRefsCla p = b.buildTableDual() return } + defer func() { + // After build the resultSetNode, need to reset it so that it can be referenced by outer level. + for _, cte := range b.outerCTEs { + cte.recursiveRef = false + } + }() return b.buildResultSetNode(ctx, from.TableRefs) } @@ -316,8 +324,12 @@ func (b *PlanBuilder) buildResultSetNode(ctx context.Context, node ast.ResultSet var isTableName bool switch v := x.Source.(type) { case *ast.SelectStmt: + ci := b.prepareCTECheckForSubQuery() + defer resetCTECheckForSubQuery(ci) p, err = b.buildSelect(ctx, v) case *ast.SetOprStmt: + ci := b.prepareCTECheckForSubQuery() + defer resetCTECheckForSubQuery(ci) p, err = b.buildSetOpr(ctx, v) case *ast.TableName: p, err = b.buildDataSource(ctx, v, &x.AsName) @@ -669,6 +681,11 @@ func (b *PlanBuilder) buildJoin(ctx context.Context, joinNode *ast.Join) (Logica return nil, err } + // The recursive part in CTE must not be on the right side of a LEFT JOIN. + if lc, ok := rightPlan.(*LogicalCTETable); ok && joinNode.Tp == ast.LeftJoin { + return nil, ErrCTERecursiveForbiddenJoinOrder.GenWithStackByArgs(lc.name) + } + handleMap1 := b.handleHelper.popMap() handleMap2 := b.handleHelper.popMap() b.handleHelper.mergeAndPush(handleMap1, handleMap2) @@ -975,6 +992,19 @@ func (b *PlanBuilder) buildSelection(ctx context.Context, p LogicalPlan, where a if len(cnfExpres) == 0 { return p, nil } + // check expr field types. + for i, expr := range cnfExpres { + if expr.GetType().EvalType() == types.ETString { + tp := &types.FieldType{ + Tp: mysql.TypeDouble, + Flag: expr.GetType().Flag, + Flen: mysql.MaxRealWidth, + Decimal: types.UnspecifiedLength, + } + types.SetBinChsClnFlag(tp) + cnfExpres[i] = expression.TryPushCastIntoControlFunctionForHybridType(b.ctx, expr, tp) + } + } selection.Conditions = cnfExpres selection.SetChildren(p) return selection, nil @@ -1103,6 +1133,7 @@ func (b *PlanBuilder) buildProjectionField(ctx context.Context, p LogicalPlan, f UniqueID: b.ctx.GetSessionVars().AllocPlanColumnID(), RetType: expr.GetType(), } + newCol.SetCoercibility(expr.Coercibility()) return newCol, name, nil } @@ -1347,12 +1378,27 @@ func (b *PlanBuilder) buildProjection4Union(ctx context.Context, u *LogicalUnion b.optFlag |= flagEliminateProjection proj := LogicalProjection{Exprs: exprs, AvoidColumnEvaluator: true}.Init(b.ctx, b.getSelectOffset()) proj.SetSchema(u.schema.Clone()) + // reset the schema type to make the "not null" flag right. + for i, expr := range exprs { + proj.schema.Columns[i].RetType = expr.GetType() + } proj.SetChildren(child) u.children[childID] = proj } } func (b *PlanBuilder) buildSetOpr(ctx context.Context, setOpr *ast.SetOprStmt) (LogicalPlan, error) { + if setOpr.With != nil { + l := len(b.outerCTEs) + defer func() { + b.outerCTEs = b.outerCTEs[:l] + }() + err := b.buildWith(ctx, setOpr.With) + if err != nil { + return nil, err + } + } + // Because INTERSECT has higher precedence than UNION and EXCEPT. We build it first. selectPlans := make([]LogicalPlan, 0, len(setOpr.SelectList.Selects)) afterSetOprs := make([]*ast.SetOprType, 0, len(setOpr.SelectList.Selects)) @@ -2239,6 +2285,16 @@ func (r *correlatedAggregateResolver) Enter(n ast.Node) (ast.Node, bool) { // ORDER BY, WHERE & GROUP BY. // Finally it restore the original SELECT stmt. func (r *correlatedAggregateResolver) resolveSelect(sel *ast.SelectStmt) (err error) { + if sel.With != nil { + l := len(r.b.outerCTEs) + defer func() { + r.b.outerCTEs = r.b.outerCTEs[:l] + }() + err := r.b.buildWith(r.ctx, sel.With) + if err != nil { + return err + } + } // collect correlated aggregate from sub-queries inside FROM clause. _, err = r.collectFromTableRefs(r.ctx, sel.From) if err != nil { @@ -2812,7 +2868,7 @@ func (b *PlanBuilder) checkOnlyFullGroupByWithGroupClause(p LogicalPlan, sel *as continue } } - checkExprInGroupByOrIsSingleValue(p, item.Expr, offset, ErrExprInOrderBy, gbyOrSingleValueColNames, gbyExprs, notInGbyOrSingleValueColNames) + checkExprInGroupByOrIsSingleValue(p, getInnerFromParenthesesAndUnaryPlus(item.Expr), offset, ErrExprInOrderBy, gbyOrSingleValueColNames, gbyExprs, notInGbyOrSingleValueColNames) } } if len(notInGbyOrSingleValueColNames) == 0 { @@ -3204,6 +3260,9 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, currentLev } func (b *PlanBuilder) popVisitInfo() { + if len(b.visitInfo) == 0 { + return + } b.visitInfo = b.visitInfo[:len(b.visitInfo)-1] } @@ -3284,6 +3343,14 @@ func (b *PlanBuilder) buildSelect(ctx context.Context, sel *ast.SelectStmt) (p L // table hints are only visible in the current SELECT statement. b.popTableHints() }() + if b.buildingRecursivePartForCTE { + if sel.Distinct || sel.OrderBy != nil || sel.Limit != nil { + return nil, ErrNotSupportedYet.GenWithStackByArgs("ORDER BY / LIMIT / SELECT DISTINCT in recursive query block of Common Table Expression") + } + if sel.GroupBy != nil { + return nil, ErrCTERecursiveForbidsAggregation.FastGenByArgs(b.genCTETableNameForError()) + } + } enableNoopFuncs := b.ctx.GetSessionVars().EnableNoopFuncs if sel.SelectStmtOpts != nil { if sel.SelectStmtOpts.CalcFoundRows && !enableNoopFuncs { @@ -3309,6 +3376,17 @@ func (b *PlanBuilder) buildSelect(ctx context.Context, sel *ast.SelectStmt) (p L b.isForUpdateRead = true } + if sel.With != nil { + l := len(b.outerCTEs) + defer func() { + b.outerCTEs = b.outerCTEs[:l] + }() + err = b.buildWith(ctx, sel.With) + if err != nil { + return nil, err + } + } + // For sub-queries, the FROM clause may have already been built in outer query when resolving correlated aggregates. // If the ResultSetNode inside FROM clause has nothing to do with correlated aggregates, we can simply get the // existing ResultSetNode from the cache. @@ -3345,6 +3423,10 @@ func (b *PlanBuilder) buildSelect(ctx context.Context, sel *ast.SelectStmt) (p L // For example: select id from t group by id WINDOW w AS (ORDER BY uids DESC) ORDER BY id; // We don't use the WINDOW w, but if the 'uids' column is not in the table t, we still need to report an error. if hasWindowFuncField || sel.WindowSpecs != nil { + if b.buildingRecursivePartForCTE { + return nil, ErrCTERecursiveForbidsAggregation.FastGenByArgs(b.genCTETableNameForError()) + } + windowAggMap, err = b.resolveWindowFunction(sel, p) if err != nil { return nil, err @@ -3393,16 +3475,21 @@ func (b *PlanBuilder) buildSelect(ctx context.Context, sel *ast.SelectStmt) (p L b.handleHelper.pushMap(nil) hasAgg := b.detectSelectAgg(sel) + needBuildAgg := hasAgg if hasAgg { + if b.buildingRecursivePartForCTE { + return nil, ErrCTERecursiveForbidsAggregation.GenWithStackByArgs(b.genCTETableNameForError()) + } + aggFuncs, totalMap = b.extractAggFuncsInSelectFields(sel.Fields.Fields) // len(aggFuncs) == 0 and sel.GroupBy == nil indicates that all the aggregate functions inside the SELECT fields // are actually correlated aggregates from the outer query, which have already been built in the outer query. // The only thing we need to do is to find them from b.correlatedAggMap in buildProjection. if len(aggFuncs) == 0 && sel.GroupBy == nil { - hasAgg = false + needBuildAgg = false } } - if hasAgg { + if needBuildAgg { var aggIndexMap map[int]int p, aggIndexMap, err = b.buildAggregation(ctx, p, aggFuncs, gbyCols, correlatedAggMap) if err != nil { @@ -3559,10 +3646,86 @@ func getStatsTable(ctx sessionctx.Context, tblInfo *model.TableInfo, pid int64) return statsTbl } +func (b *PlanBuilder) tryBuildCTE(ctx context.Context, tn *ast.TableName, asName *model.CIStr) (LogicalPlan, error) { + for i := len(b.outerCTEs) - 1; i >= 0; i-- { + cte := b.outerCTEs[i] + if cte.def.Name.L == tn.Name.L { + if cte.isBuilding { + if cte.nonRecursive { + // Can't see this CTE, try outer definition. + continue + } + + // Building the recursive part. + cte.useRecursive = true + if cte.seedLP == nil { + return nil, ErrCTERecursiveRequiresNonRecursiveFirst.FastGenByArgs(tn.Name.String()) + } + + if cte.enterSubquery || cte.recursiveRef { + return nil, ErrInvalidRequiresSingleReference.FastGenByArgs(tn.Name.String()) + } + + cte.recursiveRef = true + p := LogicalCTETable{name: cte.def.Name.String(), idForStorage: cte.storageID}.Init(b.ctx, b.getSelectOffset()) + p.SetSchema(getResultCTESchema(cte.seedLP.Schema(), b.ctx.GetSessionVars())) + p.SetOutputNames(cte.seedLP.OutputNames()) + return p, nil + } + + b.handleHelper.pushMap(nil) + + hasLimit := false + limitBeg := uint64(0) + limitEnd := uint64(0) + if cte.limitLP != nil { + hasLimit = true + switch x := cte.limitLP.(type) { + case *LogicalLimit: + limitBeg = x.Offset + limitEnd = x.Offset + x.Count + case *LogicalTableDual: + // Beg and End will both be 0. + default: + return nil, errors.Errorf("invalid type for limit plan: %v", cte.limitLP) + } + } + + var p LogicalPlan + lp := LogicalCTE{cteAsName: tn.Name, cte: &CTEClass{IsDistinct: cte.isDistinct, seedPartLogicalPlan: cte.seedLP, + recursivePartLogicalPlan: cte.recurLP, IDForStorage: cte.storageID, + optFlag: cte.optFlag, HasLimit: hasLimit, LimitBeg: limitBeg, + LimitEnd: limitEnd}}.Init(b.ctx, b.getSelectOffset()) + lp.SetSchema(getResultCTESchema(cte.seedLP.Schema(), b.ctx.GetSessionVars())) + p = lp + p.SetOutputNames(cte.seedLP.OutputNames()) + if len(asName.String()) > 0 { + lp.cteAsName = *asName + var on types.NameSlice + for _, name := range p.OutputNames() { + cpOn := *name + cpOn.TblName = *asName + on = append(on, &cpOn) + } + p.SetOutputNames(on) + } + return p, nil + } + } + + return nil, nil +} + func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, asName *model.CIStr) (LogicalPlan, error) { dbName := tn.Schema sessionVars := b.ctx.GetSessionVars() + if dbName.L == "" { + // Try CTE. + p, err := b.tryBuildCTE(ctx, tn, asName) + if err != nil || p != nil { + return p, err + } dbName = model.NewCIStr(sessionVars.CurrentDB) } @@ -3617,7 +3780,7 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as } pids[pid] = struct{}{} } - pt = tables.NewPartitionTableithGivenSets(pt, pids) + pt = tables.NewPartitionTableWithGivenSets(pt, pids) } b.partitionedTable = append(b.partitionedTable, pt) } else if len(tn.PartitionNames) != 0 { @@ -4239,20 +4402,20 @@ func (b *PlanBuilder) buildUpdate(ctx context.Context, update *ast.UpdateStmt) ( b.popTableHints() }() - // update subquery table should be forbidden - var notUpdatableTbl []string - notUpdatableTbl = extractTableSourceAsNames(update.TableRefs.TableRefs, notUpdatableTbl, true) - for _, asName := range notUpdatableTbl { - for _, assign := range update.List { - if assign.Column.Table.L == asName { - return nil, ErrNonUpdatableTable.GenWithStackByArgs(asName, "UPDATE") - } - } - } - b.inUpdateStmt = true b.isForUpdateRead = true + if update.With != nil { + l := len(b.outerCTEs) + defer func() { + b.outerCTEs = b.outerCTEs[:l] + }() + err := b.buildWith(ctx, update.With) + if err != nil { + return nil, err + } + } + p, err := b.buildResultSetNode(ctx, update.TableRefs.TableRefs) if err != nil { return nil, err @@ -4265,12 +4428,6 @@ func (b *PlanBuilder) buildUpdate(ctx context.Context, update *ast.UpdateStmt) ( if dbName == "" { dbName = b.ctx.GetSessionVars().CurrentDB } - if t.TableInfo.IsView() { - return nil, errors.Errorf("update view %s is not supported now.", t.Name.O) - } - if t.TableInfo.IsSequence() { - return nil, errors.Errorf("update sequence %s is not supported now.", t.Name.O) - } b.visitInfo = appendVisitInfo(b.visitInfo, mysql.SelectPriv, dbName, t.Name.L, "", nil) } @@ -4314,6 +4471,10 @@ func (b *PlanBuilder) buildUpdate(ctx context.Context, update *ast.UpdateStmt) ( proj.SetChildren(p) p = proj + // update subquery table should be forbidden + var notUpdatableTbl []string + notUpdatableTbl = extractTableSourceAsNames(update.TableRefs.TableRefs, notUpdatableTbl, true) + var updateTableList []*ast.TableName updateTableList = extractTableList(update.TableRefs.TableRefs, updateTableList, true) orderedList, np, allAssignmentsAreConstant, err := b.buildUpdateLists(ctx, updateTableList, update.List, p, notUpdatableTbl) @@ -4393,6 +4554,12 @@ func CheckUpdateList(assignFlags []int, updt *Update) error { return nil } +// If tl is CTE, its TableInfo will be nil. +// Only used in build plan from AST after preprocess. +func isCTE(tl *ast.TableName) bool { + return tl.TableInfo == nil +} + func (b *PlanBuilder) buildUpdateLists(ctx context.Context, tableList []*ast.TableName, list []*ast.Assignment, p LogicalPlan, notUpdatableTbl []string) (newList []*expression.Assignment, po LogicalPlan, allAssignmentsAreConstant bool, e error) { b.curClause = fieldList @@ -4417,6 +4584,21 @@ func (b *PlanBuilder) buildUpdateLists(ctx context.Context, tableList []*ast.Tab columnsIdx[assign.Column] = idx } name := p.OutputNames()[idx] + for _, tl := range tableList { + if (tl.Schema.L == "" || tl.Schema.L == name.DBName.L) && (tl.Name.L == name.TblName.L) { + if isCTE(tl) || tl.TableInfo.IsView() || tl.TableInfo.IsSequence() { + return nil, nil, false, ErrNonUpdatableTable.GenWithStackByArgs(name.TblName.O, "UPDATE") + } + // may be a subquery + if tl.Schema.L == "" { + for _, nTbl := range notUpdatableTbl { + if nTbl == name.TblName.L { + return nil, nil, false, ErrNonUpdatableTable.GenWithStackByArgs(name.TblName.O, "UPDATE") + } + } + } + } + } columnFullName := fmt.Sprintf("%s.%s.%s", name.DBName.L, name.TblName.L, name.ColName.L) // We save a flag for the column in map `modifyColumns` // This flag indicated if assign keyword `DEFAULT` to the column @@ -4439,9 +4621,10 @@ func (b *PlanBuilder) buildUpdateLists(ctx context.Context, tableList []*ast.Tab break } } - if !updatable { + if !updatable || isCTE(tn) || tn.TableInfo.IsView() || tn.TableInfo.IsSequence() { continue } + tableInfo := tn.TableInfo tableVal, found := b.is.TableByID(tableInfo.ID) if !found { @@ -4472,6 +4655,9 @@ func (b *PlanBuilder) buildUpdateLists(ctx context.Context, tableList []*ast.Tab newList = make([]*expression.Assignment, 0, p.Schema().Len()) tblDbMap := make(map[string]string, len(tableList)) for _, tbl := range tableList { + if isCTE(tbl) { + continue + } tblDbMap[tbl.Name.L] = tbl.DBInfo.Name.L } @@ -4578,6 +4764,17 @@ func (b *PlanBuilder) buildDelete(ctx context.Context, delete *ast.DeleteStmt) ( b.inDeleteStmt = true b.isForUpdateRead = true + if delete.With != nil { + l := len(b.outerCTEs) + defer func() { + b.outerCTEs = b.outerCTEs[:l] + }() + err := b.buildWith(ctx, delete.With) + if err != nil { + return nil, err + } + } + p, err := b.buildResultSetNode(ctx, delete.TableRefs.TableRefs) if err != nil { return nil, err @@ -4704,6 +4901,9 @@ func (b *PlanBuilder) buildDelete(ctx context.Context, delete *ast.DeleteStmt) ( var tableList []*ast.TableName tableList = extractTableList(delete.TableRefs.TableRefs, tableList, false) for _, v := range tableList { + if isCTE(v) { + return nil, ErrNonUpdatableTable.GenWithStackByArgs(v.Name.O, "DELETE") + } if v.TableInfo.IsView() { return nil, errors.Errorf("delete view %s is not supported now.", v.Name.O) } @@ -5326,13 +5526,27 @@ func (b *PlanBuilder) handleDefaultFrame(spec *ast.WindowSpec, windowFuncName st newSpec.Frame = nil return &newSpec, true } - // For functions that operate on the entire partition, the frame clause will be ignored. - if !needFrame && spec.Frame != nil { - specName := spec.Name.O - b.ctx.GetSessionVars().StmtCtx.AppendNote(ErrWindowFunctionIgnoresFrame.GenWithStackByArgs(windowFuncName, getWindowName(specName))) + if !needFrame { + var updated bool newSpec := *spec - newSpec.Frame = nil - return &newSpec, true + + // For functions that operate on the entire partition, the frame clause will be ignored. + if spec.Frame != nil { + specName := spec.Name.O + b.ctx.GetSessionVars().StmtCtx.AppendNote(ErrWindowFunctionIgnoresFrame.GenWithStackByArgs(windowFuncName, getWindowName(specName))) + newSpec.Frame = nil + updated = true + } + if b.ctx.GetSessionVars().EnablePipelinedWindowExec { + useDefaultFrame, defaultFrame := aggregation.UseDefaultFrame(windowFuncName) + if useDefaultFrame { + newSpec.Frame = &defaultFrame + updated = true + } + } + if updated { + return &newSpec, true + } } return spec, false } @@ -5347,11 +5561,27 @@ func appendIfAbsentWindowSpec(specs []*ast.WindowSpec, ns *ast.WindowSpec) []*as return append(specs, ns) } +func specEqual(s1, s2 *ast.WindowSpec) (equal bool, err error) { + if (s1 == nil && s2 != nil) || (s1 != nil && s2 == nil) { + return false, nil + } + var sb1, sb2 strings.Builder + ctx1 := format.NewRestoreCtx(0, &sb1) + ctx2 := format.NewRestoreCtx(0, &sb2) + if err = s1.Restore(ctx1); err != nil { + return + } + if err = s2.Restore(ctx2); err != nil { + return + } + return sb1.String() == sb2.String(), nil +} + // groupWindowFuncs groups the window functions according to the window specification name. // TODO: We can group the window function by the definition of window specification. func (b *PlanBuilder) groupWindowFuncs(windowFuncs []*ast.WindowFuncExpr) (map[*ast.WindowSpec][]*ast.WindowFuncExpr, []*ast.WindowSpec, error) { // updatedSpecMap is used to handle the specifications that have frame clause changed. - updatedSpecMap := make(map[string]*ast.WindowSpec) + updatedSpecMap := make(map[string][]*ast.WindowSpec) groupedWindow := make(map[*ast.WindowSpec][]*ast.WindowFuncExpr) orderedSpec := make([]*ast.WindowSpec, 0, len(windowFuncs)) for _, windowFunc := range windowFuncs { @@ -5383,10 +5613,26 @@ func (b *PlanBuilder) groupWindowFuncs(windowFuncs []*ast.WindowFuncExpr) (map[* groupedWindow[spec] = append(groupedWindow[spec], windowFunc) orderedSpec = appendIfAbsentWindowSpec(orderedSpec, spec) } else { + var updatedSpec *ast.WindowSpec if _, ok := updatedSpecMap[name]; !ok { - updatedSpecMap[name] = newSpec + updatedSpecMap[name] = []*ast.WindowSpec{newSpec} + updatedSpec = newSpec + } else { + for _, spec := range updatedSpecMap[name] { + eq, err := specEqual(spec, newSpec) + if err != nil { + return nil, nil, err + } + if eq { + updatedSpec = spec + break + } + } + if updatedSpec == nil { + updatedSpec = newSpec + updatedSpecMap[name] = append(updatedSpecMap[name], newSpec) + } } - updatedSpec := updatedSpecMap[name] groupedWindow[updatedSpec] = append(groupedWindow[updatedSpec], windowFunc) orderedSpec = appendIfAbsentWindowSpec(orderedSpec, updatedSpec) } @@ -5513,10 +5759,14 @@ func collectTableName(node ast.ResultSetNode, updatableName *map[string]bool, in if s, canUpdate = x.Source.(*ast.TableName); canUpdate { if name == "" { name = s.Schema.L + "." + s.Name.L + // it may be a CTE + if s.Schema.L == "" { + name = s.Name.L + } } (*info)[name] = s } - (*updatableName)[name] = canUpdate + (*updatableName)[name] = canUpdate && s.Schema.L != "" } } @@ -5596,3 +5846,299 @@ func containDifferentJoinTypes(preferJoinType uint) bool { } return cnt > 1 } + +func (b *PlanBuilder) buildCte(ctx context.Context, cte *ast.CommonTableExpression, isRecursive bool) (p LogicalPlan, err error) { + if isRecursive { + // buildingRecursivePartForCTE likes a stack. We save it before building a recursive CTE and restore it after building. + // We need a stack because we need to handle the nested recursive CTE. And buildingRecursivePartForCTE indicates the innermost CTE. + saveCheck := b.buildingRecursivePartForCTE + b.buildingRecursivePartForCTE = false + err = b.buildRecursiveCTE(ctx, cte.Query.Query) + if err != nil { + return nil, err + } + b.buildingRecursivePartForCTE = saveCheck + } else { + p, err = b.buildResultSetNode(ctx, cte.Query.Query) + if err != nil { + return nil, err + } + + p, err = b.adjustCTEPlanOutputName(p, cte) + if err != nil { + return nil, err + } + + cInfo := b.outerCTEs[len(b.outerCTEs)-1] + cInfo.seedLP = p + } + return nil, nil +} + +// buildRecursiveCTE handles the with clause `with recursive xxx as xx`. +func (b *PlanBuilder) buildRecursiveCTE(ctx context.Context, cte ast.ResultSetNode) error { + cInfo := b.outerCTEs[len(b.outerCTEs)-1] + switch x := (cte).(type) { + case *ast.SetOprStmt: + // 1. Handle the WITH clause if exists. + if x.With != nil { + l := len(b.outerCTEs) + defer func() { + b.outerCTEs = b.outerCTEs[:l] + sw := x.With + x.With = sw + }() + err := b.buildWith(ctx, x.With) + if err != nil { + return err + } + } + // Set it to nil, so that when builds the seed part, it won't build again. Reset it in defer so that the AST doesn't change after this function. + x.With = nil + + // 2. Build plans for each part of SetOprStmt. + recursive := make([]LogicalPlan, 0) + tmpAfterSetOptsForRecur := []*ast.SetOprType{nil} + + expectSeed := true + for i := 0; i < len(x.SelectList.Selects); i++ { + var p LogicalPlan + var err error + + var afterOpr *ast.SetOprType + switch y := x.SelectList.Selects[i].(type) { + case *ast.SelectStmt: + p, err = b.buildSelect(ctx, y) + afterOpr = y.AfterSetOperator + case *ast.SetOprSelectList: + p, err = b.buildSetOpr(ctx, &ast.SetOprStmt{SelectList: y, With: y.With}) + afterOpr = y.AfterSetOperator + } + + if expectSeed { + if cInfo.useRecursive { + // 3. If it fail to build a plan, it may be the recursive part. Then we build the seed part plan, and rebuild it. + if i == 0 { + return ErrCTERecursiveRequiresNonRecursiveFirst.GenWithStackByArgs(cInfo.def.Name.String()) + } + + // It's the recursive part. Build the seed part, and build this recursive part again. + // Before we build the seed part, do some checks. + if x.OrderBy != nil { + return ErrNotSupportedYet.GenWithStackByArgs("ORDER BY over UNION in recursive Common Table Expression") + } + // Limit clause is for the whole CTE instead of only for the seed part. + oriLimit := x.Limit + x.Limit = nil + + // Check union type. + if afterOpr != nil { + if *afterOpr != ast.Union && *afterOpr != ast.UnionAll { + return ErrNotSupportedYet.GenWithStackByArgs(fmt.Sprintf("%s between seed part and recursive part, hint: The operator between seed part and recursive part must bu UNION[DISTINCT] or UNION ALL", afterOpr.String())) + } + cInfo.isDistinct = *afterOpr == ast.Union + } + + expectSeed = false + cInfo.useRecursive = false + + // Build seed part plan. + saveSelect := x.SelectList.Selects + x.SelectList.Selects = x.SelectList.Selects[:i] + p, err = b.buildSetOpr(ctx, x) + if err != nil { + return err + } + x.SelectList.Selects = saveSelect + p, err = b.adjustCTEPlanOutputName(p, cInfo.def) + if err != nil { + return err + } + cInfo.seedLP = p + + // Rebuild the plan. + i-- + b.buildingRecursivePartForCTE = true + x.Limit = oriLimit + continue + } + if err != nil { + return err + } + } else { + if err != nil { + return err + } + if afterOpr != nil { + if *afterOpr != ast.Union && *afterOpr != ast.UnionAll { + return ErrNotSupportedYet.GenWithStackByArgs(fmt.Sprintf("%s between recursive part's selects, hint: The operator between recursive part's selects must bu UNION[DISTINCT] or UNION ALL", afterOpr.String())) + } + } + if !cInfo.useRecursive { + return ErrCTERecursiveRequiresNonRecursiveFirst.GenWithStackByArgs(cInfo.def.Name.String()) + } + cInfo.useRecursive = false + recursive = append(recursive, p) + tmpAfterSetOptsForRecur = append(tmpAfterSetOptsForRecur, afterOpr) + } + } + + if len(recursive) == 0 { + // In this case, even if SQL specifies "WITH RECURSIVE", the CTE is non-recursive. + p, err := b.buildSetOpr(ctx, x) + if err != nil { + return err + } + p, err = b.adjustCTEPlanOutputName(p, cInfo.def) + if err != nil { + return err + } + cInfo.seedLP = p + return nil + } + + // Build the recursive part's logical plan. + recurPart, err := b.buildUnion(ctx, recursive, tmpAfterSetOptsForRecur) + if err != nil { + return err + } + recurPart, err = b.buildProjection4CTEUnion(ctx, cInfo.seedLP, recurPart) + if err != nil { + return err + } + // 4. Finally, we get the seed part plan and recursive part plan. + cInfo.recurLP = recurPart + // Only need to handle limit if x is SetOprStmt. + if x.Limit != nil { + limit, err := b.buildLimit(cInfo.seedLP, x.Limit) + if err != nil { + return err + } + limit.SetChildren(limit.Children()[:0]...) + cInfo.limitLP = limit + } + return nil + default: + p, err := b.buildResultSetNode(ctx, x) + if err != nil { + // Refine the error message. + if errors.ErrorEqual(err, ErrCTERecursiveRequiresNonRecursiveFirst) { + err = ErrCTERecursiveRequiresUnion.GenWithStackByArgs(cInfo.def.Name.String()) + } + return err + } + p, err = b.adjustCTEPlanOutputName(p, cInfo.def) + if err != nil { + return err + } + cInfo.seedLP = p + return nil + } +} + +func (b *PlanBuilder) adjustCTEPlanOutputName(p LogicalPlan, def *ast.CommonTableExpression) (LogicalPlan, error) { + outPutNames := p.OutputNames() + for _, name := range outPutNames { + name.TblName = def.Name + name.DBName = model.NewCIStr(b.ctx.GetSessionVars().CurrentDB) + } + if len(def.ColNameList) > 0 { + if len(def.ColNameList) != len(p.OutputNames()) { + return nil, ddl.ErrViewWrongList + } + for i, n := range def.ColNameList { + outPutNames[i].ColName = n + } + } + p.SetOutputNames(outPutNames) + return p, nil +} + +// prepareCTECheckForSubQuery prepares the check that the recursive CTE can't be referenced in subQuery. It's used before building a subQuery. +// For example: with recursive cte(n) as (select 1 union select * from (select * from cte) c1) select * from cte; +func (b *PlanBuilder) prepareCTECheckForSubQuery() []*cteInfo { + modifiedCTE := make([]*cteInfo, 0) + for _, cte := range b.outerCTEs { + if cte.isBuilding && !cte.enterSubquery { + cte.enterSubquery = true + modifiedCTE = append(modifiedCTE, cte) + } + } + return modifiedCTE +} + +// resetCTECheckForSubQuery resets the related variable. It's used after leaving a subQuery. +func resetCTECheckForSubQuery(ci []*cteInfo) { + for _, cte := range ci { + cte.enterSubquery = false + } +} + +// genCTETableNameForError find the nearest CTE name. +func (b *PlanBuilder) genCTETableNameForError() string { + name := "" + for i := len(b.outerCTEs) - 1; i >= 0; i-- { + if b.outerCTEs[i].isBuilding { + name = b.outerCTEs[i].def.Name.String() + break + } + } + return name +} + +func (b *PlanBuilder) buildWith(ctx context.Context, w *ast.WithClause) error { + // Check CTE name must be unique. + nameMap := make(map[string]struct{}) + for _, cte := range w.CTEs { + if _, ok := nameMap[cte.Name.L]; ok { + return ErrNonUniqTable + } + nameMap[cte.Name.L] = struct{}{} + } + for _, cte := range w.CTEs { + b.outerCTEs = append(b.outerCTEs, &cteInfo{def: cte, nonRecursive: !w.IsRecursive, isBuilding: true, storageID: b.allocIDForCTEStorage}) + b.allocIDForCTEStorage++ + saveFlag := b.optFlag + // Init the flag to flagPrunColumns, otherwise it's missing. + b.optFlag = flagPrunColumns + _, err := b.buildCte(ctx, cte, w.IsRecursive) + if err != nil { + return err + } + b.outerCTEs[len(b.outerCTEs)-1].optFlag = b.optFlag + b.outerCTEs[len(b.outerCTEs)-1].isBuilding = false + b.optFlag = saveFlag + } + return nil +} + +func (b *PlanBuilder) buildProjection4CTEUnion(ctx context.Context, seed LogicalPlan, recur LogicalPlan) (LogicalPlan, error) { + if seed.Schema().Len() != recur.Schema().Len() { + return nil, ErrWrongNumberOfColumnsInSelect.GenWithStackByArgs() + } + exprs := make([]expression.Expression, len(seed.Schema().Columns)) + resSchema := getResultCTESchema(seed.Schema(), b.ctx.GetSessionVars()) + for i, col := range recur.Schema().Columns { + if !resSchema.Columns[i].RetType.Equal(col.RetType) { + exprs[i] = expression.BuildCastFunction4Union(b.ctx, col, resSchema.Columns[i].RetType) + } else { + exprs[i] = col + } + } + b.optFlag |= flagEliminateProjection + proj := LogicalProjection{Exprs: exprs, AvoidColumnEvaluator: true}.Init(b.ctx, b.getSelectOffset()) + proj.SetSchema(resSchema) + proj.SetChildren(recur) + return proj, nil +} + +// The recursive part/CTE's schema is nullable, and the UID should be unique. +func getResultCTESchema(seedSchema *expression.Schema, svar *variable.SessionVars) *expression.Schema { + res := seedSchema.Clone() + for _, col := range res.Columns { + col.RetType = col.RetType.Clone() + col.UniqueID = svar.AllocPlanColumnID() + col.RetType.Flag &= ^mysql.NotNullFlag + } + return res +} diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index 11a116bb4fac8..1fdf5f3ab83ca 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -457,7 +457,7 @@ func (s *testPlanSuite) TestSubquery(c *C) { stmt, err := s.ParseOneStmt(ca, "", "") c.Assert(err, IsNil, comment) - err = Preprocess(s.ctx, stmt, s.is) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil) p, _, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil) @@ -483,7 +483,7 @@ func (s *testPlanSuite) TestPlanBuilder(c *C) { c.Assert(err, IsNil, comment) s.ctx.GetSessionVars().SetHashJoinConcurrency(1) - err = Preprocess(s.ctx, stmt, s.is) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil) p, _, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil) @@ -848,7 +848,7 @@ func (s *testPlanSuite) TestValidate(c *C) { comment := Commentf("for %s", sql) stmt, err := s.ParseOneStmt(sql, "", "") c.Assert(err, IsNil, comment) - err = Preprocess(s.ctx, stmt, s.is) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil) _, _, err = BuildLogicalPlan(ctx, s.ctx, stmt, s.is) if tt.err == nil { @@ -1257,7 +1257,7 @@ func (s *testPlanSuite) TestVisitInfo(c *C) { { sql: "RESTORE DATABASE test FROM 'local:///tmp/a'", ans: []visitInfo{ - {mysql.ExtendedPriv, "", "", "", ErrSpecificAccessDenied, false, "BACKUP_ADMIN", false}, + {mysql.ExtendedPriv, "", "", "", ErrSpecificAccessDenied, false, "RESTORE_ADMIN", false}, }, }, { @@ -1269,7 +1269,7 @@ func (s *testPlanSuite) TestVisitInfo(c *C) { { sql: "SHOW RESTORES", ans: []visitInfo{ - {mysql.ExtendedPriv, "", "", "", ErrSpecificAccessDenied, false, "BACKUP_ADMIN", false}, + {mysql.ExtendedPriv, "", "", "", ErrSpecificAccessDenied, false, "RESTORE_ADMIN", false}, }, }, { @@ -1296,14 +1296,22 @@ func (s *testPlanSuite) TestVisitInfo(c *C) { {mysql.ExtendedPriv, "", "", "", ErrSpecificAccessDenied, false, "BACKUP_ADMIN", true}, }, }, + { + sql: "RENAME USER user1 to user1_tmp", + ans: []visitInfo{ + {mysql.CreateUserPriv, "", "", "", ErrSpecificAccessDenied, false, "", false}, + }, + }, } for _, tt := range tests { comment := Commentf("for %s", tt.sql) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) - // to fix, Table 'test.ttt' doesn't exist - _ = Preprocess(s.ctx, stmt, s.is) + + // TODO: to fix, Table 'test.ttt' doesn't exist + _ = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) + builder, _ := NewPlanBuilder(MockContext(), s.is, &hint.BlockHintProcessor{}) builder.ctx.GetSessionVars().SetHashJoinConcurrency(1) _, err = builder.Build(context.TODO(), stmt) @@ -1383,7 +1391,7 @@ func (s *testPlanSuite) TestUnion(c *C) { comment := Commentf("case:%v sql:%s", i, tt) stmt, err := s.ParseOneStmt(tt, "", "") c.Assert(err, IsNil, comment) - err = Preprocess(s.ctx, stmt, s.is) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil) builder, _ := NewPlanBuilder(MockContext(), s.is, &hint.BlockHintProcessor{}) plan, err := builder.Build(ctx, stmt) @@ -1416,7 +1424,7 @@ func (s *testPlanSuite) TestTopNPushDown(c *C) { comment := Commentf("case:%v sql:%s", i, tt) stmt, err := s.ParseOneStmt(tt, "", "") c.Assert(err, IsNil, comment) - err = Preprocess(s.ctx, stmt, s.is) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil) builder, _ := NewPlanBuilder(MockContext(), s.is, &hint.BlockHintProcessor{}) p, err := builder.Build(ctx, stmt) @@ -1459,7 +1467,6 @@ func (s *testPlanSuite) TestNameResolver(c *C) { {"delete a from (select * from t ) as a, t", "[planner:1288]The target table a of the DELETE is not updatable"}, {"delete b from (select * from t ) as a, t", "[planner:1109]Unknown table 'b' in MULTI DELETE"}, {"select '' as fakeCol from t group by values(fakeCol)", "[planner:1054]Unknown column '' in 'VALUES() function'"}, - {"update t, (select * from t) as b set b.a = t.a", "[planner:1288]The target table b of the UPDATE is not updatable"}, {"select row_number() over () from t group by 1", "[planner:1056]Can't group on 'row_number() over ()'"}, {"select row_number() over () as x from t group by 1", "[planner:1056]Can't group on 'x'"}, {"select sum(a) as x from t group by 1", "[planner:1056]Can't group on 'x'"}, @@ -1491,7 +1498,7 @@ func (s *testPlanSuite) TestOuterJoinEliminator(c *C) { comment := Commentf("case:%v sql:%s", i, tt) stmt, err := s.ParseOneStmt(tt, "", "") c.Assert(err, IsNil, comment) - err = Preprocess(s.ctx, stmt, s.is) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil) builder, _ := NewPlanBuilder(MockContext(), s.is, &hint.BlockHintProcessor{}) p, err := builder.Build(ctx, stmt) @@ -1528,7 +1535,7 @@ func (s *testPlanSuite) TestSelectView(c *C) { comment := Commentf("case:%v sql:%s", i, tt.sql) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) - err = Preprocess(s.ctx, stmt, s.is) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil) builder, _ := NewPlanBuilder(MockContext(), s.is, &hint.BlockHintProcessor{}) p, err := builder.Build(ctx, stmt) @@ -1600,7 +1607,7 @@ func (s *testPlanSuite) optimize(ctx context.Context, sql string) (PhysicalPlan, if err != nil { return nil, nil, err } - err = Preprocess(s.ctx, stmt, s.is) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) if err != nil { return nil, nil, err } @@ -1692,7 +1699,7 @@ func (s *testPlanSuite) TestSkylinePruning(c *C) { comment := Commentf("case:%v sql:%s", i, tt.sql) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) - err = Preprocess(s.ctx, stmt, s.is) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil) builder, _ := NewPlanBuilder(MockContext(), s.is, &hint.BlockHintProcessor{}) p, err := builder.Build(ctx, stmt) @@ -1763,7 +1770,7 @@ func (s *testPlanSuite) TestFastPlanContextTables(c *C) { for _, tt := range tests { stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil) - err = Preprocess(s.ctx, stmt, s.is) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil) s.ctx.GetSessionVars().StmtCtx.Tables = nil p := TryFastPlan(s.ctx, stmt) @@ -1787,7 +1794,7 @@ func (s *testPlanSuite) TestUpdateEQCond(c *C) { }{ { sql: "select t1.a from t t1, t t2 where t1.a = t2.a+1", - best: "Join{DataScan(t1)->DataScan(t2)->Projection}(test.t.a,Column#25)->Projection", + best: "Join{DataScan(t1)->DataScan(t2)->Projection}(test.t.a,Column#25)->Projection->Projection", }, } ctx := context.TODO() @@ -1795,7 +1802,7 @@ func (s *testPlanSuite) TestUpdateEQCond(c *C) { comment := Commentf("case:%v sql:%s", i, tt.sql) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) - err = Preprocess(s.ctx, stmt, s.is) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil) builder, _ := NewPlanBuilder(MockContext(), s.is, &hint.BlockHintProcessor{}) p, err := builder.Build(ctx, stmt) @@ -1812,7 +1819,7 @@ func (s *testPlanSuite) TestConflictedJoinTypeHints(c *C) { ctx := context.TODO() stmt, err := s.ParseOneStmt(sql, "", "") c.Assert(err, IsNil) - err = Preprocess(s.ctx, stmt, s.is) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil) builder, _ := NewPlanBuilder(MockContext(), s.is, &hint.BlockHintProcessor{}) p, err := builder.Build(ctx, stmt) @@ -1833,7 +1840,7 @@ func (s *testPlanSuite) TestSimplyOuterJoinWithOnlyOuterExpr(c *C) { ctx := context.TODO() stmt, err := s.ParseOneStmt(sql, "", "") c.Assert(err, IsNil) - err = Preprocess(s.ctx, stmt, s.is) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil) builder, _ := NewPlanBuilder(MockContext(), s.is, &hint.BlockHintProcessor{}) p, err := builder.Build(ctx, stmt) @@ -1885,7 +1892,7 @@ func (s *testPlanSuite) TestResolvingCorrelatedAggregate(c *C) { comment := Commentf("case:%v sql:%s", i, tt.sql) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) - err = Preprocess(s.ctx, stmt, s.is) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil, comment) p, _, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil, comment) @@ -1927,7 +1934,8 @@ func (s *testPlanSuite) TestFastPathInvalidBatchPointGet(c *C) { comment := Commentf("case:%v sql:%s", i, tc.sql) stmt, err := s.ParseOneStmt(tc.sql, "", "") c.Assert(err, IsNil, comment) - c.Assert(Preprocess(s.ctx, stmt, s.is), IsNil, comment) + err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) + c.Assert(err, IsNil, comment) plan := TryFastPlan(s.ctx, stmt) if tc.fastPlan { c.Assert(plan, NotNil) diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index 09f7739e8017f..7186813a0fe04 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -1169,3 +1169,47 @@ type LogicalShowDDLJobs struct { JobNumber int64 } + +// CTEClass holds the information and plan for a CTE. Most of the fields in this struct are the same as cteInfo. +// But the cteInfo is used when building the plan, and CTEClass is used also for building the executor. +type CTEClass struct { + // The union between seed part and recursive part is DISTINCT or DISTINCT ALL. + IsDistinct bool + // seedPartLogicalPlan and recursivePartLogicalPlan are the logical plans for the seed part and recursive part of this CTE. + seedPartLogicalPlan LogicalPlan + recursivePartLogicalPlan LogicalPlan + // cteTask is the physical plan for this CTE, is a wrapper of the PhysicalCTE. + cteTask task + // storageID for this CTE. + IDForStorage int + // optFlag is the optFlag for the whole CTE. + optFlag uint64 + HasLimit bool + LimitBeg uint64 + LimitEnd uint64 +} + +// LogicalCTE is for CTE. +type LogicalCTE struct { + logicalSchemaProducer + + cte *CTEClass + cteAsName model.CIStr +} + +// LogicalCTETable is for CTE table +type LogicalCTETable struct { + logicalSchemaProducer + + name string + idForStorage int +} + +// ExtractCorrelatedCols implements LogicalPlan interface. +func (p *LogicalCTE) ExtractCorrelatedCols() []*expression.CorrelatedColumn { + corCols := ExtractCorrelatedCols4LogicalPlan(p.cte.seedPartLogicalPlan) + if p.cte.recursivePartLogicalPlan != nil { + corCols = append(corCols, ExtractCorrelatedCols4LogicalPlan(p.cte.recursivePartLogicalPlan)...) + } + return corCols +} diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index 2ad7fc1136d5b..59c228767171a 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/lock" "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/privilege" @@ -102,13 +103,13 @@ func CheckPrivilege(activeRoles []*auth.RoleIdentity, pm privilege.Manager, vs [ if v.privilege == mysql.ExtendedPriv { if !pm.RequestDynamicVerification(activeRoles, v.dynamicPriv, v.dynamicWithGrant) { if v.err == nil { - return ErrPrivilegeCheckFail + return ErrPrivilegeCheckFail.GenWithStackByArgs(v.dynamicPriv) } return v.err } } else if !pm.RequestVerification(activeRoles, v.db, v.table, v.column, v.privilege) { if v.err == nil { - return ErrPrivilegeCheckFail + return ErrPrivilegeCheckFail.GenWithStackByArgs(v.privilege.String()) } return v.err } @@ -156,9 +157,34 @@ func DoOptimize(ctx context.Context, sctx sessionctx.Context, flag uint64, logic return finalPlan, cost, nil } +// mergeContinuousSelections merge continuous selections which may occur after changing plans. +func mergeContinuousSelections(p PhysicalPlan) { + if sel, ok := p.(*PhysicalSelection); ok { + for { + childSel := sel.children[0] + if tmp, ok := childSel.(*PhysicalSelection); ok { + sel.Conditions = append(sel.Conditions, tmp.Conditions...) + sel.SetChild(0, tmp.children[0]) + } else { + break + } + } + } + for _, child := range p.Children() { + mergeContinuousSelections(child) + } + // merge continuous selections in a coprocessor task of tiflash + tableReader, isTableReader := p.(*PhysicalTableReader) + if isTableReader && tableReader.StoreType == kv.TiFlash { + mergeContinuousSelections(tableReader.tablePlan) + tableReader.TablePlans = flattenPushDownPlan(tableReader.tablePlan) + } +} + func postOptimize(sctx sessionctx.Context, plan PhysicalPlan) PhysicalPlan { plan = eliminatePhysicalProjection(plan) plan = InjectExtraProjection(plan) + mergeContinuousSelections(plan) plan = eliminateUnionScanAndLock(sctx, plan) plan = enableParallelApply(sctx, plan) return plan diff --git a/planner/core/partition_pruner_test.go b/planner/core/partition_pruner_test.go index dd614239de55d..e151efdecce03 100644 --- a/planner/core/partition_pruner_test.go +++ b/planner/core/partition_pruner_test.go @@ -327,7 +327,7 @@ func (s *testPartitionPruneSuit) TestListColumnsPartitionPrunerRandom(c *C) { tk1.MustExec(insert) // Test query without condition - query := fmt.Sprintf("select * from t1 order by id,a,b") + query := "select * from t1 order by id,a,b" tk.MustQuery(query).Check(tk1.MustQuery(query).Rows()) } @@ -467,9 +467,9 @@ func (s *testPartitionPruneSuit) TestRangePartitionPredicatePruner(c *C) { tk.Se.GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeIntOnly tk.MustExec(`create table t (a int(11) default null) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin partition by range(a) ( - partition p0 values less than (1), - partition p1 values less than (2), - partition p2 values less than (3), + partition p0 values less than (1), + partition p1 values less than (2), + partition p2 values less than (3), partition p_max values less than (maxvalue));`) var input []string diff --git a/planner/core/pb_to_plan.go b/planner/core/pb_to_plan.go index 3cfaf6708affe..a453596a0f288 100644 --- a/planner/core/pb_to_plan.go +++ b/planner/core/pb_to_plan.go @@ -255,15 +255,15 @@ func (b *PBPlanBuilder) pbToKill(e *tipb.Executor) (PhysicalPlan, error) { return &PhysicalSimpleWrapper{Inner: simple}, nil } -func (b *PBPlanBuilder) predicatePushDown(p PhysicalPlan, predicates []expression.Expression) ([]expression.Expression, PhysicalPlan) { - if p == nil { - return predicates, p +func (b *PBPlanBuilder) predicatePushDown(physicalPlan PhysicalPlan, predicates []expression.Expression) ([]expression.Expression, PhysicalPlan) { + if physicalPlan == nil { + return predicates, physicalPlan } - switch p.(type) { + switch plan := physicalPlan.(type) { case *PhysicalMemTable: - memTable := p.(*PhysicalMemTable) + memTable := plan if memTable.Extractor == nil { - return predicates, p + return predicates, plan } names := make([]*types.FieldName, 0, len(memTable.Columns)) for _, col := range memTable.Columns { @@ -284,8 +284,8 @@ func (b *PBPlanBuilder) predicatePushDown(p PhysicalPlan, predicates []expressio predicates = memTable.Extractor.Extract(b.sctx, memTable.schema, names, predicates) return predicates, memTable case *PhysicalSelection: - selection := p.(*PhysicalSelection) - conditions, child := b.predicatePushDown(p.Children()[0], selection.Conditions) + selection := plan + conditions, child := b.predicatePushDown(plan.Children()[0], selection.Conditions) if len(conditions) > 0 { selection.Conditions = conditions selection.SetChildren(child) @@ -293,10 +293,10 @@ func (b *PBPlanBuilder) predicatePushDown(p PhysicalPlan, predicates []expressio } return predicates, child default: - if children := p.Children(); len(children) > 0 { + if children := plan.Children(); len(children) > 0 { _, child := b.predicatePushDown(children[0], nil) - p.SetChildren(child) + plan.SetChildren(child) } - return predicates, p + return predicates, plan } } diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 6a8993286e36a..419c12715713c 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -249,7 +249,7 @@ func (s *testPlanSuite) TestDAGPlanBuilderBasePhysicalPlan(c *C) { stmt, err := s.ParseOneStmt(tt, "", "") c.Assert(err, IsNil, comment) - err = core.Preprocess(se, stmt, s.is) + err = core.Preprocess(se, stmt, core.WithPreprocessorReturn(&core.PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil) p, _, err := planner.Optimize(context.TODO(), se, stmt, s.is) c.Assert(err, IsNil) @@ -1427,7 +1427,7 @@ func (s *testPlanSuite) TestDAGPlanBuilderSplitAvg(c *C) { stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) - err = core.Preprocess(se, stmt, s.is) + err = core.Preprocess(se, stmt, core.WithPreprocessorReturn(&core.PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil) p, _, err := planner.Optimize(context.TODO(), se, stmt, s.is) c.Assert(err, IsNil, comment) @@ -1720,8 +1720,8 @@ func (s *testPlanSuite) TestEnumIndex(c *C) { tk := testkit.NewTestKit(c, store) tk.MustExec("use test") tk.MustExec("drop table if exists t") - tk.MustExec("create table t(e enum('c','b','a'), index idx(e))") - tk.MustExec("insert into t values(1),(2),(3);") + tk.MustExec("create table t(e enum('c','b','a',''), index idx(e))") + tk.MustExec("insert ignore into t values(0),(1),(2),(3),(4);") for i, ts := range input { s.testData.OnRecord(func() { diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index 43304971b4680..d3169c865e07b 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -14,6 +14,8 @@ package core import ( + "fmt" + "strconv" "unsafe" "github.com/pingcap/errors" @@ -31,6 +33,7 @@ import ( "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/ranger" + "github.com/pingcap/tidb/util/stringutil" "github.com/pingcap/tipb/go-tipb" ) @@ -108,7 +111,10 @@ func (p *PhysicalTableReader) GetTableScan() *PhysicalTableScan { } else if chCnt == 1 { curPlan = curPlan.Children()[0] } else { - join := curPlan.(*PhysicalHashJoin) + join, ok := curPlan.(*PhysicalHashJoin) + if !ok { + return nil + } curPlan = join.children[1-join.globalChildIndex] } } @@ -883,6 +889,18 @@ type PhysicalExchangeReceiver struct { basePhysicalPlan Tasks []*kv.MPPTask + frags []*Fragment +} + +// Clone implment PhysicalPlan interface. +func (p *PhysicalExchangeReceiver) Clone() (PhysicalPlan, error) { + np := new(PhysicalExchangeReceiver) + base, err := p.basePhysicalPlan.cloneWithSelf(np) + if err != nil { + return nil, errors.Trace(err) + } + np.basePhysicalPlan = *base + return np, nil } // GetExchangeSender return the connected sender of this receiver. We assume that its child must be a receiver. @@ -897,10 +915,21 @@ type PhysicalExchangeSender struct { TargetTasks []*kv.MPPTask ExchangeType tipb.ExchangeType HashCols []*expression.Column - // Tasks is the mpp task for current PhysicalExchangeSender + // Tasks is the mpp task for current PhysicalExchangeSender. Tasks []*kv.MPPTask +} - Fragment *Fragment +// Clone implment PhysicalPlan interface. +func (p *PhysicalExchangeSender) Clone() (PhysicalPlan, error) { + np := new(PhysicalExchangeSender) + base, err := p.basePhysicalPlan.cloneWithSelf(np) + if err != nil { + return nil, errors.Trace(err) + } + np.basePhysicalPlan = *base + np.ExchangeType = p.ExchangeType + np.HashCols = p.HashCols + return np, nil } // Clone implements PhysicalPlan interface. @@ -911,9 +940,7 @@ func (p *PhysicalMergeJoin) Clone() (PhysicalPlan, error) { return nil, err } cloned.basePhysicalJoin = *base - for _, cf := range p.CompareFuncs { - cloned.CompareFuncs = append(cloned.CompareFuncs, cf) - } + cloned.CompareFuncs = append(cloned.CompareFuncs, p.CompareFuncs...) cloned.Desc = p.Desc return cloned, nil } @@ -951,6 +978,19 @@ func (p *PhysicalLimit) Clone() (PhysicalPlan, error) { // PhysicalUnionAll is the physical operator of UnionAll. type PhysicalUnionAll struct { physicalSchemaProducer + + mpp bool +} + +// Clone implements PhysicalPlan interface. +func (p *PhysicalUnionAll) Clone() (PhysicalPlan, error) { + cloned := new(PhysicalUnionAll) + base, err := p.physicalSchemaProducer.cloneWithSelf(cloned) + if err != nil { + return nil, err + } + cloned.physicalSchemaProducer = *base + return cloned, nil } // AggMppRunMode defines the running mode of aggregation in MPP @@ -1008,7 +1048,7 @@ func (p *basePhysicalAgg) numDistinctFunc() (num int) { return } -func (p *basePhysicalAgg) getAggFuncCostFactor() (factor float64) { +func (p *basePhysicalAgg) getAggFuncCostFactor(isMPP bool) (factor float64) { factor = 0.0 for _, agg := range p.AggFuncs { if fac, ok := aggFuncFactor[agg.Name]; ok { @@ -1018,7 +1058,15 @@ func (p *basePhysicalAgg) getAggFuncCostFactor() (factor float64) { } } if factor == 0 { - factor = 1.0 + if isMPP { + // The default factor 1.0 will lead to 1-phase agg in pseudo stats settings. + // But in mpp cases, 2-phase is more usual. So we change this factor. + // TODO: This is still a little tricky and might cause regression. We should + // calibrate these factors and polish our cost model in the future. + factor = aggFuncFactor[ast.AggFuncFirstRow] + } else { + factor = 1.0 + } } return } @@ -1361,3 +1409,83 @@ func NewTableSampleInfo(node *ast.TableSample, fullSchema *expression.Schema, pt Partitions: pt, } } + +// PhysicalCTE is for CTE. +type PhysicalCTE struct { + physicalSchemaProducer + + SeedPlan PhysicalPlan + RecurPlan PhysicalPlan + CTE *CTEClass + cteAsName model.CIStr +} + +// PhysicalCTETable is for CTE table. +type PhysicalCTETable struct { + physicalSchemaProducer + + IDForStorage int +} + +// ExtractCorrelatedCols implements PhysicalPlan interface. +func (p *PhysicalCTE) ExtractCorrelatedCols() []*expression.CorrelatedColumn { + corCols := ExtractCorrelatedCols4PhysicalPlan(p.SeedPlan) + if p.RecurPlan != nil { + corCols = append(corCols, ExtractCorrelatedCols4PhysicalPlan(p.RecurPlan)...) + } + return corCols +} + +// AccessObject implements physicalScan interface. +func (p *PhysicalCTE) AccessObject(normalized bool) string { + return fmt.Sprintf("CTE:%s", p.cteAsName.L) +} + +// OperatorInfo implements dataAccesser interface. +func (p *PhysicalCTE) OperatorInfo(normalized bool) string { + return fmt.Sprintf("data:%s", (*CTEDefinition)(p).ExplainID()) +} + +// ExplainInfo implements Plan interface. +func (p *PhysicalCTE) ExplainInfo() string { + return p.AccessObject(false) + ", " + p.OperatorInfo(false) +} + +// ExplainID overrides the ExplainID. +func (p *PhysicalCTE) ExplainID() fmt.Stringer { + return stringutil.MemoizeStr(func() string { + if p.ctx != nil && p.ctx.GetSessionVars().StmtCtx.IgnoreExplainIDSuffix { + return p.TP() + } + return p.TP() + "_" + strconv.Itoa(p.id) + }) +} + +// ExplainInfo overrides the ExplainInfo +func (p *PhysicalCTETable) ExplainInfo() string { + return "Scan on CTE_" + strconv.Itoa(p.IDForStorage) +} + +// CTEDefinition is CTE definition for explain. +type CTEDefinition PhysicalCTE + +// ExplainInfo overrides the ExplainInfo +func (p *CTEDefinition) ExplainInfo() string { + var res string + if p.RecurPlan != nil { + res = "Recursive CTE" + } else { + res = "Non-Recursive CTE" + } + if p.CTE.HasLimit { + res += fmt.Sprintf(", limit(offset:%v, count:%v)", p.CTE.LimitBeg, p.CTE.LimitEnd-p.CTE.LimitBeg) + } + return res +} + +// ExplainID overrides the ExplainID. +func (p *CTEDefinition) ExplainID() fmt.Stringer { + return stringutil.MemoizeStr(func() string { + return "CTE_" + strconv.Itoa(p.CTE.IDForStorage) + }) +} diff --git a/planner/core/plan.go b/planner/core/plan.go index ea3dc74cca180..3c99884a80cf1 100644 --- a/planner/core/plan.go +++ b/planner/core/plan.go @@ -282,7 +282,7 @@ type LogicalPlan interface { // It will return: // 1. All possible plans that can match the required property. // 2. Whether the SQL hint can work. Return true if there is no hint. - exhaustPhysicalPlans(*property.PhysicalProperty) (physicalPlans []PhysicalPlan, hintCanWork bool) + exhaustPhysicalPlans(*property.PhysicalProperty) (physicalPlans []PhysicalPlan, hintCanWork bool, err error) // ExtractCorrelatedCols extracts correlated columns inside the LogicalPlan. ExtractCorrelatedCols() []*expression.CorrelatedColumn @@ -408,6 +408,9 @@ func (p *basePhysicalPlan) cloneWithSelf(newSelf PhysicalPlan) (*basePhysicalPla base.children = append(base.children, cloned) } for _, prop := range p.childrenReqProps { + if prop == nil { + continue + } base.childrenReqProps = append(base.childrenReqProps, prop.CloneEssentialFields()) } return base, nil @@ -437,8 +440,8 @@ func (p *basePhysicalPlan) ExtractCorrelatedCols() []*expression.CorrelatedColum return nil } -// GetlogicalTS4TaskMap get the logical TimeStamp now to help rollback the TaskMap changes after that. -func (p *baseLogicalPlan) GetlogicalTS4TaskMap() uint64 { +// GetLogicalTS4TaskMap get the logical TimeStamp now to help rollback the TaskMap changes after that. +func (p *baseLogicalPlan) GetLogicalTS4TaskMap() uint64 { p.ctx.GetSessionVars().StmtCtx.TaskMapBakTS += 1 return p.ctx.GetSessionVars().StmtCtx.TaskMapBakTS } @@ -480,7 +483,7 @@ func (p *baseLogicalPlan) storeTask(prop *property.PhysicalProperty, task task) key := prop.HashCode() if p.ctx.GetSessionVars().StmtCtx.StmtHints.TaskMapNeedBackUp() { // Empty string for useless change. - TS := p.GetlogicalTS4TaskMap() + TS := p.GetLogicalTS4TaskMap() p.taskMapBakTS = append(p.taskMapBakTS, TS) p.taskMapBak = append(p.taskMapBak, string(key)) } diff --git a/planner/core/plan_test.go b/planner/core/plan_test.go index 53f63f25fbc18..2f395c823542d 100644 --- a/planner/core/plan_test.go +++ b/planner/core/plan_test.go @@ -171,12 +171,12 @@ func (s *testPlanNormalize) TestNormalizedPlanForDiffStore(c *C) { normalizedPlanRows := getPlanRows(normalizedPlan) c.Assert(err, IsNil) s.testData.OnRecord(func() { - output[i].Digest = digest + output[i].Digest = digest.String() output[i].Plan = normalizedPlanRows }) compareStringSlice(c, normalizedPlanRows, output[i].Plan) - c.Assert(digest != lastDigest, IsTrue) - lastDigest = digest + c.Assert(digest.String() != lastDigest, IsTrue) + lastDigest = digest.String() } } @@ -211,6 +211,20 @@ func (s *testPlanNormalize) TestEncodeDecodePlan(c *C) { c.Assert(strings.Contains(planTree, "Insert"), IsTrue) c.Assert(strings.Contains(planTree, "time"), IsTrue) c.Assert(strings.Contains(planTree, "loops"), IsTrue) + + tk.MustExec("with cte(a) as (select 1) select * from cte") + planTree = getPlanTree() + c.Assert(strings.Contains(planTree, "CTE"), IsTrue) + c.Assert(strings.Contains(planTree, "1->Column#1"), IsTrue) + c.Assert(strings.Contains(planTree, "time"), IsTrue) + c.Assert(strings.Contains(planTree, "loops"), IsTrue) + + tk.MustExec("with cte(a) as (select 2) select * from cte") + planTree = getPlanTree() + c.Assert(strings.Contains(planTree, "CTE"), IsTrue) + c.Assert(strings.Contains(planTree, "2->Column#1"), IsTrue) + c.Assert(strings.Contains(planTree, "time"), IsTrue) + c.Assert(strings.Contains(planTree, "loops"), IsTrue) } func (s *testPlanNormalize) TestNormalizedDigest(c *C) { @@ -404,10 +418,10 @@ func testNormalizeDigest(tk *testkit.TestKit, c *C, sql1, sql2 string, isSame bo comment := Commentf("sql1: %v, sql2: %v\n%v !=\n%v\n", sql1, sql2, normalized1, normalized2) if isSame { c.Assert(normalized1, Equals, normalized2, comment) - c.Assert(digest1, Equals, digest2, comment) + c.Assert(digest1.String(), Equals, digest2.String(), comment) } else { c.Assert(normalized1 != normalized2, IsTrue, comment) - c.Assert(digest1 != digest2, IsTrue, comment) + c.Assert(digest1.String() != digest2.String(), IsTrue, comment) } } diff --git a/planner/core/plan_to_pb.go b/planner/core/plan_to_pb.go index e095f78dff960..1b6ef79bbc6bc 100644 --- a/planner/core/plan_to_pb.go +++ b/planner/core/plan_to_pb.go @@ -366,7 +366,25 @@ func (p *PhysicalHashJoin) ToPB(ctx sessionctx.Context, storeType kv.StoreType) if err != nil { return nil, err } - otherConditions, err := expression.ExpressionsToPBList(sc, p.OtherConditions, client) + + var otherConditionsInJoin expression.CNFExprs + var otherEqConditionsFromIn expression.CNFExprs + if p.JoinType == AntiSemiJoin { + for _, condition := range p.OtherConditions { + if expression.IsEQCondFromIn(condition) { + otherEqConditionsFromIn = append(otherEqConditionsFromIn, condition) + } else { + otherConditionsInJoin = append(otherConditionsInJoin, condition) + } + } + } else { + otherConditionsInJoin = p.OtherConditions + } + otherConditions, err := expression.ExpressionsToPBList(sc, otherConditionsInJoin, client) + if err != nil { + return nil, err + } + otherEqConditions, err := expression.ExpressionsToPBList(sc, otherEqConditionsFromIn, client) if err != nil { return nil, err } @@ -397,17 +415,18 @@ func (p *PhysicalHashJoin) ToPB(ctx sessionctx.Context, storeType kv.StoreType) buildFiledTypes = append(buildFiledTypes, expression.ToPBFieldType(retType)) } join := &tipb.Join{ - JoinType: pbJoinType, - JoinExecType: tipb.JoinExecType_TypeHashJoin, - InnerIdx: int64(p.InnerChildIdx), - LeftJoinKeys: left, - RightJoinKeys: right, - ProbeTypes: probeFiledTypes, - BuildTypes: buildFiledTypes, - LeftConditions: leftConditions, - RightConditions: rightConditions, - OtherConditions: otherConditions, - Children: []*tipb.Executor{lChildren, rChildren}, + JoinType: pbJoinType, + JoinExecType: tipb.JoinExecType_TypeHashJoin, + InnerIdx: int64(p.InnerChildIdx), + LeftJoinKeys: left, + RightJoinKeys: right, + ProbeTypes: probeFiledTypes, + BuildTypes: buildFiledTypes, + LeftConditions: leftConditions, + RightConditions: rightConditions, + OtherConditions: otherConditions, + OtherEqConditionsFromIn: otherEqConditions, + Children: []*tipb.Executor{lChildren, rChildren}, } executorID := p.ExplainID().String() diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 6fc98bc522508..6f36d6d0a9ad8 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" + "github.com/pingcap/parser/auth" "github.com/pingcap/parser/charset" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" @@ -36,11 +37,13 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/planner/util" + "github.com/pingcap/tidb/privilege" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/store/tikv" + "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" driver "github.com/pingcap/tidb/types/parser_driver" @@ -408,6 +411,30 @@ const ( renameView ) +type cteInfo struct { + def *ast.CommonTableExpression + // nonRecursive is used to decide if a CTE is visible. If a CTE start with `WITH RECURSIVE`, then nonRecursive is false, + // so it is visible in its definition. + nonRecursive bool + // useRecursive is used to record if a subSelect in CTE's definition refer to itself. This help us to identify the seed part and recursive part. + useRecursive bool + isBuilding bool + // isDistinct indicates if the union between seed part and recursive part is distinct or not. + isDistinct bool + // seedLP is the seed part's logical plan. + seedLP LogicalPlan + // recurLP is the recursive part's logical plan. + recurLP LogicalPlan + // storageID for this CTE. + storageID int + // optFlag is the optFlag for the whole CTE. + optFlag uint64 + // enterSubquery and recursiveRef are used to check "recursive table must be referenced only once, and not in any subquery". + enterSubquery bool + recursiveRef bool + limitLP LogicalPlan +} + // PlanBuilder builds Plan from an ast.Node. // It just builds the ast node straightforwardly. type PlanBuilder struct { @@ -415,6 +442,7 @@ type PlanBuilder struct { is infoschema.InfoSchema outerSchemas []*expression.Schema outerNames [][]*types.FieldName + outerCTEs []*cteInfo // colMapper stores the column that must be pre-resolved. colMapper map[*ast.ColumnNameExpr]int // visitInfo is used for privilege check. @@ -473,7 +501,9 @@ type PlanBuilder struct { // isForUpdateRead should be true in either of the following situations // 1. use `inside insert`, `update`, `delete` or `select for update` statement // 2. isolation level is RC - isForUpdateRead bool + isForUpdateRead bool + allocIDForCTEStorage int + buildingRecursivePartForCTE bool } type handleColHelper struct { @@ -581,6 +611,7 @@ func NewPlanBuilder(sctx sessionctx.Context, is infoschema.InfoSchema, processor return &PlanBuilder{ ctx: sctx, is: is, + outerCTEs: make([]*cteInfo, 0), colMapper: make(map[*ast.ColumnNameExpr]int), handleHelper: &handleColHelper{id2HandleMapStack: make([]map[int64][]HandleCols, 0)}, hintProcessor: processor, @@ -639,8 +670,9 @@ func (b *PlanBuilder) Build(ctx context.Context, node ast.Node) (Plan, error) { case *ast.BinlogStmt, *ast.FlushStmt, *ast.UseStmt, *ast.BRIEStmt, *ast.BeginStmt, *ast.CommitStmt, *ast.RollbackStmt, *ast.CreateUserStmt, *ast.SetPwdStmt, *ast.AlterInstanceStmt, *ast.GrantStmt, *ast.DropUserStmt, *ast.AlterUserStmt, *ast.RevokeStmt, *ast.KillStmt, *ast.DropStatsStmt, - *ast.GrantRoleStmt, *ast.RevokeRoleStmt, *ast.SetRoleStmt, *ast.SetDefaultRoleStmt, *ast.ShutdownStmt: - return b.buildSimple(node.(ast.StmtNode)) + *ast.GrantRoleStmt, *ast.RevokeRoleStmt, *ast.SetRoleStmt, *ast.SetDefaultRoleStmt, *ast.ShutdownStmt, + *ast.RenameUserStmt: + return b.buildSimple(ctx, node.(ast.StmtNode)) case ast.DDLNode: return b.buildDDL(ctx, x) case *ast.CreateBindingStmt: @@ -720,6 +752,10 @@ func (b *PlanBuilder) buildSet(ctx context.Context, v *ast.SetStmt) (Plan, error err := ErrSpecificAccessDenied.GenWithStackByArgs("SUPER or SYSTEM_VARIABLES_ADMIN") b.visitInfo = appendDynamicVisitInfo(b.visitInfo, "SYSTEM_VARIABLES_ADMIN", false, err) } + if sem.IsEnabled() && sem.IsInvisibleSysVar(strings.ToLower(vars.Name)) { + err := ErrSpecificAccessDenied.GenWithStackByArgs("RESTRICTED_VARIABLES_ADMIN") + b.visitInfo = appendDynamicVisitInfo(b.visitInfo, "RESTRICTED_VARIABLES_ADMIN", false, err) + } assign := &expression.VarAssignment{ Name: vars.Name, IsGlobal: vars.IsGlobal, @@ -1648,24 +1684,35 @@ func (b *PlanBuilder) buildAnalyzeFullSamplingTask( } idxInfos = append(idxInfos, idx) } + if as.Incremental { + b.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("The version 2 stats would ignore the INCREMENTAL keyword and do full sampling")) + } for i, id := range physicalIDs { if id == tbl.TableInfo.ID { id = -1 } - info := analyzeInfo{ + info := AnalyzeInfo{ DBName: tbl.Schema.O, TableName: tbl.Name.O, PartitionName: names[i], TableID: AnalyzeTableID{TableID: tbl.TableInfo.ID, PartitionID: id}, - Incremental: as.Incremental, + Incremental: false, StatsVersion: version, } - taskSlice = append(taskSlice, AnalyzeColumnsTask{ + newTask := AnalyzeColumnsTask{ + HandleCols: BuildHandleColsForAnalyze(b.ctx, tbl.TableInfo), ColsInfo: tbl.TableInfo.Columns, - analyzeInfo: info, + AnalyzeInfo: info, TblInfo: tbl.TableInfo, Indexes: idxInfos, - }) + } + if newTask.HandleCols == nil { + extraCol := model.NewExtraHandleColInfo() + // Always place _tidb_rowid at the end of colsInfo, this is corresponding to logics in `analyzeColumnsPushdown`. + newTask.ColsInfo = append(newTask.ColsInfo, extraCol) + newTask.HandleCols = &IntHandleCols{col: colInfoToColumn(extraCol, len(newTask.ColsInfo)-1)} + } + taskSlice = append(taskSlice, newTask) } return taskSlice } @@ -1696,7 +1743,7 @@ func (b *PlanBuilder) buildAnalyzeTable(as *ast.AnalyzeTableStmt, opts map[ast.A "If you want to switch to version 2 statistics, please first disable query feedback by setting feedback-probability to 0.0 in the config file.")) } } - if version == statistics.Version3 { + if version == statistics.Version2 { p.ColTasks = b.buildAnalyzeFullSamplingTask(as, p.ColTasks, physicalIDs, names, tbl, version) continue } @@ -1711,7 +1758,7 @@ func (b *PlanBuilder) buildAnalyzeTable(as *ast.AnalyzeTableStmt, opts map[ast.A if id == tbl.TableInfo.ID { id = -1 } - info := analyzeInfo{ + info := AnalyzeInfo{ DBName: tbl.Schema.O, TableName: tbl.Name.O, PartitionName: names[i], @@ -1721,7 +1768,7 @@ func (b *PlanBuilder) buildAnalyzeTable(as *ast.AnalyzeTableStmt, opts map[ast.A } p.IdxTasks = append(p.IdxTasks, AnalyzeIndexTask{ IndexInfo: idx, - analyzeInfo: info, + AnalyzeInfo: info, TblInfo: tbl.TableInfo, }) } @@ -1732,7 +1779,7 @@ func (b *PlanBuilder) buildAnalyzeTable(as *ast.AnalyzeTableStmt, opts map[ast.A if id == tbl.TableInfo.ID { id = -1 } - info := analyzeInfo{ + info := AnalyzeInfo{ DBName: tbl.Schema.O, TableName: tbl.Name.O, PartitionName: names[i], @@ -1744,7 +1791,7 @@ func (b *PlanBuilder) buildAnalyzeTable(as *ast.AnalyzeTableStmt, opts map[ast.A HandleCols: handleCols, CommonHandleInfo: commonHandleInfo, ColsInfo: colInfo, - analyzeInfo: info, + AnalyzeInfo: info, TblInfo: tbl.TableInfo, }) } @@ -1771,8 +1818,8 @@ func (b *PlanBuilder) buildAnalyzeIndex(as *ast.AnalyzeTableStmt, opts map[ast.A } b.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("The analyze version from the session is not compatible with the existing statistics of the table. Use the existing version instead")) } - if version == statistics.Version3 { - b.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("The version 3 would collect all statistics not only the selected indexes")) + if version == statistics.Version2 { + b.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("The version 2 would collect all statistics not only the selected indexes")) return b.buildAnalyzeTable(as, opts, version) } for _, idxName := range as.IndexNames { @@ -1784,14 +1831,14 @@ func (b *PlanBuilder) buildAnalyzeIndex(as *ast.AnalyzeTableStmt, opts map[ast.A if id == tblInfo.ID { id = -1 } - info := analyzeInfo{ + info := AnalyzeInfo{ DBName: as.TableNames[0].Schema.O, TableName: as.TableNames[0].Name.O, PartitionName: names[i], TableID: AnalyzeTableID{TableID: tblInfo.ID, PartitionID: id}, Incremental: as.Incremental, StatsVersion: version, } - p.ColTasks = append(p.ColTasks, AnalyzeColumnsTask{HandleCols: handleCols, analyzeInfo: info, TblInfo: tblInfo}) + p.ColTasks = append(p.ColTasks, AnalyzeColumnsTask{HandleCols: handleCols, AnalyzeInfo: info, TblInfo: tblInfo}) } continue } @@ -1804,7 +1851,7 @@ func (b *PlanBuilder) buildAnalyzeIndex(as *ast.AnalyzeTableStmt, opts map[ast.A if id == tblInfo.ID { id = -1 } - info := analyzeInfo{ + info := AnalyzeInfo{ DBName: as.TableNames[0].Schema.O, TableName: as.TableNames[0].Name.O, PartitionName: names[i], @@ -1812,7 +1859,7 @@ func (b *PlanBuilder) buildAnalyzeIndex(as *ast.AnalyzeTableStmt, opts map[ast.A Incremental: as.Incremental, StatsVersion: version, } - p.IdxTasks = append(p.IdxTasks, AnalyzeIndexTask{IndexInfo: idx, analyzeInfo: info, TblInfo: tblInfo}) + p.IdxTasks = append(p.IdxTasks, AnalyzeIndexTask{IndexInfo: idx, AnalyzeInfo: info, TblInfo: tblInfo}) } } return p, nil @@ -1836,8 +1883,8 @@ func (b *PlanBuilder) buildAnalyzeAllIndex(as *ast.AnalyzeTableStmt, opts map[as } b.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("The analyze version from the session is not compatible with the existing statistics of the table. Use the existing version instead")) } - if version == statistics.Version3 { - b.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("The version 3 would collect all statistics not only the selected indexes")) + if version == statistics.Version2 { + b.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("The version 2 would collect all statistics not only the selected indexes")) return b.buildAnalyzeTable(as, opts, version) } for _, idx := range tblInfo.Indices { @@ -1846,7 +1893,7 @@ func (b *PlanBuilder) buildAnalyzeAllIndex(as *ast.AnalyzeTableStmt, opts map[as if id == tblInfo.ID { id = -1 } - info := analyzeInfo{ + info := AnalyzeInfo{ DBName: as.TableNames[0].Schema.O, TableName: as.TableNames[0].Name.O, PartitionName: names[i], @@ -1854,7 +1901,7 @@ func (b *PlanBuilder) buildAnalyzeAllIndex(as *ast.AnalyzeTableStmt, opts map[as Incremental: as.Incremental, StatsVersion: version, } - p.IdxTasks = append(p.IdxTasks, AnalyzeIndexTask{IndexInfo: idx, analyzeInfo: info, TblInfo: tblInfo}) + p.IdxTasks = append(p.IdxTasks, AnalyzeIndexTask{IndexInfo: idx, AnalyzeInfo: info, TblInfo: tblInfo}) } } } @@ -1864,7 +1911,7 @@ func (b *PlanBuilder) buildAnalyzeAllIndex(as *ast.AnalyzeTableStmt, opts map[as if id == tblInfo.ID { id = -1 } - info := analyzeInfo{ + info := AnalyzeInfo{ DBName: as.TableNames[0].Schema.O, TableName: as.TableNames[0].Name.O, PartitionName: names[i], @@ -1872,7 +1919,7 @@ func (b *PlanBuilder) buildAnalyzeAllIndex(as *ast.AnalyzeTableStmt, opts map[as Incremental: as.Incremental, StatsVersion: version, } - p.ColTasks = append(p.ColTasks, AnalyzeColumnsTask{HandleCols: handleCols, analyzeInfo: info, TblInfo: tblInfo}) + p.ColTasks = append(p.ColTasks, AnalyzeColumnsTask{HandleCols: handleCols, AnalyzeInfo: info, TblInfo: tblInfo}) } } return p, nil @@ -1897,10 +1944,24 @@ var analyzeOptionDefault = map[ast.AnalyzeOptionType]uint64{ ast.AnalyzeOptNumSamples: 10000, } -func handleAnalyzeOptions(opts []ast.AnalyzeOpt) (map[ast.AnalyzeOptionType]uint64, error) { +var analyzeOptionDefaultV2 = map[ast.AnalyzeOptionType]uint64{ + ast.AnalyzeOptNumBuckets: 256, + ast.AnalyzeOptNumTopN: 500, + ast.AnalyzeOptCMSketchWidth: 2048, + ast.AnalyzeOptCMSketchDepth: 5, + ast.AnalyzeOptNumSamples: 100000, +} + +func handleAnalyzeOptions(opts []ast.AnalyzeOpt, statsVer int) (map[ast.AnalyzeOptionType]uint64, error) { optMap := make(map[ast.AnalyzeOptionType]uint64, len(analyzeOptionDefault)) - for key, val := range analyzeOptionDefault { - optMap[key] = val + if statsVer == statistics.Version1 { + for key, val := range analyzeOptionDefault { + optMap[key] = val + } + } else { + for key, val := range analyzeOptionDefaultV2 { + optMap[key] = val + } } for _, opt := range opts { if opt.Type == ast.AnalyzeOptNumTopN { @@ -1926,7 +1987,7 @@ func (b *PlanBuilder) buildAnalyze(as *ast.AnalyzeTableStmt) (Plan, error) { return nil, errors.Errorf("Only support fast analyze in tikv storage.") } statsVersion := b.ctx.GetSessionVars().AnalyzeVersion - if b.ctx.GetSessionVars().EnableFastAnalyze && statsVersion == statistics.Version2 { + if b.ctx.GetSessionVars().EnableFastAnalyze && statsVersion >= statistics.Version2 { return nil, errors.Errorf("Fast analyze hasn't reached General Availability and only support analyze version 1 currently.") } for _, tbl := range as.TableNames { @@ -1939,7 +2000,7 @@ func (b *PlanBuilder) buildAnalyze(as *ast.AnalyzeTableStmt) (Plan, error) { b.visitInfo = appendVisitInfo(b.visitInfo, mysql.InsertPriv, tbl.Schema.O, tbl.Name.O, "", insertErr) b.visitInfo = appendVisitInfo(b.visitInfo, mysql.SelectPriv, tbl.Schema.O, tbl.Name.O, "", selectErr) } - opts, err := handleAnalyzeOptions(as.AnalyzeOpts) + opts, err := handleAnalyzeOptions(as.AnalyzeOpts, statsVersion) if err != nil { return nil, err } @@ -2184,9 +2245,12 @@ func (b *PlanBuilder) buildShow(ctx context.Context, show *ast.ShowStmt) (Plan, case ast.ShowCreateView: err := ErrSpecificAccessDenied.GenWithStackByArgs("SHOW VIEW") b.visitInfo = appendVisitInfo(b.visitInfo, mysql.ShowViewPriv, show.Table.Schema.L, show.Table.Name.L, "", err) - case ast.ShowBackups, ast.ShowRestores: + case ast.ShowBackups: err := ErrSpecificAccessDenied.GenWithStackByArgs("SUPER or BACKUP_ADMIN") b.visitInfo = appendDynamicVisitInfo(b.visitInfo, "BACKUP_ADMIN", false, err) + case ast.ShowRestores: + err := ErrSpecificAccessDenied.GenWithStackByArgs("SUPER or RESTORE_ADMIN") + b.visitInfo = appendDynamicVisitInfo(b.visitInfo, "RESTORE_ADMIN", false, err) case ast.ShowTableNextRowId: p := &ShowNextRowID{TableName: show.Table} p.setSchemaAndNames(buildShowNextRowID()) @@ -2199,6 +2263,14 @@ func (b *PlanBuilder) buildShow(ctx context.Context, show *ast.ShowStmt) (Plan, err = ErrDBaccessDenied.GenWithStackByArgs(user.AuthUsername, user.AuthHostname, mysql.SystemDB) } b.visitInfo = appendVisitInfo(b.visitInfo, mysql.SelectPriv, mysql.SystemDB, "", "", err) + case ast.ShowRegions: + tableInfo, err := b.is.TableByName(show.Table.Schema, show.Table.Name) + if err != nil { + return nil, err + } + if tableInfo.Meta().TempTableType != model.TempTableNone { + return nil, ErrOptOnTemporaryTable.GenWithStackByArgs("show table regions") + } } schema, names := buildShowSchema(show, isView, isSequence) p.SetSchema(schema) @@ -2252,7 +2324,7 @@ func (b *PlanBuilder) buildShow(ctx context.Context, show *ast.ShowStmt) (Plan, return np, nil } -func (b *PlanBuilder) buildSimple(node ast.StmtNode) (Plan, error) { +func (b *PlanBuilder) buildSimple(ctx context.Context, node ast.StmtNode) (Plan, error) { p := &Simple{Statement: node} switch raw := node.(type) { @@ -2262,7 +2334,7 @@ func (b *PlanBuilder) buildSimple(node ast.StmtNode) (Plan, error) { case *ast.AlterInstanceStmt: err := ErrSpecificAccessDenied.GenWithStack("SUPER") b.visitInfo = appendVisitInfo(b.visitInfo, mysql.SuperPriv, "", "", "", err) - case *ast.AlterUserStmt: + case *ast.AlterUserStmt, *ast.RenameUserStmt: err := ErrSpecificAccessDenied.GenWithStackByArgs("CREATE USER") b.visitInfo = appendVisitInfo(b.visitInfo, mysql.CreateUserPriv, "", "", "", err) case *ast.GrantStmt: @@ -2274,11 +2346,23 @@ func (b *PlanBuilder) buildSimple(node ast.StmtNode) (Plan, error) { b.visitInfo = collectVisitInfoFromGrantStmt(b.ctx, b.visitInfo, raw) case *ast.BRIEStmt: p.setSchemaAndNames(buildBRIESchema()) - err := ErrSpecificAccessDenied.GenWithStackByArgs("SUPER or BACKUP_ADMIN") - b.visitInfo = appendDynamicVisitInfo(b.visitInfo, "BACKUP_ADMIN", false, err) - case *ast.GrantRoleStmt, *ast.RevokeRoleStmt: + if raw.Kind == ast.BRIEKindRestore { + err := ErrSpecificAccessDenied.GenWithStackByArgs("SUPER or RESTORE_ADMIN") + b.visitInfo = appendDynamicVisitInfo(b.visitInfo, "RESTORE_ADMIN", false, err) + } else { + err := ErrSpecificAccessDenied.GenWithStackByArgs("SUPER or BACKUP_ADMIN") + b.visitInfo = appendDynamicVisitInfo(b.visitInfo, "BACKUP_ADMIN", false, err) + } + case *ast.GrantRoleStmt: + err := ErrSpecificAccessDenied.GenWithStackByArgs("SUPER or ROLE_ADMIN") + b.visitInfo = appendDynamicVisitInfo(b.visitInfo, "ROLE_ADMIN", false, err) + case *ast.RevokeRoleStmt: err := ErrSpecificAccessDenied.GenWithStackByArgs("SUPER or ROLE_ADMIN") b.visitInfo = appendDynamicVisitInfo(b.visitInfo, "ROLE_ADMIN", false, err) + // Check if any of the users are RESTRICTED + for _, user := range raw.Users { + b.visitInfo = appendVisitInfoIsRestrictedUser(b.visitInfo, b.ctx, user, "RESTRICTED_USER_ADMIN") + } case *ast.RevokeStmt: b.visitInfo = collectVisitInfoFromRevokeStmt(b.ctx, b.visitInfo, raw) case *ast.KillStmt: @@ -2292,18 +2376,60 @@ func (b *PlanBuilder) buildSimple(node ast.StmtNode) (Plan, error) { err := ErrSpecificAccessDenied.GenWithStackByArgs("SUPER or CONNECTION_ADMIN") b.visitInfo = appendDynamicVisitInfo(b.visitInfo, "CONNECTION_ADMIN", false, err) } + b.visitInfo = appendVisitInfoIsRestrictedUser(b.visitInfo, b.ctx, &auth.UserIdentity{Username: pi.User, Hostname: pi.Host}, "RESTRICTED_CONNECTION_ADMIN") } } case *ast.UseStmt: if raw.DBName == "" { return nil, ErrNoDB } + case *ast.DropUserStmt: + // The main privilege checks for DROP USER are currently performed in executor/simple.go + // because they use complex OR conditions (not supported by visitInfo). + for _, user := range raw.UserList { + b.visitInfo = appendVisitInfoIsRestrictedUser(b.visitInfo, b.ctx, user, "RESTRICTED_USER_ADMIN") + } + case *ast.SetPwdStmt: + if raw.User != nil { + b.visitInfo = appendVisitInfoIsRestrictedUser(b.visitInfo, b.ctx, raw.User, "RESTRICTED_USER_ADMIN") + } case *ast.ShutdownStmt: b.visitInfo = appendVisitInfo(b.visitInfo, mysql.ShutdownPriv, "", "", "", nil) + case *ast.BeginStmt: + readTS := b.ctx.GetSessionVars().TxnReadTS.UseTxnReadTS() + if raw.AsOf != nil { + startTS, err := calculateTsExpr(b.ctx, raw.AsOf) + if err != nil { + return nil, err + } + p.StaleTxnStartTS = startTS + } else if readTS > 0 { + p.StaleTxnStartTS = readTS + } } return p, nil } +// calculateTsExpr calculates the TsExpr of AsOfClause to get a StartTS. +func calculateTsExpr(sctx sessionctx.Context, asOfClause *ast.AsOfClause) (uint64, error) { + tsVal, err := evalAstExpr(sctx, asOfClause.TsExpr) + if err != nil { + return 0, err + } + toTypeTimestamp := types.NewFieldType(mysql.TypeTimestamp) + // We need at least the millionsecond here, so set fsp to 3. + toTypeTimestamp.Decimal = 3 + tsTimestamp, err := tsVal.ConvertTo(sctx.GetSessionVars().StmtCtx, toTypeTimestamp) + if err != nil { + return 0, err + } + tsTime, err := tsTimestamp.GetMysqlTime().GoTime(sctx.GetSessionVars().TimeZone) + if err != nil { + return 0, err + } + return oracle.GoTimeToTS(tsTime), nil +} + func collectVisitInfoFromRevokeStmt(sctx sessionctx.Context, vi []visitInfo, stmt *ast.RevokeStmt) []visitInfo { // To use REVOKE, you must have the GRANT OPTION privilege, // and you must have the privileges that you are granting. @@ -2339,6 +2465,10 @@ func collectVisitInfoFromRevokeStmt(sctx sessionctx.Context, vi []visitInfo, stm for _, priv := range allPrivs { vi = appendVisitInfo(vi, priv, dbName, tableName, "", nil) } + for _, u := range stmt.Users { + // For SEM, make sure the users are not restricted + vi = appendVisitInfoIsRestrictedUser(vi, sctx, u.User, "RESTRICTED_USER_ADMIN") + } if nonDynamicPrivilege { // Dynamic privileges use their own GRANT OPTION. If there were any non-dynamic privilege requests, // we need to attach the "GLOBAL" version of the GRANT OPTION. @@ -2347,6 +2477,20 @@ func collectVisitInfoFromRevokeStmt(sctx sessionctx.Context, vi []visitInfo, stm return vi } +// appendVisitInfoIsRestrictedUser appends additional visitInfo if the user has a +// special privilege called "RESTRICTED_USER_ADMIN". It only applies when SEM is enabled. +func appendVisitInfoIsRestrictedUser(visitInfo []visitInfo, sctx sessionctx.Context, user *auth.UserIdentity, priv string) []visitInfo { + if !sem.IsEnabled() { + return visitInfo + } + checker := privilege.GetPrivilegeManager(sctx) + if checker != nil && checker.RequestDynamicVerificationWithUser("RESTRICTED_USER_ADMIN", false, user) { + err := ErrSpecificAccessDenied.GenWithStackByArgs(priv) + visitInfo = appendDynamicVisitInfo(visitInfo, priv, false, err) + } + return visitInfo +} + func collectVisitInfoFromGrantStmt(sctx sessionctx.Context, vi []visitInfo, stmt *ast.GrantStmt) []visitInfo { // To use GRANT, you must have the GRANT OPTION privilege, // and you must have the privileges that you are granting. @@ -2517,7 +2661,7 @@ func (b *PlanBuilder) buildInsert(ctx context.Context, insert *ast.InsertStmt) ( givenPartitionSets[id] = struct{}{} } pt := tableInPlan.(table.PartitionedTable) - insertPlan.Table = tables.NewPartitionTableithGivenSets(pt, givenPartitionSets) + insertPlan.Table = tables.NewPartitionTableWithGivenSets(pt, givenPartitionSets) } else if len(insert.PartitionNames) != 0 { return nil, ErrPartitionClauseOnNonpartitioned } @@ -3027,6 +3171,9 @@ func (b *PlanBuilder) buildIndexAdvise(node *ast.IndexAdviseStmt) Plan { } func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) { + if node.Table.TableInfo.TempTableType != model.TempTableNone { + return nil, ErrOptOnTemporaryTable.GenWithStackByArgs("split table") + } if node.SplitSyntaxOpt != nil && node.SplitSyntaxOpt.HasPartition && node.Table.TableInfo.Partition == nil { return nil, ErrPartitionClauseOnNonpartitioned } @@ -3275,7 +3422,11 @@ func (b *PlanBuilder) buildDDL(ctx context.Context, node ast.DDLNode) (Plan, err authErr = ErrTableaccessDenied.GenWithStackByArgs("ALTER", b.ctx.GetSessionVars().User.AuthUsername, b.ctx.GetSessionVars().User.AuthHostname, v.Table.Name.L) } - b.visitInfo = appendVisitInfo(b.visitInfo, mysql.AlterPriv, v.Table.Schema.L, + dbName := v.Table.Schema.L + if dbName == "" { + dbName = b.ctx.GetSessionVars().CurrentDB + } + b.visitInfo = appendVisitInfo(b.visitInfo, mysql.AlterPriv, dbName, v.Table.Name.L, "", authErr) for _, spec := range v.Specs { if spec.Tp == ast.AlterTableRenameTable || spec.Tp == ast.AlterTableExchangePartition { @@ -3283,21 +3434,21 @@ func (b *PlanBuilder) buildDDL(ctx context.Context, node ast.DDLNode) (Plan, err authErr = ErrTableaccessDenied.GenWithStackByArgs("DROP", b.ctx.GetSessionVars().User.AuthUsername, b.ctx.GetSessionVars().User.AuthHostname, v.Table.Name.L) } - b.visitInfo = appendVisitInfo(b.visitInfo, mysql.DropPriv, v.Table.Schema.L, + b.visitInfo = appendVisitInfo(b.visitInfo, mysql.DropPriv, dbName, v.Table.Name.L, "", authErr) if b.ctx.GetSessionVars().User != nil { authErr = ErrTableaccessDenied.GenWithStackByArgs("CREATE", b.ctx.GetSessionVars().User.AuthUsername, b.ctx.GetSessionVars().User.AuthHostname, spec.NewTable.Name.L) } - b.visitInfo = appendVisitInfo(b.visitInfo, mysql.CreatePriv, spec.NewTable.Schema.L, + b.visitInfo = appendVisitInfo(b.visitInfo, mysql.CreatePriv, dbName, spec.NewTable.Name.L, "", authErr) if b.ctx.GetSessionVars().User != nil { authErr = ErrTableaccessDenied.GenWithStackByArgs("INSERT", b.ctx.GetSessionVars().User.AuthUsername, b.ctx.GetSessionVars().User.AuthHostname, spec.NewTable.Name.L) } - b.visitInfo = appendVisitInfo(b.visitInfo, mysql.InsertPriv, spec.NewTable.Schema.L, + b.visitInfo = appendVisitInfo(b.visitInfo, mysql.InsertPriv, dbName, spec.NewTable.Name.L, "", authErr) } else if spec.Tp == ast.AlterTableDropPartition { if b.ctx.GetSessionVars().User != nil { @@ -3353,6 +3504,13 @@ func (b *PlanBuilder) buildDDL(ctx context.Context, node ast.DDLNode) (Plan, err b.visitInfo = appendVisitInfo(b.visitInfo, mysql.IndexPriv, v.Table.Schema.L, v.Table.Name.L, "", authErr) case *ast.CreateTableStmt: + if v.TemporaryKeyword != ast.TemporaryNone { + for _, cons := range v.Constraints { + if cons.Tp == ast.ConstraintForeignKey { + return nil, infoschema.ErrCannotAddForeign + } + } + } if b.ctx.GetSessionVars().User != nil { authErr = ErrTableaccessDenied.GenWithStackByArgs("CREATE", b.ctx.GetSessionVars().User.AuthUsername, b.ctx.GetSessionVars().User.AuthHostname, v.Table.Name.L) diff --git a/planner/core/point_get_plan.go b/planner/core/point_get_plan.go index e2d74d92376ad..37747f33b0167 100644 --- a/planner/core/point_get_plan.go +++ b/planner/core/point_get_plan.go @@ -17,6 +17,7 @@ import ( "bytes" "fmt" math2 "math" + "sort" "github.com/pingcap/errors" "github.com/pingcap/parser/ast" @@ -246,8 +247,8 @@ func (p *PointGetPlan) GetCost(cols []*expression.Column) float64 { } else { rowSize = p.stats.HistColl.GetIndexAvgRowSize(p.ctx, cols, p.IndexInfo.Unique) } - cost += rowSize * sessVars.NetworkFactor - cost += sessVars.SeekFactor + cost += rowSize * sessVars.GetNetworkFactor(p.TblInfo) + cost += sessVars.GetSeekFactor(p.TblInfo) cost /= float64(sessVars.DistSQLScanConcurrency()) return cost } @@ -269,6 +270,7 @@ type BatchPointGetPlan struct { IdxCols []*expression.Column IdxColLens []int PartitionColPos int + PartitionExpr *tables.PartitionExpr KeepOrder bool Desc bool Lock bool @@ -426,8 +428,8 @@ func (p *BatchPointGetPlan) GetCost(cols []*expression.Column) float64 { rowCount = float64(len(p.IndexValues)) rowSize = p.stats.HistColl.GetIndexAvgRowSize(p.ctx, cols, p.IndexInfo.Unique) } - cost += rowCount * rowSize * sessVars.NetworkFactor - cost += rowCount * sessVars.SeekFactor + cost += rowCount * rowSize * sessVars.GetNetworkFactor(p.TblInfo) + cost += rowCount * sessVars.GetSeekFactor(p.TblInfo) cost /= float64(sessVars.DistSQLScanConcurrency()) return cost } @@ -531,10 +533,10 @@ func newBatchPointGetPlan( names []*types.FieldName, whereColNames []string, indexHints []*ast.IndexHint, ) *BatchPointGetPlan { statsInfo := &property.StatsInfo{RowCount: float64(len(patternInExpr.List))} - var partitionColName *ast.ColumnName + var partitionExpr *tables.PartitionExpr if tbl.GetPartitionInfo() != nil { - partitionColName = getHashPartitionColumnName(ctx, tbl) - if partitionColName == nil { + partitionExpr = getPartitionExpr(ctx, tbl) + if partitionExpr == nil { return nil } } @@ -576,9 +578,10 @@ func newBatchPointGetPlan( handleParams[i] = param } return BatchPointGetPlan{ - TblInfo: tbl, - Handles: handles, - HandleParams: handleParams, + TblInfo: tbl, + Handles: handles, + HandleParams: handleParams, + PartitionExpr: partitionExpr, }.Init(ctx, statsInfo, schema, names, 0) } @@ -625,6 +628,12 @@ func newBatchPointGetPlan( if matchIdxInfo == nil { return nil } + + pos, err := getPartitionColumnPos(matchIdxInfo, partitionExpr, tbl) + if err != nil { + return nil + } + indexValues := make([][]types.Datum, len(patternInExpr.List)) indexValueParams := make([][]*driver.ParamMarkerExpr, len(patternInExpr.List)) for i, item := range patternInExpr.List { @@ -690,7 +699,8 @@ func newBatchPointGetPlan( IndexInfo: matchIdxInfo, IndexValues: indexValues, IndexValueParams: indexValueParams, - PartitionColPos: getPartitionColumnPos(matchIdxInfo, partitionColName), + PartitionColPos: pos, + PartitionExpr: partitionExpr, }.Init(ctx, statsInfo, schema, names, 0) } @@ -810,9 +820,7 @@ func tryPointGetPlan(ctx sessionctx.Context, selStmt *ast.SelectStmt, check bool return nil } pi := tbl.GetPartitionInfo() - if pi != nil && pi.Type != model.PartitionTypeHash { - return nil - } + for _, col := range tbl.Columns { // Do not handle generated columns. if col.IsGenerated() { @@ -841,7 +849,12 @@ func tryPointGetPlan(ctx sessionctx.Context, selStmt *ast.SelectStmt, check bool var partitionInfo *model.PartitionDefinition var pos int if pi != nil { - partitionInfo, pos = getPartitionInfo(ctx, tbl, pairs) + partitionInfo, pos, isTableDual = getPartitionInfo(ctx, tbl, pairs) + if isTableDual { + p := newPointGetPlan(ctx, tblName.Schema.O, schema, tbl, names) + p.IsTableDual = true + return p + } if partitionInfo == nil { return nil } @@ -994,7 +1007,7 @@ func checkFastPlanPrivilege(ctx sessionctx.Context, dbName, tableName string, ch var visitInfos []visitInfo for _, checkType := range checkTypes { if pm != nil && !pm.RequestVerification(ctx.GetSessionVars().ActiveRoles, dbName, tableName, "", checkType) { - return errors.New("privilege check fail") + return ErrPrivilegeCheckFail.GenWithStackByArgs(checkType.String()) } // This visitInfo is only for table lock check, so we do not need column field, // just fill it empty string. @@ -1007,7 +1020,7 @@ func checkFastPlanPrivilege(ctx sessionctx.Context, dbName, tableName string, ch }) } - infoSchema := infoschema.GetInfoSchema(ctx) + infoSchema := ctx.GetInfoSchema().(infoschema.InfoSchema) return CheckTableLock(ctx, infoSchema, visitInfos) } @@ -1313,7 +1326,7 @@ func buildPointUpdatePlan(ctx sessionctx.Context, pointPlan PhysicalPlan, dbName VirtualAssignmentsOffset: len(orderedList), }.Init(ctx) updatePlan.names = pointPlan.OutputNames() - is := infoschema.GetInfoSchema(ctx) + is := ctx.GetInfoSchema().(infoschema.InfoSchema) t, _ := is.TableByID(tbl.ID) updatePlan.tblID2Table = map[int64]table.Table{ tbl.ID: t, @@ -1333,7 +1346,7 @@ func buildPointUpdatePlan(ctx sessionctx.Context, pointPlan PhysicalPlan, dbName } pids[pid] = struct{}{} } - pt = tables.NewPartitionTableithGivenSets(pt, pids) + pt = tables.NewPartitionTableWithGivenSets(pt, pids) } updatePlan.PartitionedTable = append(updatePlan.PartitionedTable, pt) } @@ -1463,20 +1476,80 @@ func buildHandleCols(ctx sessionctx.Context, tbl *model.TableInfo, schema *expre return &IntHandleCols{col: handleCol} } -func getPartitionInfo(ctx sessionctx.Context, tbl *model.TableInfo, pairs []nameValuePair) (*model.PartitionDefinition, int) { - partitionColName := getHashPartitionColumnName(ctx, tbl) - if partitionColName == nil { - return nil, 0 +func getPartitionInfo(ctx sessionctx.Context, tbl *model.TableInfo, pairs []nameValuePair) (*model.PartitionDefinition, int, bool) { + partitionExpr := getPartitionExpr(ctx, tbl) + if partitionExpr == nil { + return nil, 0, false } - pi := tbl.Partition - for i, pair := range pairs { - if partitionColName.Name.L == pair.colName { - val := pair.value.GetInt64() - pos := math.Abs(val % int64(pi.Num)) - return &pi.Definitions[pos], i + + pi := tbl.GetPartitionInfo() + if pi == nil { + return nil, 0, false + } + + switch pi.Type { + case model.PartitionTypeHash: + expr := partitionExpr.OrigExpr + col, ok := expr.(*ast.ColumnNameExpr) + if !ok { + return nil, 0, false + } + + partitionColName := col.Name + if partitionColName == nil { + return nil, 0, false + } + + for i, pair := range pairs { + if partitionColName.Name.L == pair.colName { + val := pair.value.GetInt64() + pos := math.Abs(val % int64(pi.Num)) + return &pi.Definitions[pos], i, false + } + } + case model.PartitionTypeRange: + // left range columns partition for future development + if len(pi.Columns) == 0 { + if col, ok := partitionExpr.Expr.(*expression.Column); ok { + colInfo := findColNameByColID(tbl.Columns, col) + for i, pair := range pairs { + if colInfo.Name.L == pair.colName { + val := pair.value.GetInt64() // val cannot be Null, we've check this in func getNameValuePairs + unsigned := mysql.HasUnsignedFlag(col.GetType().Flag) + ranges := partitionExpr.ForRangePruning + length := len(ranges.LessThan) + pos := sort.Search(length, func(i int) bool { + return ranges.Compare(i, val, unsigned) > 0 + }) + if pos >= 0 && pos < length { + return &pi.Definitions[pos], i, false + } + return nil, 0, true + } + } + } + } + case model.PartitionTypeList: + // left list columns partition for future development + if partitionExpr.ForListPruning.ColPrunes == nil { + locateExpr := partitionExpr.ForListPruning.LocateExpr + if locateExpr, ok := locateExpr.(*expression.Column); ok { + colInfo := findColNameByColID(tbl.Columns, locateExpr) + for i, pair := range pairs { + if colInfo.Name.L == pair.colName { + val := pair.value.GetInt64() // val cannot be Null, we've check this in func getNameValuePairs + isNull := false + pos := partitionExpr.ForListPruning.LocatePartition(val, isNull) + if pos >= 0 { + return &pi.Definitions[pos], i, false + } + return nil, 0, true + } + } + } } } - return nil, 0 + return nil, 0, false } func findPartitionIdx(idxInfo *model.IndexInfo, pos int, pairs []nameValuePair) int { @@ -1488,8 +1561,58 @@ func findPartitionIdx(idxInfo *model.IndexInfo, pos int, pairs []nameValuePair) return 0 } -// getPartitionColumnPos gets the partition column's position in the index. -func getPartitionColumnPos(idx *model.IndexInfo, partitionColName *ast.ColumnName) int { +// getPartitionColumnPos gets the partition column's position in the unique index. +func getPartitionColumnPos(idx *model.IndexInfo, partitionExpr *tables.PartitionExpr, tbl *model.TableInfo) (int, error) { + // regular table + if partitionExpr == nil { + return 0, nil + } + pi := tbl.GetPartitionInfo() + if pi == nil { + return 0, nil + } + + var partitionName model.CIStr + switch pi.Type { + case model.PartitionTypeHash: + if col, ok := partitionExpr.OrigExpr.(*ast.ColumnNameExpr); ok { + partitionName = col.Name.Name + } else { + return 0, errors.Errorf("unsupported partition type in BatchGet") + } + case model.PartitionTypeRange: + // left range columns partition for future development + if len(pi.Columns) == 0 { + if col, ok := partitionExpr.Expr.(*expression.Column); ok { + colInfo := findColNameByColID(tbl.Columns, col) + partitionName = colInfo.Name + } + } else { + return 0, errors.Errorf("unsupported partition type in BatchGet") + } + case model.PartitionTypeList: + // left list columns partition for future development + if partitionExpr.ForListPruning.ColPrunes == nil { + locateExpr := partitionExpr.ForListPruning.LocateExpr + if locateExpr, ok := locateExpr.(*expression.Column); ok { + colInfo := findColNameByColID(tbl.Columns, locateExpr) + partitionName = colInfo.Name + } + } else { + return 0, errors.Errorf("unsupported partition type in BatchGet") + } + } + + for i, idxCol := range idx.Columns { + if partitionName.L == idxCol.Name.L { + return i, nil + } + } + panic("unique index must include all partition columns") +} + +// getHashPartitionColumnPos gets the hash partition column's position in the unique index. +func getHashPartitionColumnPos(idx *model.IndexInfo, partitionColName *ast.ColumnName) int { if partitionColName == nil { return 0 } @@ -1501,6 +1624,27 @@ func getPartitionColumnPos(idx *model.IndexInfo, partitionColName *ast.ColumnNam panic("unique index must include all partition columns") } +func getPartitionExpr(ctx sessionctx.Context, tbl *model.TableInfo) *tables.PartitionExpr { + is := ctx.GetInfoSchema().(infoschema.InfoSchema) + table, ok := is.TableByID(tbl.ID) + if !ok { + return nil + } + + partTable, ok := table.(partitionTable) + if !ok { + return nil + } + + // PartitionExpr don't need columns and names for hash partition. + partitionExpr, err := partTable.PartitionExpr() + if err != nil { + return nil + } + + return partitionExpr +} + func getHashPartitionColumnName(ctx sessionctx.Context, tbl *model.TableInfo) *ast.ColumnName { pi := tbl.GetPartitionInfo() if pi == nil { @@ -1509,7 +1653,7 @@ func getHashPartitionColumnName(ctx sessionctx.Context, tbl *model.TableInfo) *a if pi.Type != model.PartitionTypeHash { return nil } - is := infoschema.GetInfoSchema(ctx) + is := ctx.GetInfoSchema().(infoschema.InfoSchema) table, ok := is.TableByID(tbl.ID) if !ok { return nil @@ -1526,3 +1670,12 @@ func getHashPartitionColumnName(ctx sessionctx.Context, tbl *model.TableInfo) *a } return col.Name } + +func findColNameByColID(cols []*model.ColumnInfo, col *expression.Column) *model.ColumnInfo { + for _, c := range cols { + if c.ID == col.ID { + return c + } + } + return nil +} diff --git a/planner/core/point_get_plan_test.go b/planner/core/point_get_plan_test.go index dfece394a5390..1306da4bed42c 100644 --- a/planner/core/point_get_plan_test.go +++ b/planner/core/point_get_plan_test.go @@ -316,10 +316,10 @@ func (s *testPointGetSuite) TestPointGetId(c *C) { c.Assert(err, IsNil) c.Assert(stmts, HasLen, 1) stmt := stmts[0] - is := domain.GetDomain(ctx).InfoSchema() - err = core.Preprocess(ctx, stmt, is) + ret := &core.PreprocessorReturn{} + err = core.Preprocess(ctx, stmt, core.WithPreprocessorReturn(ret)) c.Assert(err, IsNil) - p, _, err := planner.Optimize(context.TODO(), ctx, stmt, is) + p, _, err := planner.Optimize(context.TODO(), ctx, stmt, ret.InfoSchema) c.Assert(err, IsNil) // Test explain format = 'brief' result is useless, plan id will be reset when running `explain`. c.Assert(p.ID(), Equals, 1) diff --git a/planner/core/prepare_test.go b/planner/core/prepare_test.go index 537f1e06468bd..1a8d8a496d38b 100644 --- a/planner/core/prepare_test.go +++ b/planner/core/prepare_test.go @@ -201,7 +201,7 @@ func (s *testPlanSerialSuite) TestPrepareCacheDeferredFunction(c *C) { for i := 0; i < 2; i++ { stmt, err := s.ParseOneStmt(sql1, "", "") c.Check(err, IsNil) - is := tk.Se.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema) + is := tk.Se.GetInfoSchema().(infoschema.InfoSchema) builder, _ := core.NewPlanBuilder(tk.Se, is, &hint.BlockHintProcessor{}) p, err := builder.Build(ctx, stmt) c.Check(err, IsNil) @@ -278,9 +278,6 @@ func (s *testPrepareSerialSuite) TestPrepareOverMaxPreparedStmtCount(c *C) { tk.MustExec("set @@global.max_prepared_stmt_count = 2") tk.MustQuery("select @@global.max_prepared_stmt_count").Check(testkit.Rows("2")) - // Disable global variable cache, so load global session variable take effect immediate. - dom.GetGlobalVarsCache().Disable() - // test close session to give up all prepared stmt tk.MustExec(`prepare stmt2 from "select 1"`) prePrepared = readGaugeInt(metrics.PreparedStmtGauge) @@ -1338,3 +1335,174 @@ func (s *testPlanSerialSuite) TestPartitionTable(c *C) { } } } + +func (s *testPlanSerialSuite) TestPartitionWithVariedDatasources(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + // enable plan cache + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + tk := testkit.NewTestKit(c, store) + orgEnable := core.PreparedPlanCacheEnabled() + defer func() { + dom.Close() + err = store.Close() + c.Assert(err, IsNil) + core.SetPreparedPlanCache(orgEnable) + }() + core.SetPreparedPlanCache(true) + tk.Se, err = session.CreateSession4TestWithOpt(store, &session.Opt{ + PreparedPlanCache: kvcache.NewSimpleLRUCache(100, 0.1, math.MaxUint64), + }) + c.Assert(err, IsNil) + + // enable partition table dynamic mode + tk.MustExec("create database test_plan_cache2") + tk.MustExec("use test_plan_cache2") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + // prepare tables + tk.MustExec(`create table trangePK (a int primary key, b int) partition by range (a) ( + partition p0 values less than (10000), + partition p1 values less than (20000), + partition p2 values less than (30000), + partition p3 values less than (40000))`) + tk.MustExec(`create table thashPK (a int primary key, b int) partition by hash (a) partitions 4`) + tk.MustExec(`create table tnormalPK (a int primary key, b int)`) + tk.MustExec(`create table trangeIdx (a int unique key, b int) partition by range (a) ( + partition p0 values less than (10000), + partition p1 values less than (20000), + partition p2 values less than (30000), + partition p3 values less than (40000))`) + tk.MustExec(`create table thashIdx (a int unique key, b int) partition by hash (a) partitions 4`) + tk.MustExec(`create table tnormalIdx (a int unique key, b int)`) + uniqueVals := make(map[int]struct{}) + vals := make([]string, 0, 1000) + for len(vals) < 1000 { + a := rand.Intn(40000) + if _, ok := uniqueVals[a]; ok { + continue + } + uniqueVals[a] = struct{}{} + b := rand.Intn(40000) + vals = append(vals, fmt.Sprintf("(%v, %v)", a, b)) + } + for _, tbl := range []string{"trangePK", "thashPK", "tnormalPK", "trangeIdx", "thashIdx", "tnormalIdx"} { + tk.MustExec(fmt.Sprintf(`insert into %v values %v`, tbl, strings.Join(vals, ", "))) + } + + // TableReader, PointGet on PK, BatchGet on PK + for _, tbl := range []string{`trangePK`, `thashPK`, `tnormalPK`} { + tk.MustExec(fmt.Sprintf(`prepare stmt%v_tablescan from 'select * from %v use index(primary) where a > ? and a < ?'`, tbl, tbl)) + tk.MustExec(fmt.Sprintf(`prepare stmt%v_pointget from 'select * from %v use index(primary) where a = ?'`, tbl, tbl)) + tk.MustExec(fmt.Sprintf(`prepare stmt%v_batchget from 'select * from %v use index(primary) where a in (?, ?, ?)'`, tbl, tbl)) + } + for i := 0; i < 100; i++ { + mina, maxa := rand.Intn(40000), rand.Intn(40000) + if mina > maxa { + mina, maxa = maxa, mina + } + tk.MustExec(fmt.Sprintf(`set @mina=%v, @maxa=%v`, mina, maxa)) + tk.MustExec(fmt.Sprintf(`set @pointa=%v`, rand.Intn(40000))) + tk.MustExec(fmt.Sprintf(`set @a0=%v, @a1=%v, @a2=%v`, rand.Intn(40000), rand.Intn(40000), rand.Intn(40000))) + + var rscan, rpoint, rbatch [][]interface{} + for id, tbl := range []string{`trangePK`, `thashPK`, `tnormalPK`} { + scan := tk.MustQuery(fmt.Sprintf(`execute stmt%v_tablescan using @mina, @maxa`, tbl)).Sort() + if i > 0 { + tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) + } + if id == 0 { + rscan = scan.Rows() + } else { + scan.Check(rscan) + } + + point := tk.MustQuery(fmt.Sprintf(`execute stmt%v_pointget using @pointa`, tbl)).Sort() + if tbl == `tnormalPK` && i > 0 { + // PlanCache cannot support PointGet now since we haven't relocated partition after rebuilding range. + // Please see Execute.rebuildRange for more details. + tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) + } + if id == 0 { + rpoint = point.Rows() + } else { + point.Check(rpoint) + } + + batch := tk.MustQuery(fmt.Sprintf(`execute stmt%v_batchget using @a0, @a1, @a2`, tbl)).Sort() + if i > 0 { + tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) + } + if id == 0 { + rbatch = batch.Rows() + } else { + batch.Check(rbatch) + } + } + } + + // IndexReader, IndexLookUp, PointGet on Idx, BatchGet on Idx + for _, tbl := range []string{"trangeIdx", "thashIdx", "tnormalIdx"} { + tk.MustExec(fmt.Sprintf(`prepare stmt%v_indexscan from 'select a from %v use index(a) where a > ? and a < ?'`, tbl, tbl)) + tk.MustExec(fmt.Sprintf(`prepare stmt%v_indexlookup from 'select * from %v use index(a) where a > ? and a < ?'`, tbl, tbl)) + tk.MustExec(fmt.Sprintf(`prepare stmt%v_pointget_idx from 'select * from %v use index(a) where a = ?'`, tbl, tbl)) + tk.MustExec(fmt.Sprintf(`prepare stmt%v_batchget_idx from 'select * from %v use index(a) where a in (?, ?, ?)'`, tbl, tbl)) + } + for i := 0; i < 100; i++ { + mina, maxa := rand.Intn(40000), rand.Intn(40000) + if mina > maxa { + mina, maxa = maxa, mina + } + tk.MustExec(fmt.Sprintf(`set @mina=%v, @maxa=%v`, mina, maxa)) + tk.MustExec(fmt.Sprintf(`set @pointa=%v`, rand.Intn(40000))) + tk.MustExec(fmt.Sprintf(`set @a0=%v, @a1=%v, @a2=%v`, rand.Intn(40000), rand.Intn(40000), rand.Intn(40000))) + + var rscan, rlookup, rpoint, rbatch [][]interface{} + for id, tbl := range []string{"trangeIdx", "thashIdx", "tnormalIdx"} { + scan := tk.MustQuery(fmt.Sprintf(`execute stmt%v_indexscan using @mina, @maxa`, tbl)).Sort() + if i > 0 { + tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) + } + if id == 0 { + rscan = scan.Rows() + } else { + scan.Check(rscan) + } + + lookup := tk.MustQuery(fmt.Sprintf(`execute stmt%v_indexlookup using @mina, @maxa`, tbl)).Sort() + if i > 0 { + tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) + } + if id == 0 { + rlookup = lookup.Rows() + } else { + lookup.Check(rlookup) + } + + point := tk.MustQuery(fmt.Sprintf(`execute stmt%v_pointget_idx using @pointa`, tbl)).Sort() + if tbl == `tnormalPK` && i > 0 { + // PlanCache cannot support PointGet now since we haven't relocated partition after rebuilding range. + // Please see Execute.rebuildRange for more details. + tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("0")) + } + if id == 0 { + rpoint = point.Rows() + } else { + point.Check(rpoint) + } + + batch := tk.MustQuery(fmt.Sprintf(`execute stmt%v_batchget_idx using @a0, @a1, @a2`, tbl)).Sort() + if i > 0 { + tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) + } + if id == 0 { + rbatch = batch.Rows() + } else { + batch.Check(rbatch) + } + } + } +} diff --git a/planner/core/preprocess.go b/planner/core/preprocess.go index a3719fe4c4b0b..70e1391d3361c 100644 --- a/planner/core/preprocess.go +++ b/planner/core/preprocess.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/meta/autoid" @@ -51,6 +52,13 @@ func InTxnRetry(p *preprocessor) { p.flag |= inTxnRetry } +// WithPreprocessorReturn returns a PreprocessOpt to initialize the PreprocesorReturn. +func WithPreprocessorReturn(ret *PreprocessorReturn) PreprocessOpt { + return func(p *preprocessor) { + p.PreprocessorReturn = ret + } +} + // TryAddExtraLimit trys to add an extra limit for SELECT or UNION statement when sql_select_limit is set. func TryAddExtraLimit(ctx sessionctx.Context, node ast.StmtNode) ast.StmtNode { if ctx.GetSessionVars().SelectLimit == math.MaxUint64 || ctx.GetSessionVars().InRestrictedSQL { @@ -82,12 +90,25 @@ func TryAddExtraLimit(ctx sessionctx.Context, node ast.StmtNode) ast.StmtNode { } // Preprocess resolves table names of the node, and checks some statements validation. -func Preprocess(ctx sessionctx.Context, node ast.Node, is infoschema.InfoSchema, preprocessOpt ...PreprocessOpt) error { - v := preprocessor{is: is, ctx: ctx, tableAliasInJoin: make([]map[string]interface{}, 0)} +// prepreocssReturn used to extract the infoschema for the tableName and the timestamp from the asof clause. +func Preprocess(ctx sessionctx.Context, node ast.Node, preprocessOpt ...PreprocessOpt) error { + v := preprocessor{ctx: ctx, tableAliasInJoin: make([]map[string]interface{}, 0), withName: make(map[string]interface{})} for _, optFn := range preprocessOpt { optFn(&v) } + // PreprocessorReturn must be non-nil before preprocessing + if v.PreprocessorReturn == nil { + v.PreprocessorReturn = &PreprocessorReturn{} + } node.Accept(&v) + readTS := ctx.GetSessionVars().TxnReadTS.UseTxnReadTS() + if readTS > 0 { + v.PreprocessorReturn.SnapshotTS = readTS + } + // InfoSchema must be non-nil after preprocessing + if v.InfoSchema == nil { + v.ensureInfoSchema() + } return errors.Trace(v.err) } @@ -109,22 +130,34 @@ const ( inSequenceFunction ) +// PreprocessorReturn is used to retain information obtained in the preprocessor. +type PreprocessorReturn struct { + SnapshotTS uint64 + ExplicitStaleness bool + InfoSchema infoschema.InfoSchema +} + // preprocessor is an ast.Visitor that preprocess // ast Nodes parsed from parser. type preprocessor struct { - is infoschema.InfoSchema ctx sessionctx.Context - err error flag preprocessorFlag stmtTp byte // tableAliasInJoin is a stack that keeps the table alias names for joins. // len(tableAliasInJoin) may bigger than 1 because the left/right child of join may be subquery that contains `JOIN` tableAliasInJoin []map[string]interface{} + withName map[string]interface{} + + // values that may be returned + *PreprocessorReturn + err error } func (p *preprocessor) Enter(in ast.Node) (out ast.Node, skipChildren bool) { switch node := in.(type) { + case *ast.AdminStmt: + p.checkAdminCheckTableGrammar(node) case *ast.DeleteStmt: p.stmtTp = TypeDelete case *ast.SelectStmt: @@ -133,6 +166,9 @@ func (p *preprocessor) Enter(in ast.Node) (out ast.Node, skipChildren bool) { p.stmtTp = TypeUpdate case *ast.InsertStmt: p.stmtTp = TypeInsert + // handle the insert table name imminently + // insert into t with t ..., the insert can not see t here. We should hand it before the CTE statement + p.handleTableName(node.Table.TableRefs.Left.(*ast.TableSource).Source.(*ast.TableName)) case *ast.CreateTableStmt: p.stmtTp = TypeCreate p.flag |= inCreateOrDropTable @@ -215,6 +251,7 @@ func (p *preprocessor) Enter(in ast.Node) (out ast.Node, skipChildren bool) { if node.FnName.L == ast.NextVal || node.FnName.L == ast.LastVal || node.FnName.L == ast.SetVal { p.flag |= inSequenceFunction } + case *ast.BRIEStmt: if node.Kind == ast.BRIEKindRestore { p.flag |= inCreateOrDropTable @@ -233,6 +270,10 @@ func (p *preprocessor) Enter(in ast.Node) (out ast.Node, skipChildren bool) { } case *ast.GroupByClause: p.checkGroupBy(node) + case *ast.WithClause: + for _, cte := range node.CTEs { + p.withName[cte.Name.L] = struct{}{} + } default: p.flag &= ^parentIsJoin } @@ -557,7 +598,60 @@ func (p *preprocessor) checkDropDatabaseGrammar(stmt *ast.DropDatabaseStmt) { } } +func (p *preprocessor) checkAdminCheckTableGrammar(stmt *ast.AdminStmt) { + for _, table := range stmt.Tables { + currentDB := p.ctx.GetSessionVars().CurrentDB + if table.Schema.String() != "" { + currentDB = table.Schema.L + } + if currentDB == "" { + p.err = errors.Trace(ErrNoDB) + return + } + sName := model.NewCIStr(currentDB) + tName := table.Name + tableInfo, err := p.ensureInfoSchema().TableByName(sName, tName) + if err != nil { + p.err = err + return + } + tempTableType := tableInfo.Meta().TempTableType + if (stmt.Tp == ast.AdminCheckTable || stmt.Tp == ast.AdminChecksumTable) && tempTableType != model.TempTableNone { + if stmt.Tp == ast.AdminChecksumTable { + p.err = ErrOptOnTemporaryTable.GenWithStackByArgs("admin checksum table") + } else { + p.err = ErrOptOnTemporaryTable.GenWithStackByArgs("admin check table") + } + return + } + } +} + func (p *preprocessor) checkCreateTableGrammar(stmt *ast.CreateTableStmt) { + if stmt.ReferTable != nil { + schema := model.NewCIStr(p.ctx.GetSessionVars().CurrentDB) + if stmt.ReferTable.Schema.String() != "" { + schema = stmt.ReferTable.Schema + } + // get the infoschema from the context. + tableInfo, err := p.ensureInfoSchema().TableByName(schema, stmt.ReferTable.Name) + if err != nil { + p.err = err + return + } + if tableInfo.Meta().TempTableType != model.TempTableNone { + p.err = ErrOptOnTemporaryTable.GenWithStackByArgs("create table like") + return + } + } + if stmt.TemporaryKeyword != ast.TemporaryNone { + for _, opt := range stmt.Options { + if opt.Tp == ast.TableOptionShardRowID { + p.err = ErrOptOnTemporaryTable.GenWithStackByArgs("shard_row_id_bits") + return + } + } + } tName := stmt.Table.Name.String() if isIncorrectName(tName) { p.err = ddl.ErrWrongTableName.GenWithStackByArgs(tName) @@ -574,7 +668,7 @@ func (p *preprocessor) checkCreateTableGrammar(stmt *ast.CreateTableStmt) { p.err = err return } - isPrimary, err := checkColumnOptions(colDef.Options) + isPrimary, err := checkColumnOptions(stmt.TemporaryKeyword != ast.TemporaryNone, colDef.Options) if err != nil { p.err = err return @@ -731,7 +825,7 @@ func isTableAliasDuplicate(node ast.ResultSetNode, tableAliases map[string]inter return nil } -func checkColumnOptions(ops []*ast.ColumnOption) (int, error) { +func checkColumnOptions(isTempTable bool, ops []*ast.ColumnOption) (int, error) { isPrimary, isGenerated, isStored := 0, 0, false for _, op := range ops { @@ -741,6 +835,10 @@ func checkColumnOptions(ops []*ast.ColumnOption) (int, error) { case ast.ColumnOptionGenerated: isGenerated = 1 isStored = op.Stored + case ast.ColumnOptionAutoRandom: + if isTempTable { + return isPrimary, ErrOptOnTemporaryTable.GenWithStackByArgs("auto_random") + } } } @@ -1112,6 +1210,10 @@ func (p *preprocessor) stmtType() string { func (p *preprocessor) handleTableName(tn *ast.TableName) { if tn.Schema.L == "" { + if _, ok := p.withName[tn.Name.L]; ok { + return + } + currentDB := p.ctx.GetSessionVars().CurrentDB if currentDB == "" { p.err = errors.Trace(ErrNoDB) @@ -1119,6 +1221,7 @@ func (p *preprocessor) handleTableName(tn *ast.TableName) { } tn.Schema = model.NewCIStr(currentDB) } + if p.flag&inCreateOrDropTable > 0 { // The table may not exist in create table or drop table statement. if p.flag&inRepairTable > 0 { @@ -1138,7 +1241,12 @@ func (p *preprocessor) handleTableName(tn *ast.TableName) { return } - table, err := p.is.TableByName(tn.Schema, tn.Name) + p.handleAsOf(tn.AsOf) + if p.err != nil { + return + } + + table, err := p.ensureInfoSchema().TableByName(tn.Schema, tn.Name) if err != nil { // We should never leak that the table doesn't exist (i.e. attach ErrTableNotExists) // unless we know that the user has permissions to it, should it exist. @@ -1160,7 +1268,7 @@ func (p *preprocessor) handleTableName(tn *ast.TableName) { return } tableInfo := table.Meta() - dbInfo, _ := p.is.SchemaByName(tn.Schema) + dbInfo, _ := p.ensureInfoSchema().SchemaByName(tn.Schema) // tableName should be checked as sequence object. if p.flag&inSequenceFunction > 0 { if !tableInfo.IsSequence() { @@ -1285,3 +1393,50 @@ func (p *preprocessor) checkFuncCastExpr(node *ast.FuncCastExpr) { } } } + +// handleAsOf tries to validate the timestamp. +// If it is not nil, timestamp is used to get the history infoschema from the infocache. +func (p *preprocessor) handleAsOf(node *ast.AsOfClause) { + readTS := p.ctx.GetSessionVars().TxnReadTS.PeakTxnReadTS() + if readTS > 0 && node != nil { + p.err = ErrAsOf.FastGenWithCause("can't use select as of while already set transaction as of") + return + } + dom := domain.GetDomain(p.ctx) + ts := uint64(0) + if node != nil { + if p.ctx.GetSessionVars().InTxn() { + p.err = ErrAsOf.FastGenWithCause("as of timestamp can't be set in transaction.") + return + } + ts, p.err = calculateTsExpr(p.ctx, node) + if p.err != nil { + return + } + } + if ts != 0 && p.InfoSchema == nil { + is, err := dom.GetSnapshotInfoSchema(ts) + if err != nil { + p.err = err + return + } + p.SnapshotTS = ts + p.ExplicitStaleness = true + p.InfoSchema = is + } + if p.SnapshotTS != ts { + p.err = ErrDifferentAsOf.GenWithStack("can not set different time in the as of") + } +} + +// ensureInfoSchema get the infoschema from the preprecessor. +// there some situations: +// - the stmt specifies the schema version. +// - session variable +// - transcation context +func (p *preprocessor) ensureInfoSchema() infoschema.InfoSchema { + if p.InfoSchema == nil { + p.InfoSchema = p.ctx.GetInfoSchema().(infoschema.InfoSchema) + } + return p.InfoSchema +} diff --git a/planner/core/preprocess_test.go b/planner/core/preprocess_test.go index 14b006c836ca9..d9f053f509e92 100644 --- a/planner/core/preprocess_test.go +++ b/planner/core/preprocess_test.go @@ -67,7 +67,7 @@ func (s *testValidatorSuite) runSQL(c *C, sql string, inPrepare bool, terr error if inPrepare { opts = append(opts, core.InPrepare) } - err := core.Preprocess(s.ctx, stmt, s.is, opts...) + err := core.Preprocess(s.ctx, stmt, append(opts, core.WithPreprocessorReturn(&core.PreprocessorReturn{InfoSchema: s.is}))...) c.Assert(terror.ErrorEqual(err, terr), IsTrue, Commentf("sql: %s, err:%v", sql, err)) } diff --git a/planner/core/property_cols_prune.go b/planner/core/property_cols_prune.go index 91a9f34fb9017..9cd83adc412de 100644 --- a/planner/core/property_cols_prune.go +++ b/planner/core/property_cols_prune.go @@ -148,21 +148,22 @@ func (p *LogicalProjection) PreparePossibleProperties(schema *expression.Schema, } } tmpSchema := expression.NewSchema(oldCols...) - for i := len(childProperties) - 1; i >= 0; i-- { - for j, col := range childProperties[i] { + newProperties := make([][]*expression.Column, 0, len(childProperties)) + for _, childProperty := range childProperties { + newChildProperty := make([]*expression.Column, 0, len(childProperty)) + for _, col := range childProperty { pos := tmpSchema.ColumnIndex(col) if pos >= 0 { - childProperties[i][j] = newCols[pos] + newChildProperty = append(newChildProperty, newCols[pos]) } else { - childProperties[i] = childProperties[i][:j] break } } - if len(childProperties[i]) == 0 { - childProperties = append(childProperties[:i], childProperties[i+1:]...) + if len(newChildProperty) != 0 { + newProperties = append(newProperties, newChildProperty) } } - return childProperties + return newProperties } func clonePossibleProperties(props [][]*expression.Column) [][]*expression.Column { diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go index 4b31853c138d0..8a627792ecc7f 100644 --- a/planner/core/rule_column_pruning.go +++ b/planner/core/rule_column_pruning.go @@ -88,7 +88,11 @@ func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column) child := la.children[0] used := expression.GetUsedList(parentUsedCols, la.Schema()) + allFirstRow := true for i := len(used) - 1; i >= 0; i-- { + if la.AggFuncs[i].Name != ast.AggFuncFirstRow { + allFirstRow = false + } if !used[i] { la.schema.Columns = append(la.schema.Columns[:i], la.schema.Columns[i+1:]...) la.AggFuncs = append(la.AggFuncs[:i], la.AggFuncs[i+1:]...) @@ -103,15 +107,24 @@ func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column) selfUsedCols = append(selfUsedCols, cols...) } if len(la.AggFuncs) == 0 { - // If all the aggregate functions are pruned, we should add an aggregate function to keep the correctness. - one, err := aggregation.NewAggFuncDesc(la.ctx, ast.AggFuncFirstRow, []expression.Expression{expression.NewOne()}, false) + // If all the aggregate functions are pruned, we should add an aggregate function to maintain the info of row numbers. + // For all the aggregate functions except `first_row`, if we have an empty table defined as t(a,b), + // `select agg(a) from t` would always return one row, while `select agg(a) from t group by b` would return empty. + // For `first_row` which is only used internally by tidb, `first_row(a)` would always return empty for empty input now. + var err error + var newAgg *aggregation.AggFuncDesc + if allFirstRow { + newAgg, err = aggregation.NewAggFuncDesc(la.ctx, ast.AggFuncFirstRow, []expression.Expression{expression.NewOne()}, false) + } else { + newAgg, err = aggregation.NewAggFuncDesc(la.ctx, ast.AggFuncCount, []expression.Expression{expression.NewOne()}, false) + } if err != nil { return err } - la.AggFuncs = []*aggregation.AggFuncDesc{one} + la.AggFuncs = []*aggregation.AggFuncDesc{newAgg} col := &expression.Column{ UniqueID: la.ctx.GetSessionVars().AllocPlanColumnID(), - RetType: one.RetTp, + RetType: newAgg.RetTp, } la.schema.Columns = []*expression.Column{col} } diff --git a/planner/core/rule_generate_column_substitute.go b/planner/core/rule_generate_column_substitute.go index bc4a31e0b88e8..670753d0833ec 100644 --- a/planner/core/rule_generate_column_substitute.go +++ b/planner/core/rule_generate_column_substitute.go @@ -119,6 +119,12 @@ func (gc *gcSubstituter) substitute(ctx context.Context, lp LogicalPlan, exprToC tryToSubstituteExpr(expr, lp.SCtx(), candidateExpr, tp, x.Schema(), column) } } + case ast.Like: + expr := &sf.GetArgs()[0] + tp = sf.GetArgs()[1].GetType().EvalType() + for candidateExpr, column := range exprToColumn { + tryToSubstituteExpr(expr, lp.SCtx(), candidateExpr, tp, x.Schema(), column) + } } } case *LogicalProjection: diff --git a/planner/core/rule_inject_extra_projection.go b/planner/core/rule_inject_extra_projection.go index 2896a1dade0ff..968917a4fef2e 100644 --- a/planner/core/rule_inject_extra_projection.go +++ b/planner/core/rule_inject_extra_projection.go @@ -14,6 +14,7 @@ package core import ( + "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/expression/aggregation" "github.com/pingcap/tidb/kv" @@ -62,10 +63,42 @@ func (pe *projInjector) inject(plan PhysicalPlan) PhysicalPlan { plan = InjectProjBelowSort(p, p.ByItems) case *NominalSort: plan = TurnNominalSortIntoProj(p, p.OnlyColumn, p.ByItems) + case *PhysicalUnionAll: + plan = injectProjBelowUnion(p) } return plan } +func injectProjBelowUnion(un *PhysicalUnionAll) *PhysicalUnionAll { + if !un.mpp { + return un + } + for i, ch := range un.children { + exprs := make([]expression.Expression, len(ch.Schema().Columns)) + needChange := false + for i, dstCol := range un.schema.Columns { + dstType := dstCol.RetType + srcCol := ch.Schema().Columns[i] + srcType := srcCol.RetType + if !srcType.Equal(dstType) || !(mysql.HasNotNullFlag(dstType.Flag) == mysql.HasNotNullFlag(srcType.Flag)) { + exprs[i] = expression.BuildCastFunction4Union(un.ctx, srcCol, dstType) + needChange = true + } else { + exprs[i] = srcCol + } + } + if needChange { + proj := PhysicalProjection{ + Exprs: exprs, + }.Init(un.ctx, ch.statsInfo(), 0) + proj.SetSchema(un.schema.Clone()) + proj.SetChildren(ch) + un.children[i] = proj + } + } + return un +} + // wrapCastForAggFunc wraps the args of an aggregate function with a cast function. // If the mode is FinalMode or Partial2Mode, we do not need to wrap cast upon the args, // since the types of the args are already the expected. diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index 57858679b7795..90a03991864ab 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -129,11 +129,11 @@ func (s *partitionProcessor) findUsedPartitions(ctx sessionctx.Context, tbl tabl partIdx[i].Index = i colLen = append(colLen, types.UnspecifiedLength) } - datchedResult, err := ranger.DetachCondAndBuildRangeForPartition(ctx, conds, partIdx, colLen) + detachedResult, err := ranger.DetachCondAndBuildRangeForPartition(ctx, conds, partIdx, colLen) if err != nil { return nil, nil, err } - ranges := datchedResult.Ranges + ranges := detachedResult.Ranges used := make([]int, 0, len(ranges)) for _, r := range ranges { if r.IsPointNullable(ctx.GetSessionVars().StmtCtx) { @@ -143,7 +143,10 @@ func (s *partitionProcessor) findUsedPartitions(ctx sessionctx.Context, tbl tabl break } } - pos, isNull, err := pe.EvalInt(ctx, chunk.MutRowFromDatums(r.HighVal).ToRow()) + highLowVals := make([]types.Datum, 0, len(r.HighVal)+len(r.LowVal)) + highLowVals = append(highLowVals, r.HighVal...) + highLowVals = append(highLowVals, r.LowVal...) + pos, isNull, err := pe.EvalInt(ctx, chunk.MutRowFromDatums(highLowVals).ToRow()) if err != nil { return nil, nil, err } @@ -225,7 +228,7 @@ func (s *partitionProcessor) findUsedPartitions(ctx sessionctx.Context, tbl tabl ret = append(ret, used[i]) } } - return ret, datchedResult.RemainedConds, nil + return ret, detachedResult.RemainedConds, nil } func (s *partitionProcessor) convertToIntSlice(or partitionRangeOR, pi *model.PartitionInfo, partitionNames []model.CIStr) []int { @@ -297,7 +300,7 @@ func (s *partitionProcessor) reconstructTableColNames(ds *DataSource) ([]*types. }) continue } - return nil, errors.New(fmt.Sprintf("information of column %v is not found", colExpr.String())) + return nil, fmt.Errorf("information of column %v is not found", colExpr.String()) } return names, nil } @@ -1345,9 +1348,9 @@ func appendWarnForUnknownPartitions(ctx sessionctx.Context, hintName string, unk if len(unknownPartitions) == 0 { return } - ctx.GetSessionVars().StmtCtx.AppendWarning( - errors.New(fmt.Sprintf("Unknown partitions (%s) in optimizer hint %s", - strings.Join(unknownPartitions, ","), hintName))) + + warning := fmt.Errorf("Unknown partitions (%s) in optimizer hint %s", strings.Join(unknownPartitions, ","), hintName) + ctx.GetSessionVars().StmtCtx.AppendWarning(warning) } func (s *partitionProcessor) checkHintsApplicable(ds *DataSource, partitionSet set.StringSet) { diff --git a/planner/core/rule_predicate_push_down.go b/planner/core/rule_predicate_push_down.go index 936c4720cbc87..b591343870ce4 100644 --- a/planner/core/rule_predicate_push_down.go +++ b/planner/core/rule_predicate_push_down.go @@ -42,6 +42,12 @@ func addSelection(p LogicalPlan, child LogicalPlan, conditions []expression.Expr p.Children()[chIdx] = dual return } + + conditions = DeleteTrueExprs(p, conditions) + if len(conditions) == 0 { + p.Children()[chIdx] = child + return + } selection := LogicalSelection{Conditions: conditions}.Init(p.SCtx(), p.SelectBlockOffset()) selection.SetChildren(child) p.Children()[chIdx] = selection @@ -73,6 +79,8 @@ func splitSetGetVarFunc(filters []expression.Expression) ([]expression.Expressio // PredicatePushDown implements LogicalPlan PredicatePushDown interface. func (p *LogicalSelection) PredicatePushDown(predicates []expression.Expression) ([]expression.Expression, LogicalPlan) { + predicates = DeleteTrueExprs(p, predicates) + p.Conditions = DeleteTrueExprs(p, p.Conditions) canBePushDown, canNotBePushDown := splitSetGetVarFunc(p.Conditions) retConditions, child := p.children[0].PredicatePushDown(append(canBePushDown, predicates...)) retConditions = append(retConditions, canNotBePushDown...) @@ -100,6 +108,7 @@ func (p *LogicalUnionScan) PredicatePushDown(predicates []expression.Expression) // PredicatePushDown implements LogicalPlan PredicatePushDown interface. func (ds *DataSource) PredicatePushDown(predicates []expression.Expression) ([]expression.Expression, LogicalPlan) { predicates = expression.PropagateConstant(ds.ctx, predicates) + predicates = DeleteTrueExprs(ds, predicates) ds.allConds = predicates ds.pushedDownConds, predicates = expression.PushDownExprs(ds.ctx.GetSessionVars().StmtCtx, predicates, ds.ctx.GetClient(), kv.UnSpecified) return predicates, ds @@ -199,7 +208,6 @@ func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression) (ret addSelection(p, lCh, leftRet, 0) addSelection(p, rCh, rightRet, 1) p.updateEQCond() - p.mergeSchema() buildKeyInfo(p) return ret, p.self } @@ -532,6 +540,28 @@ func Conds2TableDual(p LogicalPlan, conds []expression.Expression) LogicalPlan { return nil } +// DeleteTrueExprs deletes the surely true expressions +func DeleteTrueExprs(p LogicalPlan, conds []expression.Expression) []expression.Expression { + newConds := make([]expression.Expression, 0, len(conds)) + for _, cond := range conds { + con, ok := cond.(*expression.Constant) + if !ok { + newConds = append(newConds, cond) + continue + } + if expression.ContainMutableConst(p.SCtx(), []expression.Expression{con}) { + newConds = append(newConds, cond) + continue + } + sc := p.SCtx().GetSessionVars().StmtCtx + if isTrue, err := con.Value.ToBool(sc); err == nil && isTrue == 1 { + continue + } + newConds = append(newConds, cond) + } + return newConds +} + // outerJoinPropConst propagates constant equal and column equal conditions over outer join. func (p *LogicalJoin) outerJoinPropConst(predicates []expression.Expression) []expression.Expression { outerTable := p.children[0] diff --git a/planner/core/stats_test.go b/planner/core/stats_test.go index d74f1ba1df1d8..8999d3208621f 100644 --- a/planner/core/stats_test.go +++ b/planner/core/stats_test.go @@ -69,15 +69,16 @@ func (s *testStatsSuite) TestGroupNDVs(c *C) { AggInput string JoinInput string } - is := dom.InfoSchema() s.testData.GetTestCases(c, &input, &output) for i, tt := range input { comment := Commentf("case:%v sql: %s", i, tt) stmt, err := s.ParseOneStmt(tt, "", "") c.Assert(err, IsNil, comment) - err = core.Preprocess(tk.Se, stmt, is) + ret := &core.PreprocessorReturn{} + err = core.Preprocess(tk.Se, stmt, core.WithPreprocessorReturn(ret)) c.Assert(err, IsNil) - builder, _ := core.NewPlanBuilder(tk.Se, is, &hint.BlockHintProcessor{}) + tk.Se.GetSessionVars().PlanColumnID = 0 + builder, _ := core.NewPlanBuilder(tk.Se, ret.InfoSchema, &hint.BlockHintProcessor{}) p, err := builder.Build(ctx, stmt) c.Assert(err, IsNil, comment) p, err = core.LogicalOptimize(ctx, builder.GetOptFlag(), p.(core.LogicalPlan)) diff --git a/planner/core/stringer.go b/planner/core/stringer.go index 4c63f6ff244b9..346b5b50e5742 100644 --- a/planner/core/stringer.go +++ b/planner/core/stringer.go @@ -270,31 +270,31 @@ func toString(in Plan, strs []string, idxs []int) ([]string, []int) { case *PhysicalShuffleReceiverStub: str = fmt.Sprintf("PartitionReceiverStub(%s)", x.ExplainInfo()) case *PointGetPlan: - str = fmt.Sprintf("PointGet(") + str = "PointGet(" if x.IndexInfo != nil { str += fmt.Sprintf("Index(%s.%s)%v)", x.TblInfo.Name.L, x.IndexInfo.Name.L, x.IndexValues) } else { str += fmt.Sprintf("Handle(%s.%s)%v)", x.TblInfo.Name.L, x.TblInfo.GetPkName().L, x.Handle) } case *BatchPointGetPlan: - str = fmt.Sprintf("BatchPointGet(") + str = "BatchPointGet(" if x.IndexInfo != nil { str += fmt.Sprintf("Index(%s.%s)%v)", x.TblInfo.Name.L, x.IndexInfo.Name.L, x.IndexValues) } else { str += fmt.Sprintf("Handle(%s.%s)%v)", x.TblInfo.Name.L, x.TblInfo.GetPkName().L, x.Handles) } case *PhysicalExchangeReceiver: - str = fmt.Sprintf("Recv(") + str = "Recv(" for _, task := range x.Tasks { str += fmt.Sprintf("%d, ", task.ID) } - str += fmt.Sprintf(")") + str += ")" case *PhysicalExchangeSender: - str = fmt.Sprintf("Send(") + str = "Send(" for _, task := range x.TargetTasks { str += fmt.Sprintf("%d, ", task.ID) } - str += fmt.Sprintf(")") + str += ")" default: str = fmt.Sprintf("%T", in) } diff --git a/planner/core/task.go b/planner/core/task.go index 205f5eb77b08a..eacc5dbf73e6d 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/charset" + "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/expression" @@ -150,19 +151,24 @@ func (t *copTask) finishIndexPlan() { cnt := t.count() t.indexPlanFinished = true sessVars := t.indexPlan.SCtx().GetSessionVars() + var tableInfo *model.TableInfo + if t.tablePlan != nil { + ts := t.tablePlan.(*PhysicalTableScan) + ts.stats = t.indexPlan.statsInfo() + tableInfo = ts.Table + } // Network cost of transferring rows of index scan to TiDB. - t.cst += cnt * sessVars.NetworkFactor * t.tblColHists.GetAvgRowSize(t.indexPlan.SCtx(), t.indexPlan.Schema().Columns, true, false) - + t.cst += cnt * sessVars.GetNetworkFactor(tableInfo) * t.tblColHists.GetAvgRowSize(t.indexPlan.SCtx(), t.indexPlan.Schema().Columns, true, false) if t.tablePlan == nil { return } + // Calculate the IO cost of table scan here because we cannot know its stats until we finish index plan. - t.tablePlan.(*PhysicalTableScan).stats = t.indexPlan.statsInfo() var p PhysicalPlan for p = t.indexPlan; len(p.Children()) > 0; p = p.Children()[0] { } rowSize := t.tblColHists.GetIndexAvgRowSize(t.indexPlan.SCtx(), t.tblCols, p.(*PhysicalIndexScan).Index.Unique) - t.cst += cnt * rowSize * sessVars.ScanFactor + t.cst += cnt * rowSize * sessVars.GetScanFactor(tableInfo) } func (t *copTask) getStoreType() kv.StoreType { @@ -935,7 +941,7 @@ func (t *copTask) convertToRootTaskImpl(ctx sessionctx.Context) *rootTask { var prevSchema *expression.Schema // Network cost of transferring rows of table scan to TiDB. if t.tablePlan != nil { - t.cst += t.count() * sessVars.NetworkFactor * t.tblColHists.GetAvgRowSize(ctx, t.tablePlan.Schema().Columns, false, false) + t.cst += t.count() * sessVars.GetNetworkFactor(nil) * t.tblColHists.GetAvgRowSize(ctx, t.tablePlan.Schema().Columns, false, false) tp := t.tablePlan for len(tp.Children()) > 0 { @@ -1052,8 +1058,10 @@ func setTableScanToTableRowIDScan(p PhysicalPlan) { // rootTask is the final sink node of a plan graph. It should be a single goroutine on tidb. type rootTask struct { - p PhysicalPlan - cst float64 + p PhysicalPlan + cst float64 + isEmpty bool // isEmpty indicates if this task contains a dual table and returns empty data. + // TODO: The flag 'isEmpty' is only checked by Projection and UnionAll. We should support more cases in the future. } func (t *rootTask) copy() task { @@ -1100,6 +1108,15 @@ func (p *PhysicalLimit) attach2Task(tasks ...task) task { } t = cop.convertToRootTask(p.ctx) sunk = p.sinkIntoIndexLookUp(t) + } else if mpp, ok := t.(*mppTask); ok { + newCount := p.Offset + p.Count + childProfile := mpp.plan().statsInfo() + stats := deriveLimitStats(childProfile, float64(newCount)) + pushedDownLimit := PhysicalLimit{Count: newCount}.Init(p.ctx, stats, p.blockOffset) + mpp = attachPlan2Task(pushedDownLimit, mpp).(*mppTask) + pushedDownLimit.SetSchema(pushedDownLimit.children[0].Schema()) + pushedDownLimit.cost = mpp.cost() + t = mpp.convertToRootTask(p.ctx) } p.cost = t.cost() if sunk { @@ -1121,6 +1138,16 @@ func (p *PhysicalLimit) sinkIntoIndexLookUp(t task) bool { return false } } + + // If this happens, some Projection Operator must be inlined into this Limit. (issues/14428) + // For example, if the original plan is `IndexLookUp(col1, col2) -> Limit(col1, col2) -> Project(col1)`, + // then after inlining the Project, it will be `IndexLookUp(col1, col2) -> Limit(col1)` here. + // If the Limit is sunk into the IndexLookUp, the IndexLookUp's schema needs to be updated as well, + // but updating it here is not safe, so do not sink Limit into this IndexLookUp in this case now. + if p.Schema().Len() != reader.Schema().Len() { + return false + } + // We can sink Limit into IndexLookUpReader only if tablePlan contains no Selection. ts, isTableScan := reader.tablePlan.(*PhysicalTableScan) if !isTableScan { @@ -1297,10 +1324,44 @@ func (p *PhysicalProjection) attach2Task(tasks ...task) task { t = attachPlan2Task(p, t) t.addCost(p.GetCost(t.count())) p.cost = t.cost() + if root, ok := tasks[0].(*rootTask); ok && root.isEmpty { + t.(*rootTask).isEmpty = true + } + return t +} + +func (p *PhysicalUnionAll) attach2MppTasks(tasks ...task) task { + t := &mppTask{p: p} + childPlans := make([]PhysicalPlan, 0, len(tasks)) + var childMaxCost float64 + for _, tk := range tasks { + if mpp, ok := tk.(*mppTask); ok && !tk.invalid() { + childCost := mpp.cost() + if childCost > childMaxCost { + childMaxCost = childCost + } + childPlans = append(childPlans, mpp.plan()) + } else if root, ok := tk.(*rootTask); ok && root.isEmpty { + continue + } else { + return invalidTask + } + } + if len(childPlans) == 0 { + return invalidTask + } + p.SetChildren(childPlans...) + t.cst = childMaxCost + p.cost = t.cost() return t } func (p *PhysicalUnionAll) attach2Task(tasks ...task) task { + for _, t := range tasks { + if _, ok := t.(*mppTask); ok { + return p.attach2MppTasks(tasks...) + } + } t := &rootTask{p: p} childPlans := make([]PhysicalPlan, 0, len(tasks)) var childMaxCost float64 @@ -1780,7 +1841,7 @@ func (p *PhysicalStreamAgg) attach2Task(tasks ...task) task { // GetCost computes cost of stream aggregation considering CPU/memory. func (p *PhysicalStreamAgg) GetCost(inputRows float64, isRoot bool) float64 { - aggFuncFactor := p.getAggFuncCostFactor() + aggFuncFactor := p.getAggFuncCostFactor(false) var cpuCost float64 sessVars := p.ctx.GetSessionVars() if isRoot { @@ -1827,7 +1888,7 @@ func (p *PhysicalHashAgg) attach2TaskForMpp(tasks ...task) task { if proj != nil { attachPlan2Task(proj, mpp) } - mpp.addCost(p.GetCost(inputRows, false)) + mpp.addCost(p.GetCost(inputRows, false, true)) p.cost = mpp.cost() return mpp case Mpp2Phase: @@ -1860,7 +1921,7 @@ func (p *PhysicalHashAgg) attach2TaskForMpp(tasks ...task) task { attachPlan2Task(proj, newMpp) } // TODO: how to set 2-phase cost? - newMpp.addCost(p.GetCost(inputRows, false)) + newMpp.addCost(p.GetCost(inputRows, false, true)) finalAgg.SetCost(mpp.cost()) if proj != nil { proj.SetCost(mpp.cost()) @@ -1871,14 +1932,14 @@ func (p *PhysicalHashAgg) attach2TaskForMpp(tasks ...task) task { if partialAgg != nil { attachPlan2Task(partialAgg, mpp) } - mpp.addCost(p.GetCost(inputRows, false)) + mpp.addCost(p.GetCost(inputRows, false, true)) if partialAgg != nil { partialAgg.SetCost(mpp.cost()) } t = mpp.convertToRootTask(p.ctx) inputRows = t.count() attachPlan2Task(finalAgg, t) - t.addCost(p.GetCost(inputRows, true)) + t.addCost(p.GetCost(inputRows, true, false)) finalAgg.SetCost(t.cost()) return t default: @@ -1909,7 +1970,7 @@ func (p *PhysicalHashAgg) attach2Task(tasks ...task) task { partialAgg.SetChildren(cop.indexPlan) cop.indexPlan = partialAgg } - cop.addCost(p.GetCost(inputRows, false)) + cop.addCost(p.GetCost(inputRows, false, false)) } // In `newPartialAggregate`, we are using stats of final aggregation as stats // of `partialAgg`, so the network cost of transferring result rows of `partialAgg` @@ -1942,16 +2003,16 @@ func (p *PhysicalHashAgg) attach2Task(tasks ...task) task { // hash aggregation, it would cause under-estimation as the reason mentioned in comment above. // To make it simple, we also treat 2-phase parallel hash aggregation in TiDB layer as // 1-phase when computing cost. - t.addCost(p.GetCost(inputRows, true)) + t.addCost(p.GetCost(inputRows, true, false)) p.cost = t.cost() return t } // GetCost computes the cost of hash aggregation considering CPU/memory. -func (p *PhysicalHashAgg) GetCost(inputRows float64, isRoot bool) float64 { +func (p *PhysicalHashAgg) GetCost(inputRows float64, isRoot bool, isMPP bool) float64 { cardinality := p.statsInfo().RowCount numDistinctFunc := p.numDistinctFunc() - aggFuncFactor := p.getAggFuncCostFactor() + aggFuncFactor := p.getAggFuncCostFactor(isMPP) var cpuCost float64 sessVars := p.ctx.GetSessionVars() if isRoot { @@ -2026,11 +2087,16 @@ func (t *mppTask) convertToRootTaskImpl(ctx sessionctx.Context) *rootTask { StoreType: kv.TiFlash, }.Init(ctx, t.p.SelectBlockOffset()) p.stats = t.p.statsInfo() + + cst := t.cst + t.count()*ctx.GetSessionVars().GetNetworkFactor(nil) + p.cost = cst / p.ctx.GetSessionVars().CopTiFlashConcurrencyFactor + if p.ctx.GetSessionVars().IsMPPEnforced() { + p.cost = 0 + } rt := &rootTask{ p: p, - cst: t.cst / p.ctx.GetSessionVars().CopTiFlashConcurrencyFactor, + cst: p.cost, } - p.cost = rt.cost() return rt } @@ -2083,7 +2149,7 @@ func (t *mppTask) enforceExchangerImpl(prop *property.PhysicalProperty) *mppTask sender.SetChildren(t.p) receiver := PhysicalExchangeReceiver{}.Init(ctx, t.p.statsInfo()) receiver.SetChildren(sender) - cst := t.cst + t.count()*ctx.GetSessionVars().NetworkFactor + cst := t.cst + t.count()*ctx.GetSessionVars().GetNetworkFactor(nil) sender.cost = cst receiver.cost = cst return &mppTask{ diff --git a/planner/core/testdata/analyze_suite_out.json b/planner/core/testdata/analyze_suite_out.json index bdd31cd56f8cd..cb0dd2137c515 100644 --- a/planner/core/testdata/analyze_suite_out.json +++ b/planner/core/testdata/analyze_suite_out.json @@ -435,14 +435,14 @@ { "Name": "TestAnalyze", "Cases": [ - "Analyze{Index(a),Table(a, b)}", + "Analyze{Table(_tidb_rowid, a, b, _tidb_rowid)}", "TableReader(Table(t)->Sel([le(test.t.a, 2)]))", "IndexReader(Index(t.b)[[-inf,2)])", "TableReader(Table(t)->Sel([eq(test.t.a, 1) le(test.t.b, 2)]))", "TableReader(Table(t1)->Sel([le(test.t1.a, 2)]))", "IndexLookUp(Index(t1.a)[[1,1]], Table(t1)->Sel([le(test.t1.b, 2)]))", "TableReader(Table(t2)->Sel([le(test.t2.a, 2)]))", - "Analyze{Index(a),Index(b)}", + "Analyze{Table(_tidb_rowid, a, b, _tidb_rowid)}", "PartitionUnionAll{TableReader(Table(t4)->Sel([le(test.t4.a, 2)]))->TableReader(Table(t4)->Sel([le(test.t4.a, 2)]))}", "PartitionUnionAll{IndexReader(Index(t4.b)[[-inf,2)])->IndexReader(Index(t4.b)[[-inf,2)])}", "TableReader(Table(t4)->Sel([eq(test.t4.a, 1) le(test.t4.b, 2)]))" diff --git a/planner/core/testdata/integration_serial_suite_in.json b/planner/core/testdata/integration_serial_suite_in.json index 3234652e5d000..267f5e532ddc6 100644 --- a/planner/core/testdata/integration_serial_suite_in.json +++ b/planner/core/testdata/integration_serial_suite_in.json @@ -52,7 +52,32 @@ "explain format = 'brief' select count(*) from fact_t where exists (select 1 from d1_t where d1_k = fact_t.d1_k)", "explain format = 'brief' select count(*) from fact_t where exists (select 1 from d1_t where d1_k = fact_t.d1_k and value > fact_t.col1)", "explain format = 'brief' select count(*) from fact_t where not exists (select 1 from d1_t where d1_k = fact_t.d1_k)", - "explain format = 'brief' select count(*) from fact_t where not exists (select 1 from d1_t where d1_k = fact_t.d1_k and value > fact_t.col1)" + "explain format = 'brief' select count(*) from fact_t where not exists (select 1 from d1_t where d1_k = fact_t.d1_k and value > fact_t.col1)", + "explain format = 'brief' select count(*) from fact_t join d1_t on fact_t.d1_k > d1_t.d1_k", + "explain format = 'brief' select count(*) from fact_t left join d1_t on fact_t.d1_k > d1_t.d1_k", + "explain format = 'brief' select count(*) from fact_t right join d1_t on fact_t.d1_k > d1_t.d1_k", + "explain format = 'brief' select count(*) from fact_t where d1_k not in (select d1_k from d1_t)" + ] + }, + { + "name": "TestMPPOuterJoinBuildSideForBroadcastJoin", + "cases": [ + "explain format = 'brief' select count(*) from a left join b on a.id = b.id", + "explain format = 'brief' select count(*) from b right join a on a.id = b.id" + ] + }, + { + "name": "TestMPPOuterJoinBuildSideForShuffleJoinWithFixedBuildSide", + "cases": [ + "explain format = 'brief' select count(*) from a left join b on a.id = b.id", + "explain format = 'brief' select count(*) from b right join a on a.id = b.id" + ] + }, + { + "name": "TestMPPOuterJoinBuildSideForShuffleJoin", + "cases": [ + "explain format = 'brief' select count(*) from a left join b on a.id = b.id", + "explain format = 'brief' select count(*) from b right join a on a.id = b.id" ] }, { @@ -76,26 +101,6 @@ "explain format = 'brief' select count(*) from fact_t where not exists (select 1 from d1_t where d1_k = fact_t.d1_k and value > fact_t.col1)" ] }, - { - "name": "TestBroadcastJoin", - "cases": [ - "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k", - "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t,d2_t,d3_t) */ count(*) from fact_t, d1_t, d2_t, d3_t where fact_t.d1_k = d1_t.d1_k and fact_t.d2_k = d2_t.d2_k and fact_t.d3_k = d3_t.d3_k", - "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t), broadcast_join_local(d1_t) */ count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k", - "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t,d2_t,d3_t), broadcast_join_local(d2_t) */ count(*) from fact_t, d1_t, d2_t, d3_t where fact_t.d1_k = d1_t.d1_k and fact_t.d2_k = d2_t.d2_k and fact_t.d3_k = d3_t.d3_k", - "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k", - "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k", - "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col1 > d1_t.value", - "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col1 > 10", - "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col2 > 10 and fact_t.col1 > d1_t.value", - "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k and d1_t.value > 10", - "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k and d1_t.value > 10 and fact_t.col1 > d1_t.value", - "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t where exists (select 1 from d1_t where d1_k = fact_t.d1_k)", - "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t where exists (select 1 from d1_t where d1_k = fact_t.d1_k and value > fact_t.col1)", - "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t where not exists (select 1 from d1_t where d1_k = fact_t.d1_k)", - "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t where not exists (select 1 from d1_t where d1_k = fact_t.d1_k and value > fact_t.col1)" - ] - }, { "name": "TestJoinNotSupportedByTiFlash", "cases": [ @@ -225,6 +230,16 @@ "desc format = 'brief' select id from t as A where not exists (select 1 from t where t.id=A.id)" ] }, + { + "name": "TestMppUnionAll", + "cases": [ + "explain format = 'brief' select count(*) from (select a , b from t union all select a , b from t1) tt", + "explain format = 'brief' select count(*) from (select a , b from t union all select a , b from t1 union all select a, b from t where false) tt", + "explain format = 'brief' select count(*) from (select a , b from t union all select a , c from t1) tt", + "explain format = 'brief' select count(*) from (select a , b from t union all select a , c from t1 where false) tt", + "explain format = 'brief' select count(*) from (select a , b from t where false union all select a , c from t1 where false) tt" + ] + }, { "name": "TestMppJoinDecimal", "cases": [ @@ -276,7 +291,10 @@ "desc format = 'brief' select * from (select id from t group by id) C join (select sum(b),id from (select t.id, t1.id as b from t join (select id, count(*) as c from t group by id) t1 on t.id=t1.id)A group by id)B on C.id=b.id", "desc format = 'brief' select * from t join t t1 on t.id = t1.id order by t.value limit 1", "desc format = 'brief' select * from t join t t1 on t.id = t1.id order by t.value % 100 limit 1", - "desc format = 'brief' select count(*) from (select t.id, t.value v1 from t join t t1 on t.id = t1.id order by t.value limit 20) v group by v.v1" + "desc format = 'brief' select count(*) from (select t.id, t.value v1 from t join t t1 on t.id = t1.id order by t.value limit 20) v group by v.v1", + "desc format = 'brief' select * from t join t t1 on t.id = t1.id limit 1", + "desc format = 'brief' select * from t join t t1 on t.id = t1.id limit 1", + "desc format = 'brief' select count(*) from (select t.id, t.value v1 from t join t t1 on t.id = t1.id limit 20) v group by v.v1" ] }, { @@ -300,5 +318,11 @@ "cases": [ "select (2) in (select b from t) from (select t.a < (select t.a from t t1 limit 1) from t) t" ] + }, + { + "name": "TestMergeContinuousSelections", + "cases": [ + "desc format = 'brief' SELECT table2 . `col_char_64` AS field1 FROM `ts` AS table2 INNER JOIN (SELECT DISTINCT SUBQUERY3_t1 . * FROM `ts` AS SUBQUERY3_t1 LEFT OUTER JOIN `ts` AS SUBQUERY3_t2 ON SUBQUERY3_t2 . `col_varchar_64_not_null` = SUBQUERY3_t1 . `col_varchar_key`) AS table3 ON (table3 . `col_varchar_key` = table2 . `col_varchar_64`) WHERE table3 . `col_char_64_not_null` >= SOME (SELECT SUBQUERY4_t1 . `col_varchar_64` AS SUBQUERY4_field1 FROM `ts` AS SUBQUERY4_t1) GROUP BY field1 ;" + ] } ] diff --git a/planner/core/testdata/integration_serial_suite_out.json b/planner/core/testdata/integration_serial_suite_out.json index 8b67310894d4b..48afc8171f401 100644 --- a/planner/core/testdata/integration_serial_suite_out.json +++ b/planner/core/testdata/integration_serial_suite_out.json @@ -13,8 +13,8 @@ { "SQL": "explain format = 'brief' select * from t where cast(t.a as float) + 3 = 5.1", "Plan": [ - "Selection 10000.00 root eq(plus(cast(test.t.a, float BINARY), 3), 5.1)", - "└─TableReader 10000.00 root data:TableFullScan", + "TableReader 8000.00 root data:Selection", + "└─Selection 8000.00 cop[tiflash] eq(plus(cast(test.t.a, float BINARY), 3), 5.1)", " └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" ] }, @@ -102,7 +102,7 @@ { "SQL": "explain format = 'verbose' select count(*) from t2 group by a", "Plan": [ - "TableReader_24 3.00 3.21 root data:ExchangeSender_23", + "TableReader_24 3.00 3.33 root data:ExchangeSender_23", "└─ExchangeSender_23 3.00 77.00 batchCop[tiflash] ExchangeType: PassThrough", " └─Projection_22 3.00 0.00 batchCop[tiflash] Column#4", " └─HashAgg_8 3.00 77.00 batchCop[tiflash] group by:test.t2.a, funcs:count(1)->Column#4", @@ -152,8 +152,8 @@ { "SQL": "explain format = 'verbose' select count(*) from t1 join t2 on t1.a = t2.a", "Plan": [ - "StreamAgg_12 1.00 18.81 root funcs:count(1)->Column#7", - "└─TableReader_44 3.00 9.81 root data:ExchangeSender_43", + "StreamAgg_12 1.00 18.93 root funcs:count(1)->Column#7", + "└─TableReader_44 3.00 9.93 root data:ExchangeSender_43", " └─ExchangeSender_43 3.00 235.38 cop[tiflash] ExchangeType: PassThrough", " └─HashJoin_40 3.00 235.38 cop[tiflash] inner join, equal:[eq(test.t1.a, test.t2.a)]", " ├─ExchangeReceiver_19(Build) 3.00 77.00 cop[tiflash] ", @@ -167,11 +167,11 @@ { "SQL": "explain format = 'verbose' select count(*) from t1 join t2 on t1.a = t2.a join t3 on t1.b = t3.b", "Plan": [ - "StreamAgg_15 1.00 60.48 root funcs:count(1)->Column#10", - "└─HashJoin_65 3.00 51.48 root inner join, equal:[eq(test.t1.b, test.t3.b)]", + "StreamAgg_15 1.00 60.60 root funcs:count(1)->Column#10", + "└─HashJoin_65 3.00 51.60 root inner join, equal:[eq(test.t1.b, test.t3.b)]", " ├─IndexReader_53(Build) 3.00 11.66 root index:IndexFullScan_52", " │ └─IndexFullScan_52 3.00 150.50 cop[tikv] table:t3, index:c(b) keep order:false", - " └─TableReader_39(Probe) 3.00 11.02 root data:ExchangeSender_38", + " └─TableReader_39(Probe) 3.00 11.14 root data:ExchangeSender_38", " └─ExchangeSender_38 3.00 264.38 cop[tiflash] ExchangeType: PassThrough", " └─HashJoin_29 3.00 264.38 cop[tiflash] inner join, equal:[eq(test.t1.a, test.t2.a)]", " ├─ExchangeReceiver_35(Build) 3.00 106.00 cop[tiflash] ", @@ -185,21 +185,22 @@ { "SQL": "explain format = 'verbose' select (2) in (select count(*) from t1) from (select t.b < (select t.b from t2 limit 1 ) from t3 t) t", "Plan": [ - "HashJoin_19 3.00 133.41 root CARTESIAN left outer semi join", - "├─Selection_39(Build) 0.80 11.18 root eq(2, Column#18)", - "│ └─StreamAgg_60 1.00 69.50 root funcs:count(Column#32)->Column#18", - "│ └─TableReader_61 1.00 5.17 root data:StreamAgg_44", - "│ └─StreamAgg_44 1.00 8.18 batchCop[tiflash] funcs:count(1)->Column#32", - "│ └─TableFullScan_59 3.00 60.50 batchCop[tiflash] table:t1 keep order:false", - "└─Projection_20(Probe) 3.00 101.83 root 1->Column#28", - " └─Apply_22 3.00 82.03 root CARTESIAN left outer join", + "HashJoin_19 3.00 127.40 root CARTESIAN left outer semi join", + "├─Selection_44(Build) 0.80 11.18 root eq(2, Column#18)", + "│ └─StreamAgg_65 1.00 69.50 root funcs:count(Column#32)->Column#18", + "│ └─TableReader_66 1.00 5.17 root data:StreamAgg_49", + "│ └─StreamAgg_49 1.00 8.18 batchCop[tiflash] funcs:count(1)->Column#32", + "│ └─TableFullScan_64 3.00 60.50 batchCop[tiflash] table:t1 keep order:false", + "└─Projection_20(Probe) 3.00 95.82 root 1->Column#28", + " └─Apply_22 3.00 76.02 root CARTESIAN left outer join", " ├─TableReader_24(Build) 3.00 10.16 root data:TableFullScan_23", " │ └─TableFullScan_23 3.00 128.00 cop[tikv] table:t keep order:false", - " └─Projection_27(Probe) 1.00 23.96 root 1->Column#26", - " └─Limit_28 1.00 5.36 root offset:0, count:1", - " └─TableReader_34 1.00 5.36 root data:Limit_33", - " └─Limit_33 1.00 56.00 cop[tikv] offset:0, count:1", - " └─TableFullScan_31 1.00 56.00 cop[tikv] table:t2 keep order:false" + " └─Projection_27(Probe) 1.00 21.95 root 1->Column#26", + " └─Limit_31 1.00 3.35 root offset:0, count:1", + " └─TableReader_43 1.00 3.35 root data:ExchangeSender_42", + " └─ExchangeSender_42 1.00 79.50 cop[tiflash] ExchangeType: PassThrough", + " └─Limit_41 1.00 79.50 cop[tiflash] offset:0, count:1", + " └─TableFullScan_40 1.00 79.50 cop[tiflash] table:t2 keep order:false" ] }, { @@ -460,6 +461,172 @@ " │ └─TableFullScan 2.00 cop[tiflash] table:d1_t keep order:false", " └─TableFullScan(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" ] + }, + { + "SQL": "explain format = 'brief' select count(*) from fact_t join d1_t on fact_t.d1_k > d1_t.d1_k", + "Plan": [ + "HashAgg 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader 1.00 root data:ExchangeSender", + " └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin 16.00 batchCop[tiflash] CARTESIAN inner join, other cond:gt(test.fact_t.d1_k, test.d1_t.d1_k)", + " ├─ExchangeReceiver(Build) 2.00 batchCop[tiflash] ", + " │ └─ExchangeSender 2.00 batchCop[tiflash] ExchangeType: Broadcast", + " │ └─Selection 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false", + " └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false" + ] + }, + { + "SQL": "explain format = 'brief' select count(*) from fact_t left join d1_t on fact_t.d1_k > d1_t.d1_k", + "Plan": [ + "HashAgg 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader 1.00 root data:ExchangeSender", + " └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin 16.00 batchCop[tiflash] CARTESIAN left outer join, other cond:gt(test.fact_t.d1_k, test.d1_t.d1_k)", + " ├─ExchangeReceiver(Build) 2.00 batchCop[tiflash] ", + " │ └─ExchangeSender 2.00 batchCop[tiflash] ExchangeType: Broadcast", + " │ └─Selection 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false", + " └─TableFullScan(Probe) 8.00 batchCop[tiflash] table:fact_t keep order:false" + ] + }, + { + "SQL": "explain format = 'brief' select count(*) from fact_t right join d1_t on fact_t.d1_k > d1_t.d1_k", + "Plan": [ + "HashAgg 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader 1.00 root data:ExchangeSender", + " └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", + " └─HashJoin 16.00 batchCop[tiflash] CARTESIAN right outer join, other cond:gt(test.fact_t.d1_k, test.d1_t.d1_k)", + " ├─ExchangeReceiver(Build) 8.00 batchCop[tiflash] ", + " │ └─ExchangeSender 8.00 batchCop[tiflash] ExchangeType: Broadcast", + " │ └─Selection 8.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k))", + " │ └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false", + " └─TableFullScan(Probe) 2.00 batchCop[tiflash] table:d1_t keep order:false" + ] + }, + { + "SQL": "explain format = 'brief' select count(*) from fact_t where d1_k not in (select d1_k from d1_t)", + "Plan": [ + "StreamAgg 1.00 root funcs:count(1)->Column#11", + "└─TableReader 6.40 root data:ExchangeSender", + " └─ExchangeSender 6.40 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin 6.40 cop[tiflash] CARTESIAN anti semi join, other cond:eq(test.fact_t.d1_k, test.d1_t.d1_k)", + " ├─ExchangeReceiver(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender 2.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─TableFullScan 2.00 cop[tiflash] table:d1_t keep order:false", + " └─TableFullScan(Probe) 8.00 cop[tiflash] table:fact_t keep order:false" + ] + } + ] + }, + { + "Name": "TestMPPOuterJoinBuildSideForBroadcastJoin", + "Cases": [ + { + "SQL": "explain format = 'brief' select count(*) from a left join b on a.id = b.id", + "Plan": [ + "StreamAgg 1.00 root funcs:count(1)->Column#7", + "└─TableReader 2.00 root data:ExchangeSender", + " └─ExchangeSender 2.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin 2.00 cop[tiflash] left outer join, equal:[eq(test.a.id, test.b.id)]", + " ├─ExchangeReceiver(Build) 3.00 cop[tiflash] ", + " │ └─ExchangeSender 3.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection 3.00 cop[tiflash] not(isnull(test.b.id))", + " │ └─TableFullScan 3.00 cop[tiflash] table:b keep order:false", + " └─TableFullScan(Probe) 2.00 cop[tiflash] table:a keep order:false" + ] + }, + { + "SQL": "explain format = 'brief' select count(*) from b right join a on a.id = b.id", + "Plan": [ + "StreamAgg 1.00 root funcs:count(1)->Column#7", + "└─TableReader 2.00 root data:ExchangeSender", + " └─ExchangeSender 2.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin 2.00 cop[tiflash] right outer join, equal:[eq(test.b.id, test.a.id)]", + " ├─ExchangeReceiver(Build) 3.00 cop[tiflash] ", + " │ └─ExchangeSender 3.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection 3.00 cop[tiflash] not(isnull(test.b.id))", + " │ └─TableFullScan 3.00 cop[tiflash] table:b keep order:false", + " └─TableFullScan(Probe) 2.00 cop[tiflash] table:a keep order:false" + ] + } + ] + }, + { + "Name": "TestMPPOuterJoinBuildSideForShuffleJoinWithFixedBuildSide", + "Cases": [ + { + "SQL": "explain format = 'brief' select count(*) from a left join b on a.id = b.id", + "Plan": [ + "StreamAgg 1.00 root funcs:count(1)->Column#7", + "└─TableReader 2.00 root data:ExchangeSender", + " └─ExchangeSender 2.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin 2.00 cop[tiflash] left outer join, equal:[eq(test.a.id, test.b.id)]", + " ├─ExchangeReceiver(Build) 3.00 cop[tiflash] ", + " │ └─ExchangeSender 3.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.b.id", + " │ └─Selection 3.00 cop[tiflash] not(isnull(test.b.id))", + " │ └─TableFullScan 3.00 cop[tiflash] table:b keep order:false", + " └─ExchangeReceiver(Probe) 2.00 cop[tiflash] ", + " └─ExchangeSender 2.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.a.id", + " └─TableFullScan 2.00 cop[tiflash] table:a keep order:false" + ] + }, + { + "SQL": "explain format = 'brief' select count(*) from b right join a on a.id = b.id", + "Plan": [ + "StreamAgg 1.00 root funcs:count(1)->Column#7", + "└─TableReader 2.00 root data:ExchangeSender", + " └─ExchangeSender 2.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin 2.00 cop[tiflash] right outer join, equal:[eq(test.b.id, test.a.id)]", + " ├─ExchangeReceiver(Build) 3.00 cop[tiflash] ", + " │ └─ExchangeSender 3.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.b.id", + " │ └─Selection 3.00 cop[tiflash] not(isnull(test.b.id))", + " │ └─TableFullScan 3.00 cop[tiflash] table:b keep order:false", + " └─ExchangeReceiver(Probe) 2.00 cop[tiflash] ", + " └─ExchangeSender 2.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.a.id", + " └─TableFullScan 2.00 cop[tiflash] table:a keep order:false" + ] + } + ] + }, + { + "Name": "TestMPPOuterJoinBuildSideForShuffleJoin", + "Cases": [ + { + "SQL": "explain format = 'brief' select count(*) from a left join b on a.id = b.id", + "Plan": [ + "StreamAgg 1.00 root funcs:count(1)->Column#7", + "└─TableReader 2.00 root data:ExchangeSender", + " └─ExchangeSender 2.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin 2.00 cop[tiflash] left outer join, equal:[eq(test.a.id, test.b.id)]", + " ├─ExchangeReceiver(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender 2.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.a.id", + " │ └─TableFullScan 2.00 cop[tiflash] table:a keep order:false", + " └─ExchangeReceiver(Probe) 3.00 cop[tiflash] ", + " └─ExchangeSender 3.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.b.id", + " └─Selection 3.00 cop[tiflash] not(isnull(test.b.id))", + " └─TableFullScan 3.00 cop[tiflash] table:b keep order:false" + ] + }, + { + "SQL": "explain format = 'brief' select count(*) from b right join a on a.id = b.id", + "Plan": [ + "StreamAgg 1.00 root funcs:count(1)->Column#7", + "└─TableReader 2.00 root data:ExchangeSender", + " └─ExchangeSender 2.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin 2.00 cop[tiflash] right outer join, equal:[eq(test.b.id, test.a.id)]", + " ├─ExchangeReceiver(Build) 2.00 cop[tiflash] ", + " │ └─ExchangeSender 2.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.a.id", + " │ └─TableFullScan 2.00 cop[tiflash] table:a keep order:false", + " └─ExchangeReceiver(Probe) 3.00 cop[tiflash] ", + " └─ExchangeSender 3.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.b.id", + " └─Selection 3.00 cop[tiflash] not(isnull(test.b.id))", + " └─TableFullScan 3.00 cop[tiflash] table:b keep order:false" + ] } ] }, @@ -587,13 +754,13 @@ " └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough", " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", " └─HashJoin 32.00 batchCop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─ExchangeReceiver(Build) 16.00 batchCop[tiflash] ", - " │ └─ExchangeSender 16.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " │ └─Selection 16.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k))", - " │ └─TableFullScan 16.00 batchCop[tiflash] table:fact_t keep order:false", - " └─ExchangeReceiver(Probe) 4.00 batchCop[tiflash] ", - " └─ExchangeSender 4.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " └─TableFullScan 4.00 batchCop[tiflash] table:d1_t keep order:false" + " ├─ExchangeReceiver(Build) 4.00 batchCop[tiflash] ", + " │ └─ExchangeSender 4.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " │ └─TableFullScan 4.00 batchCop[tiflash] table:d1_t keep order:false", + " └─ExchangeReceiver(Probe) 16.00 batchCop[tiflash] ", + " └─ExchangeSender 16.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " └─Selection 16.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan 16.00 batchCop[tiflash] table:fact_t keep order:false" ] }, { @@ -681,13 +848,13 @@ " └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough", " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", " └─HashJoin 32.00 batchCop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10)", - " ├─ExchangeReceiver(Build) 16.00 batchCop[tiflash] ", - " │ └─ExchangeSender 16.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", - " │ └─Selection 16.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k))", - " │ └─TableFullScan 16.00 batchCop[tiflash] table:fact_t keep order:false", - " └─ExchangeReceiver(Probe) 4.00 batchCop[tiflash] ", - " └─ExchangeSender 4.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", - " └─TableFullScan 4.00 batchCop[tiflash] table:d1_t keep order:false" + " ├─ExchangeReceiver(Build) 4.00 batchCop[tiflash] ", + " │ └─ExchangeSender 4.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.d1_t.d1_k", + " │ └─TableFullScan 4.00 batchCop[tiflash] table:d1_t keep order:false", + " └─ExchangeReceiver(Probe) 16.00 batchCop[tiflash] ", + " └─ExchangeSender 16.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.fact_t.d1_k", + " └─Selection 16.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan 16.00 batchCop[tiflash] table:fact_t keep order:false" ] }, { @@ -777,208 +944,6 @@ } ] }, - { - "Name": "TestBroadcastJoin", - "Cases": [ - { - "SQL": "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k", - "Plan": [ - "HashAgg 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin 8.00 batchCop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─Selection(Build) 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false, global read", - " └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false" - ] - }, - { - "SQL": "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t,d2_t,d3_t) */ count(*) from fact_t, d1_t, d2_t, d3_t where fact_t.d1_k = d1_t.d1_k and fact_t.d2_k = d2_t.d2_k and fact_t.d3_k = d3_t.d3_k", - "Plan": [ - "HashAgg 1.00 root funcs:count(Column#18)->Column#17", - "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#18", - " └─HashJoin 8.00 batchCop[tiflash] inner join, equal:[eq(test.fact_t.d3_k, test.d3_t.d3_k)]", - " ├─Selection(Build) 2.00 batchCop[tiflash] not(isnull(test.d3_t.d3_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d3_t keep order:false, global read", - " └─HashJoin(Probe) 8.00 batchCop[tiflash] inner join, equal:[eq(test.fact_t.d2_k, test.d2_t.d2_k)]", - " ├─Selection(Build) 2.00 batchCop[tiflash] not(isnull(test.d2_t.d2_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d2_t keep order:false, global read", - " └─HashJoin(Probe) 8.00 batchCop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─Selection(Build) 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false, global read", - " └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k)), not(isnull(test.fact_t.d2_k)), not(isnull(test.fact_t.d3_k))", - " └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false" - ] - }, - { - "SQL": "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t), broadcast_join_local(d1_t) */ count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k", - "Plan": [ - "HashAgg 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin 8.00 batchCop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─Selection(Build) 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false", - " └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false, global read" - ] - }, - { - "SQL": "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t,d2_t,d3_t), broadcast_join_local(d2_t) */ count(*) from fact_t, d1_t, d2_t, d3_t where fact_t.d1_k = d1_t.d1_k and fact_t.d2_k = d2_t.d2_k and fact_t.d3_k = d3_t.d3_k", - "Plan": [ - "HashAgg 1.00 root funcs:count(Column#18)->Column#17", - "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#18", - " └─HashJoin 8.00 batchCop[tiflash] inner join, equal:[eq(test.fact_t.d3_k, test.d3_t.d3_k)]", - " ├─Selection(Build) 2.00 batchCop[tiflash] not(isnull(test.d3_t.d3_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d3_t keep order:false, global read", - " └─HashJoin(Probe) 8.00 batchCop[tiflash] inner join, equal:[eq(test.fact_t.d2_k, test.d2_t.d2_k)]", - " ├─Selection(Build) 2.00 batchCop[tiflash] not(isnull(test.d2_t.d2_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d2_t keep order:false", - " └─HashJoin(Probe) 8.00 batchCop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─Selection(Build) 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false, global read", - " └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k)), not(isnull(test.fact_t.d2_k)), not(isnull(test.fact_t.d3_k))", - " └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false, global read" - ] - }, - { - "SQL": "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k", - "Plan": [ - "HashAgg 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin 8.00 batchCop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─Selection(Build) 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false, global read", - " └─TableFullScan(Probe) 8.00 batchCop[tiflash] table:fact_t keep order:false" - ] - }, - { - "SQL": "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k", - "Plan": [ - "HashAgg 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin 8.00 batchCop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─TableFullScan(Build) 2.00 batchCop[tiflash] table:d1_t keep order:false", - " └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false, global read" - ] - }, - { - "SQL": "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col1 > d1_t.value", - "Plan": [ - "HashAgg 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin 8.00 batchCop[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.fact_t.col1, test.d1_t.value)", - " ├─Selection(Build) 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false, global read", - " └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", - " └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false" - ] - }, - { - "SQL": "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col1 > 10", - "Plan": [ - "HashAgg 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin 8.00 batchCop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col1, 10)]", - " ├─Selection(Build) 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false, global read", - " └─TableFullScan(Probe) 8.00 batchCop[tiflash] table:fact_t keep order:false" - ] - }, - { - "SQL": "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col2 > 10 and fact_t.col1 > d1_t.value", - "Plan": [ - "HashAgg 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin 8.00 batchCop[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col2, 10)], other cond:gt(test.fact_t.col1, test.d1_t.value)", - " ├─Selection(Build) 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false, global read", - " └─TableFullScan(Probe) 8.00 batchCop[tiflash] table:fact_t keep order:false" - ] - }, - { - "SQL": "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k and d1_t.value > 10", - "Plan": [ - "HashAgg 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin 8.00 batchCop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10)", - " ├─TableFullScan(Build) 2.00 batchCop[tiflash] table:d1_t keep order:false", - " └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false, global read" - ] - }, - { - "SQL": "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k and d1_t.value > 10 and fact_t.col1 > d1_t.value", - "Plan": [ - "HashAgg 1.00 root funcs:count(Column#12)->Column#11", - "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#12", - " └─HashJoin 8.00 batchCop[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10), other cond:gt(test.fact_t.col1, test.d1_t.value)", - " ├─Selection(Build) 8.00 batchCop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", - " │ └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false, global read", - " └─TableFullScan(Probe) 2.00 batchCop[tiflash] table:d1_t keep order:false" - ] - }, - { - "SQL": "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t where exists (select 1 from d1_t where d1_k = fact_t.d1_k)", - "Plan": [ - "HashAgg 1.00 root funcs:count(Column#13)->Column#12", - "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#13", - " └─HashJoin 6.40 batchCop[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─Selection(Build) 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false, global read", - " └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false" - ] - }, - { - "SQL": "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t where exists (select 1 from d1_t where d1_k = fact_t.d1_k and value > fact_t.col1)", - "Plan": [ - "HashAgg 1.00 root funcs:count(Column#13)->Column#12", - "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#13", - " └─HashJoin 6.40 batchCop[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", - " ├─Selection(Build) 2.00 batchCop[tiflash] not(isnull(test.d1_t.d1_k)), not(isnull(test.d1_t.value))", - " │ └─TableFullScan 2.00 batchCop[tiflash] table:d1_t keep order:false, global read", - " └─Selection(Probe) 8.00 batchCop[tiflash] not(isnull(test.fact_t.col1)), not(isnull(test.fact_t.d1_k))", - " └─TableFullScan 8.00 batchCop[tiflash] table:fact_t keep order:false" - ] - }, - { - "SQL": "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t where not exists (select 1 from d1_t where d1_k = fact_t.d1_k)", - "Plan": [ - "HashAgg 1.00 root funcs:count(Column#13)->Column#12", - "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#13", - " └─HashJoin 6.40 batchCop[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─TableFullScan(Build) 2.00 batchCop[tiflash] table:d1_t keep order:false, global read", - " └─TableFullScan(Probe) 8.00 batchCop[tiflash] table:fact_t keep order:false" - ] - }, - { - "SQL": "explain format = 'brief' select /*+ broadcast_join(fact_t,d1_t) */ count(*) from fact_t where not exists (select 1 from d1_t where d1_k = fact_t.d1_k and value > fact_t.col1)", - "Plan": [ - "HashAgg 1.00 root funcs:count(Column#13)->Column#12", - "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#13", - " └─HashJoin 6.40 batchCop[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", - " ├─TableFullScan(Build) 2.00 batchCop[tiflash] table:d1_t keep order:false, global read", - " └─TableFullScan(Probe) 8.00 batchCop[tiflash] table:fact_t keep order:false" - ] - } - ] - }, { "Name": "TestJoinNotSupportedByTiFlash", "Cases": [ @@ -1796,6 +1761,78 @@ } ] }, + { + "Name": "TestMppUnionAll", + "Cases": [ + { + "SQL": "explain format = 'brief' select count(*) from (select a , b from t union all select a , b from t1) tt", + "Plan": [ + "HashAgg 1.00 root funcs:count(Column#12)->Column#11", + "└─TableReader 1.00 root data:ExchangeSender", + " └─ExchangeSender 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg 1.00 cop[tiflash] funcs:count(1)->Column#12", + " └─Union 20000.00 cop[tiflash] ", + " ├─Projection 10000.00 cop[tiflash] cast(test.t.a, int(11) BINARY)->Column#9, test.t.b", + " │ └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + " └─Projection 10000.00 cop[tiflash] test.t1.a, cast(test.t1.b, int(11) BINARY)->Column#10", + " └─TableFullScan 10000.00 cop[tiflash] table:t1 keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format = 'brief' select count(*) from (select a , b from t union all select a , b from t1 union all select a, b from t where false) tt", + "Plan": [ + "HashAgg 1.00 root funcs:count(Column#16)->Column#15", + "└─TableReader 1.00 root data:ExchangeSender", + " └─ExchangeSender 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg 1.00 cop[tiflash] funcs:count(1)->Column#16", + " └─Union 20000.00 cop[tiflash] ", + " ├─Projection 10000.00 cop[tiflash] cast(test.t.a, int(11) BINARY)->Column#13, test.t.b", + " │ └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + " └─Projection 10000.00 cop[tiflash] test.t1.a, cast(test.t1.b, int(11) BINARY)->Column#14", + " └─TableFullScan 10000.00 cop[tiflash] table:t1 keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format = 'brief' select count(*) from (select a , b from t union all select a , c from t1) tt", + "Plan": [ + "HashAgg 1.00 root funcs:count(Column#14)->Column#11", + "└─TableReader 1.00 root data:ExchangeSender", + " └─ExchangeSender 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─HashAgg 1.00 cop[tiflash] funcs:count(1)->Column#14", + " └─Union 20000.00 cop[tiflash] ", + " ├─Projection 10000.00 cop[tiflash] cast(Column#9, int(11) BINARY)->Column#9, Column#10", + " │ └─Projection 10000.00 cop[tiflash] test.t.a, cast(test.t.b, double BINARY)->Column#10", + " │ └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + " └─Projection 10000.00 cop[tiflash] test.t1.a, cast(test.t1.c, double BINARY)->Column#10", + " └─TableFullScan 10000.00 cop[tiflash] table:t1 keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format = 'brief' select count(*) from (select a , b from t union all select a , c from t1 where false) tt", + "Plan": [ + "HashAgg 1.00 root funcs:count(Column#14)->Column#11", + "└─TableReader 1.00 root data:ExchangeSender", + " └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#14", + " └─Union 10000.00 batchCop[tiflash] ", + " └─Projection 10000.00 batchCop[tiflash] cast(Column#9, int(11) BINARY)->Column#9, Column#10", + " └─Projection 10000.00 batchCop[tiflash] test.t.a, cast(test.t.b, double BINARY)->Column#10", + " └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format = 'brief' select count(*) from (select a , b from t where false union all select a , c from t1 where false) tt", + "Plan": [ + "StreamAgg 1.00 root funcs:count(1)->Column#11", + "└─Union 0.00 root ", + " ├─Projection 0.00 root test.t.a, cast(test.t.b, double BINARY)->Column#10", + " │ └─TableDual 0.00 root rows:0", + " └─Projection 0.00 root test.t1.a, cast(test.t1.c, double BINARY)->Column#10", + " └─TableDual 0.00 root rows:0" + ] + } + ] + }, { "Name": "TestMppJoinDecimal", "Cases": [ @@ -1926,27 +1963,27 @@ "└─TableReader 19492.21 root data:ExchangeSender", " └─ExchangeSender 19492.21 cop[tiflash] ExchangeType: PassThrough", " └─HashJoin 19492.21 cop[tiflash] right outer join, equal:[eq(test.t.c3, test.t.c4)]", - " ├─Projection(Build) 15593.77 cop[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5", - " │ └─HashJoin 15593.77 cop[tiflash] inner join, equal:[eq(test.t.c5, test.t.c3)]", - " │ ├─ExchangeReceiver(Build) 10000.00 cop[tiflash] ", - " │ │ └─ExchangeSender 10000.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: Column#25", - " │ │ └─Projection 10000.00 cop[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, cast(test.t.c3, decimal(40,20))->Column#25", - " │ │ └─TableFullScan 10000.00 cop[tiflash] table:t3 keep order:false, stats:pseudo", - " │ └─ExchangeReceiver(Probe) 12475.01 cop[tiflash] ", - " │ └─ExchangeSender 12475.01 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.c5", - " │ └─HashJoin 12475.01 cop[tiflash] inner join, equal:[eq(test.t.c2, test.t.c1)]", - " │ ├─ExchangeReceiver(Build) 9980.01 cop[tiflash] ", - " │ │ └─ExchangeSender 9980.01 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.c2", - " │ │ └─Selection 9980.01 cop[tiflash] not(isnull(test.t.c2)), not(isnull(test.t.c5))", - " │ │ └─TableFullScan 10000.00 cop[tiflash] table:t2 keep order:false, stats:pseudo", - " │ └─ExchangeReceiver(Probe) 9990.00 cop[tiflash] ", - " │ └─ExchangeSender 9990.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.c1", - " │ └─Selection 9990.00 cop[tiflash] not(isnull(test.t.c1))", - " │ └─TableFullScan 10000.00 cop[tiflash] table:t1 keep order:false, stats:pseudo", - " └─ExchangeReceiver(Probe) 10000.00 cop[tiflash] ", - " └─ExchangeSender 10000.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: Column#27", - " └─Projection 10000.00 cop[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, cast(test.t.c4, decimal(40,20))->Column#27", - " └─TableFullScan 10000.00 cop[tiflash] table:t4 keep order:false, stats:pseudo" + " ├─ExchangeReceiver(Build) 10000.00 cop[tiflash] ", + " │ └─ExchangeSender 10000.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: Column#27", + " │ └─Projection 10000.00 cop[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, cast(test.t.c4, decimal(40,20))->Column#27", + " │ └─TableFullScan 10000.00 cop[tiflash] table:t4 keep order:false, stats:pseudo", + " └─Projection(Probe) 15593.77 cop[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5", + " └─HashJoin 15593.77 cop[tiflash] inner join, equal:[eq(test.t.c5, test.t.c3)]", + " ├─ExchangeReceiver(Build) 10000.00 cop[tiflash] ", + " │ └─ExchangeSender 10000.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: Column#25", + " │ └─Projection 10000.00 cop[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, cast(test.t.c3, decimal(40,20))->Column#25", + " │ └─TableFullScan 10000.00 cop[tiflash] table:t3 keep order:false, stats:pseudo", + " └─ExchangeReceiver(Probe) 12475.01 cop[tiflash] ", + " └─ExchangeSender 12475.01 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.c5", + " └─HashJoin 12475.01 cop[tiflash] inner join, equal:[eq(test.t.c2, test.t.c1)]", + " ├─ExchangeReceiver(Build) 9980.01 cop[tiflash] ", + " │ └─ExchangeSender 9980.01 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.c2", + " │ └─Selection 9980.01 cop[tiflash] not(isnull(test.t.c2)), not(isnull(test.t.c5))", + " │ └─TableFullScan 10000.00 cop[tiflash] table:t2 keep order:false, stats:pseudo", + " └─ExchangeReceiver(Probe) 9990.00 cop[tiflash] ", + " └─ExchangeSender 9990.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.c1", + " └─Selection 9990.00 cop[tiflash] not(isnull(test.t.c1))", + " └─TableFullScan 10000.00 cop[tiflash] table:t1 keep order:false, stats:pseudo" ] }, { @@ -1956,23 +1993,24 @@ "└─ExchangeSender 8000.00 batchCop[tiflash] ExchangeType: PassThrough", " └─Projection 8000.00 batchCop[tiflash] test.tt.col_varchar_64, test.tt.col_char_64_not_null", " └─HashAgg 8000.00 batchCop[tiflash] group by:test.tt.col_char_64_not_null, test.tt.col_varchar_64, funcs:firstrow(test.tt.col_varchar_64)->test.tt.col_varchar_64, funcs:firstrow(test.tt.col_char_64_not_null)->test.tt.col_char_64_not_null", - " └─ExchangeReceiver 15609.38 batchCop[tiflash] ", - " └─ExchangeSender 15609.38 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.tt.col_varchar_64, test.tt.col_char_64_not_null", - " └─HashJoin 15609.38 batchCop[tiflash] inner join, equal:[eq(test.tt.col_char_64_not_null, test.tt.col_varchar_64)]", - " ├─ExchangeReceiver(Build) 10000.00 batchCop[tiflash] ", - " │ └─ExchangeSender 10000.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: Column#29", - " │ └─Projection 10000.00 batchCop[tiflash] test.tt.col_varchar_64, test.tt.col_char_64_not_null, cast(test.tt.col_char_64_not_null, varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#29", - " │ └─TableFullScan 10000.00 batchCop[tiflash] table:t1 keep order:false, stats:pseudo", - " └─HashJoin(Probe) 12487.50 batchCop[tiflash] inner join, equal:[eq(test.tt.col_varchar_key, test.tt.col_varchar_64) eq(Column#19, test.tt.col_decimal_30_10_key)]", - " ├─ExchangeReceiver(Build) 9990.00 batchCop[tiflash] ", - " │ └─ExchangeSender 9990.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.tt.col_varchar_key", - " │ └─Projection 9990.00 batchCop[tiflash] test.tt.col_varchar_key, cast(test.tt.col_tinyint, decimal(20,0) BINARY)->Column#19", - " │ └─Selection 9990.00 batchCop[tiflash] not(isnull(test.tt.col_varchar_key))", - " │ └─TableFullScan 10000.00 batchCop[tiflash] table:t2 keep order:false, stats:pseudo", - " └─ExchangeReceiver(Probe) 9990.00 batchCop[tiflash] ", - " └─ExchangeSender 9990.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.tt.col_varchar_64", - " └─Selection 9990.00 batchCop[tiflash] not(isnull(test.tt.col_varchar_64))", - " └─TableFullScan 10000.00 batchCop[tiflash] table:t3 keep order:false, stats:pseudo" + " └─ExchangeReceiver 8000.00 batchCop[tiflash] ", + " └─ExchangeSender 8000.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.tt.col_varchar_64, test.tt.col_char_64_not_null", + " └─HashAgg 8000.00 batchCop[tiflash] group by:test.tt.col_char_64_not_null, test.tt.col_varchar_64, ", + " └─HashJoin 15609.38 batchCop[tiflash] inner join, equal:[eq(test.tt.col_char_64_not_null, test.tt.col_varchar_64)]", + " ├─ExchangeReceiver(Build) 10000.00 batchCop[tiflash] ", + " │ └─ExchangeSender 10000.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: Column#30", + " │ └─Projection 10000.00 batchCop[tiflash] test.tt.col_varchar_64, test.tt.col_char_64_not_null, cast(test.tt.col_char_64_not_null, varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin)->Column#30", + " │ └─TableFullScan 10000.00 batchCop[tiflash] table:t1 keep order:false, stats:pseudo", + " └─HashJoin(Probe) 12487.50 batchCop[tiflash] inner join, equal:[eq(test.tt.col_varchar_key, test.tt.col_varchar_64) eq(Column#19, test.tt.col_decimal_30_10_key)]", + " ├─ExchangeReceiver(Build) 9990.00 batchCop[tiflash] ", + " │ └─ExchangeSender 9990.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.tt.col_varchar_key", + " │ └─Projection 9990.00 batchCop[tiflash] test.tt.col_varchar_key, cast(test.tt.col_tinyint, decimal(20,0) BINARY)->Column#19", + " │ └─Selection 9990.00 batchCop[tiflash] not(isnull(test.tt.col_varchar_key))", + " │ └─TableFullScan 10000.00 batchCop[tiflash] table:t2 keep order:false, stats:pseudo", + " └─ExchangeReceiver(Probe) 9990.00 batchCop[tiflash] ", + " └─ExchangeSender 9990.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.tt.col_varchar_64", + " └─Selection 9990.00 batchCop[tiflash] not(isnull(test.tt.col_varchar_64))", + " └─TableFullScan 10000.00 batchCop[tiflash] table:t3 keep order:false, stats:pseudo" ] } ] @@ -2181,20 +2219,22 @@ " └─HashJoin 7992.00 batchCop[tiflash] inner join, equal:[eq(test.t.id, test.t.id)]", " ├─Projection(Build) 7992.00 batchCop[tiflash] test.t.id", " │ └─HashAgg 7992.00 batchCop[tiflash] group by:test.t.id, funcs:firstrow(test.t.id)->test.t.id", - " │ └─ExchangeReceiver 9990.00 batchCop[tiflash] ", - " │ └─ExchangeSender 9990.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.id", - " │ └─Selection 9990.00 batchCop[tiflash] not(isnull(test.t.id))", - " │ └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo", + " │ └─ExchangeReceiver 7992.00 batchCop[tiflash] ", + " │ └─ExchangeSender 7992.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.id", + " │ └─HashAgg 7992.00 batchCop[tiflash] group by:test.t.id, ", + " │ └─Selection 9990.00 batchCop[tiflash] not(isnull(test.t.id))", + " │ └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo", " └─Projection(Probe) 7992.00 batchCop[tiflash] Column#11, test.t.id", " └─HashAgg 7992.00 batchCop[tiflash] group by:Column#32, funcs:sum(Column#30)->Column#11, funcs:firstrow(Column#31)->test.t.id", " └─Projection 9990.00 batchCop[tiflash] cast(test.t.id, decimal(32,0) BINARY)->Column#30, test.t.id, test.t.id", " └─HashJoin 9990.00 batchCop[tiflash] inner join, equal:[eq(test.t.id, test.t.id)]", " ├─Projection(Build) 7992.00 batchCop[tiflash] test.t.id", " │ └─HashAgg 7992.00 batchCop[tiflash] group by:test.t.id, funcs:firstrow(test.t.id)->test.t.id", - " │ └─ExchangeReceiver 9990.00 batchCop[tiflash] ", - " │ └─ExchangeSender 9990.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.id", - " │ └─Selection 9990.00 batchCop[tiflash] not(isnull(test.t.id))", - " │ └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo", + " │ └─ExchangeReceiver 7992.00 batchCop[tiflash] ", + " │ └─ExchangeSender 7992.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.id", + " │ └─HashAgg 7992.00 batchCop[tiflash] group by:test.t.id, ", + " │ └─Selection 9990.00 batchCop[tiflash] not(isnull(test.t.id))", + " │ └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo", " └─ExchangeReceiver(Probe) 9990.00 batchCop[tiflash] ", " └─ExchangeSender 9990.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.id", " └─Selection 9990.00 batchCop[tiflash] not(isnull(test.t.id))", @@ -2271,10 +2311,11 @@ " └─Projection 6400.00 batchCop[tiflash] cast(test.t.id, decimal(32,0) BINARY)->Column#21, test.t.value", " └─Projection 6400.00 batchCop[tiflash] test.t.id, test.t.value", " └─HashAgg 6400.00 batchCop[tiflash] group by:test.t.id, test.t.value, funcs:firstrow(test.t.id)->test.t.id, funcs:firstrow(test.t.value)->test.t.value", - " └─ExchangeReceiver 8000.00 batchCop[tiflash] ", - " └─ExchangeSender 8000.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.value", - " └─Selection 8000.00 batchCop[tiflash] gt(cast(test.t.id, decimal(20,0) BINARY), test.t.value)", - " └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" + " └─ExchangeReceiver 6400.00 batchCop[tiflash] ", + " └─ExchangeSender 6400.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.value", + " └─HashAgg 6400.00 batchCop[tiflash] group by:test.t.id, test.t.value, ", + " └─Selection 8000.00 batchCop[tiflash] gt(cast(test.t.id, decimal(20,0) BINARY), test.t.value)", + " └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" ] }, { @@ -2408,10 +2449,11 @@ " │ └─ExchangeSender 7992.00 batchCop[tiflash] ExchangeType: Broadcast", " │ └─Projection 7992.00 batchCop[tiflash] test.t.id", " │ └─HashAgg 7992.00 batchCop[tiflash] group by:test.t.id, funcs:firstrow(test.t.id)->test.t.id", - " │ └─ExchangeReceiver 9990.00 batchCop[tiflash] ", - " │ └─ExchangeSender 9990.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.id", - " │ └─Selection 9990.00 batchCop[tiflash] not(isnull(test.t.id))", - " │ └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo", + " │ └─ExchangeReceiver 7992.00 batchCop[tiflash] ", + " │ └─ExchangeSender 7992.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.id", + " │ └─HashAgg 7992.00 batchCop[tiflash] group by:test.t.id, ", + " │ └─Selection 9990.00 batchCop[tiflash] not(isnull(test.t.id))", + " │ └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo", " └─Projection(Probe) 7992.00 batchCop[tiflash] Column#7, test.t.id", " └─HashAgg 7992.00 batchCop[tiflash] group by:test.t.id, funcs:sum(Column#9)->Column#7, funcs:firstrow(test.t.id)->test.t.id", " └─ExchangeReceiver 7992.00 batchCop[tiflash] ", @@ -2431,10 +2473,11 @@ " │ └─ExchangeSender 7992.00 batchCop[tiflash] ExchangeType: Broadcast", " │ └─Projection 7992.00 batchCop[tiflash] test.t.id", " │ └─HashAgg 7992.00 batchCop[tiflash] group by:test.t.id, funcs:firstrow(test.t.id)->test.t.id", - " │ └─ExchangeReceiver 9990.00 batchCop[tiflash] ", - " │ └─ExchangeSender 9990.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.id", - " │ └─Selection 9990.00 batchCop[tiflash] not(isnull(test.t.id))", - " │ └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo", + " │ └─ExchangeReceiver 7992.00 batchCop[tiflash] ", + " │ └─ExchangeSender 7992.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.id", + " │ └─HashAgg 7992.00 batchCop[tiflash] group by:test.t.id, ", + " │ └─Selection 9990.00 batchCop[tiflash] not(isnull(test.t.id))", + " │ └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo", " └─Projection(Probe) 7992.00 batchCop[tiflash] Column#11, test.t.id", " └─HashAgg 7992.00 batchCop[tiflash] group by:test.t.id, funcs:sum(Column#14)->Column#11, funcs:firstrow(test.t.id)->test.t.id", " └─ExchangeReceiver 7992.00 batchCop[tiflash] ", @@ -2446,10 +2489,11 @@ " │ └─ExchangeSender 7992.00 batchCop[tiflash] ExchangeType: Broadcast", " │ └─Projection 7992.00 batchCop[tiflash] test.t.id", " │ └─HashAgg 7992.00 batchCop[tiflash] group by:test.t.id, funcs:firstrow(test.t.id)->test.t.id", - " │ └─ExchangeReceiver 9990.00 batchCop[tiflash] ", - " │ └─ExchangeSender 9990.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.id", - " │ └─Selection 9990.00 batchCop[tiflash] not(isnull(test.t.id))", - " │ └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo", + " │ └─ExchangeReceiver 7992.00 batchCop[tiflash] ", + " │ └─ExchangeSender 7992.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.id", + " │ └─HashAgg 7992.00 batchCop[tiflash] group by:test.t.id, ", + " │ └─Selection 9990.00 batchCop[tiflash] not(isnull(test.t.id))", + " │ └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo", " └─Selection(Probe) 9990.00 batchCop[tiflash] not(isnull(test.t.id))", " └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" ] @@ -2503,6 +2547,55 @@ " └─Selection(Probe) 9990.00 batchCop[tiflash] not(isnull(test.t.id))", " └─TableFullScan 10000.00 batchCop[tiflash] table:t1 keep order:false, stats:pseudo" ] + }, + { + "SQL": "desc format = 'brief' select * from t join t t1 on t.id = t1.id limit 1", + "Plan": [ + "Limit 1.00 root offset:0, count:1", + "└─TableReader 1.00 root data:ExchangeSender", + " └─ExchangeSender 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─Limit 1.00 cop[tiflash] offset:0, count:1", + " └─HashJoin 12487.50 cop[tiflash] inner join, equal:[eq(test.t.id, test.t.id)]", + " ├─ExchangeReceiver(Build) 9990.00 cop[tiflash] ", + " │ └─ExchangeSender 9990.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 cop[tiflash] not(isnull(test.t.id))", + " │ └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 cop[tiflash] not(isnull(test.t.id))", + " └─TableFullScan 0.80 cop[tiflash] table:t1 keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select * from t join t t1 on t.id = t1.id limit 1", + "Plan": [ + "Limit 1.00 root offset:0, count:1", + "└─TableReader 1.00 root data:ExchangeSender", + " └─ExchangeSender 1.00 cop[tiflash] ExchangeType: PassThrough", + " └─Limit 1.00 cop[tiflash] offset:0, count:1", + " └─HashJoin 12487.50 cop[tiflash] inner join, equal:[eq(test.t.id, test.t.id)]", + " ├─ExchangeReceiver(Build) 9990.00 cop[tiflash] ", + " │ └─ExchangeSender 9990.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 cop[tiflash] not(isnull(test.t.id))", + " │ └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 cop[tiflash] not(isnull(test.t.id))", + " └─TableFullScan 0.80 cop[tiflash] table:t1 keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select count(*) from (select t.id, t.value v1 from t join t t1 on t.id = t1.id limit 20) v group by v.v1", + "Plan": [ + "HashAgg 20.00 root group by:test.t.value, funcs:count(1)->Column#7", + "└─Limit 20.00 root offset:0, count:20", + " └─TableReader 20.00 root data:ExchangeSender", + " └─ExchangeSender 20.00 cop[tiflash] ExchangeType: PassThrough", + " └─Limit 20.00 cop[tiflash] offset:0, count:20", + " └─HashJoin 12487.50 cop[tiflash] inner join, equal:[eq(test.t.id, test.t.id)]", + " ├─ExchangeReceiver(Build) 9990.00 cop[tiflash] ", + " │ └─ExchangeSender 9990.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 cop[tiflash] not(isnull(test.t.id))", + " │ └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 cop[tiflash] not(isnull(test.t.id))", + " └─TableFullScan 16.02 cop[tiflash] table:t1 keep order:false, stats:pseudo" + ] } ] }, @@ -2608,5 +2701,33 @@ ] } ] + }, + { + "Name": "TestMergeContinuousSelections", + "Cases": [ + { + "SQL": "desc format = 'brief' SELECT table2 . `col_char_64` AS field1 FROM `ts` AS table2 INNER JOIN (SELECT DISTINCT SUBQUERY3_t1 . * FROM `ts` AS SUBQUERY3_t1 LEFT OUTER JOIN `ts` AS SUBQUERY3_t2 ON SUBQUERY3_t2 . `col_varchar_64_not_null` = SUBQUERY3_t1 . `col_varchar_key`) AS table3 ON (table3 . `col_varchar_key` = table2 . `col_varchar_64`) WHERE table3 . `col_char_64_not_null` >= SOME (SELECT SUBQUERY4_t1 . `col_varchar_64` AS SUBQUERY4_field1 FROM `ts` AS SUBQUERY4_t1) GROUP BY field1 ;", + "Plan": [ + "HashAgg 7992.00 root group by:test.ts.col_char_64, funcs:firstrow(test.ts.col_char_64)->test.ts.col_char_64", + "└─HashJoin 9990.00 root CARTESIAN inner join, other cond:or(ge(test.ts.col_char_64_not_null, Column#25), if(ne(Column#26, 0), NULL, 0))", + " ├─Selection(Build) 0.80 root ne(Column#27, 0)", + " │ └─HashAgg 1.00 root funcs:min(Column#36)->Column#25, funcs:sum(Column#37)->Column#26, funcs:count(Column#38)->Column#27", + " │ └─TableReader 1.00 root data:ExchangeSender", + " │ └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough", + " │ └─HashAgg 1.00 batchCop[tiflash] funcs:min(Column#42)->Column#36, funcs:sum(Column#43)->Column#37, funcs:count(1)->Column#38", + " │ └─Projection 10000.00 batchCop[tiflash] test.ts.col_varchar_64, cast(isnull(test.ts.col_varchar_64), decimal(22,0) BINARY)->Column#43", + " │ └─TableFullScan 10000.00 batchCop[tiflash] table:SUBQUERY4_t1 keep order:false, stats:pseudo", + " └─TableReader(Probe) 12487.50 root data:ExchangeSender", + " └─ExchangeSender 12487.50 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin 12487.50 cop[tiflash] inner join, equal:[eq(test.ts.col_varchar_64, test.ts.col_varchar_key)]", + " ├─ExchangeReceiver(Build) 9990.00 cop[tiflash] ", + " │ └─ExchangeSender 9990.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 cop[tiflash] not(isnull(test.ts.col_varchar_64))", + " │ └─TableFullScan 10000.00 cop[tiflash] table:table2 keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 cop[tiflash] not(isnull(test.ts.col_varchar_key))", + " └─TableFullScan 10000.00 cop[tiflash] table:SUBQUERY3_t1 keep order:false, stats:pseudo" + ] + } + ] } ] diff --git a/planner/core/testdata/integration_suite_in.json b/planner/core/testdata/integration_suite_in.json index 087b32110e18f..63b866ad3badd 100644 --- a/planner/core/testdata/integration_suite_in.json +++ b/planner/core/testdata/integration_suite_in.json @@ -19,6 +19,21 @@ "explain format = 'brief' select * from t t1 left join t t2 on t1.a=t2.a where from_unixtime(t2.b);" ] }, + { + "name": "TestAggColumnPrune", + "cases": [ + "select count(1) from t join (select count(1) from t where false) as tmp", + "select count(1) from t join (select max(a) from t where false) as tmp", + "select count(1) from t join (select min(a) from t where false) as tmp", + "select count(1) from t join (select sum(a) from t where false) as tmp", + "select count(1) from t join (select avg(a) from t where false) as tmp", + "select count(1) from t join (select count(1) from t where false group by a) as tmp", + "select count(1) from t join (select max(a) from t where false group by a) as tmp", + "select count(1) from t join (select min(a) from t where false group by a) as tmp", + "select count(1) from t join (select sum(a) from t where false group by a) as tmp", + "select count(1) from t join (select avg(a) from t where false group by a) as tmp" + ] + }, { "name": "TestIndexJoinInnerIndexNDV", "cases": [ diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index 7c735fcb5657c..81458b413da90 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -14,11 +14,12 @@ { "SQL": "explain format = 'brief' select * from tbl use index(idx_b_c) where b > 1 order by b desc limit 2,1", "Plan": [ - "Projection 1.00 root test.tbl.a, test.tbl.b, test.tbl.c", - "└─IndexLookUp 1.00 root limit embedded(offset:2, count:1)", - " ├─Limit(Build) 3.00 cop[tikv] offset:0, count:3", - " │ └─IndexRangeScan 3.00 cop[tikv] table:tbl, index:idx_b_c(b, c) range:(1,+inf], keep order:true, desc", - " └─TableRowIDScan(Probe) 1.00 cop[tikv] table:tbl keep order:false, stats:pseudo" + "Limit 1.00 root offset:2, count:1", + "└─Projection 3.00 root test.tbl.a, test.tbl.b, test.tbl.c", + " └─IndexLookUp 3.00 root ", + " ├─Limit(Build) 3.00 cop[tikv] offset:0, count:3", + " │ └─IndexRangeScan 3.00 cop[tikv] table:tbl, index:idx_b_c(b, c) range:(1,+inf], keep order:true, desc", + " └─TableRowIDScan(Probe) 3.00 cop[tikv] table:tbl keep order:false, stats:pseudo" ] }, { @@ -63,6 +64,71 @@ } ] }, + { + "Name": "TestAggColumnPrune", + "Cases": [ + { + "SQL": "select count(1) from t join (select count(1) from t where false) as tmp", + "Res": [ + "2" + ] + }, + { + "SQL": "select count(1) from t join (select max(a) from t where false) as tmp", + "Res": [ + "2" + ] + }, + { + "SQL": "select count(1) from t join (select min(a) from t where false) as tmp", + "Res": [ + "2" + ] + }, + { + "SQL": "select count(1) from t join (select sum(a) from t where false) as tmp", + "Res": [ + "2" + ] + }, + { + "SQL": "select count(1) from t join (select avg(a) from t where false) as tmp", + "Res": [ + "2" + ] + }, + { + "SQL": "select count(1) from t join (select count(1) from t where false group by a) as tmp", + "Res": [ + "0" + ] + }, + { + "SQL": "select count(1) from t join (select max(a) from t where false group by a) as tmp", + "Res": [ + "0" + ] + }, + { + "SQL": "select count(1) from t join (select min(a) from t where false group by a) as tmp", + "Res": [ + "0" + ] + }, + { + "SQL": "select count(1) from t join (select sum(a) from t where false group by a) as tmp", + "Res": [ + "0" + ] + }, + { + "SQL": "select count(1) from t join (select avg(a) from t where false group by a) as tmp", + "Res": [ + "0" + ] + } + ] + }, { "Name": "TestIndexJoinInnerIndexNDV", "Cases": [ @@ -912,8 +978,8 @@ { "SQL": "select * from t1 where t1.a = 1 and t1.b < \"333\"", "Plan": [ - "TableReader 0.67 root data:TableRangeScan", - "└─TableRangeScan 0.67 cop[tikv] table:t1 range:[1 -inf,1 \"333\"), keep order:false" + "TableReader 0.82 root data:TableRangeScan", + "└─TableRangeScan 0.82 cop[tikv] table:t1 range:[1 -inf,1 \"333\"), keep order:false" ], "Res": [ "1 111 1.1000000000 11" diff --git a/planner/core/testdata/partition_pruner_out.json b/planner/core/testdata/partition_pruner_out.json index 5593ab52bee74..c1085f5fb30fe 100644 --- a/planner/core/testdata/partition_pruner_out.json +++ b/planner/core/testdata/partition_pruner_out.json @@ -598,9 +598,7 @@ "SQL": "select * from t1 where a = 100", "Result": null, "Plan": [ - "TableReader 10.00 root partition:dual data:Selection", - "└─Selection 10.00 cop[tikv] eq(test_partition.t1.a, 100)", - " └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo" + "TableDual 0.00 root rows:0" ] }, { @@ -714,13 +712,13 @@ "SQL": "select * from t1 join t2 on true where t1.a=5 and t2.a in (6,7,8) and t1.a-t2.a=1 and t2.b = 6", "Result": null, "Plan": [ - "Projection 80.00 root test_partition.t1.id, test_partition.t1.a, test_partition.t1.b, test_partition.t2.a, test_partition.t2.id, test_partition.t2.b", - "└─HashJoin 80.00 root CARTESIAN inner join", - " ├─TableReader(Build) 8.00 root partition:p1 data:Selection", - " │ └─Selection 8.00 cop[tikv] 1, eq(minus(5, test_partition.t2.a), 1), eq(test_partition.t2.b, 6), in(test_partition.t2.a, 6, 7, 8)", + "Projection 0.24 root test_partition.t1.id, test_partition.t1.a, test_partition.t1.b, test_partition.t2.a, test_partition.t2.id, test_partition.t2.b", + "└─HashJoin 0.24 root CARTESIAN inner join", + " ├─TableReader(Build) 0.02 root partition:p1 data:Selection", + " │ └─Selection 0.02 cop[tikv] eq(minus(5, test_partition.t2.a), 1), eq(test_partition.t2.b, 6), in(test_partition.t2.a, 6, 7, 8)", " │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", " └─TableReader(Probe) 10.00 root partition:p0 data:Selection", - " └─Selection 10.00 cop[tikv] 1, eq(test_partition.t1.a, 5)", + " └─Selection 10.00 cop[tikv] eq(test_partition.t1.a, 5)", " └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo" ] }, @@ -1129,25 +1127,24 @@ "3 3 3 7 7 7" ], "Plan": [ - "Sort 80.16 root test_partition.t1.id, test_partition.t1.a", - "└─Projection 80.16 root test_partition.t1.id, test_partition.t1.a, test_partition.t1.b, test_partition.t2.id, test_partition.t2.a, test_partition.t2.b", - " └─HashJoin 80.16 root CARTESIAN inner join", - " ├─TableReader(Build) 8.00 root partition:p1 data:Selection", - " │ └─Selection 8.00 cop[tikv] 1, eq(test_partition.t2.b, 7), eq(test_partition.t2.id, 7), in(test_partition.t2.a, 6, 7, 8)", + "Sort 0.00 root test_partition.t1.id, test_partition.t1.a", + "└─Projection 0.00 root test_partition.t1.id, test_partition.t1.a, test_partition.t1.b, test_partition.t2.id, test_partition.t2.a, test_partition.t2.b", + " └─HashJoin 0.00 root CARTESIAN inner join", + " ├─TableReader(Build) 0.00 root partition:p1 data:Selection", + " │ └─Selection 0.00 cop[tikv] eq(test_partition.t2.b, 7), eq(test_partition.t2.id, 7), in(test_partition.t2.a, 6, 7, 8)", " │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", " └─TableReader(Probe) 10.02 root partition:p0 data:Selection", - " └─Selection 10.02 cop[tikv] 1, or(eq(test_partition.t1.a, 1), and(eq(test_partition.t1.a, 3), in(test_partition.t1.b, 3, 5)))", + " └─Selection 10.02 cop[tikv] or(eq(test_partition.t1.a, 1), and(eq(test_partition.t1.a, 3), in(test_partition.t1.b, 3, 5)))", " └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo" ], "IndexPlan": [ - "Sort 0.05 root test_partition_1.t1.id, test_partition_1.t1.a", - "└─HashJoin 0.05 root CARTESIAN inner join", - " ├─IndexReader(Build) 0.02 root partition:p0 index:Selection", - " │ └─Selection 0.02 cop[tikv] or(eq(test_partition_1.t1.a, 1), and(eq(test_partition_1.t1.a, 3), in(test_partition_1.t1.b, 3, 5)))", - " │ └─IndexRangeScan 20.00 cop[tikv] table:t1, index:a(a, b, id) range:[1,1], [3,3], keep order:false, stats:pseudo", - " └─IndexReader(Probe) 2.40 root partition:p1 index:Selection", - " └─Selection 2.40 cop[tikv] 1", - " └─IndexRangeScan 3.00 cop[tikv] table:t2, index:a(a, b, id) range:[6 7 7,6 7 7], [7 7 7,7 7 7], [8 7 7,8 7 7], keep order:false, stats:pseudo" + "Sort 30.60 root test_partition_1.t1.id, test_partition_1.t1.a", + "└─Projection 30.60 root test_partition_1.t1.id, test_partition_1.t1.a, test_partition_1.t1.b, test_partition_1.t2.id, test_partition_1.t2.a, test_partition_1.t2.b", + " └─HashJoin 30.60 root CARTESIAN inner join", + " ├─IndexReader(Build) 3.00 root partition:p1 index:IndexRangeScan", + " │ └─IndexRangeScan 3.00 cop[tikv] table:t2, index:a(a, b, id) range:[6 7 7,6 7 7], [7 7 7,7 7 7], [8 7 7,8 7 7], keep order:false, stats:pseudo", + " └─IndexReader(Probe) 10.20 root partition:p0 index:IndexRangeScan", + " └─IndexRangeScan 10.20 cop[tikv] table:t1, index:a(a, b, id) range:[1,1], [3 3,3 3], [3 5,3 5], keep order:false, stats:pseudo" ] }, { @@ -1734,22 +1731,22 @@ "5 5 5 6 6 6" ], "Plan": [ - "Projection 80.00 root test_partition.t1.id, test_partition.t1.a, test_partition.t1.b, test_partition.t2.id, test_partition.t2.a, test_partition.t2.b", - "└─HashJoin 80.00 root CARTESIAN inner join", - " ├─TableReader(Build) 8.00 root partition:p1 data:Selection", - " │ └─Selection 8.00 cop[tikv] 1, eq(test_partition.t2.b, 6), in(test_partition.t2.a, 6, 7, 8)", + "Projection 0.30 root test_partition.t1.id, test_partition.t1.a, test_partition.t1.b, test_partition.t2.id, test_partition.t2.a, test_partition.t2.b", + "└─HashJoin 0.30 root CARTESIAN inner join", + " ├─TableReader(Build) 0.03 root partition:p1 data:Selection", + " │ └─Selection 0.03 cop[tikv] eq(test_partition.t2.b, 6), in(test_partition.t2.a, 6, 7, 8)", " │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", " └─TableReader(Probe) 10.00 root partition:p0 data:Selection", - " └─Selection 10.00 cop[tikv] 1, eq(test_partition.t1.a, 5)", + " └─Selection 10.00 cop[tikv] eq(test_partition.t1.a, 5)", " └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo" ], "IndexPlan": [ - "Projection 300.00 root test_partition_1.t1.id, test_partition_1.t1.a, test_partition_1.t1.b, test_partition_1.t2.id, test_partition_1.t2.a, test_partition_1.t2.b", - "└─HashJoin 300.00 root CARTESIAN inner join", - " ├─IndexReader(Build) 3.00 root partition:p1 index:IndexRangeScan", - " │ └─IndexRangeScan 3.00 cop[tikv] table:t2, index:a(a, b, id) range:[6 6 NULL,6 6 +inf], [7 6 NULL,7 6 +inf], [8 6 NULL,8 6 +inf], keep order:false, stats:pseudo", - " └─IndexReader(Probe) 100.00 root partition:p0 index:IndexRangeScan", - " └─IndexRangeScan 100.00 cop[tikv] table:t1, index:a(a, b, id) range:[5 NULL,5 +inf], keep order:false, stats:pseudo" + "Projection 3.00 root test_partition_1.t1.id, test_partition_1.t1.a, test_partition_1.t1.b, test_partition_1.t2.id, test_partition_1.t2.a, test_partition_1.t2.b", + "└─HashJoin 3.00 root CARTESIAN inner join", + " ├─IndexReader(Build) 0.30 root partition:p1 index:IndexRangeScan", + " │ └─IndexRangeScan 0.30 cop[tikv] table:t2, index:a(a, b, id) range:[6 6,6 6], [7 6,7 6], [8 6,8 6], keep order:false, stats:pseudo", + " └─IndexReader(Probe) 10.00 root partition:p0 index:IndexRangeScan", + " └─IndexRangeScan 10.00 cop[tikv] table:t1, index:a(a, b, id) range:[5,5], keep order:false, stats:pseudo" ] }, { @@ -2127,25 +2124,25 @@ "3 3 3 7 7 7" ], "Plan": [ - "Sort 675761.06 root test_partition.t1.id, test_partition.t1.a", - "└─Projection 675761.06 root test_partition.t1.id, test_partition.t1.a, test_partition.t1.b, test_partition.t2.id, test_partition.t2.a, test_partition.t2.b", - " └─HashJoin 675761.06 root CARTESIAN inner join", - " ├─TableReader(Build) 200.00 root partition:p1 data:Selection", - " │ └─Selection 200.00 cop[tikv] 1, ge(test_partition.t2.a, 6), ge(test_partition.t2.b, 7), ge(test_partition.t2.id, 7), le(test_partition.t2.a, 8)", + "Sort 93855.70 root test_partition.t1.id, test_partition.t1.a", + "└─Projection 93855.70 root test_partition.t1.id, test_partition.t1.a, test_partition.t1.b, test_partition.t2.id, test_partition.t2.a, test_partition.t2.b", + " └─HashJoin 93855.70 root CARTESIAN inner join", + " ├─TableReader(Build) 27.78 root partition:p1 data:Selection", + " │ └─Selection 27.78 cop[tikv] ge(test_partition.t2.a, 6), ge(test_partition.t2.b, 7), ge(test_partition.t2.id, 7), le(test_partition.t2.a, 8)", " │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", " └─TableReader(Probe) 3378.81 root partition:p0 data:Selection", - " └─Selection 3378.81 cop[tikv] 1, or(le(test_partition.t1.a, 1), and(le(test_partition.t1.a, 3), and(ge(test_partition.t1.b, 3), le(test_partition.t1.b, 5))))", + " └─Selection 3378.81 cop[tikv] or(le(test_partition.t1.a, 1), and(le(test_partition.t1.a, 3), and(ge(test_partition.t1.b, 3), le(test_partition.t1.b, 5))))", " └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo" ], "IndexPlan": [ - "Sort 224577.93 root test_partition_1.t1.id, test_partition_1.t1.a", - "└─Projection 224577.93 root test_partition_1.t1.id, test_partition_1.t1.a, test_partition_1.t1.b, test_partition_1.t2.id, test_partition_1.t2.a, test_partition_1.t2.b", - " └─HashJoin 224577.93 root CARTESIAN inner join", - " ├─IndexReader(Build) 200.00 root partition:p1 index:Selection", - " │ └─Selection 200.00 cop[tikv] ge(test_partition_1.t2.b, 7), ge(test_partition_1.t2.id, 7)", + "Sort 73851.85 root test_partition_1.t1.id, test_partition_1.t1.a", + "└─Projection 73851.85 root test_partition_1.t1.id, test_partition_1.t1.a, test_partition_1.t1.b, test_partition_1.t2.id, test_partition_1.t2.a, test_partition_1.t2.b", + " └─HashJoin 73851.85 root CARTESIAN inner join", + " ├─IndexReader(Build) 27.78 root partition:p1 index:Selection", + " │ └─Selection 27.78 cop[tikv] ge(test_partition_1.t2.b, 7), ge(test_partition_1.t2.id, 7)", " │ └─IndexRangeScan 250.00 cop[tikv] table:t2, index:a(a, b, id) range:[6,8], keep order:false, stats:pseudo", - " └─IndexReader(Probe) 1122.89 root partition:p0 index:Selection", - " └─Selection 1122.89 cop[tikv] or(le(test_partition_1.t1.a, 1), and(le(test_partition_1.t1.a, 3), and(ge(test_partition_1.t1.b, 3), le(test_partition_1.t1.b, 5))))", + " └─IndexReader(Probe) 2658.67 root partition:p0 index:Selection", + " └─Selection 2658.67 cop[tikv] or(le(test_partition_1.t1.a, 1), and(le(test_partition_1.t1.a, 3), and(ge(test_partition_1.t1.b, 3), le(test_partition_1.t1.b, 5))))", " └─IndexRangeScan 3323.33 cop[tikv] table:t1, index:a(a, b, id) range:[-inf,3], keep order:false, stats:pseudo" ] }, @@ -2761,20 +2758,20 @@ "5 5 5 8 8 8" ], "Plan": [ - "HashJoin 2000.00 root CARTESIAN inner join", + "HashJoin 833.33 root CARTESIAN inner join", "├─TableReader(Build) 10.00 root partition:p0 data:Selection", - "│ └─Selection 10.00 cop[tikv] 1, eq(test_partition.t1.a, 5)", + "│ └─Selection 10.00 cop[tikv] eq(test_partition.t1.a, 5)", "│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo", - "└─TableReader(Probe) 200.00 root partition:p1 data:Selection", - " └─Selection 200.00 cop[tikv] 1, ge(test_partition.t2.a, 6), ge(test_partition.t2.b, 6), le(test_partition.t2.a, 8)", + "└─TableReader(Probe) 83.33 root partition:p1 data:Selection", + " └─Selection 83.33 cop[tikv] ge(test_partition.t2.a, 6), ge(test_partition.t2.b, 6), le(test_partition.t2.a, 8)", " └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo" ], "IndexPlan": [ - "HashJoin 20000.00 root CARTESIAN inner join", - "├─IndexReader(Build) 100.00 root partition:p0 index:IndexRangeScan", - "│ └─IndexRangeScan 100.00 cop[tikv] table:t1, index:a(a, b, id) range:[5 NULL,5 +inf], keep order:false, stats:pseudo", - "└─IndexReader(Probe) 200.00 root partition:p1 index:Selection", - " └─Selection 200.00 cop[tikv] ge(test_partition_1.t2.b, 6)", + "HashJoin 833.33 root CARTESIAN inner join", + "├─IndexReader(Build) 10.00 root partition:p0 index:IndexRangeScan", + "│ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:a(a, b, id) range:[5,5], keep order:false, stats:pseudo", + "└─IndexReader(Probe) 83.33 root partition:p1 index:Selection", + " └─Selection 83.33 cop[tikv] ge(test_partition_1.t2.b, 6)", " └─IndexRangeScan 250.00 cop[tikv] table:t2, index:a(a, b, id) range:[6,8], keep order:false, stats:pseudo" ] }, diff --git a/planner/core/testdata/plan_suite_in.json b/planner/core/testdata/plan_suite_in.json index 64a5c973e89cd..dc420972aaa50 100644 --- a/planner/core/testdata/plan_suite_in.json +++ b/planner/core/testdata/plan_suite_in.json @@ -697,7 +697,11 @@ "select e from t where e > ''", "select e from t where e > 'd'", "select e from t where e > -1", - "select e from t where e > 5" + "select e from t where e > 5", + + // zero-value + "select e from t where e = ''", + "select e from t where e != ''" ] } ] diff --git a/planner/core/testdata/plan_suite_out.json b/planner/core/testdata/plan_suite_out.json index a0125e601c94a..9e2d5b248a7ac 100644 --- a/planner/core/testdata/plan_suite_out.json +++ b/planner/core/testdata/plan_suite_out.json @@ -818,7 +818,7 @@ }, { "SQL": "select (select count(1) k from t s where s.a = t.a having k != 0) from t", - "Best": "LeftHashJoin{IndexReader(Index(t.f)[[NULL,+inf]])->IndexReader(Index(t.f)[[NULL,+inf]]->Sel([1]))->Projection}(test.t.a,test.t.a)->Projection" + "Best": "LeftHashJoin{IndexReader(Index(t.f)[[NULL,+inf]])->IndexReader(Index(t.f)[[NULL,+inf]])->Projection}(test.t.a,test.t.a)->Projection" }, { "SQL": "select sum(to_base64(e)) from t group by e,d,c order by c", @@ -1233,12 +1233,12 @@ }, { "SQL": "select /*+ HASH_AGG() */ t1.a from t t1 where t1.a < any(select t2.b from t t2)", - "Best": "LeftHashJoin{IndexReader(Index(t.f)[[NULL,+inf]]->Sel([1]))->TableReader(Table(t)->Sel([1])->HashAgg)->HashAgg->Sel([ne(Column#27, 0) 1])}", + "Best": "LeftHashJoin{IndexReader(Index(t.f)[[NULL,+inf]])->TableReader(Table(t)->HashAgg)->HashAgg->Sel([ne(Column#27, 0)])}", "Warning": "" }, { "SQL": "select /*+ hash_agg() */ t1.a from t t1 where t1.a != any(select t2.b from t t2)", - "Best": "LeftHashJoin{IndexReader(Index(t.f)[[NULL,+inf]]->Sel([1]))->TableReader(Table(t)->Sel([1]))->HashAgg->Sel([ne(Column#28, 0) 1])}", + "Best": "LeftHashJoin{IndexReader(Index(t.f)[[NULL,+inf]])->TableReader(Table(t))->HashAgg->Sel([ne(Column#28, 0)])}", "Warning": "" }, { @@ -2297,10 +2297,12 @@ { "SQL": "select e from t where e != 'b'", "Plan": [ - "IndexReader 20.00 root index:IndexRangeScan", - "└─IndexRangeScan 20.00 cop[tikv] table:t, index:idx(e) range:[\"c\",\"c\"], [\"a\",\"a\"], keep order:false, stats:pseudo" + "IndexReader 40.00 root index:IndexRangeScan", + "└─IndexRangeScan 40.00 cop[tikv] table:t, index:idx(e) range:[\"\",\"\"], [\"c\",\"c\"], [\"a\",\"a\"], [\"\",\"\"], keep order:false, stats:pseudo" ], "Result": [ + "", + "", "a", "c" ] @@ -2329,20 +2331,24 @@ { "SQL": "select e from t where e < 'b'", "Plan": [ - "IndexReader 10.00 root index:IndexRangeScan", - "└─IndexRangeScan 10.00 cop[tikv] table:t, index:idx(e) range:[\"a\",\"a\"], keep order:false, stats:pseudo" + "IndexReader 30.00 root index:IndexRangeScan", + "└─IndexRangeScan 30.00 cop[tikv] table:t, index:idx(e) range:[\"\",\"\"], [\"a\",\"a\"], [\"\",\"\"], keep order:false, stats:pseudo" ], "Result": [ + "", + "", "a" ] }, { "SQL": "select e from t where e <= 'b'", "Plan": [ - "IndexReader 20.00 root index:IndexRangeScan", - "└─IndexRangeScan 20.00 cop[tikv] table:t, index:idx(e) range:[\"b\",\"b\"], [\"a\",\"a\"], keep order:false, stats:pseudo" + "IndexReader 40.00 root index:IndexRangeScan", + "└─IndexRangeScan 40.00 cop[tikv] table:t, index:idx(e) range:[\"\",\"\"], [\"b\",\"b\"], [\"a\",\"a\"], [\"\",\"\"], keep order:false, stats:pseudo" ], "Result": [ + "", + "", "a", "b" ] @@ -2364,6 +2370,8 @@ "└─IndexRangeScan 6656.67 cop[tikv] table:t, index:idx(e) range:[-inf,\"b\"), (\"b\",+inf], keep order:false, stats:pseudo" ], "Result": [ + "", + "", "a", "c" ] @@ -2375,6 +2383,7 @@ "└─IndexRangeScan 3333.33 cop[tikv] table:t, index:idx(e) range:(\"b\",+inf], keep order:false, stats:pseudo" ], "Result": [ + "", "a" ] }, @@ -2385,6 +2394,7 @@ "└─IndexRangeScan 3333.33 cop[tikv] table:t, index:idx(e) range:[\"b\",+inf], keep order:false, stats:pseudo" ], "Result": [ + "", "a", "b" ] @@ -2396,6 +2406,7 @@ "└─IndexRangeScan 3323.33 cop[tikv] table:t, index:idx(e) range:[-inf,\"b\"), keep order:false, stats:pseudo" ], "Result": [ + "", "c" ] }, @@ -2406,6 +2417,7 @@ "└─IndexRangeScan 3323.33 cop[tikv] table:t, index:idx(e) range:[-inf,\"b\"], keep order:false, stats:pseudo" ], "Result": [ + "", "b", "c" ] @@ -2436,6 +2448,8 @@ "└─IndexRangeScan 3333.33 cop[tikv] table:t, index:idx(e) range:[\"\",+inf], keep order:false, stats:pseudo" ], "Result": [ + "", + "", "a", "b", "c" @@ -2445,9 +2459,32 @@ "SQL": "select e from t where e > 5", "Plan": [ "IndexReader 3333.33 root index:IndexRangeScan", - "└─IndexRangeScan 3333.33 cop[tikv] table:t, index:idx(e) range:(\"a\",+inf], keep order:false, stats:pseudo" + "└─IndexRangeScan 3333.33 cop[tikv] table:t, index:idx(e) range:(\"\",+inf], keep order:false, stats:pseudo" ], "Result": null + }, + { + "SQL": "select e from t where e = ''", + "Plan": [ + "IndexReader 20.00 root index:IndexRangeScan", + "└─IndexRangeScan 20.00 cop[tikv] table:t, index:idx(e) range:[\"\",\"\"], [\"\",\"\"], keep order:false, stats:pseudo" + ], + "Result": [ + "", + "" + ] + }, + { + "SQL": "select e from t where e != ''", + "Plan": [ + "IndexReader 30.00 root index:IndexRangeScan", + "└─IndexRangeScan 30.00 cop[tikv] table:t, index:idx(e) range:[\"c\",\"c\"], [\"b\",\"b\"], [\"a\",\"a\"], keep order:false, stats:pseudo" + ], + "Result": [ + "a", + "b", + "c" + ] } ] } diff --git a/planner/core/testdata/point_get_plan_out.json b/planner/core/testdata/point_get_plan_out.json index b0053fde6044d..9f9ac9e27deb9 100644 --- a/planner/core/testdata/point_get_plan_out.json +++ b/planner/core/testdata/point_get_plan_out.json @@ -15,8 +15,7 @@ { "SQL": "select b, c from t where t.b = 2 and t.c = 2 and t.b+1=3", "Plan": [ - "Selection 0.80 root 1", - "└─Point_Get 1.00 root table:t, index:b(b, c) " + "Point_Get 1.00 root table:t, index:b(b, c) " ], "Res": [ "2 2" diff --git a/planner/core/testdata/stats_suite_out.json b/planner/core/testdata/stats_suite_out.json index c782b4658d542..ee73ce1a878a4 100644 --- a/planner/core/testdata/stats_suite_out.json +++ b/planner/core/testdata/stats_suite_out.json @@ -10,16 +10,16 @@ { "SQL": "select * from t1, t2 where t1.a = t2.a and t1.b = t2.b", "AggInput": "", - "JoinInput": "[{[5 6] 4}];[{[8 9] 9}]" + "JoinInput": "[{[1 2] 4}];[{[4 5] 9}]" }, { "SQL": "select count(1) from t1 where a > 0 group by a, b", - "AggInput": "[{[11 12] 4}]", + "AggInput": "[{[1 2] 4}]", "JoinInput": "" }, { "SQL": "select count(1) from t1 where b > 0 group by a, b", - "AggInput": "[{[15 16] 4}]", + "AggInput": "[{[1 2] 4}]", "JoinInput": "" }, { @@ -29,7 +29,7 @@ }, { "SQL": "select count(c3) from (select a as c1, b as c2, a+1 as c3 from t1) as tmp group by c2, c1", - "AggInput": "[{[23 24] 4}]", + "AggInput": "[{[1 2] 4}]", "JoinInput": "" }, { @@ -39,7 +39,7 @@ }, { "SQL": "select count(tmp.cmp) from (select t1.a as a, t1.b as b, (t1.b > (select t2.b from t2 where t2.a = t1.a)) as cmp from t1) tmp group by tmp.a, tmp.b", - "AggInput": "[{[37 38] 4}]", + "AggInput": "[{[1 2] 4}]", "JoinInput": "" }, { @@ -49,12 +49,12 @@ }, { "SQL": "select count(tmp.cmp) from (select t1.a as a, t1.b as b, (t1.b in (select t2.b from t2 where t2.a = t1.a limit 3)) as cmp from t1) tmp group by tmp.a, tmp.b", - "AggInput": "[{[56 57] 4}]", + "AggInput": "[{[1 2] 4}]", "JoinInput": "" }, { "SQL": "select count(tmp.cmp) from (select t1.a as a, t1.b as b, (t1.b not in (select t2.b from t2 where t2.a = t1.a limit 3)) as cmp from t1) tmp group by tmp.a, tmp.b", - "AggInput": "[{[67 68] 4}]", + "AggInput": "[{[1 2] 4}]", "JoinInput": "" }, { @@ -74,8 +74,8 @@ }, { "SQL": "select count(1) from t1 left join t2 on t1.a = t2.a group by t1.a, t1.b", - "AggInput": "[{[99 100] 4}]", - "JoinInput": "[{[99 100] 4}];[]" + "AggInput": "[{[1 2] 4}]", + "JoinInput": "[{[1 2] 4}];[]" }, { "SQL": "select count(1) from t1 left join t2 on t1.a = t2.a group by t2.a, t2.b", @@ -89,18 +89,18 @@ }, { "SQL": "select count(1) from t1 right join t2 on t1.a = t2.a group by t2.a, t2.b", - "AggInput": "[{[123 124] 9}]", - "JoinInput": "[];[{[123 124] 9}]" + "AggInput": "[{[4 5] 9}]", + "JoinInput": "[];[{[4 5] 9}]" }, { "SQL": "select count(tmp.cmp) from (select t1.a as a, t1.b as b, (t1.b in (select t2.b from t2 where t2.a > t1.a)) as cmp from t1) tmp group by tmp.a, tmp.b", - "AggInput": "[{[127 128] 4}]", - "JoinInput": "[{[127 128] 4}];[]" + "AggInput": "[{[1 2] 4}]", + "JoinInput": "[{[1 2] 4}];[]" }, { "SQL": "select count(tmp.cmp) from (select t1.a as a, t1.b as b, (t1.b not in (select t2.b from t2 where t2.a > t1.a)) as cmp from t1) tmp group by tmp.a, tmp.b", - "AggInput": "[{[138 139] 4}]", - "JoinInput": "[{[138 139] 4}];[]" + "AggInput": "[{[1 2] 4}]", + "JoinInput": "[{[1 2] 4}];[]" }, { "SQL": "select count(1) from (select t1.a as a, t1.b as b from t1 where t1.b in (select t2.b from t2 where t2.a > t1.a)) tmp group by tmp.a, tmp.b", @@ -114,8 +114,8 @@ }, { "SQL": "select * from t1 left join (select t2.a as a, t2.b as b, count(1) as cnt from t2 group by t2.a, t2.b) as tmp on t1.a = tmp.a and t1.b = tmp.b", - "AggInput": "[{[166 167] 9}]", - "JoinInput": "[{[163 164] 4}];[{[166 167] 9}]" + "AggInput": "[{[4 5] 9}]", + "JoinInput": "[{[1 2] 4}];[{[4 5] 9}]" }, { "SQL": "select count(1) from (select t1.a as a, t1.b as b from t1 limit 3) tmp group by tmp.a, tmp.b", @@ -124,7 +124,7 @@ }, { "SQL": "select count(tmp.a_sum) from (select t1.a as a, t1.b as b, sum(a) over() as a_sum from t1) tmp group by tmp.a, tmp.b", - "AggInput": "[{[174 175] 4}]", + "AggInput": "[{[1 2] 4}]", "JoinInput": "" } ] diff --git a/planner/core/util.go b/planner/core/util.go index 753445f07ee42..19ce0a47673a2 100644 --- a/planner/core/util.go +++ b/planner/core/util.go @@ -291,7 +291,7 @@ func extractStringFromStringSet(set set.StringSet) string { l = append(l, fmt.Sprintf(`"%s"`, k)) } sort.Strings(l) - return fmt.Sprintf("%s", strings.Join(l, ",")) + return strings.Join(l, ",") } func tableHasDirtyContent(ctx sessionctx.Context, tableInfo *model.TableInfo) bool { diff --git a/planner/implementation/datasource.go b/planner/implementation/datasource.go index 1135ddbc1569d..c18d9ae64f586 100644 --- a/planner/implementation/datasource.go +++ b/planner/implementation/datasource.go @@ -74,7 +74,9 @@ func (impl *TableReaderImpl) CalcCost(outCount float64, children ...memo.Impleme reader := impl.plan.(*plannercore.PhysicalTableReader) width := impl.tblColHists.GetAvgRowSize(impl.plan.SCtx(), reader.Schema().Columns, false, false) sessVars := reader.SCtx().GetSessionVars() - networkCost := outCount * sessVars.NetworkFactor * width + // TableReaderImpl don't have tableInfo property, so using nil to replace it. + // Todo add the tableInfo property for the TableReaderImpl. + networkCost := outCount * sessVars.GetNetworkFactor(nil) * width // copTasks are run in parallel, to make the estimated cost closer to execution time, we amortize // the cost to cop iterator workers. According to `CopClient::Send`, the concurrency // is Min(DistSQLScanConcurrency, numRegionsInvolvedInScan), since we cannot infer @@ -118,9 +120,9 @@ func (impl *TableScanImpl) CalcCost(outCount float64, children ...memo.Implement ts := impl.plan.(*plannercore.PhysicalTableScan) width := impl.tblColHists.GetTableAvgRowSize(impl.plan.SCtx(), impl.tblCols, kv.TiKV, true) sessVars := ts.SCtx().GetSessionVars() - impl.cost = outCount * sessVars.ScanFactor * width + impl.cost = outCount * sessVars.GetScanFactor(ts.Table) * width if ts.Desc { - impl.cost = outCount * sessVars.DescScanFactor * width + impl.cost = outCount * sessVars.GetDescScanFactor(ts.Table) * width } return impl.cost } @@ -146,7 +148,7 @@ func (impl *IndexReaderImpl) GetCostLimit(costLimit float64, children ...memo.Im func (impl *IndexReaderImpl) CalcCost(outCount float64, children ...memo.Implementation) float64 { reader := impl.plan.(*plannercore.PhysicalIndexReader) sessVars := reader.SCtx().GetSessionVars() - networkCost := outCount * sessVars.NetworkFactor * impl.tblColHists.GetAvgRowSize(reader.SCtx(), children[0].GetPlan().Schema().Columns, true, false) + networkCost := outCount * sessVars.GetNetworkFactor(nil) * impl.tblColHists.GetAvgRowSize(reader.SCtx(), children[0].GetPlan().Schema().Columns, true, false) copIterWorkers := float64(sessVars.DistSQLScanConcurrency()) impl.cost = (networkCost + children[0].GetCost()) / copIterWorkers return impl.cost @@ -171,11 +173,11 @@ func (impl *IndexScanImpl) CalcCost(outCount float64, children ...memo.Implement is := impl.plan.(*plannercore.PhysicalIndexScan) sessVars := is.SCtx().GetSessionVars() rowSize := impl.tblColHists.GetIndexAvgRowSize(is.SCtx(), is.Schema().Columns, is.Index.Unique) - cost := outCount * rowSize * sessVars.ScanFactor + cost := outCount * rowSize * sessVars.GetScanFactor(is.Table) if is.Desc { - cost = outCount * rowSize * sessVars.DescScanFactor + cost = outCount * rowSize * sessVars.GetDescScanFactor(is.Table) } - cost += float64(len(is.Ranges)) * sessVars.SeekFactor + cost += float64(len(is.Ranges)) * sessVars.GetSeekFactor(is.Table) impl.cost = cost return impl.cost } diff --git a/planner/implementation/simple_plans.go b/planner/implementation/simple_plans.go index cb49fd0e10225..0a727576b3950 100644 --- a/planner/implementation/simple_plans.go +++ b/planner/implementation/simple_plans.go @@ -85,7 +85,7 @@ type TiDBHashAggImpl struct { // CalcCost implements Implementation CalcCost interface. func (agg *TiDBHashAggImpl) CalcCost(outCount float64, children ...memo.Implementation) float64 { hashAgg := agg.plan.(*plannercore.PhysicalHashAgg) - selfCost := hashAgg.GetCost(children[0].GetPlan().Stats().RowCount, true) + selfCost := hashAgg.GetCost(children[0].GetPlan().Stats().RowCount, true, false) agg.cost = selfCost + children[0].GetCost() return agg.cost } @@ -110,7 +110,7 @@ type TiKVHashAggImpl struct { // CalcCost implements Implementation CalcCost interface. func (agg *TiKVHashAggImpl) CalcCost(outCount float64, children ...memo.Implementation) float64 { hashAgg := agg.plan.(*plannercore.PhysicalHashAgg) - selfCost := hashAgg.GetCost(children[0].GetPlan().Stats().RowCount, false) + selfCost := hashAgg.GetCost(children[0].GetPlan().Stats().RowCount, false, false) agg.cost = selfCost + children[0].GetCost() return agg.cost } diff --git a/planner/optimize.go b/planner/optimize.go index 4e0f7334ae5ae..6d87d6ffac5ed 100644 --- a/planner/optimize.go +++ b/planner/optimize.go @@ -36,7 +36,6 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx/variable" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/hint" "github.com/pingcap/tidb/util/logutil" @@ -45,7 +44,7 @@ import ( ) // GetPreparedStmt extract the prepared statement from the execute statement. -func GetPreparedStmt(stmt *ast.ExecuteStmt, vars *variable.SessionVars) (ast.StmtNode, error) { +func GetPreparedStmt(stmt *ast.ExecuteStmt, vars *variable.SessionVars) (*plannercore.CachedPrepareStmt, error) { var ok bool execID := stmt.ExecID if stmt.Name != "" { @@ -58,7 +57,7 @@ func GetPreparedStmt(stmt *ast.ExecuteStmt, vars *variable.SessionVars) (ast.Stm if !ok { return nil, errors.Errorf("invalid CachedPrepareStmt type") } - return preparedObj.PreparedAst.Stmt, nil + return preparedObj, nil } return nil, plannercore.ErrStmtNotFound } @@ -66,12 +65,12 @@ func GetPreparedStmt(stmt *ast.ExecuteStmt, vars *variable.SessionVars) (ast.Stm // IsReadOnly check whether the ast.Node is a read only statement. func IsReadOnly(node ast.Node, vars *variable.SessionVars) bool { if execStmt, isExecStmt := node.(*ast.ExecuteStmt); isExecStmt { - s, err := GetPreparedStmt(execStmt, vars) + prepareStmt, err := GetPreparedStmt(execStmt, vars) if err != nil { logutil.BgLogger().Warn("GetPreparedStmt failed", zap.Error(err)) return false } - return ast.IsReadOnly(s) + return ast.IsReadOnly(prepareStmt.PreparedAst.Stmt) } return ast.IsReadOnly(node) } @@ -307,7 +306,7 @@ func extractSelectAndNormalizeDigest(stmtNode ast.StmtNode, specifiledDB string) normalizeSQL := parser.Normalize(utilparser.RestoreWithDefaultDB(x.Stmt, specifiledDB, x.Text())) normalizeSQL = plannercore.EraseLastSemicolonInSQL(normalizeSQL) hash := parser.DigestNormalized(normalizeSQL) - return x.Stmt, normalizeSQL, hash, nil + return x.Stmt, normalizeSQL, hash.String(), nil case *ast.SetOprStmt: plannercore.EraseLastSemicolon(x) var normalizeExplainSQL string @@ -323,7 +322,7 @@ func extractSelectAndNormalizeDigest(stmtNode ast.StmtNode, specifiledDB string) } normalizeSQL := normalizeExplainSQL[idx:] hash := parser.DigestNormalized(normalizeSQL) - return x.Stmt, normalizeSQL, hash, nil + return x.Stmt, normalizeSQL, hash.String(), nil } case *ast.SelectStmt, *ast.SetOprStmt, *ast.DeleteStmt, *ast.UpdateStmt, *ast.InsertStmt: plannercore.EraseLastSemicolon(x) @@ -336,7 +335,7 @@ func extractSelectAndNormalizeDigest(stmtNode ast.StmtNode, specifiledDB string) return x, "", "", nil } normalizedSQL, hash := parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(x, specifiledDB, x.Text())) - return x, normalizedSQL, hash, nil + return x, normalizedSQL, hash.String(), nil } return nil, "", "", nil } @@ -533,7 +532,7 @@ func handleStmtHints(hints []*ast.TableOptimizerHint) (stmtHints stmtctx.StmtHin warns = append(warns, warn) } stmtHints.HasReplicaReadHint = true - stmtHints.ReplicaRead = byte(tikvstore.ReplicaReadFollower) + stmtHints.ReplicaRead = byte(kv.ReplicaReadFollower) } // Handle MAX_EXECUTION_TIME if maxExecutionTimeCnt != 0 { diff --git a/plugin/conn_ip_example/conn_ip_example.go b/plugin/conn_ip_example/conn_ip_example.go index bae1b3ff37497..24d0bf04b0309 100644 --- a/plugin/conn_ip_example/conn_ip_example.go +++ b/plugin/conn_ip_example/conn_ip_example.go @@ -62,19 +62,14 @@ func OnGeneralEvent(ctx context.Context, sctx *variable.SessionVars, event plugi switch event { case plugin.Log: fmt.Println("---- event: Log") - break case plugin.Error: fmt.Println("---- event: Error") - break case plugin.Result: fmt.Println("---- event: Result") - break case plugin.Status: fmt.Println("---- event: Status") - break default: fmt.Println("---- event: unrecognized") - break } fmt.Printf("---- cmd: %s\n", cmd) } diff --git a/privilege/privilege.go b/privilege/privilege.go index cf59ce9b0314e..f732d9da1199b 100644 --- a/privilege/privilege.go +++ b/privilege/privilege.go @@ -50,6 +50,9 @@ type Manager interface { // Dynamic privileges are only assignable globally, and have their own grantable attribute. RequestDynamicVerification(activeRoles []*auth.RoleIdentity, privName string, grantable bool) bool + // RequestDynamicVerification verifies a DYNAMIC privilege for a specific user. + RequestDynamicVerificationWithUser(privName string, grantable bool, user *auth.UserIdentity) bool + // ConnectionVerification verifies user privilege for connection. ConnectionVerification(user, host string, auth, salt []byte, tlsState *tls.ConnectionState) (string, string, bool) diff --git a/privilege/privileges/privileges.go b/privilege/privileges/privileges.go index 5b7917e802aac..29bf61d9a25bf 100644 --- a/privilege/privileges/privileges.go +++ b/privilege/privileges/privileges.go @@ -40,11 +40,14 @@ var SkipWithGrant = false var _ privilege.Manager = (*UserPrivileges)(nil) var dynamicPrivs = []string{ "BACKUP_ADMIN", + "RESTORE_ADMIN", "SYSTEM_VARIABLES_ADMIN", "ROLE_ADMIN", "CONNECTION_ADMIN", - "RESTRICTED_TABLES_ADMIN", // Can see system tables when SEM is enabled - "RESTRICTED_STATUS_ADMIN", // Can see all status vars when SEM is enabled. + "RESTRICTED_TABLES_ADMIN", // Can see system tables when SEM is enabled + "RESTRICTED_STATUS_ADMIN", // Can see all status vars when SEM is enabled. + "RESTRICTED_VARIABLES_ADMIN", // Can see all variables when SEM is enabled + "RESTRICTED_USER_ADMIN", // User can not have their access revoked by SUPER users. } var dynamicPrivLock sync.Mutex @@ -56,6 +59,21 @@ type UserPrivileges struct { *Handle } +// RequestDynamicVerificationWithUser implements the Manager interface. +func (p *UserPrivileges) RequestDynamicVerificationWithUser(privName string, grantable bool, user *auth.UserIdentity) bool { + if SkipWithGrant { + return true + } + + if user == nil { + return false + } + + mysqlPriv := p.Handle.Get() + roles := mysqlPriv.getDefaultRoles(user.Username, user.Hostname) + return mysqlPriv.RequestDynamicVerification(roles, user.Username, user.Hostname, privName, grantable) +} + // RequestDynamicVerification implements the Manager interface. func (p *UserPrivileges) RequestDynamicVerification(activeRoles []*auth.RoleIdentity, privName string, grantable bool) bool { if SkipWithGrant { @@ -140,7 +158,8 @@ func (p *UserPrivileges) RequestVerificationWithUser(db, table, column string, p } mysqlPriv := p.Handle.Get() - return mysqlPriv.RequestVerification(nil, user.Username, user.Hostname, db, table, column, priv) + roles := mysqlPriv.getDefaultRoles(user.Username, user.Hostname) + return mysqlPriv.RequestVerification(roles, user.Username, user.Hostname, db, table, column, priv) } // GetEncodedPassword implements the Manager interface. @@ -153,11 +172,16 @@ func (p *UserPrivileges) GetEncodedPassword(user, host string) string { return "" } pwd := record.AuthenticationString - if len(pwd) != 0 && len(pwd) != mysql.PWDHashLen+1 { - logutil.BgLogger().Error("user password from system DB not like sha1sum", zap.String("user", user)) - return "" - } - return pwd + switch len(pwd) { + case 0: + return pwd + case mysql.PWDHashLen + 1: // mysql_native_password + return pwd + case 70: // caching_sha2_password + return pwd + } + logutil.BgLogger().Error("user password from system DB not like a known hash format", zap.String("user", user), zap.Int("hash_length", len(pwd))) + return "" } // GetAuthWithoutVerification implements the Manager interface. @@ -513,7 +537,8 @@ func (p *UserPrivileges) GetAllRoles(user, host string) []*auth.RoleIdentity { } // IsDynamicPrivilege returns true if the DYNAMIC privilege is built-in or has been registered by a plugin -func (p *UserPrivileges) IsDynamicPrivilege(privNameInUpper string) bool { +func (p *UserPrivileges) IsDynamicPrivilege(privName string) bool { + privNameInUpper := strings.ToUpper(privName) for _, priv := range dynamicPrivs { if privNameInUpper == priv { return true @@ -523,7 +548,11 @@ func (p *UserPrivileges) IsDynamicPrivilege(privNameInUpper string) bool { } // RegisterDynamicPrivilege is used by plugins to add new privileges to TiDB -func RegisterDynamicPrivilege(privNameInUpper string) error { +func RegisterDynamicPrivilege(privName string) error { + privNameInUpper := strings.ToUpper(privName) + if len(privNameInUpper) > 32 { + return errors.New("privilege name is longer than 32 characters") + } dynamicPrivLock.Lock() defer dynamicPrivLock.Unlock() for _, priv := range dynamicPrivs { @@ -534,3 +563,14 @@ func RegisterDynamicPrivilege(privNameInUpper string) error { dynamicPrivs = append(dynamicPrivs, privNameInUpper) return nil } + +// GetDynamicPrivileges returns the list of registered DYNAMIC privileges +// for use in meta data commands (i.e. SHOW PRIVILEGES) +func GetDynamicPrivileges() []string { + dynamicPrivLock.Lock() + defer dynamicPrivLock.Unlock() + + privCopy := make([]string, len(dynamicPrivs)) + copy(privCopy, dynamicPrivs) + return privCopy +} diff --git a/privilege/privileges/privileges_test.go b/privilege/privileges/privileges_test.go index 816fe5a59d0bd..1bb69be14c826 100644 --- a/privilege/privileges/privileges_test.go +++ b/privilege/privileges/privileges_test.go @@ -39,6 +39,7 @@ import ( "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/sem" + "github.com/pingcap/tidb/util/sqlexec" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testleak" "github.com/pingcap/tidb/util/testutil" @@ -843,6 +844,12 @@ func (s *testPrivilegeSuite) TestRevokePrivileges(c *C) { c.Assert(se.Auth(&auth.UserIdentity{Username: "hasgrant", Hostname: "localhost", AuthUsername: "hasgrant", AuthHostname: "%"}, nil, nil), IsTrue) mustExec(c, se, "REVOKE SELECT ON mysql.* FROM 'withoutgrant'") mustExec(c, se, "REVOKE ALL ON mysql.* FROM withoutgrant") + + // For issue https://github.com/pingcap/tidb/issues/23850 + mustExec(c, se, "CREATE USER u4") + mustExec(c, se, "GRANT ALL ON *.* TO u4 WITH GRANT OPTION") + c.Assert(se.Auth(&auth.UserIdentity{Username: "u4", Hostname: "localhost", AuthUsername: "u4", AuthHostname: "%"}, nil, nil), IsTrue) + mustExec(c, se, "REVOKE ALL ON *.* FROM CURRENT_USER()") } func (s *testPrivilegeSuite) TestSetGlobal(c *C) { @@ -1006,14 +1013,14 @@ func (s *testPrivilegeSuite) TestSystemSchema(c *C) { _, err = se.ExecuteInternal(context.Background(), "drop table information_schema.tables") c.Assert(strings.Contains(err.Error(), "denied to user"), IsTrue) _, err = se.ExecuteInternal(context.Background(), "update information_schema.tables set table_name = 'tst' where table_name = 'mysql'") - c.Assert(strings.Contains(err.Error(), "privilege check fail"), IsTrue) + c.Assert(strings.Contains(err.Error(), "privilege check"), IsTrue) // Test performance_schema. mustExec(c, se, `select * from performance_schema.events_statements_summary_by_digest`) _, err = se.ExecuteInternal(context.Background(), "drop table performance_schema.events_statements_summary_by_digest") c.Assert(strings.Contains(err.Error(), "denied to user"), IsTrue) _, err = se.ExecuteInternal(context.Background(), "update performance_schema.events_statements_summary_by_digest set schema_name = 'tst'") - c.Assert(strings.Contains(err.Error(), "privilege check fail"), IsTrue) + c.Assert(strings.Contains(err.Error(), "privilege check"), IsTrue) _, err = se.ExecuteInternal(context.Background(), "delete from performance_schema.events_statements_summary_by_digest") c.Assert(strings.Contains(err.Error(), "DELETE command denied to user"), IsTrue) _, err = se.ExecuteInternal(context.Background(), "create table performance_schema.t(a int)") @@ -1025,7 +1032,7 @@ func (s *testPrivilegeSuite) TestSystemSchema(c *C) { _, err = se.ExecuteInternal(context.Background(), "drop table metrics_schema.tidb_query_duration") c.Assert(strings.Contains(err.Error(), "denied to user"), IsTrue) _, err = se.ExecuteInternal(context.Background(), "update metrics_schema.tidb_query_duration set instance = 'tst'") - c.Assert(strings.Contains(err.Error(), "privilege check fail"), IsTrue) + c.Assert(strings.Contains(err.Error(), "privilege check"), IsTrue) _, err = se.ExecuteInternal(context.Background(), "delete from metrics_schema.tidb_query_duration") c.Assert(strings.Contains(err.Error(), "DELETE command denied to user"), IsTrue) _, err = se.ExecuteInternal(context.Background(), "create table metric_schema.t(a int)") @@ -1041,9 +1048,9 @@ func (s *testPrivilegeSuite) TestAdminCommand(c *C) { c.Assert(se.Auth(&auth.UserIdentity{Username: "test_admin", Hostname: "localhost"}, nil, nil), IsTrue) _, err := se.ExecuteInternal(context.Background(), "ADMIN SHOW DDL JOBS") - c.Assert(strings.Contains(err.Error(), "privilege check fail"), IsTrue) + c.Assert(strings.Contains(err.Error(), "privilege check"), IsTrue) _, err = se.ExecuteInternal(context.Background(), "ADMIN CHECK TABLE t") - c.Assert(strings.Contains(err.Error(), "privilege check fail"), IsTrue) + c.Assert(strings.Contains(err.Error(), "privilege check"), IsTrue) c.Assert(se.Auth(&auth.UserIdentity{Username: "root", Hostname: "localhost"}, nil, nil), IsTrue) _, err = se.ExecuteInternal(context.Background(), "ADMIN SHOW DDL JOBS") @@ -1246,11 +1253,9 @@ func (s *testPrivilegeSuite) TestDynamicPrivs(c *C) { mustExec(c, rootSe, "CREATE USER notsuper") mustExec(c, rootSe, "CREATE USER otheruser") mustExec(c, rootSe, "CREATE ROLE anyrolename") - mustExec(c, rootSe, "SET tidb_enable_dynamic_privileges=1") se := newSession(c, s.store, s.dbName) c.Assert(se.Auth(&auth.UserIdentity{Username: "notsuper", Hostname: "%"}, nil, nil), IsTrue) - mustExec(c, se, "SET tidb_enable_dynamic_privileges=1") // test SYSTEM_VARIABLES_ADMIN _, err := se.ExecuteInternal(context.Background(), "SET GLOBAL wait_timeout = 86400") @@ -1291,20 +1296,17 @@ func (s *testPrivilegeSuite) TestDynamicGrantOption(c *C) { mustExec(c, rootSe, "CREATE USER varuser1") mustExec(c, rootSe, "CREATE USER varuser2") mustExec(c, rootSe, "CREATE USER varuser3") - mustExec(c, rootSe, "SET tidb_enable_dynamic_privileges=1") mustExec(c, rootSe, "GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO varuser1") mustExec(c, rootSe, "GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO varuser2 WITH GRANT OPTION") se1 := newSession(c, s.store, s.dbName) - mustExec(c, se1, "SET tidb_enable_dynamic_privileges=1") c.Assert(se1.Auth(&auth.UserIdentity{Username: "varuser1", Hostname: "%"}, nil, nil), IsTrue) _, err := se1.ExecuteInternal(context.Background(), "GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO varuser3") c.Assert(err.Error(), Equals, "[planner:1227]Access denied; you need (at least one of) the GRANT OPTION privilege(s) for this operation") se2 := newSession(c, s.store, s.dbName) - mustExec(c, se2, "SET tidb_enable_dynamic_privileges=1") c.Assert(se2.Auth(&auth.UserIdentity{Username: "varuser2", Hostname: "%"}, nil, nil), IsTrue) mustExec(c, se2, "GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO varuser3") @@ -1314,14 +1316,12 @@ func (s *testPrivilegeSuite) TestSecurityEnhancedModeRestrictedTables(c *C) { // This provides an integration test of the tests in util/security/security_test.go cloudAdminSe := newSession(c, s.store, s.dbName) mustExec(c, cloudAdminSe, "CREATE USER cloudadmin") - mustExec(c, cloudAdminSe, "SET tidb_enable_dynamic_privileges=1") mustExec(c, cloudAdminSe, "GRANT RESTRICTED_TABLES_ADMIN, SELECT ON *.* to cloudadmin") mustExec(c, cloudAdminSe, "GRANT CREATE ON mysql.* to cloudadmin") mustExec(c, cloudAdminSe, "CREATE USER uroot") mustExec(c, cloudAdminSe, "GRANT ALL ON *.* to uroot WITH GRANT OPTION") // A "MySQL" all powerful user. c.Assert(cloudAdminSe.Auth(&auth.UserIdentity{Username: "cloudadmin", Hostname: "%"}, nil, nil), IsTrue) urootSe := newSession(c, s.store, s.dbName) - mustExec(c, urootSe, "SET tidb_enable_dynamic_privileges=1") c.Assert(urootSe.Auth(&auth.UserIdentity{Username: "uroot", Hostname: "%"}, nil, nil), IsTrue) sem.Enable() @@ -1345,7 +1345,6 @@ func (s *testPrivilegeSuite) TestSecurityEnhancedModeInfoschema(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("CREATE USER uroot1, uroot2, uroot3") tk.MustExec("GRANT SUPER ON *.* to uroot1 WITH GRANT OPTION") // super not process - tk.MustExec("SET tidb_enable_dynamic_privileges=1") tk.MustExec("GRANT SUPER, PROCESS, RESTRICTED_TABLES_ADMIN ON *.* to uroot2 WITH GRANT OPTION") tk.Se.Auth(&auth.UserIdentity{ Username: "uroot1", @@ -1384,7 +1383,6 @@ func (s *testPrivilegeSuite) TestSecurityEnhancedModeStatusVars(c *C) { // and verify if it appears. tk := testkit.NewTestKit(c, s.store) tk.MustExec("CREATE USER unostatus, ustatus") - tk.MustExec("SET tidb_enable_dynamic_privileges=1") tk.MustExec("GRANT RESTRICTED_STATUS_ADMIN ON *.* to ustatus") tk.Se.Auth(&auth.UserIdentity{ Username: "unostatus", @@ -1393,3 +1391,210 @@ func (s *testPrivilegeSuite) TestSecurityEnhancedModeStatusVars(c *C) { AuthHostname: "%", }, nil, nil) } + +func (s *testPrivilegeSuite) TestRenameUser(c *C) { + rootSe := newSession(c, s.store, s.dbName) + mustExec(c, rootSe, "DROP USER IF EXISTS 'ru1'@'localhost'") + mustExec(c, rootSe, "DROP USER IF EXISTS ru3") + mustExec(c, rootSe, "DROP USER IF EXISTS ru6@localhost") + mustExec(c, rootSe, "CREATE USER 'ru1'@'localhost'") + mustExec(c, rootSe, "CREATE USER ru3") + mustExec(c, rootSe, "CREATE USER ru6@localhost") + se1 := newSession(c, s.store, s.dbName) + c.Assert(se1.Auth(&auth.UserIdentity{Username: "ru1", Hostname: "localhost"}, nil, nil), IsTrue) + + // Check privileges (need CREATE USER) + _, err := se1.ExecuteInternal(context.Background(), "RENAME USER ru3 TO ru4") + c.Assert(err, ErrorMatches, ".*Access denied; you need .at least one of. the CREATE USER privilege.s. for this operation") + mustExec(c, rootSe, "GRANT UPDATE ON mysql.user TO 'ru1'@'localhost'") + _, err = se1.ExecuteInternal(context.Background(), "RENAME USER ru3 TO ru4") + c.Assert(err, ErrorMatches, ".*Access denied; you need .at least one of. the CREATE USER privilege.s. for this operation") + mustExec(c, rootSe, "GRANT CREATE USER ON *.* TO 'ru1'@'localhost'") + _, err = se1.ExecuteInternal(context.Background(), "RENAME USER ru3 TO ru4") + c.Assert(err, IsNil) + + // Test a few single rename (both Username and Hostname) + _, err = se1.ExecuteInternal(context.Background(), "RENAME USER 'ru4'@'%' TO 'ru3'@'localhost'") + c.Assert(err, IsNil) + _, err = se1.ExecuteInternal(context.Background(), "RENAME USER 'ru3'@'localhost' TO 'ru3'@'%'") + c.Assert(err, IsNil) + // Including negative tests, i.e. non existing from user and existing to user + _, err = rootSe.ExecuteInternal(context.Background(), "RENAME USER ru3 TO ru1@localhost") + c.Assert(err, ErrorMatches, ".*Operation RENAME USER failed for ru3@%.*") + _, err = se1.ExecuteInternal(context.Background(), "RENAME USER ru4 TO ru5@localhost") + c.Assert(err, ErrorMatches, ".*Operation RENAME USER failed for ru4@%.*") + _, err = se1.ExecuteInternal(context.Background(), "RENAME USER ru3 TO ru3") + c.Assert(err, ErrorMatches, ".*Operation RENAME USER failed for ru3@%.*") + _, err = se1.ExecuteInternal(context.Background(), "RENAME USER ru3 TO ru5@localhost, ru4 TO ru7") + c.Assert(err, ErrorMatches, ".*Operation RENAME USER failed for ru4@%.*") + _, err = se1.ExecuteInternal(context.Background(), "RENAME USER ru3 TO ru5@localhost, ru6@localhost TO ru1@localhost") + c.Assert(err, ErrorMatches, ".*Operation RENAME USER failed for ru6@localhost.*") + + // Test multi rename, this is a full swap of ru3 and ru6, i.e. need to read its previous state in the same transaction. + _, err = se1.ExecuteInternal(context.Background(), "RENAME USER 'ru3' TO 'ru3_tmp', ru6@localhost TO ru3, 'ru3_tmp' to ru6@localhost") + c.Assert(err, IsNil) + + // Cleanup + mustExec(c, rootSe, "DROP USER ru6@localhost") + mustExec(c, rootSe, "DROP USER ru3") + mustExec(c, rootSe, "DROP USER 'ru1'@'localhost'") +} + +func (s *testPrivilegeSuite) TestSecurityEnhancedModeSysVars(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("CREATE USER svroot1, svroot2") + tk.MustExec("GRANT SUPER ON *.* to svroot1 WITH GRANT OPTION") + tk.MustExec("GRANT SUPER, RESTRICTED_VARIABLES_ADMIN ON *.* to svroot2") + + sem.Enable() + defer sem.Disable() + + // svroot1 has SUPER but in SEM will be restricted + tk.Se.Auth(&auth.UserIdentity{ + Username: "svroot1", + Hostname: "localhost", + AuthUsername: "uroot", + AuthHostname: "%", + }, nil, nil) + + tk.MustQuery(`SHOW VARIABLES LIKE 'tidb_force_priority'`).Check(testkit.Rows()) + tk.MustQuery(`SHOW GLOBAL VARIABLES LIKE 'tidb_enable_telemetry'`).Check(testkit.Rows()) + + _, err := tk.Exec("SET tidb_force_priority = 'NO_PRIORITY'") + c.Assert(err.Error(), Equals, "[planner:1227]Access denied; you need (at least one of) the RESTRICTED_VARIABLES_ADMIN privilege(s) for this operation") + _, err = tk.Exec("SET GLOBAL tidb_enable_telemetry = OFF") + c.Assert(err.Error(), Equals, "[planner:1227]Access denied; you need (at least one of) the RESTRICTED_VARIABLES_ADMIN privilege(s) for this operation") + + _, err = tk.Exec("SELECT @@session.tidb_force_priority") + c.Assert(err.Error(), Equals, "[planner:1227]Access denied; you need (at least one of) the RESTRICTED_VARIABLES_ADMIN privilege(s) for this operation") + _, err = tk.Exec("SELECT @@global.tidb_enable_telemetry") + c.Assert(err.Error(), Equals, "[planner:1227]Access denied; you need (at least one of) the RESTRICTED_VARIABLES_ADMIN privilege(s) for this operation") + + tk.Se.Auth(&auth.UserIdentity{ + Username: "svroot2", + Hostname: "localhost", + AuthUsername: "uroot", + AuthHostname: "%", + }, nil, nil) + + tk.MustQuery(`SHOW VARIABLES LIKE 'tidb_force_priority'`).Check(testkit.Rows("tidb_force_priority NO_PRIORITY")) + tk.MustQuery(`SHOW GLOBAL VARIABLES LIKE 'tidb_enable_telemetry'`).Check(testkit.Rows("tidb_enable_telemetry ON")) + + // should not actually make any change. + tk.MustExec("SET tidb_force_priority = 'NO_PRIORITY'") + tk.MustExec("SET GLOBAL tidb_enable_telemetry = ON") + + tk.MustQuery(`SELECT @@session.tidb_force_priority`).Check(testkit.Rows("NO_PRIORITY")) + tk.MustQuery(`SELECT @@global.tidb_enable_telemetry`).Check(testkit.Rows("1")) +} + +// TestViewDefiner tests that default roles are correctly applied in the algorithm definer +// See: https://github.com/pingcap/tidb/issues/24414 +func (s *testPrivilegeSuite) TestViewDefiner(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("CREATE DATABASE issue24414") + tk.MustExec("USE issue24414") + tk.MustExec(`create table table1( + col1 int, + col2 int, + col3 int + )`) + tk.MustExec(`insert into table1 values (1,1,1),(2,2,2)`) + tk.MustExec(`CREATE ROLE 'ACL-mobius-admin'`) + tk.MustExec(`CREATE USER 'mobius-admin'`) + tk.MustExec(`CREATE USER 'mobius-admin-no-role'`) + tk.MustExec(`GRANT Select,Insert,Update,Delete,Create,Drop,Alter,Index,Create View,Show View ON issue24414.* TO 'ACL-mobius-admin'@'%'`) + tk.MustExec(`GRANT Select,Insert,Update,Delete,Create,Drop,Alter,Index,Create View,Show View ON issue24414.* TO 'mobius-admin-no-role'@'%'`) + tk.MustExec(`GRANT 'ACL-mobius-admin'@'%' to 'mobius-admin'@'%'`) + tk.MustExec(`SET DEFAULT ROLE ALL TO 'mobius-admin'`) + // create tables + tk.MustExec(`CREATE ALGORITHM = UNDEFINED DEFINER = 'mobius-admin'@'127.0.0.1' SQL SECURITY DEFINER VIEW test_view (col1 , col2 , col3) AS SELECT * from table1`) + tk.MustExec(`CREATE ALGORITHM = UNDEFINED DEFINER = 'mobius-admin-no-role'@'127.0.0.1' SQL SECURITY DEFINER VIEW test_view2 (col1 , col2 , col3) AS SELECT * from table1`) + + // all examples should work + tk.MustExec("select * from test_view") + tk.MustExec("select * from test_view2") +} + +func (s *testPrivilegeSuite) TestSecurityEnhancedModeRestrictedUsers(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("CREATE USER ruroot1, ruroot2, ruroot3") + tk.MustExec("CREATE ROLE notimportant") + tk.MustExec("GRANT SUPER, CREATE USER ON *.* to ruroot1 WITH GRANT OPTION") + tk.MustExec("GRANT SUPER, RESTRICTED_USER_ADMIN, CREATE USER ON *.* to ruroot2 WITH GRANT OPTION") + tk.MustExec("GRANT RESTRICTED_USER_ADMIN ON *.* to ruroot3") + tk.MustExec("GRANT notimportant TO ruroot2, ruroot3") + + sem.Enable() + defer sem.Disable() + + stmts := []string{ + "SET PASSWORD for ruroot3 = 'newpassword'", + "REVOKE notimportant FROM ruroot3", + "REVOKE SUPER ON *.* FROM ruroot3", + "DROP USER ruroot3", + } + + // ruroot1 has SUPER but in SEM will be restricted + tk.Se.Auth(&auth.UserIdentity{ + Username: "ruroot1", + Hostname: "localhost", + AuthUsername: "uroot", + AuthHostname: "%", + }, nil, nil) + + for _, stmt := range stmts { + err := tk.ExecToErr(stmt) + c.Assert(err.Error(), Equals, "[planner:1227]Access denied; you need (at least one of) the RESTRICTED_USER_ADMIN privilege(s) for this operation") + } + + // Switch to ruroot2, it should be permitted + tk.Se.Auth(&auth.UserIdentity{ + Username: "ruroot2", + Hostname: "localhost", + AuthUsername: "uroot", + AuthHostname: "%", + }, nil, nil) + + for _, stmt := range stmts { + err := tk.ExecToErr(stmt) + c.Assert(err, IsNil) + } +} + +func (s *testPrivilegeSuite) TestDynamicPrivsRegistration(c *C) { + se := newSession(c, s.store, s.dbName) + pm := privilege.GetPrivilegeManager(se) + + count := len(privileges.GetDynamicPrivileges()) + + c.Assert(pm.IsDynamicPrivilege("ACDC_ADMIN"), IsFalse) + c.Assert(privileges.RegisterDynamicPrivilege("ACDC_ADMIN"), IsNil) + c.Assert(pm.IsDynamicPrivilege("ACDC_ADMIN"), IsTrue) + c.Assert(len(privileges.GetDynamicPrivileges()), Equals, count+1) + + c.Assert(pm.IsDynamicPrivilege("iAmdynamIC"), IsFalse) + c.Assert(privileges.RegisterDynamicPrivilege("IAMdynamic"), IsNil) + c.Assert(pm.IsDynamicPrivilege("IAMdyNAMIC"), IsTrue) + c.Assert(len(privileges.GetDynamicPrivileges()), Equals, count+2) + + c.Assert(privileges.RegisterDynamicPrivilege("THIS_PRIVILEGE_NAME_IS_TOO_LONG_THE_MAX_IS_32_CHARS").Error(), Equals, "privilege name is longer than 32 characters") + c.Assert(pm.IsDynamicPrivilege("THIS_PRIVILEGE_NAME_IS_TOO_LONG_THE_MAX_IS_32_CHARS"), IsFalse) + + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("CREATE USER privassigntest") + + // Check that all privileges registered are assignable to users, + // including the recently registered ACDC_ADMIN + for _, priv := range privileges.GetDynamicPrivileges() { + sqlGrant, err := sqlexec.EscapeSQL("GRANT %n ON *.* TO privassigntest", priv) + c.Assert(err, IsNil) + tk.MustExec(sqlGrant) + } + // Check that all privileges registered are revokable + for _, priv := range privileges.GetDynamicPrivileges() { + sqlGrant, err := sqlexec.EscapeSQL("REVOKE %n ON *.* FROM privassigntest", priv) + c.Assert(err, IsNil) + tk.MustExec(sqlGrant) + } +} diff --git a/server/conn.go b/server/conn.go index 29c87bd0dfd86..76b4f8d32dd1d 100644 --- a/server/conn.go +++ b/server/conn.go @@ -39,6 +39,7 @@ import ( "context" "crypto/tls" "encoding/binary" + goerr "errors" "fmt" "io" "net" @@ -52,8 +53,6 @@ import ( "time" "unsafe" - goerr "errors" - "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/failpoint" @@ -63,7 +62,6 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" - "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/infoschema" @@ -76,9 +74,10 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx/variable" - txndriver "github.com/pingcap/tidb/store/driver/txn" + storeerr "github.com/pingcap/tidb/store/driver/error" "github.com/pingcap/tidb/store/tikv/util" "github.com/pingcap/tidb/tablecodec" + tidbutil "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/arena" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/execdetails" @@ -907,14 +906,6 @@ func (cc *clientConn) ShutdownOrNotify() bool { return false } -func queryStrForLog(query string) string { - const size = 4096 - if len(query) > size { - return query[:size] + fmt.Sprintf("(len: %d)", len(query)) - } - return query -} - func errStrForLog(err error, enableRedactLog bool) string { if enableRedactLog { // currently, only ErrParse is considered when enableRedactLog because it may contain sensitive information like @@ -1026,6 +1017,9 @@ func (cc *clientConn) dispatch(ctx context.Context, data []byte) error { cc.lastPacket = data cmd := data[0] data = data[1:] + if variable.TopSQLEnabled() { + defer pprof.SetGoroutineLabels(ctx) + } if variable.EnablePProfSQLCPU.Load() { label := getLastStmtInConn{cc}.PProfLabel() if len(label) > 0 { @@ -1569,7 +1563,7 @@ func (cc *clientConn) handleQuery(ctx context.Context, sql string) (err error) { retryable, err = cc.handleStmt(ctx, stmt, parserWarns, i == len(stmts)-1) if err != nil { _, allowTiFlashFallback := cc.ctx.GetSessionVars().AllowFallbackToTiKV[kv.TiFlash] - if allowTiFlashFallback && errors.ErrorEqual(err, txndriver.ErrTiFlashServerTimeout) && retryable { + if allowTiFlashFallback && errors.ErrorEqual(err, storeerr.ErrTiFlashServerTimeout) && retryable { // When the TiFlash server seems down, we append a warning to remind the user to check the status of the TiFlash // server and fallback to TiKV. warns := append(parserWarns, stmtctx.SQLWarn{Level: stmtctx.WarnLevelError, Err: err}) @@ -1621,11 +1615,11 @@ func (cc *clientConn) prefetchPointPlanKeys(ctx context.Context, stmts []ast.Stm pointPlans := make([]plannercore.Plan, len(stmts)) var idxKeys []kv.Key var rowKeys []kv.Key - is := domain.GetDomain(cc.ctx).InfoSchema() sc := vars.StmtCtx for i, stmt := range stmts { // TODO: the preprocess is run twice, we should find some way to avoid do it again. - if err = plannercore.Preprocess(cc.ctx, stmt, is); err != nil { + // TODO: handle the PreprocessorReturn. + if err = plannercore.Preprocess(cc.ctx, stmt); err != nil { return nil, err } p := plannercore.TryFastPlan(cc.ctx.Session, stmt) @@ -1870,10 +1864,10 @@ func (cc *clientConn) writeChunks(ctx context.Context, rs ResultSet, binary bool failpoint.Inject("fetchNextErr", func(value failpoint.Value) { switch value.(string) { case "firstNext": - failpoint.Return(firstNext, txndriver.ErrTiFlashServerTimeout) + failpoint.Return(firstNext, storeerr.ErrTiFlashServerTimeout) case "secondNext": if !firstNext { - failpoint.Return(firstNext, txndriver.ErrTiFlashServerTimeout) + failpoint.Return(firstNext, storeerr.ErrTiFlashServerTimeout) } } }) @@ -2126,10 +2120,10 @@ func (cc getLastStmtInConn) String() string { if cc.ctx.GetSessionVars().EnableRedactLog { sql = parser.Normalize(sql) } - return queryStrForLog(sql) + return tidbutil.QueryStrForLog(sql) case mysql.ComStmtExecute, mysql.ComStmtFetch: stmtID := binary.LittleEndian.Uint32(data[0:4]) - return queryStrForLog(cc.preparedStmt2String(stmtID)) + return tidbutil.QueryStrForLog(cc.preparedStmt2String(stmtID)) case mysql.ComStmtClose, mysql.ComStmtReset: stmtID := binary.LittleEndian.Uint32(data[0:4]) return mysql.Command2Str[cmd] + " " + strconv.Itoa(int(stmtID)) @@ -2157,10 +2151,10 @@ func (cc getLastStmtInConn) PProfLabel() string { case mysql.ComStmtReset: return "ResetStmt" case mysql.ComQuery, mysql.ComStmtPrepare: - return parser.Normalize(queryStrForLog(string(hack.String(data)))) + return parser.Normalize(tidbutil.QueryStrForLog(string(hack.String(data)))) case mysql.ComStmtExecute, mysql.ComStmtFetch: stmtID := binary.LittleEndian.Uint32(data[0:4]) - return queryStrForLog(cc.preparedStmt2StringNoArgs(stmtID)) + return tidbutil.QueryStrForLog(cc.preparedStmt2StringNoArgs(stmtID)) default: return "" } diff --git a/server/conn_stmt.go b/server/conn_stmt.go index 242b0df80fc83..df85f7ce45f52 100644 --- a/server/conn_stmt.go +++ b/server/conn_stmt.go @@ -50,11 +50,13 @@ import ( "github.com/pingcap/tidb/metrics" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx/stmtctx" - txndriver "github.com/pingcap/tidb/store/driver/txn" + "github.com/pingcap/tidb/sessionctx/variable" + storeerr "github.com/pingcap/tidb/store/driver/error" "github.com/pingcap/tidb/store/tikv/util" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/hack" + "github.com/pingcap/tidb/util/topsql" ) func (cc *clientConn) handleStmtPrepare(ctx context.Context, sql string) error { @@ -128,6 +130,13 @@ func (cc *clientConn) handleStmtExecute(ctx context.Context, data []byte) (err e stmtID := binary.LittleEndian.Uint32(data[0:4]) pos += 4 + if variable.TopSQLEnabled() { + preparedStmt, _ := cc.preparedStmtID2CachePreparedStmt(stmtID) + if preparedStmt != nil && preparedStmt.SQLDigest != nil { + ctx = topsql.AttachSQLInfo(ctx, preparedStmt.NormalizedSQL, preparedStmt.SQLDigest, "", nil) + } + } + stmt := cc.ctx.GetStatement(int(stmtID)) if stmt == nil { return mysql.NewErr(mysql.ErrUnknownStmtHandler, @@ -198,7 +207,7 @@ func (cc *clientConn) handleStmtExecute(ctx context.Context, data []byte) (err e ctx = context.WithValue(ctx, util.ExecDetailsKey, &util.ExecDetails{}) retryable, err := cc.executePreparedStmtAndWriteResult(ctx, stmt, args, useCursor) _, allowTiFlashFallback := cc.ctx.GetSessionVars().AllowFallbackToTiKV[kv.TiFlash] - if allowTiFlashFallback && err != nil && errors.ErrorEqual(err, txndriver.ErrTiFlashServerTimeout) && retryable { + if allowTiFlashFallback && err != nil && errors.ErrorEqual(err, storeerr.ErrTiFlashServerTimeout) && retryable { // When the TiFlash server seems down, we append a warning to remind the user to check the status of the TiFlash // server and fallback to TiKV. prevErr := err @@ -265,6 +274,12 @@ func (cc *clientConn) handleStmtFetch(ctx context.Context, data []byte) (err err return errors.Annotate(mysql.NewErr(mysql.ErrUnknownStmtHandler, strconv.FormatUint(uint64(stmtID), 10), "stmt_fetch"), cc.preparedStmt2String(stmtID)) } + if variable.TopSQLEnabled() { + prepareObj, _ := cc.preparedStmtID2CachePreparedStmt(stmtID) + if prepareObj != nil && prepareObj.SQLDigest != nil { + ctx = topsql.AttachSQLInfo(ctx, prepareObj.NormalizedSQL, prepareObj.SQLDigest, "", nil) + } + } sql := "" if prepared, ok := cc.ctx.GetStatement(int(stmtID)).(*TiDBStatement); ok { sql = prepared.sql @@ -680,14 +695,30 @@ func (cc *clientConn) preparedStmt2StringNoArgs(stmtID uint32) string { if sv == nil { return "" } + preparedObj, invalid := cc.preparedStmtID2CachePreparedStmt(stmtID) + if invalid { + return "invalidate CachedPrepareStmt type, ID: " + strconv.FormatUint(uint64(stmtID), 10) + } + if preparedObj == nil { + return "prepared statement not found, ID: " + strconv.FormatUint(uint64(stmtID), 10) + } + return preparedObj.PreparedAst.Stmt.Text() +} + +func (cc *clientConn) preparedStmtID2CachePreparedStmt(stmtID uint32) (_ *plannercore.CachedPrepareStmt, invalid bool) { + sv := cc.ctx.GetSessionVars() + if sv == nil { + return nil, false + } preparedPointer, ok := sv.PreparedStmts[stmtID] if !ok { - return "prepared statement not found, ID: " + strconv.FormatUint(uint64(stmtID), 10) + // not found + return nil, false } preparedObj, ok := preparedPointer.(*plannercore.CachedPrepareStmt) if !ok { - return "invalidate CachedPrepareStmt type, ID: " + strconv.FormatUint(uint64(stmtID), 10) + // invalid cache. should never happen. + return nil, true } - preparedAst := preparedObj.PreparedAst - return preparedAst.Stmt.Text() + return preparedObj, false } diff --git a/server/conn_test.go b/server/conn_test.go index 0ed451a4fde8e..c2347e95e4bdd 100644 --- a/server/conn_test.go +++ b/server/conn_test.go @@ -764,11 +764,22 @@ func (ts *ConnTestSuite) TestTiFlashFallback(c *C) { tb := testGetTableByName(c, tk.Se, "test", "t") err := domain.GetDomain(tk.Se).DDL().UpdateTableReplicaInfo(tk.Se, tb.Meta().ID, true) c.Assert(err, IsNil) + + dml := "insert into t values" for i := 0; i < 50; i++ { - tk.MustExec(fmt.Sprintf("insert into t values(%v, 0)", i)) + dml += fmt.Sprintf("(%v, 0)", i) + if i != 49 { + dml += "," + } } + tk.MustExec(dml) tk.MustQuery("select count(*) from t").Check(testkit.Rows("50")) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/copr/ReduceCopNextMaxBackoff", `return(true)`), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/copr/ReduceCopNextMaxBackoff"), IsNil) + }() + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/unistore/BatchCopRpcErrtiflash0", "return(\"tiflash0\")"), IsNil) // test COM_STMT_EXECUTE ctx := context.Background() @@ -777,6 +788,7 @@ func (ts *ConnTestSuite) TestTiFlashFallback(c *C) { c.Assert(cc.handleStmtPrepare(ctx, "select sum(a) from t"), IsNil) c.Assert(cc.handleStmtExecute(ctx, []byte{0x1, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0}), IsNil) tk.MustQuery("show warnings").Check(testkit.Rows("Error 9012 TiFlash server timeout")) + // test COM_STMT_FETCH (cursor mode) c.Assert(cc.handleStmtExecute(ctx, []byte{0x1, 0x0, 0x0, 0x0, 0x1, 0x1, 0x0, 0x0, 0x0}), IsNil) c.Assert(cc.handleStmtFetch(ctx, []byte{0x1, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0}), NotNil) diff --git a/server/http_handler.go b/server/http_handler.go index 67babd1f05e8d..023bebfbf7226 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -34,6 +34,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/log" "github.com/pingcap/parser/model" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" @@ -45,14 +46,12 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/session" - "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/binloginfo" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/gcworker" "github.com/pingcap/tidb/store/helper" "github.com/pingcap/tidb/store/tikv" - "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/tablecodec" @@ -60,10 +59,10 @@ import ( "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/admin" "github.com/pingcap/tidb/util/codec" + "github.com/pingcap/tidb/util/deadlockhistory" "github.com/pingcap/tidb/util/gcutil" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/pdapi" - log "github.com/sirupsen/logrus" "go.uber.org/zap" ) @@ -199,71 +198,7 @@ func (t *tikvHandlerTool) getHandle(tb table.PhysicalTable, params map[string]st return handle, nil } -func (t *tikvHandlerTool) getMvccByStartTs(startTS uint64, startKey, endKey kv.Key) (*mvccKV, error) { - bo := tikv.NewBackofferWithVars(context.Background(), 5000, nil) - for { - curRegion, err := t.RegionCache.LocateKey(bo, startKey) - if err != nil { - logutil.BgLogger().Error("get MVCC by startTS failed", zap.Uint64("txnStartTS", startTS), - zap.Stringer("startKey", startKey), zap.Error(err)) - return nil, errors.Trace(err) - } - - tikvReq := tikvrpc.NewRequest(tikvrpc.CmdMvccGetByStartTs, &kvrpcpb.MvccGetByStartTsRequest{ - StartTs: startTS, - }) - tikvReq.Context.Priority = kvrpcpb.CommandPri_Low - kvResp, err := t.Store.SendReq(bo, tikvReq, curRegion.Region, time.Hour) - if err != nil { - logutil.BgLogger().Error("get MVCC by startTS failed", - zap.Uint64("txnStartTS", startTS), - zap.Stringer("startKey", startKey), - zap.Reflect("region", curRegion.Region), - zap.Stringer("curRegion", curRegion), - zap.Reflect("kvResp", kvResp), - zap.Error(err)) - return nil, errors.Trace(err) - } - data := kvResp.Resp.(*kvrpcpb.MvccGetByStartTsResponse) - if err := data.GetRegionError(); err != nil { - logutil.BgLogger().Warn("get MVCC by startTS failed", - zap.Uint64("txnStartTS", startTS), - zap.Stringer("startKey", startKey), - zap.Reflect("region", curRegion.Region), - zap.Stringer("curRegion", curRegion), - zap.Reflect("kvResp", kvResp), - zap.Stringer("error", err)) - continue - } - - if len(data.GetError()) > 0 { - logutil.BgLogger().Error("get MVCC by startTS failed", - zap.Uint64("txnStartTS", startTS), - zap.Stringer("startKey", startKey), - zap.Reflect("region", curRegion.Region), - zap.Stringer("curRegion", curRegion), - zap.Reflect("kvResp", kvResp), - zap.String("error", data.GetError())) - return nil, errors.New(data.GetError()) - } - - key := data.GetKey() - if len(key) > 0 { - resp := &kvrpcpb.MvccGetByKeyResponse{Info: data.Info, RegionError: data.RegionError, Error: data.Error} - return &mvccKV{Key: strings.ToUpper(hex.EncodeToString(key)), Value: resp, RegionID: curRegion.Region.GetID()}, nil - } - - if len(endKey) > 0 && curRegion.Contains(endKey) { - return nil, nil - } - if len(curRegion.EndKey) == 0 { - return nil, nil - } - startKey = kv.Key(curRegion.EndKey) - } -} - -func (t *tikvHandlerTool) getMvccByIdxValue(idx table.Index, values url.Values, idxCols []*model.ColumnInfo, handle kv.Handle) (*mvccKV, error) { +func (t *tikvHandlerTool) getMvccByIdxValue(idx table.Index, values url.Values, idxCols []*model.ColumnInfo, handle kv.Handle) (*helper.MvccKV, error) { sc := new(stmtctx.StatementContext) // HTTP request is not a database session, set timezone to UTC directly here. // See https://github.com/pingcap/tidb/blob/master/docs/tidb_http_api.md for more details. @@ -284,7 +219,7 @@ func (t *tikvHandlerTool) getMvccByIdxValue(idx table.Index, values url.Values, if err != nil { return nil, err } - return &mvccKV{strings.ToUpper(hex.EncodeToString(encodedKey)), regionID, data}, err + return &helper.MvccKV{Key: strings.ToUpper(hex.EncodeToString(encodedKey)), RegionID: regionID, Value: data}, err } // formValue2DatumRow converts URL query string to a Datum Row. @@ -355,11 +290,11 @@ func (t *tikvHandlerTool) getPartition(tableVal table.Table, partitionName strin } func (t *tikvHandlerTool) schema() (infoschema.InfoSchema, error) { - session, err := session.CreateSession(t.Store) + dom, err := session.GetDomain(t.Store) if err != nil { - return nil, errors.Trace(err) + return nil, err } - return domain.GetDomain(session.(sessionctx.Context)).InfoSchema(), nil + return dom.InfoSchema(), nil } func (t *tikvHandlerTool) handleMvccGetByHex(params map[string]string) (*mvccKV, error) { @@ -691,13 +626,6 @@ func (h settingsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - l, err1 := log.ParseLevel(levelStr) - if err1 != nil { - writeError(w, err1) - return - } - log.SetLevel(l) - config.GetGlobalConfig().Log.Level = levelStr } if generalLog := req.Form.Get("tidb_general_log"); generalLog != "" { @@ -712,14 +640,13 @@ func (h settingsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } if asyncCommit := req.Form.Get("tidb_enable_async_commit"); asyncCommit != "" { - s, err := session.CreateSession(h.Store.(kv.Storage)) + s, err := session.CreateSession(h.Store) if err != nil { writeError(w, err) return } - if s != nil { - defer s.Close() - } + defer s.Close() + switch asyncCommit { case "0": err = s.GetSessionVars().GlobalVarsAccessor.SetGlobalSysVar(variable.TiDBEnableAsyncCommit, variable.Off) @@ -735,14 +662,13 @@ func (h settingsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } if onePC := req.Form.Get("tidb_enable_1pc"); onePC != "" { - s, err := session.CreateSession(h.Store.(kv.Storage)) + s, err := session.CreateSession(h.Store) if err != nil { writeError(w, err) return } - if s != nil { - defer s.Close() - } + defer s.Close() + switch onePC { case "0": err = s.GetSessionVars().GlobalVarsAccessor.SetGlobalSysVar(variable.TiDBEnable1PC, variable.Off) @@ -778,6 +704,20 @@ func (h settingsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } } + if deadlockHistoryCapacity := req.Form.Get("tidb_deadlock_history_capacity"); deadlockHistoryCapacity != "" { + capacity, err := strconv.Atoi(deadlockHistoryCapacity) + if err != nil { + writeError(w, errors.New("illegal argument")) + return + } else if capacity < 0 || capacity > 10000 { + writeError(w, errors.New("tidb_deadlock_history_capacity out of range, should be in 0 to 10000")) + return + } + cfg := config.GetGlobalConfig() + cfg.PessimisticTxn.DeadlockHistoryCapacity = uint(capacity) + config.StoreGlobalConfig(cfg) + deadlockhistory.GlobalDeadlockHistory.Resize(uint(capacity)) + } } else { writeData(w, config.GetGlobalConfig()) } @@ -878,14 +818,11 @@ func (h flashReplicaHandler) getTiFlashReplicaInfo(tblInfo *model.TableInfo, rep } func (h flashReplicaHandler) getDropOrTruncateTableTiflash(currentSchema infoschema.InfoSchema) ([]*tableFlashReplicaInfo, error) { - s, err := session.CreateSession(h.Store.(kv.Storage)) + s, err := session.CreateSession(h.Store) if err != nil { return nil, errors.Trace(err) } - - if s != nil { - defer s.Close() - } + defer s.Close() store := domain.GetDomain(s).Store() txn, err := store.Begin() @@ -948,16 +885,18 @@ func (h flashReplicaHandler) handleStatusReport(w http.ResponseWriter, req *http writeError(w, err) return } - do, err := session.GetDomain(h.Store.(kv.Storage)) + do, err := session.GetDomain(h.Store) if err != nil { writeError(w, err) return } - s, err := session.CreateSession(h.Store.(kv.Storage)) + s, err := session.CreateSession(h.Store) if err != nil { writeError(w, err) return } + defer s.Close() + available := status.checkTableFlashReplicaAvailable() err = do.DDL().UpdateTableReplicaInfo(s, status.ID, available) if err != nil { @@ -1123,18 +1062,7 @@ func (h ddlHistoryJobHandler) ServeHTTP(w http.ResponseWriter, req *http.Request } func (h ddlHistoryJobHandler) getAllHistoryDDL() ([]*model.Job, error) { - s, err := session.CreateSession(h.Store.(kv.Storage)) - if err != nil { - return nil, errors.Trace(err) - } - - if s != nil { - defer s.Close() - } - - store := domain.GetDomain(s.(sessionctx.Context)).Store() - txn, err := store.Begin() - + txn, err := h.Store.Begin() if err != nil { return nil, errors.Trace(err) } @@ -1170,7 +1098,7 @@ func (h ddlResignOwnerHandler) ServeHTTP(w http.ResponseWriter, req *http.Reques err := h.resignDDLOwner() if err != nil { - log.Error(err) + log.Error("failed to resign DDL owner", zap.Error(err)) writeError(w, err) return } @@ -1214,7 +1142,7 @@ func (h tableHandler) addScatterSchedule(startKey, endKey []byte, name string) e return err } if err := resp.Body.Close(); err != nil { - log.Error(err) + log.Error("failed to close response body", zap.Error(err)) } return nil } @@ -1234,7 +1162,7 @@ func (h tableHandler) deleteScatterSchedule(name string) error { return err } if err := resp.Body.Close(); err != nil { - log.Error(err) + log.Error("failed to close response body", zap.Error(err)) } return nil } @@ -1661,7 +1589,7 @@ func (h mvccTxnHandler) handleMvccGetByKey(params map[string]string, values url. if err != nil { return nil, err } - resp := &mvccKV{Key: strings.ToUpper(hex.EncodeToString(encodedKey)), Value: data, RegionID: regionID} + resp := &helper.MvccKV{Key: strings.ToUpper(hex.EncodeToString(encodedKey)), Value: data, RegionID: regionID} if len(values.Get("decode")) == 0 { return resp, nil } @@ -1728,7 +1656,7 @@ func (h *mvccTxnHandler) handleMvccGetByTxn(params map[string]string) (interface } startKey := tablecodec.EncodeTablePrefix(tableID) endKey := tablecodec.EncodeRowKeyWithHandle(tableID, kv.IntHandle(math.MaxInt64)) - return h.getMvccByStartTs(uint64(startTS), startKey, endKey) + return h.GetMvccByStartTs(uint64(startTS), startKey, endKey) } // serverInfo is used to report the servers info when do http request. @@ -1741,17 +1669,17 @@ type serverInfo struct { // ServeHTTP handles request of ddl server info. func (h serverInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - do, err := session.GetDomain(h.Store.(kv.Storage)) + do, err := session.GetDomain(h.Store) if err != nil { writeError(w, errors.New("create session error")) - log.Error(err) + log.Error("failed to get session domain", zap.Error(err)) return } info := serverInfo{} info.ServerInfo, err = infosync.GetServerInfo() if err != nil { writeError(w, err) - log.Error(err) + log.Error("failed to get server info", zap.Error(err)) return } info.IsOwner = do.DDL().OwnerManager().IsOwner() @@ -1771,17 +1699,17 @@ type clusterServerInfo struct { // ServeHTTP handles request of all ddl servers info. func (h allServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - do, err := session.GetDomain(h.Store.(kv.Storage)) + do, err := session.GetDomain(h.Store) if err != nil { writeError(w, errors.New("create session error")) - log.Error(err) + log.Error("failed to get session domain", zap.Error(err)) return } ctx := context.Background() allServersInfo, err := infosync.GetAllServerInfo(ctx) if err != nil { writeError(w, errors.New("ddl server information not found")) - log.Error(err) + log.Error("failed to get all server info", zap.Error(err)) return } ctx, cancel := context.WithTimeout(ctx, 3*time.Second) @@ -1789,7 +1717,7 @@ func (h allServerInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request cancel() if err != nil { writeError(w, errors.New("ddl server information not found")) - log.Error(err) + log.Error("failed to get owner id", zap.Error(err)) return } allVersionsMap := map[infosync.ServerVersionInfo]struct{}{} @@ -1872,6 +1800,8 @@ func (h profileHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { writeError(w, err) return } + defer sctx.Close() + var start, end time.Time if req.FormValue("end") != "" { end, err = time.ParseInLocation(time.RFC3339, req.FormValue("end"), sctx.GetSessionVars().Location()) @@ -1983,13 +1913,13 @@ func (h ddlHookHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { dom, err := session.GetDomain(h.store) if err != nil { - log.Error(err) + log.Error("failed to get session domain", zap.Error(err)) writeError(w, err) } newCallbackFunc, err := ddl.GetCustomizedHook(req.FormValue("ddl_hook")) if err != nil { - log.Error(err) + log.Error("failed to get customized hook", zap.Error(err)) writeError(w, err) } callback := newCallbackFunc(dom) diff --git a/server/http_handler_test.go b/server/http_handler_test.go index a7495b987f24a..abb466f897080 100644 --- a/server/http_handler_test.go +++ b/server/http_handler_test.go @@ -23,7 +23,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net" "net/http" "net/http/httputil" @@ -37,7 +36,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/kvrpcpb" - zaplog "github.com/pingcap/log" + "github.com/pingcap/log" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/config" @@ -56,9 +55,9 @@ import ( "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/codec" + "github.com/pingcap/tidb/util/deadlockhistory" "github.com/pingcap/tidb/util/rowcodec" "github.com/pingcap/tidb/util/versioninfo" - log "github.com/sirupsen/logrus" "go.uber.org/zap" ) @@ -560,7 +559,7 @@ partition by range (a) func decodeKeyMvcc(closer io.ReadCloser, c *C, valid bool) { decoder := json.NewDecoder(closer) - var data mvccKV + var data helper.MvccKV err := decoder.Decode(&data) c.Assert(err, IsNil) if valid { @@ -578,10 +577,10 @@ func (ts *HTTPHandlerTestSuite) TestGetTableMVCC(c *C) { ts.prepareData(c) defer ts.stopServer(c) - resp, err := ts.fetchStatus(fmt.Sprintf("/mvcc/key/tidb/test/1")) + resp, err := ts.fetchStatus("/mvcc/key/tidb/test/1") c.Assert(err, IsNil) decoder := json.NewDecoder(resp.Body) - var data mvccKV + var data helper.MvccKV err = decoder.Decode(&data) c.Assert(err, IsNil) c.Assert(data.Value, NotNil) @@ -602,7 +601,7 @@ func (ts *HTTPHandlerTestSuite) TestGetTableMVCC(c *C) { resp, err = ts.fetchStatus(fmt.Sprintf("/mvcc/txn/%d/tidb/test", startTs)) c.Assert(err, IsNil) - var p2 mvccKV + var p2 helper.MvccKV decoder = json.NewDecoder(resp.Body) err = decoder.Decode(&p2) c.Assert(err, IsNil) @@ -616,12 +615,12 @@ func (ts *HTTPHandlerTestSuite) TestGetTableMVCC(c *C) { resp, err = ts.fetchStatus("/mvcc/hex/" + hexKey) c.Assert(err, IsNil) decoder = json.NewDecoder(resp.Body) - var data2 mvccKV + var data2 helper.MvccKV err = decoder.Decode(&data2) c.Assert(err, IsNil) c.Assert(data2, DeepEquals, data) - resp, err = ts.fetchStatus(fmt.Sprintf("/mvcc/key/tidb/test/1?decode=true")) + resp, err = ts.fetchStatus("/mvcc/key/tidb/test/1?decode=true") c.Assert(err, IsNil) decoder = json.NewDecoder(resp.Body) var data3 map[string]interface{} @@ -667,10 +666,10 @@ func (ts *HTTPHandlerTestSuite) TestGetMVCCNotFound(c *C) { ts.startServer(c) ts.prepareData(c) defer ts.stopServer(c) - resp, err := ts.fetchStatus(fmt.Sprintf("/mvcc/key/tidb/test/1234")) + resp, err := ts.fetchStatus("/mvcc/key/tidb/test/1234") c.Assert(err, IsNil) decoder := json.NewDecoder(resp.Body) - var data mvccKV + var data helper.MvccKV err = decoder.Decode(&data) c.Assert(err, IsNil) c.Assert(data.Value.Info.Lock, IsNil) @@ -740,7 +739,7 @@ func (ts *HTTPHandlerTestSuite) TestTiFlashReplica(c *C) { resp, err = ts.postStatus("/tiflash/replica", "application/json", bytes.NewBuffer([]byte(`{"id":84,"region_count":3,"flash_region_count":3}`))) c.Assert(err, IsNil) c.Assert(resp, NotNil) - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) c.Assert(err, IsNil) c.Assert(string(body), Equals, "[schema:1146]Table which ID = 84 does not exist.") @@ -750,7 +749,7 @@ func (ts *HTTPHandlerTestSuite) TestTiFlashReplica(c *C) { resp, err = ts.postStatus("/tiflash/replica", "application/json", bytes.NewBuffer([]byte(req))) c.Assert(err, IsNil) c.Assert(resp, NotNil) - body, err = ioutil.ReadAll(resp.Body) + body, err = io.ReadAll(resp.Body) c.Assert(err, IsNil) c.Assert(string(body), Equals, "") @@ -975,14 +974,14 @@ func (ts *HTTPHandlerTestSuite) TestGetIndexMVCC(c *C) { resp, err = ts.fetchStatus("/mvcc/index/tidb/test/idx1/1?a=1") c.Assert(err, IsNil) decoder := json.NewDecoder(resp.Body) - var data1 mvccKV + var data1 helper.MvccKV err = decoder.Decode(&data1) c.Assert(err, NotNil) resp, err = ts.fetchStatus("/mvcc/index/tidb/test/idx2/1?a=1") c.Assert(err, IsNil) decoder = json.NewDecoder(resp.Body) - var data2 mvccKV + var data2 helper.MvccKV err = decoder.Decode(&data2) c.Assert(err, NotNil) @@ -1141,7 +1140,11 @@ func (ts *HTTPHandlerTestSuite) TestAllHistory(c *C) { c.Assert(jobs, DeepEquals, data) } -func (ts *HTTPHandlerTestSuite) TestPostSettings(c *C) { +func dummyRecord() *deadlockhistory.DeadlockRecord { + return &deadlockhistory.DeadlockRecord{} +} + +func (ts *HTTPHandlerTestSerialSuite) TestPostSettings(c *C) { ts.startServer(c) ts.prepareData(c) defer ts.stopServer(c) @@ -1156,8 +1159,7 @@ func (ts *HTTPHandlerTestSuite) TestPostSettings(c *C) { resp, err := ts.formStatus("/settings", form) c.Assert(err, IsNil) c.Assert(resp.StatusCode, Equals, http.StatusOK) - c.Assert(log.GetLevel(), Equals, log.ErrorLevel) - c.Assert(zaplog.GetLevel(), Equals, zap.ErrorLevel) + c.Assert(log.GetLevel(), Equals, zap.ErrorLevel) c.Assert(config.GetGlobalConfig().Log.Level, Equals, "error") c.Assert(variable.ProcessGeneralLog.Load(), IsTrue) val, err := variable.GetGlobalSystemVar(se.GetSessionVars(), variable.TiDBEnableAsyncCommit) @@ -1176,8 +1178,7 @@ func (ts *HTTPHandlerTestSuite) TestPostSettings(c *C) { c.Assert(err, IsNil) c.Assert(resp.StatusCode, Equals, http.StatusOK) c.Assert(variable.ProcessGeneralLog.Load(), IsFalse) - c.Assert(log.GetLevel(), Equals, log.FatalLevel) - c.Assert(zaplog.GetLevel(), Equals, zap.FatalLevel) + c.Assert(log.GetLevel(), Equals, zap.FatalLevel) c.Assert(config.GetGlobalConfig().Log.Level, Equals, "fatal") val, err = variable.GetGlobalSystemVar(se.GetSessionVars(), variable.TiDBEnableAsyncCommit) c.Assert(err, IsNil) @@ -1228,6 +1229,31 @@ func (ts *HTTPHandlerTestSuite) TestPostSettings(c *C) { c.Assert(resp.StatusCode, Equals, http.StatusOK) c.Assert(config.GetGlobalConfig().CheckMb4ValueInUTF8, Equals, false) dbt.mustExec("insert t2 values (unhex('f09f8c80'));") + + // test tidb_deadlock_history_capacity + deadlockhistory.GlobalDeadlockHistory.Resize(10) + for i := 0; i < 10; i++ { + deadlockhistory.GlobalDeadlockHistory.Push(dummyRecord()) + } + form = make(url.Values) + form.Set("tidb_deadlock_history_capacity", "5") + resp, err = ts.formStatus("/settings", form) + c.Assert(len(deadlockhistory.GlobalDeadlockHistory.GetAll()), Equals, 5) + c.Assert(deadlockhistory.GlobalDeadlockHistory.GetAll()[0].ID, Equals, uint64(6)) + c.Assert(deadlockhistory.GlobalDeadlockHistory.GetAll()[4].ID, Equals, uint64(10)) + deadlockhistory.GlobalDeadlockHistory.Push(dummyRecord()) + c.Assert(len(deadlockhistory.GlobalDeadlockHistory.GetAll()), Equals, 5) + c.Assert(deadlockhistory.GlobalDeadlockHistory.GetAll()[0].ID, Equals, uint64(7)) + c.Assert(deadlockhistory.GlobalDeadlockHistory.GetAll()[4].ID, Equals, uint64(11)) + form = make(url.Values) + form.Set("tidb_deadlock_history_capacity", "6") + resp, err = ts.formStatus("/settings", form) + deadlockhistory.GlobalDeadlockHistory.Push(dummyRecord()) + c.Assert(len(deadlockhistory.GlobalDeadlockHistory.GetAll()), Equals, 6) + c.Assert(deadlockhistory.GlobalDeadlockHistory.GetAll()[0].ID, Equals, uint64(7)) + c.Assert(deadlockhistory.GlobalDeadlockHistory.GetAll()[5].ID, Equals, uint64(12)) + // restore original value. + config.GetGlobalConfig().CheckMb4ValueInUTF8 = true } func (ts *HTTPHandlerTestSuite) TestPprof(c *C) { @@ -1237,7 +1263,7 @@ func (ts *HTTPHandlerTestSuite) TestPprof(c *C) { for retry := 0; retry < retryTime; retry++ { resp, err := ts.fetchStatus("/debug/pprof/heap") if err == nil { - _, err = ioutil.ReadAll(resp.Body) + _, err = io.ReadAll(resp.Body) c.Assert(err, IsNil) err = resp.Body.Close() c.Assert(err, IsNil) @@ -1245,7 +1271,7 @@ func (ts *HTTPHandlerTestSuite) TestPprof(c *C) { } time.Sleep(time.Millisecond * 10) } - zaplog.Fatal("failed to get profile for %d retries in every 10 ms", zap.Int("retryTime", retryTime)) + log.Fatal("failed to get profile for %d retries in every 10 ms", zap.Int("retryTime", retryTime)) } func (ts *HTTPHandlerTestSuite) TestServerInfo(c *C) { @@ -1398,7 +1424,7 @@ func (ts *HTTPHandlerTestSuite) TestZipInfoForSQL(c *C) { resp, err = ts.formStatus("/debug/sub-optimal-plan", urlValues) c.Assert(err, IsNil) c.Assert(resp.StatusCode, Equals, http.StatusInternalServerError) - b, err = ioutil.ReadAll(resp.Body) + b, err = io.ReadAll(resp.Body) c.Assert(err, IsNil) c.Assert(string(b), Equals, "use database non_exists_db failed, err: [schema:1049]Unknown database 'non_exists_db'\n") c.Assert(resp.Body.Close(), IsNil) @@ -1420,7 +1446,7 @@ func (ts *HTTPHandlerTestSuite) TestFailpointHandler(c *C) { resp, err = ts.fetchStatus("/fail/") c.Assert(err, IsNil) c.Assert(resp.StatusCode, Equals, http.StatusOK) - b, err := ioutil.ReadAll(resp.Body) + b, err := io.ReadAll(resp.Body) c.Assert(err, IsNil) c.Assert(strings.Contains(string(b), "github.com/pingcap/tidb/server/enableTestAPI=return"), IsTrue) c.Assert(resp.Body.Close(), IsNil) @@ -1488,7 +1514,7 @@ func (ts *HTTPHandlerTestSuite) TestDDLHookHandler(c *C) { resp, err = ts.postStatus("/test/ddl/hook", "application/x-www-form-urlencoded", bytes.NewBuffer([]byte(`ddl_hook=ctc_hook`))) c.Assert(err, IsNil) c.Assert(resp, NotNil) - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) c.Assert(err, IsNil) c.Assert(string(body), Equals, "\"success!\"") c.Assert(resp.StatusCode, Equals, http.StatusOK) @@ -1496,7 +1522,7 @@ func (ts *HTTPHandlerTestSuite) TestDDLHookHandler(c *C) { resp, err = ts.postStatus("/test/ddl/hook", "application/x-www-form-urlencoded", bytes.NewBuffer([]byte(`ddl_hook=default_hook`))) c.Assert(err, IsNil) c.Assert(resp, NotNil) - body, err = ioutil.ReadAll(resp.Body) + body, err = io.ReadAll(resp.Body) c.Assert(err, IsNil) c.Assert(string(body), Equals, "\"success!\"") c.Assert(resp.StatusCode, Equals, http.StatusOK) diff --git a/server/http_status.go b/server/http_status.go index b385ea0c45890..67eace56562fe 100644 --- a/server/http_status.go +++ b/server/http_status.go @@ -21,7 +21,7 @@ import ( "crypto/x509" "encoding/json" "fmt" - "io/ioutil" + "io" "net" "net/http" "net/http/pprof" @@ -43,6 +43,7 @@ import ( "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/printer" + "github.com/pingcap/tidb/util/topsql/tracecpu" "github.com/pingcap/tidb/util/versioninfo" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/soheilhy/cmux" @@ -184,7 +185,7 @@ func (s *Server) startHTTPServer() { serverMux.HandleFunc("/debug/pprof/", pprof.Index) serverMux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - serverMux.HandleFunc("/debug/pprof/profile", pprof.Profile) + serverMux.HandleFunc("/debug/pprof/profile", tracecpu.ProfileHTTPHandler) serverMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) serverMux.HandleFunc("/debug/pprof/trace", pprof.Trace) serverMux.HandleFunc("/debug/gogc", func(w http.ResponseWriter, r *http.Request) { @@ -193,7 +194,7 @@ func (s *Server) startHTTPServer() { _, err := w.Write([]byte(strconv.Itoa(util.GetGOGC()))) terror.Log(err) case http.MethodPost: - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { terror.Log(err) return @@ -251,7 +252,7 @@ func (s *Server) startHTTPServer() { serveError(w, http.StatusInternalServerError, fmt.Sprintf("Create zipped %s fail: %v", "profile", err)) return } - if err := rpprof.StartCPUProfile(fw); err != nil { + if err := tracecpu.StartCPUProfile(fw); err != nil { serveError(w, http.StatusInternalServerError, fmt.Sprintf("Could not enable CPU profiling: %s", err)) return @@ -261,7 +262,11 @@ func (s *Server) startHTTPServer() { sec = 10 } sleepWithCtx(r.Context(), time.Duration(sec)*time.Second) - rpprof.StopCPUProfile() + err = tracecpu.StopCPUProfile() + if err != nil { + serveError(w, http.StatusInternalServerError, fmt.Sprintf("Create zipped %s fail: %v", "config", err)) + return + } // dump config fw, err = zw.Create("config") diff --git a/server/server.go b/server/server.go index f7a6021a11221..935abbb7bc693 100644 --- a/server/server.go +++ b/server/server.go @@ -37,7 +37,6 @@ import ( "math/rand" "net" "net/http" - "unsafe" // For pprof _ "net/http/pprof" @@ -46,6 +45,7 @@ import ( "sync" "sync/atomic" "time" + "unsafe" "github.com/blacktear23/go-proxyprotocol" "github.com/pingcap/errors" @@ -54,10 +54,11 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/errno" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/plugin" + "github.com/pingcap/tidb/session/txninfo" "github.com/pingcap/tidb/sessionctx/variable" - "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/dbterror" "github.com/pingcap/tidb/util/fastrand" @@ -310,9 +311,9 @@ func setSSLVariable(ca, key, cert string) { func setTxnScope() { variable.SetSysVar("txn_scope", func() string { if isGlobal, _ := config.GetTxnScopeFromConfig(); isGlobal { - return oracle.GlobalTxnScope + return kv.GlobalTxnScope } - return oracle.LocalTxnScope + return kv.LocalTxnScope }()) } @@ -557,6 +558,22 @@ func (s *Server) ShowProcessList() map[uint64]*util.ProcessInfo { return rs } +// ShowTxnList shows all txn info for displaying in `TIDB_TRX` +func (s *Server) ShowTxnList() []*txninfo.TxnInfo { + s.rwlock.RLock() + defer s.rwlock.RUnlock() + rs := make([]*txninfo.TxnInfo, 0, len(s.clients)) + for _, client := range s.clients { + if client.ctx.Session != nil { + info := client.ctx.Session.TxnInfo() + if info != nil { + rs = append(rs, info) + } + } + } + return rs +} + // GetProcessInfo implements the SessionManager interface. func (s *Server) GetProcessInfo(id uint64) (*util.ProcessInfo, bool) { s.rwlock.RLock() diff --git a/server/server_test.go b/server/server_test.go index 20c0c2b508213..461b59a070bb1 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -19,7 +19,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "math/rand" "net/http" "net/url" @@ -499,14 +498,14 @@ func (cli *testServerClient) runTestLoadDataForSlowLog(c *C, server *Server) { } // Test for record slow log for load data statement. - rows := dbt.mustQuery(fmt.Sprintf("select plan from information_schema.slow_query where query like 'load data local infile %% into table t_slow;' order by time desc limit 1")) + rows := dbt.mustQuery("select plan from information_schema.slow_query where query like 'load data local infile % into table t_slow;' order by time desc limit 1") expectedPlan := ".*LoadData.* time.* loops.* prepare.* check_insert.* mem_insert_time:.* prefetch.* rpc.* commit_txn.*" checkPlan(rows, expectedPlan) // Test for record statements_summary for load data statement. - rows = dbt.mustQuery(fmt.Sprintf("select plan from information_schema.STATEMENTS_SUMMARY where QUERY_SAMPLE_TEXT like 'load data local infile %%' limit 1")) + rows = dbt.mustQuery("select plan from information_schema.STATEMENTS_SUMMARY where QUERY_SAMPLE_TEXT like 'load data local infile %' limit 1") checkPlan(rows, expectedPlan) // Test log normal statement after executing load date. - rows = dbt.mustQuery(fmt.Sprintf("select plan from information_schema.slow_query where query = 'insert ignore into t_slow values (1,1);' order by time desc limit 1")) + rows = dbt.mustQuery("select plan from information_schema.slow_query where query = 'insert ignore into t_slow values (1,1);' order by time desc limit 1") expectedPlan = ".*Insert.* time.* loops.* prepare.* check_insert.* mem_insert_time:.* prefetch.* rpc.*" checkPlan(rows, expectedPlan) }) @@ -1913,7 +1912,7 @@ func (cli *testServerClient) runTestSumAvg(c *C) { func (cli *testServerClient) getMetrics(t *C) []byte { resp, err := cli.fetchStatus("/metrics") t.Assert(err, IsNil) - content, err := ioutil.ReadAll(resp.Body) + content, err := io.ReadAll(resp.Body) t.Assert(err, IsNil) err = resp.Body.Close() t.Assert(err, IsNil) @@ -1955,7 +1954,7 @@ func (cli *testServerClient) waitUntilServerOnline() { // fetch http status resp, err := cli.fetchStatus("/status") if err == nil { - _, err = ioutil.ReadAll(resp.Body) + _, err = io.ReadAll(resp.Body) if err != nil { panic(err) } @@ -2058,6 +2057,7 @@ func (cli *testServerClient) runTestInfoschemaClientErrors(t *C) { if rows.Next() { rows.Scan(&errors, &warnings) } + rows.Close() if test.incrementErrors { errors++ @@ -2065,14 +2065,22 @@ func (cli *testServerClient) runTestInfoschemaClientErrors(t *C) { if test.incrementWarnings { warnings++ } - - dbt.db.Query(test.stmt) // ignore results and errors (query table) + var err error + rows, err = dbt.db.Query(test.stmt) + if err == nil { + // make sure to read the result since the error/warnings are populated in the network send code. + if rows.Next() { + var fake string + rows.Scan(&fake) + } + rows.Close() + } var newErrors, newWarnings int rows = dbt.mustQuery("SELECT SUM(error_count), SUM(warning_count) FROM information_schema."+tbl+" WHERE error_number = ? GROUP BY error_number", test.errCode) if rows.Next() { rows.Scan(&newErrors, &newWarnings) } - + rows.Close() dbt.Check(newErrors, Equals, errors) dbt.Check(newWarnings, Equals, warnings) } diff --git a/server/sql_info_fetcher.go b/server/sql_info_fetcher.go index a7be33ea00154..57f51f544b90b 100644 --- a/server/sql_info_fetcher.go +++ b/server/sql_info_fetcher.go @@ -20,7 +20,6 @@ import ( "encoding/json" "fmt" "net/http" - "runtime/pprof" "strconv" "strings" "time" @@ -35,6 +34,7 @@ import ( "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/statistics/handle" "github.com/pingcap/tidb/util/sqlexec" + "github.com/pingcap/tidb/util/topsql/tracecpu" ) type sqlInfoFetcher struct { @@ -81,6 +81,7 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) return } defer sh.s.Close() + sh.do = domain.GetDomain(sh.s) reqCtx := r.Context() sql := r.FormValue("sql") @@ -274,13 +275,13 @@ func (sh *sqlInfoFetcher) getExplainAnalyze(ctx context.Context, sql string, res } func (sh *sqlInfoFetcher) catchCPUProfile(ctx context.Context, sec int, buf *bytes.Buffer, errChan chan<- error) { - if err := pprof.StartCPUProfile(buf); err != nil { + if err := tracecpu.StartCPUProfile(buf); err != nil { errChan <- err return } sleepWithCtx(ctx, time.Duration(sec)*time.Second) - pprof.StopCPUProfile() - errChan <- nil + err := tracecpu.StopCPUProfile() + errChan <- err } func (sh *sqlInfoFetcher) getStatsForTable(pair tableNamePair) (*handle.JSONTable, error) { diff --git a/server/statistics_handler.go b/server/statistics_handler.go index 733a0559f4943..55e9e4f16df18 100644 --- a/server/statistics_handler.go +++ b/server/statistics_handler.go @@ -92,6 +92,8 @@ func (sh StatsHistoryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request writeError(w, err) return } + defer se.Close() + se.GetSessionVars().StmtCtx.TimeZone = time.Local t, err := types.ParseTime(se.GetSessionVars().StmtCtx, params[pSnapshot], mysql.TypeTimestamp, 6) if err != nil { diff --git a/server/statistics_handler_test.go b/server/statistics_handler_test.go index 2388392db57c8..4eb1e35f89752 100644 --- a/server/statistics_handler_test.go +++ b/server/statistics_handler_test.go @@ -16,7 +16,7 @@ package server import ( "database/sql" "fmt" - "io/ioutil" + "io" "os" "time" @@ -106,7 +106,7 @@ func (ds *testDumpStatsSuite) TestDumpStatsAPI(c *C) { c.Assert(os.Remove(path), IsNil) }() - js, err := ioutil.ReadAll(resp.Body) + js, err := io.ReadAll(resp.Body) c.Assert(err, IsNil) _, err = fp.Write(js) c.Assert(err, IsNil) @@ -123,7 +123,7 @@ func (ds *testDumpStatsSuite) TestDumpStatsAPI(c *C) { resp1, err := ds.fetchStatus("/stats/dump/tidb/test") c.Assert(err, IsNil) defer resp1.Body.Close() - js, err = ioutil.ReadAll(resp1.Body) + js, err = io.ReadAll(resp1.Body) c.Assert(err, IsNil) c.Assert(string(js), Equals, "null") @@ -139,7 +139,7 @@ func (ds *testDumpStatsSuite) TestDumpStatsAPI(c *C) { resp1, err = ds.fetchStatus("/stats/dump/tidb/test/" + snapshot) c.Assert(err, IsNil) - js, err = ioutil.ReadAll(resp1.Body) + js, err = io.ReadAll(resp1.Body) c.Assert(err, IsNil) _, err = fp1.Write(js) c.Assert(err, IsNil) diff --git a/server/tidb_test.go b/server/tidb_test.go index b9d46b3d2bf00..a5649033d1e06 100644 --- a/server/tidb_test.go +++ b/server/tidb_test.go @@ -15,24 +15,29 @@ package server import ( + "bytes" "context" "crypto/rand" "crypto/rsa" "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "database/sql" "encoding/pem" - "io/ioutil" + "fmt" "math/big" "net/http" "os" "path/filepath" + "regexp" + "strings" "sync/atomic" "time" "github.com/go-sql-driver/mysql" . "github.com/pingcap/check" "github.com/pingcap/errors" + "github.com/pingcap/parser" tmysql "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/domain" @@ -44,7 +49,10 @@ import ( "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/collate" "github.com/pingcap/tidb/util/logutil" + "github.com/pingcap/tidb/util/plancodec" "github.com/pingcap/tidb/util/testkit" + "github.com/pingcap/tidb/util/topsql/tracecpu" + "github.com/pingcap/tidb/util/topsql/tracecpu/mock" ) type tidbTestSuite struct { @@ -55,6 +63,10 @@ type tidbTestSerialSuite struct { *tidbTestSuiteBase } +type tidbTestTopSQLSuite struct { + *tidbTestSuiteBase +} + type tidbTestSuiteBase struct { *testServerClient tidbdrv *TiDBDriver @@ -71,12 +83,18 @@ func newTiDBTestSuiteBase() *tidbTestSuiteBase { var _ = Suite(&tidbTestSuite{newTiDBTestSuiteBase()}) var _ = SerialSuites(&tidbTestSerialSuite{newTiDBTestSuiteBase()}) +var _ = SerialSuites(&tidbTestTopSQLSuite{newTiDBTestSuiteBase()}) func (ts *tidbTestSuite) SetUpSuite(c *C) { metrics.RegisterMetrics() ts.tidbTestSuiteBase.SetUpSuite(c) } +func (ts *tidbTestTopSQLSuite) SetUpSuite(c *C) { + ts.tidbTestSuiteBase.SetUpSuite(c) + tracecpu.GlobalSQLCPUProfiler.Run() +} + func (ts *tidbTestSuiteBase) SetUpSuite(c *C) { var err error ts.store, err = mockstore.NewMockStore() @@ -342,7 +360,7 @@ func (ts *tidbTestSuite) TestStatusAPIWithTLSCNCheck(c *C) { func newTLSHttpClient(c *C, caFile, certFile, keyFile string) *http.Client { cert, err := tls.LoadX509KeyPair(certFile, keyFile) c.Assert(err, IsNil) - caCert, err := ioutil.ReadFile(caFile) + caCert, err := os.ReadFile(caFile) c.Assert(err, IsNil) caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) @@ -500,7 +518,7 @@ func generateCert(sn int, commonName string, parentCert *x509.Certificate, paren // See https://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig for details. func registerTLSConfig(configName string, caCertPath string, clientCertPath string, clientKeyPath string, serverName string, verifyServer bool) error { rootCertPool := x509.NewCertPool() - data, err := ioutil.ReadFile(caCertPath) + data, err := os.ReadFile(caCertPath) if err != nil { return err } @@ -1154,3 +1172,217 @@ func (ts *tidbTestSerialSuite) TestPrepareCount(c *C) { c.Assert(err, IsNil) c.Assert(atomic.LoadInt64(&variable.PreparedStmtCount), Equals, prepareCnt) } + +func (ts *tidbTestTopSQLSuite) TestTopSQLCPUProfile(c *C) { + c.Skip("unstable") + db, err := sql.Open("mysql", ts.getDSN()) + c.Assert(err, IsNil, Commentf("Error connecting")) + defer func() { + err := db.Close() + c.Assert(err, IsNil) + }() + collector := mock.NewTopSQLCollector() + tracecpu.GlobalSQLCPUProfiler.SetCollector(collector) + + dbt := &DBTest{c, db} + dbt.mustExec("drop database if exists topsql") + dbt.mustExec("create database topsql") + dbt.mustExec("use topsql;") + dbt.mustExec("create table t (a int auto_increment, b int, unique index idx(a));") + dbt.mustExec("create table t1 (a int auto_increment, b int, unique index idx(a));") + dbt.mustExec("create table t2 (a int auto_increment, b int, unique index idx(a));") + dbt.mustExec("set @@global.tidb_enable_top_sql='On';") + dbt.mustExec("set @@global.tidb_top_sql_agent_address='127.0.0.1:4001';") + dbt.mustExec("set @@global.tidb_top_sql_precision_seconds=1;") + + // Test case 1: DML query: insert/update/replace/delete/select + cases1 := []struct { + sql string + planRegexp string + cancel func() + }{ + {sql: "insert into t () values (),(),(),(),(),(),();", planRegexp: ""}, + {sql: "insert into t (b) values (1),(1),(1),(1),(1),(1),(1),(1);", planRegexp: ""}, + {sql: "replace into t (b) values (1),(1),(1),(1),(1),(1),(1),(1);", planRegexp: ""}, + {sql: "update t set b=a where b is null limit 1;", planRegexp: ".*Limit.*TableReader.*"}, + {sql: "delete from t where b is null limit 2;", planRegexp: ".*Limit.*TableReader.*"}, + {sql: "select * from t use index(idx) where a>0;", planRegexp: ".*IndexLookUp.*"}, + {sql: "select * from t ignore index(idx) where a>0;", planRegexp: ".*TableReader.*"}, + {sql: "select /*+ HASH_JOIN(t1, t2) */ * from t t1 join t t2 on t1.a=t2.a where t1.b is not null;", planRegexp: ".*HashJoin.*"}, + {sql: "select /*+ INL_HASH_JOIN(t1, t2) */ * from t t1 join t t2 on t1.a=t2.a where t1.b is not null;", planRegexp: ".*IndexHashJoin.*"}, + {sql: "select * from t where a=1;", planRegexp: ".*Point_Get.*"}, + {sql: "select * from t where a in (1,2,3,4)", planRegexp: ".*Batch_Point_Get.*"}, + } + for i, ca := range cases1 { + ctx, cancel := context.WithCancel(context.Background()) + cases1[i].cancel = cancel + sqlStr := ca.sql + go ts.loopExec(ctx, c, func(db *sql.DB) { + dbt := &DBTest{c, db} + if strings.HasPrefix(sqlStr, "select") { + rows := dbt.mustQuery(sqlStr) + for rows.Next() { + } + } else { + // Ignore error here since the error may be write conflict. + db.Exec(sqlStr) + } + }) + } + + // Test case 2: prepare/execute sql + cases2 := []struct { + prepare string + args []interface{} + planRegexp string + cancel func() + }{ + {prepare: "insert into t1 (b) values (?);", args: []interface{}{1}, planRegexp: ""}, + {prepare: "replace into t1 (b) values (?);", args: []interface{}{1}, planRegexp: ""}, + {prepare: "update t1 set b=a where b is null limit ?;", args: []interface{}{1}, planRegexp: ".*Limit.*TableReader.*"}, + {prepare: "delete from t1 where b is null limit ?;", args: []interface{}{1}, planRegexp: ".*Limit.*TableReader.*"}, + {prepare: "select * from t1 use index(idx) where a>?;", args: []interface{}{1}, planRegexp: ".*IndexLookUp.*"}, + {prepare: "select * from t1 ignore index(idx) where a>?;", args: []interface{}{1}, planRegexp: ".*TableReader.*"}, + {prepare: "select /*+ HASH_JOIN(t1, t2) */ * from t1 t1 join t1 t2 on t1.a=t2.a where t1.b is not null;", args: nil, planRegexp: ".*HashJoin.*"}, + {prepare: "select /*+ INL_HASH_JOIN(t1, t2) */ * from t1 t1 join t1 t2 on t1.a=t2.a where t1.b is not null;", args: nil, planRegexp: ".*IndexHashJoin.*"}, + {prepare: "select * from t1 where a=?;", args: []interface{}{1}, planRegexp: ".*Point_Get.*"}, + {prepare: "select * from t1 where a in (?,?,?,?)", args: []interface{}{1, 2, 3, 4}, planRegexp: ".*Batch_Point_Get.*"}, + } + for i, ca := range cases2 { + ctx, cancel := context.WithCancel(context.Background()) + cases2[i].cancel = cancel + prepare, args := ca.prepare, ca.args + go ts.loopExec(ctx, c, func(db *sql.DB) { + stmt, err := db.Prepare(prepare) + c.Assert(err, IsNil) + if strings.HasPrefix(prepare, "select") { + rows, err := stmt.Query(args...) + c.Assert(err, IsNil) + for rows.Next() { + } + } else { + // Ignore error here since the error may be write conflict. + stmt.Exec(args...) + } + }) + } + + // Test case 3: prepare, execute stmt using @val... + cases3 := []struct { + prepare string + args []interface{} + planRegexp string + cancel func() + }{ + {prepare: "insert into t2 (b) values (?);", args: []interface{}{1}, planRegexp: ""}, + {prepare: "replace into t2 (b) values (?);", args: []interface{}{1}, planRegexp: ""}, + {prepare: "update t2 set b=a where b is null limit ?;", args: []interface{}{1}, planRegexp: ".*Limit.*TableReader.*"}, + {prepare: "delete from t2 where b is null limit ?;", args: []interface{}{1}, planRegexp: ".*Limit.*TableReader.*"}, + {prepare: "select * from t2 use index(idx) where a>?;", args: []interface{}{1}, planRegexp: ".*IndexLookUp.*"}, + {prepare: "select * from t2 ignore index(idx) where a>?;", args: []interface{}{1}, planRegexp: ".*TableReader.*"}, + {prepare: "select /*+ HASH_JOIN(t1, t2) */ * from t2 t1 join t2 t2 on t1.a=t2.a where t1.b is not null;", args: nil, planRegexp: ".*HashJoin.*"}, + {prepare: "select /*+ INL_HASH_JOIN(t1, t2) */ * from t2 t1 join t2 t2 on t1.a=t2.a where t1.b is not null;", args: nil, planRegexp: ".*IndexHashJoin.*"}, + {prepare: "select * from t2 where a=?;", args: []interface{}{1}, planRegexp: ".*Point_Get.*"}, + {prepare: "select * from t2 where a in (?,?,?,?)", args: []interface{}{1, 2, 3, 4}, planRegexp: ".*Batch_Point_Get.*"}, + } + for i, ca := range cases3 { + ctx, cancel := context.WithCancel(context.Background()) + cases3[i].cancel = cancel + prepare, args := ca.prepare, ca.args + go ts.loopExec(ctx, c, func(db *sql.DB) { + _, err := db.Exec(fmt.Sprintf("prepare stmt from '%v'", prepare)) + c.Assert(err, IsNil) + sqlBuf := bytes.NewBuffer(nil) + sqlBuf.WriteString("execute stmt ") + for i := range args { + _, err = db.Exec(fmt.Sprintf("set @%c=%v", 'a'+i, args[i])) + c.Assert(err, IsNil) + if i == 0 { + sqlBuf.WriteString("using ") + } else { + sqlBuf.WriteByte(',') + } + sqlBuf.WriteByte('@') + sqlBuf.WriteByte('a' + byte(i)) + } + if strings.HasPrefix(prepare, "select") { + rows, err := db.Query(sqlBuf.String()) + c.Assert(err, IsNil, Commentf("%v", sqlBuf.String())) + for rows.Next() { + } + } else { + // Ignore error here since the error may be write conflict. + db.Exec(sqlBuf.String()) + } + }) + } + + // Wait the top sql collector to collect profile data. + collector.WaitCollectCnt(1) + + checkFn := func(sql, planRegexp string) { + commentf := Commentf("sql: %v", sql) + stats := collector.GetSQLStatsBySQLWithRetry(sql, len(planRegexp) > 0) + // since 1 sql may has many plan, check `len(stats) > 0` instead of `len(stats) == 1`. + c.Assert(len(stats) > 0, IsTrue, commentf) + + match := false + for _, s := range stats { + sqlStr := collector.GetSQL(s.SQLDigest) + encodedPlan := collector.GetPlan(s.PlanDigest) + // Normalize the user SQL before check. + normalizedSQL := parser.Normalize(sql) + c.Assert(sqlStr, Equals, normalizedSQL, commentf) + // decode plan before check. + normalizedPlan, err := plancodec.DecodeNormalizedPlan(encodedPlan) + c.Assert(err, IsNil) + // remove '\n' '\t' before do regexp match. + normalizedPlan = strings.Replace(normalizedPlan, "\n", " ", -1) + normalizedPlan = strings.Replace(normalizedPlan, "\t", " ", -1) + ok, err := regexp.MatchString(planRegexp, normalizedPlan) + c.Assert(err, IsNil, commentf) + if ok { + match = true + break + } + } + c.Assert(match, IsTrue, commentf) + } + + // Check result of test case 1. + for _, ca := range cases1 { + checkFn(ca.sql, ca.planRegexp) + ca.cancel() + } + + // Check result of test case 2. + for _, ca := range cases2 { + checkFn(ca.prepare, ca.planRegexp) + ca.cancel() + } + + // Check result of test case 3. + for _, ca := range cases3 { + checkFn(ca.prepare, ca.planRegexp) + ca.cancel() + } +} + +func (ts *tidbTestTopSQLSuite) loopExec(ctx context.Context, c *C, fn func(db *sql.DB)) { + db, err := sql.Open("mysql", ts.getDSN()) + c.Assert(err, IsNil, Commentf("Error connecting")) + defer func() { + err := db.Close() + c.Assert(err, IsNil) + }() + dbt := &DBTest{c, db} + dbt.mustExec("use topsql;") + for { + select { + case <-ctx.Done(): + return + default: + } + fn(db) + } +} diff --git a/session/bootstrap.go b/session/bootstrap.go index 34d6748ae38c1..fd448bd0eeef8 100644 --- a/session/bootstrap.go +++ b/session/bootstrap.go @@ -55,6 +55,7 @@ const ( Host CHAR(64), User CHAR(32), authentication_string TEXT, + plugin CHAR(64), Select_priv ENUM('N','Y') NOT NULL DEFAULT 'N', Insert_priv ENUM('N','Y') NOT NULL DEFAULT 'N', Update_priv ENUM('N','Y') NOT NULL DEFAULT 'N', @@ -489,11 +490,13 @@ const ( version68 = 68 // version69 adds mysql.global_grants for DYNAMIC privileges version69 = 69 + // version70 adds mysql.user.plugin to allow multiple authentication plugins + version70 = 70 ) // currentBootstrapVersion is defined as a variable, so we can modify its value for testing. // please make sure this is the largest version -var currentBootstrapVersion int64 = version69 +var currentBootstrapVersion int64 = version70 var ( bootstrapVersion = []func(Session, int64){ @@ -566,6 +569,7 @@ var ( upgradeToVer67, upgradeToVer68, upgradeToVer69, + upgradeToVer70, } ) @@ -1403,6 +1407,12 @@ func updateBindInfo(iter *chunk.Iterator4Chunk, p *parser.Parser, bindMap map[st for row := iter.Begin(); row != iter.End(); row = iter.Next() { bind := row.GetString(0) db := row.GetString(1) + status := row.GetString(2) + + if status != "using" && status != "builtin" { + continue + } + charset := row.GetString(4) collation := row.GetString(5) stmt, err := p.ParseOneStmt(bind, charset, collation) @@ -1421,7 +1431,7 @@ func updateBindInfo(iter *chunk.Iterator4Chunk, p *parser.Parser, bindMap map[st } bindMap[originWithDB] = bindInfo{ bindSQL: utilparser.RestoreWithDefaultDB(stmt, db, bind), - status: row.GetString(2), + status: status, createTime: row.GetTime(3), charset: charset, collation: collation, @@ -1489,6 +1499,14 @@ func upgradeToVer69(s Session, ver int64) { doReentrantDDL(s, CreateGlobalGrantsTable) } +func upgradeToVer70(s Session, ver int64) { + if ver >= version70 { + return + } + doReentrantDDL(s, "ALTER TABLE mysql.user ADD COLUMN plugin CHAR(64) AFTER authentication_string", infoschema.ErrColumnExists) + mustExecute(s, "UPDATE HIGH_PRIORITY mysql.user SET plugin='mysql_native_password'") +} + func writeOOMAction(s Session) { comment := "oom-action is `log` by default in v3.0.x, `cancel` by default in v4.0.11+" mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, %?) ON DUPLICATE KEY UPDATE VARIABLE_VALUE= %?`, @@ -1577,7 +1595,7 @@ func doDMLWorks(s Session) { // Insert a default user with empty password. mustExecute(s, `INSERT HIGH_PRIORITY INTO mysql.user VALUES - ("%", "root", "", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "N", "Y", "Y", "Y", "Y", "Y", "Y", "Y")`) + ("%", "root", "", "mysql_native_password", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "N", "Y", "Y", "Y", "Y", "Y", "Y", "Y")`) // Init global system variables table. values := make([]string, 0, len(variable.GetSysVars())) @@ -1592,7 +1610,7 @@ func doDMLWorks(s Session) { vVal = strconv.Itoa(variable.DefTiDBRowFormatV2) } if v.Name == variable.TiDBPartitionPruneMode { - vVal = string(variable.Static) + vVal = variable.DefTiDBPartitionPruneMode if flag.Lookup("test.v") != nil || flag.Lookup("check.v") != nil || config.CheckTableBeforeDrop { // enable Dynamic Prune by default in test case. vVal = string(variable.Dynamic) diff --git a/session/bootstrap_test.go b/session/bootstrap_test.go index ffa9ba22e5723..35c4495f3b74e 100644 --- a/session/bootstrap_test.go +++ b/session/bootstrap_test.go @@ -57,7 +57,7 @@ func (s *testBootstrapSuite) TestBootstrap(c *C) { c.Assert(err, IsNil) c.Assert(req.NumRows() == 0, IsFalse) datums := statistics.RowToDatums(req.GetRow(0), r.Fields()) - match(c, datums, `%`, "root", "", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "N", "Y", "Y", "Y", "Y", "Y", "Y", "Y") + match(c, datums, `%`, "root", "", "mysql_native_password", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "N", "Y", "Y", "Y", "Y", "Y", "Y", "Y") c.Assert(se.Auth(&auth.UserIdentity{Username: "root", Hostname: "anyhost"}, []byte(""), []byte("")), IsTrue) mustExecSQL(c, se, "USE test;") @@ -162,7 +162,7 @@ func (s *testBootstrapSuite) TestBootstrapWithError(c *C) { c.Assert(req.NumRows() == 0, IsFalse) row := req.GetRow(0) datums := statistics.RowToDatums(row, r.Fields()) - match(c, datums, `%`, "root", "", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "N", "Y", "Y", "Y", "Y", "Y", "Y", "Y") + match(c, datums, `%`, "root", "", "mysql_native_password", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "Y", "N", "Y", "Y", "Y", "Y", "Y", "Y", "Y") c.Assert(r.Close(), IsNil) mustExecSQL(c, se, "USE test;") @@ -592,18 +592,36 @@ func (s *testBootstrapSuite) TestUpdateDuplicateBindInfo(c *C) { // The latest one. mustExecSQL(c, se, `insert into mysql.bind_info values('select * from test . t', 'select /*+ use_index(t, idx_b)*/ * from test.t', 'test', 'using', '2021-01-04 14:50:58.257', '2021-01-09 14:50:58.257', 'utf8', 'utf8_general_ci', 'manual')`) + mustExecSQL(c, se, `insert into mysql.bind_info values('select * from t where a < ?', 'select * from t use index(idx) where a < 1', 'test', 'deleted', '2021-06-04 17:04:43.333', '2021-06-04 17:04:43.335', 'utf8', 'utf8_general_ci', 'manual')`) + mustExecSQL(c, se, `insert into mysql.bind_info values('select * from t where a < ?', 'select * from t ignore index(idx) where a < 1', 'test', 'using', '2021-06-04 17:04:43.335', '2021-06-04 17:04:43.335', 'utf8', 'utf8_general_ci', 'manual')`) + mustExecSQL(c, se, `insert into mysql.bind_info values('select * from test . t where a <= ?', 'select * from test.t use index(idx) where a <= 1', '', 'deleted', '2021-06-04 17:04:43.345', '2021-06-04 17:04:45.334', 'utf8', 'utf8_general_ci', 'manual')`) + mustExecSQL(c, se, `insert into mysql.bind_info values('select * from test . t where a <= ?', 'select * from test.t ignore index(idx) where a <= 1', '', 'using', '2021-06-04 17:04:45.334', '2021-06-04 17:04:45.334', 'utf8', 'utf8_general_ci', 'manual')`) + upgradeToVer67(se, version66) - r := mustExecSQL(c, se, `select original_sql, bind_sql, default_db, status, create_time from mysql.bind_info where source != 'builtin'`) + r := mustExecSQL(c, se, `select original_sql, bind_sql, default_db, status, create_time from mysql.bind_info where source != 'builtin' order by create_time`) req := r.NewChunk() c.Assert(r.Next(ctx, req), IsNil) - c.Assert(req.NumRows(), Equals, 1) + c.Assert(req.NumRows(), Equals, 3) row := req.GetRow(0) c.Assert(row.GetString(0), Equals, "select * from `test` . `t`") c.Assert(row.GetString(1), Equals, "SELECT /*+ use_index(`t` `idx_b`)*/ * FROM `test`.`t`") c.Assert(row.GetString(2), Equals, "") c.Assert(row.GetString(3), Equals, "using") c.Assert(row.GetTime(4).String(), Equals, "2021-01-04 14:50:58.257") + row = req.GetRow(1) + c.Assert(row.GetString(0), Equals, "select * from `test` . `t` where `a` < ?") + c.Assert(row.GetString(1), Equals, "SELECT * FROM `test`.`t` IGNORE INDEX (`idx`) WHERE `a` < 1") + c.Assert(row.GetString(2), Equals, "") + c.Assert(row.GetString(3), Equals, "using") + c.Assert(row.GetTime(4).String(), Equals, "2021-06-04 17:04:43.335") + row = req.GetRow(2) + c.Assert(row.GetString(0), Equals, "select * from `test` . `t` where `a` <= ?") + c.Assert(row.GetString(1), Equals, "SELECT * FROM `test`.`t` IGNORE INDEX (`idx`) WHERE `a` <= 1") + c.Assert(row.GetString(2), Equals, "") + c.Assert(row.GetString(3), Equals, "using") + c.Assert(row.GetTime(4).String(), Equals, "2021-06-04 17:04:45.334") + c.Assert(r.Close(), IsNil) mustExecSQL(c, se, "delete from mysql.bind_info where original_sql = 'select * from test . t'") } diff --git a/session/clustered_index_test.go b/session/clustered_index_test.go index fd40cfd567f11..b7e529f29fe0e 100644 --- a/session/clustered_index_test.go +++ b/session/clustered_index_test.go @@ -14,11 +14,16 @@ package session_test import ( + "fmt" + "math/rand" + "strings" + . "github.com/pingcap/check" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/util/collate" + "github.com/pingcap/tidb/util/israce" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testutil" ) @@ -578,6 +583,50 @@ func (s *testClusteredSerialSuite) TestPrefixClusteredIndexAddIndexAndRecover(c tk1.MustExec("admin check table t") } +func (s *testClusteredSerialSuite) TestPartitionTable(c *C) { + if israce.RaceEnabled { + c.Skip("exhaustive types test, skip race test") + } + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_view") + tk.MustExec("use test_view") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + + tk.MustExec(`create table thash (a int, b int, c varchar(32), primary key(a, b) clustered) partition by hash(a) partitions 4`) + tk.MustExec(`create table trange (a int, b int, c varchar(32), primary key(a, b) clustered) partition by range columns(a) ( + partition p0 values less than (3000), + partition p1 values less than (6000), + partition p2 values less than (9000), + partition p3 values less than (10000))`) + tk.MustExec(`create table tnormal (a int, b int, c varchar(32), primary key(a, b))`) + + vals := make([]string, 0, 4000) + existedPK := make(map[string]struct{}, 4000) + for i := 0; i < 4000; { + a := rand.Intn(10000) + b := rand.Intn(10000) + pk := fmt.Sprintf("%v, %v", a, b) + if _, ok := existedPK[pk]; ok { + continue + } + existedPK[pk] = struct{}{} + i++ + vals = append(vals, fmt.Sprintf(`(%v, %v, '%v')`, a, b, rand.Intn(10000))) + } + + tk.MustExec("insert into thash values " + strings.Join(vals, ", ")) + tk.MustExec("insert into trange values " + strings.Join(vals, ", ")) + tk.MustExec("insert into tnormal values " + strings.Join(vals, ", ")) + + for i := 0; i < 200; i++ { + cond := fmt.Sprintf("where a in (%v, %v, %v) and b < %v", rand.Intn(10000), rand.Intn(10000), rand.Intn(10000), rand.Intn(10000)) + result := tk.MustQuery("select * from tnormal " + cond).Sort().Rows() + tk.MustQuery("select * from thash use index(primary) " + cond).Sort().Check(result) + tk.MustQuery("select * from trange use index(primary) " + cond).Sort().Check(result) + } +} + // https://github.com/pingcap/tidb/issues/23106 func (s *testClusteredSerialSuite) TestClusteredIndexDecodeRestoredDataV5(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) @@ -616,3 +665,18 @@ func (s *testClusteredSerialSuite) TestPrefixedClusteredIndexUniqueKeyWithNewCol tk.MustExec("admin check table t;") tk.MustExec("drop table t;") } + +func (s *testClusteredSerialSuite) TestClusteredIndexNewCollationWithOldRowFormat(c *C) { + // This case maybe not useful, because newCollation isn't convenience to run on TiKV(it's required serialSuit) + // but unistore doesn't support old row format. + defer collate.SetNewCollationEnabledForTest(false) + collate.SetNewCollationEnabledForTest(true) + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("use test;") + tk.Se.GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeOn + tk.Se.GetSessionVars().RowEncoder.Enable = false + tk.MustExec("drop table if exists t2") + tk.MustExec("create table t2(col_1 varchar(132) CHARACTER SET utf8 COLLATE utf8_unicode_ci, primary key(col_1) clustered)") + tk.MustExec("insert into t2 select 'aBc'") + tk.MustQuery("select col_1 from t2 where col_1 = 'aBc'").Check(testkit.Rows("aBc")) +} diff --git a/session/pessimistic_test.go b/session/pessimistic_test.go index 05a93e3aee30d..775fa89cb3f28 100644 --- a/session/pessimistic_test.go +++ b/session/pessimistic_test.go @@ -16,6 +16,7 @@ package session_test import ( "context" "fmt" + "strconv" "strings" "sync" "sync/atomic" @@ -24,6 +25,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/errors" "github.com/pingcap/failpoint" + "github.com/pingcap/parser" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" @@ -31,12 +33,15 @@ import ( plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx/variable" + storeerr "github.com/pingcap/tidb/store/driver/error" "github.com/pingcap/tidb/store/tikv" - tikverr "github.com/pingcap/tidb/store/tikv/error" + "github.com/pingcap/tidb/store/tikv/mockstore" "github.com/pingcap/tidb/store/tikv/oracle" + "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/codec" + "github.com/pingcap/tidb/util/deadlockhistory" "github.com/pingcap/tidb/util/testkit" ) @@ -63,11 +68,13 @@ func (s *testPessimisticSuite) SetUpSuite(c *C) { // Set it to 300ms for testing lock resolve. atomic.StoreUint64(&tikv.ManagedLockTTL, 300) tikv.PrewriteMaxBackoff = 500 + tikv.VeryLongMaxBackoff = 500 } func (s *testPessimisticSuite) TearDownSuite(c *C) { s.testSessionSuiteBase.TearDownSuite(c) tikv.PrewriteMaxBackoff = 20000 + tikv.VeryLongMaxBackoff = 600000 } func (s *testPessimisticSuite) TestPessimisticTxn(c *C) { @@ -171,27 +178,34 @@ func (s *testPessimisticSuite) TestTxnMode(c *C) { } func (s *testPessimisticSuite) TestDeadlock(c *C) { - tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustExec("drop table if exists deadlock") - tk.MustExec("create table deadlock (k int primary key, v int)") - tk.MustExec("insert into deadlock values (1, 1), (2, 1)") + deadlockhistory.GlobalDeadlockHistory.Clear() + deadlockhistory.GlobalDeadlockHistory.Resize(10) + + tk1 := testkit.NewTestKitWithInit(c, s.store) + tk1.MustExec("drop table if exists deadlock") + tk1.MustExec("create table deadlock (k int primary key, v int)") + tk1.MustExec("insert into deadlock values (1, 1), (2, 1)") + tk1.MustExec("begin pessimistic") + tk1.MustExec("update deadlock set v = v + 1 where k = 1") + ts1, err := strconv.ParseUint(tk1.MustQuery("select @@tidb_current_ts").Rows()[0][0].(string), 10, 64) + c.Assert(err, IsNil) + + tk2 := testkit.NewTestKitWithInit(c, s.store) + tk2.MustExec("begin pessimistic") + ts2, err := strconv.ParseUint(tk2.MustQuery("select @@tidb_current_ts").Rows()[0][0].(string), 10, 64) + c.Assert(err, IsNil) syncCh := make(chan error) go func() { - tk1 := testkit.NewTestKitWithInit(c, s.store) - tk1.MustExec("begin pessimistic") - tk1.MustExec("update deadlock set v = v + 1 where k = 2") + tk2.MustExec("update deadlock set v = v + 1 where k = 2") syncCh <- nil - _, err := tk1.Exec("update deadlock set v = v + 1 where k = 1") + _, err := tk2.Exec("update deadlock set v = v + 1 where k = 1") syncCh <- err }() - tk.MustExec("begin pessimistic") - tk.MustExec("update deadlock set v = v + 1 where k = 1") <-syncCh - _, err1 := tk.Exec("update deadlock set v = v + 1 where k = 2") + _, err1 := tk1.Exec("update deadlock set v = v + 1 where k = 2") err2 := <-syncCh // Either err1 or err2 is deadlock error. - var err error if err1 != nil { c.Assert(err2, IsNil) err = err1 @@ -201,10 +215,25 @@ func (s *testPessimisticSuite) TestDeadlock(c *C) { e, ok := errors.Cause(err).(*terror.Error) c.Assert(ok, IsTrue) c.Assert(int(e.Code()), Equals, mysql.ErrLockDeadlock) + + _, digest := parser.NormalizeDigest("update deadlock set v = v + 1 where k = 1") + + expectedDeadlockInfo := []string{ + fmt.Sprintf("%v %v %v", ts1, ts2, digest), + fmt.Sprintf("%v %v %v", ts2, ts1, digest), + } + // The last one is the transaction that encountered the deadlock error. + if err1 != nil { + // Swap the two to match the correct order. + expectedDeadlockInfo[0], expectedDeadlockInfo[1] = expectedDeadlockInfo[1], expectedDeadlockInfo[0] + } + res := tk1.MustQuery("select deadlock_id, try_lock_trx_id, trx_holding_lock, current_sql_digest from information_schema.deadlocks") + res.CheckAt([]int{1, 2, 3}, testkit.Rows(expectedDeadlockInfo...)) + c.Assert(res.Rows()[0][0], Equals, res.Rows()[1][0]) } func (s *testPessimisticSuite) TestSingleStatementRollback(c *C) { - if *withTiKV { + if *mockstore.WithTiKV { c.Skip("skip with tikv because cluster manipulate is not available") } tk := testkit.NewTestKitWithInit(c, s.store) @@ -611,7 +640,7 @@ func (s *testPessimisticSuite) TestWaitLockKill(c *C) { _, err := tk2.Exec("update test_kill set c = c + 1 where id = 1") wg.Done() c.Assert(err, NotNil) - c.Assert(terror.ErrorEqual(err, tikverr.ErrQueryInterrupted), IsTrue) + c.Assert(terror.ErrorEqual(err, storeerr.ErrQueryInterrupted), IsTrue) tk.MustExec("rollback") } @@ -733,10 +762,10 @@ func (s *testPessimisticSuite) TestInnodbLockWaitTimeout(c *C) { timeoutErr := <-timeoutErrCh c.Assert(timeoutErr, NotNil) - c.Assert(timeoutErr.Error(), Equals, tikverr.ErrLockWaitTimeout.Error()) + c.Assert(timeoutErr.Error(), Equals, storeerr.ErrLockWaitTimeout.Error()) timeoutErr = <-timeoutErrCh c.Assert(timeoutErr, NotNil) - c.Assert(timeoutErr.Error(), Equals, tikverr.ErrLockWaitTimeout.Error()) + c.Assert(timeoutErr.Error(), Equals, storeerr.ErrLockWaitTimeout.Error()) // tk4 lock c1 = 2 tk4.MustExec("begin pessimistic") @@ -749,7 +778,7 @@ func (s *testPessimisticSuite) TestInnodbLockWaitTimeout(c *C) { _, err := tk2.Exec("delete from tk where c1 = 2") c.Check(time.Since(start), GreaterEqual, 1000*time.Millisecond) c.Check(time.Since(start), Less, 3000*time.Millisecond) // unit test diff should not be too big - c.Check(err.Error(), Equals, tikverr.ErrLockWaitTimeout.Error()) + c.Check(err.Error(), Equals, storeerr.ErrLockWaitTimeout.Error()) tk4.MustExec("commit") @@ -767,7 +796,7 @@ func (s *testPessimisticSuite) TestInnodbLockWaitTimeout(c *C) { _, err = tk2.Exec("delete from tk where c1 = 3") // tk2 tries to lock c1 = 3 fail, this delete should be rollback, but previous update should be keeped c.Check(time.Since(start), GreaterEqual, 1000*time.Millisecond) c.Check(time.Since(start), Less, 3000*time.Millisecond) // unit test diff should not be too big - c.Check(err.Error(), Equals, tikverr.ErrLockWaitTimeout.Error()) + c.Check(err.Error(), Equals, storeerr.ErrLockWaitTimeout.Error()) tk2.MustExec("commit") tk3.MustExec("commit") @@ -841,7 +870,7 @@ func (s *testPessimisticSuite) TestInnodbLockWaitTimeoutWaitStart(c *C) { c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/PessimisticLockErrWriteConflict"), IsNil) waitErr := <-done c.Assert(waitErr, NotNil) - c.Check(waitErr.Error(), Equals, tikverr.ErrLockWaitTimeout.Error()) + c.Check(waitErr.Error(), Equals, storeerr.ErrLockWaitTimeout.Error()) c.Check(duration, GreaterEqual, 1000*time.Millisecond) c.Check(duration, LessEqual, 3000*time.Millisecond) tk2.MustExec("rollback") @@ -1131,11 +1160,11 @@ func (s *testPessimisticSuite) TestPessimisticLockNonExistsKey(c *C) { tk1.MustExec("begin pessimistic") err := tk1.ExecToErr("select * from t where k = 2 for update nowait") - c.Check(tikverr.ErrLockAcquireFailAndNoWaitSet.Equal(err), IsTrue) + c.Check(storeerr.ErrLockAcquireFailAndNoWaitSet.Equal(err), IsTrue) err = tk1.ExecToErr("select * from t where k = 4 for update nowait") - c.Check(tikverr.ErrLockAcquireFailAndNoWaitSet.Equal(err), IsTrue) + c.Check(storeerr.ErrLockAcquireFailAndNoWaitSet.Equal(err), IsTrue) err = tk1.ExecToErr("select * from t where k = 7 for update nowait") - c.Check(tikverr.ErrLockAcquireFailAndNoWaitSet.Equal(err), IsTrue) + c.Check(storeerr.ErrLockAcquireFailAndNoWaitSet.Equal(err), IsTrue) tk.MustExec("rollback") tk1.MustExec("rollback") @@ -1147,9 +1176,9 @@ func (s *testPessimisticSuite) TestPessimisticLockNonExistsKey(c *C) { tk1.MustExec("begin pessimistic") err = tk1.ExecToErr("select * from t where k = 2 for update nowait") - c.Check(tikverr.ErrLockAcquireFailAndNoWaitSet.Equal(err), IsTrue) + c.Check(storeerr.ErrLockAcquireFailAndNoWaitSet.Equal(err), IsTrue) err = tk1.ExecToErr("select * from t where k = 6 for update nowait") - c.Check(tikverr.ErrLockAcquireFailAndNoWaitSet.Equal(err), IsTrue) + c.Check(storeerr.ErrLockAcquireFailAndNoWaitSet.Equal(err), IsTrue) tk.MustExec("rollback") tk1.MustExec("rollback") } @@ -1279,10 +1308,10 @@ func (s *testPessimisticSuite) TestBatchPointGetLockIndex(c *C) { tk2.MustExec("begin pessimistic") err := tk2.ExecToErr("insert into t1 values(2, 2, 2)") c.Assert(err, NotNil) - c.Assert(tikverr.ErrLockWaitTimeout.Equal(err), IsTrue) + c.Assert(storeerr.ErrLockWaitTimeout.Equal(err), IsTrue) err = tk2.ExecToErr("select * from t1 where c2 = 3 for update nowait") c.Assert(err, NotNil) - c.Assert(tikverr.ErrLockAcquireFailAndNoWaitSet.Equal(err), IsTrue) + c.Assert(storeerr.ErrLockAcquireFailAndNoWaitSet.Equal(err), IsTrue) tk.MustExec("rollback") tk2.MustExec("rollback") } @@ -1429,12 +1458,12 @@ func (s *testPessimisticSuite) TestGenerateColPointGet(c *C) { tk2.MustExec("begin pessimistic") err := tk2.ExecToErr("select * from tu where z = 3 for update nowait") c.Assert(err, NotNil) - c.Assert(terror.ErrorEqual(err, tikverr.ErrLockAcquireFailAndNoWaitSet), IsTrue) + c.Assert(terror.ErrorEqual(err, storeerr.ErrLockAcquireFailAndNoWaitSet), IsTrue) tk.MustExec("begin pessimistic") tk.MustExec("insert into tu(x, y) values(2, 2);") err = tk2.ExecToErr("select * from tu where z = 4 for update nowait") c.Assert(err, NotNil) - c.Assert(terror.ErrorEqual(err, tikverr.ErrLockAcquireFailAndNoWaitSet), IsTrue) + c.Assert(terror.ErrorEqual(err, storeerr.ErrLockAcquireFailAndNoWaitSet), IsTrue) // test batch point get lock tk.MustExec("begin pessimistic") @@ -1443,12 +1472,12 @@ func (s *testPessimisticSuite) TestGenerateColPointGet(c *C) { tk2.MustExec("begin pessimistic") err = tk2.ExecToErr("select x from tu where z in (3, 7, 9) for update nowait") c.Assert(err, NotNil) - c.Assert(terror.ErrorEqual(err, tikverr.ErrLockAcquireFailAndNoWaitSet), IsTrue) + c.Assert(terror.ErrorEqual(err, storeerr.ErrLockAcquireFailAndNoWaitSet), IsTrue) tk.MustExec("begin pessimistic") tk.MustExec("insert into tu(x, y) values(5, 6);") err = tk2.ExecToErr("select * from tu where z = 11 for update nowait") c.Assert(err, NotNil) - c.Assert(terror.ErrorEqual(err, tikverr.ErrLockAcquireFailAndNoWaitSet), IsTrue) + c.Assert(terror.ErrorEqual(err, storeerr.ErrLockAcquireFailAndNoWaitSet), IsTrue) tk.MustExec("commit") tk2.MustExec("commit") @@ -1996,11 +2025,11 @@ func (s *testPessimisticSuite) TestSelectForUpdateWaitSeconds(c *C) { waitErr2 := <-errCh waitErr3 := <-errCh c.Assert(waitErr, NotNil) - c.Check(waitErr.Error(), Equals, tikverr.ErrLockWaitTimeout.Error()) + c.Check(waitErr.Error(), Equals, storeerr.ErrLockWaitTimeout.Error()) c.Assert(waitErr2, NotNil) - c.Check(waitErr2.Error(), Equals, tikverr.ErrLockWaitTimeout.Error()) + c.Check(waitErr2.Error(), Equals, storeerr.ErrLockWaitTimeout.Error()) c.Assert(waitErr3, NotNil) - c.Check(waitErr3.Error(), Equals, tikverr.ErrLockWaitTimeout.Error()) + c.Check(waitErr3.Error(), Equals, storeerr.ErrLockWaitTimeout.Error()) c.Assert(time.Since(start).Seconds(), Less, 45.0) tk2.MustExec("commit") tk3.MustExec("rollback") @@ -2029,14 +2058,14 @@ func (s *testPessimisticSuite) TestSelectForUpdateConflictRetry(c *C) { tsCh := make(chan uint64) go func() { tk3.MustExec("update tk set c2 = c2 + 1 where c1 = 1") - lastTS, err := s.store.GetOracle().GetLowResolutionTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) + lastTS, err := s.store.GetOracle().GetLowResolutionTimestamp(context.Background(), &oracle.Option{TxnScope: kv.GlobalTxnScope}) c.Assert(err, IsNil) tsCh <- lastTS tk3.MustExec("commit") tsCh <- lastTS }() // tk2LastTS should be its forUpdateTS - tk2LastTS, err := s.store.GetOracle().GetLowResolutionTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) + tk2LastTS, err := s.store.GetOracle().GetLowResolutionTimestamp(context.Background(), &oracle.Option{TxnScope: kv.GlobalTxnScope}) c.Assert(err, IsNil) tk2.MustExec("commit") @@ -2050,7 +2079,7 @@ func (s *testPessimisticSuite) TestSelectForUpdateConflictRetry(c *C) { func (s *testPessimisticSuite) TestAsyncCommitWithSchemaChange(c *C) { // TODO: implement commit_ts calculation in unistore - if !*withTiKV { + if !*mockstore.WithTiKV { return } @@ -2124,7 +2153,7 @@ func (s *testPessimisticSuite) TestAsyncCommitWithSchemaChange(c *C) { func (s *testPessimisticSuite) Test1PCWithSchemaChange(c *C) { // TODO: implement commit_ts calculation in unistore - if !*withTiKV { + if !*mockstore.WithTiKV { return } @@ -2298,17 +2327,13 @@ func (s *testPessimisticSuite) TestAmendForUniqueIndex(c *C) { err = <-errCh c.Assert(err, Equals, nil) tk.MustExec("commit") - tk2.MustExec("admin check table t") + tk.MustExec("admin check table t") err = <-errCh c.Assert(err, Equals, nil) } func (s *testPessimisticSuite) TestAmendWithColumnTypeChange(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustExec("set global tidb_enable_change_column_type = 1;") - defer func() { - tk.MustExec("set global tidb_enable_change_column_type = 0;") - }() tk2 := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("drop database if exists test_db") tk.MustExec("create database test_db") diff --git a/session/schema_amender.go b/session/schema_amender.go index 19c7ac5a9667f..08216637e9f74 100644 --- a/session/schema_amender.go +++ b/session/schema_amender.go @@ -21,7 +21,7 @@ import ( "reflect" "github.com/pingcap/errors" - pb "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/ddl" @@ -170,7 +170,7 @@ func colChangeAmendable(colAtStart *model.ColumnInfo, colAtCommit *model.ColumnI if colAtStart.Charset != colAtCommit.Charset || colAtStart.Collate != colAtCommit.Collate { return errors.Trace(errors.Errorf("charset or collate is not matched for column=%v", colAtCommit.Name.String())) } - _, _, err := ddl.CheckModifyTypeCompatible(&colAtStart.FieldType, &colAtCommit.FieldType) + _, err := ddl.CheckModifyTypeCompatible(&colAtStart.FieldType, &colAtCommit.FieldType) if err != nil { return errors.Trace(err) } @@ -283,13 +283,13 @@ func (a *amendCollector) collectTblAmendOps(sctx sessionctx.Context, phyTblID in } // mayGenDelIndexRowKeyOp returns if the row key op could generate Op_Del index key mutations. -func mayGenDelIndexRowKeyOp(keyOp pb.Op) bool { - return keyOp == pb.Op_Del || keyOp == pb.Op_Put +func mayGenDelIndexRowKeyOp(keyOp kvrpcpb.Op) bool { + return keyOp == kvrpcpb.Op_Del || keyOp == kvrpcpb.Op_Put } // mayGenPutIndexRowKeyOp returns if the row key op could generate Op_Put/Op_Insert index key mutations. -func mayGenPutIndexRowKeyOp(keyOp pb.Op) bool { - return keyOp == pb.Op_Put || keyOp == pb.Op_Insert +func mayGenPutIndexRowKeyOp(keyOp kvrpcpb.Op) bool { + return keyOp == kvrpcpb.Op_Put || keyOp == kvrpcpb.Op_Insert } // amendOp is an amend operation for a specific schema change, new mutations will be generated using input ones. @@ -394,9 +394,9 @@ func (a *amendOperationAddIndex) genMutations(ctx context.Context, sctx sessionc } for i := 0; i < len(insertedMutations.GetKeys()); i++ { key := insertedMutations.GetKeys()[i] - destKeyOp := pb.Op_Insert + destKeyOp := kvrpcpb.Op_Insert if _, ok := a.deletedOldIndexKeys[string(key)]; ok { - destKeyOp = pb.Op_Put + destKeyOp = kvrpcpb.Op_Put } resAddMutations.Push(destKeyOp, key, insertedMutations.GetValues()[i], insertedMutations.GetPessimisticFlags()[i]) } @@ -477,14 +477,14 @@ func (a *amendOperationAddIndex) genNewIdxKey(ctx context.Context, sctx sessionc if err != nil { return nil, errors.Trace(err) } - newIndexOp := pb.Op_Put + newIndexOp := kvrpcpb.Op_Put isPessimisticLock := false if _, ok := a.insertedNewIndexKeys[string(newIdxKey)]; ok { return nil, errors.Trace(errors.Errorf("amend process key same key=%v found for index=%v in table=%v", newIdxKey, a.info.indexInfoAtCommit.Meta().Name, a.info.tblInfoAtCommit.Meta().Name)) } if a.info.indexInfoAtCommit.Meta().Unique { - newIndexOp = pb.Op_Insert + newIndexOp = kvrpcpb.Op_Insert isPessimisticLock = true } a.insertedNewIndexKeys[string(newIdxKey)] = struct{}{} @@ -515,7 +515,7 @@ func (a *amendOperationAddIndex) genOldIdxKey(ctx context.Context, sctx sessionc isPessimisticLock = true } a.deletedOldIndexKeys[string(newIdxKey)] = struct{}{} - return &tikv.PlainMutation{KeyOp: pb.Op_Del, Key: newIdxKey, Value: emptyVal, IsPessimisticLock: isPessimisticLock}, nil + return &tikv.PlainMutation{KeyOp: kvrpcpb.Op_Del, Key: newIdxKey, Value: emptyVal, IsPessimisticLock: isPessimisticLock}, nil } return nil, nil } @@ -540,12 +540,12 @@ func (s *SchemaAmender) getAmendableKeys(commitMutations tikv.CommitterMutations } keyOp := commitMutations.GetOp(i) switch keyOp { - case pb.Op_Put: + case kvrpcpb.Op_Put: addKeys = append(addKeys, byteKey) removeKeys = append(removeKeys, byteKey) - case pb.Op_Insert: + case kvrpcpb.Op_Insert: addKeys = append(addKeys, byteKey) - case pb.Op_Del: + case kvrpcpb.Op_Del: removeKeys = append(removeKeys, byteKey) } } @@ -594,7 +594,7 @@ func (s *SchemaAmender) prepareKvMap(ctx context.Context, commitMutations tikv.C func (s *SchemaAmender) checkDupKeys(ctx context.Context, mutations tikv.CommitterMutations) error { // Check if there are duplicate key entries. - checkMap := make(map[string]pb.Op) + checkMap := make(map[string]kvrpcpb.Op) for i := 0; i < mutations.Len(); i++ { key := mutations.GetKey(i) keyOp := mutations.GetOp(i) diff --git a/session/schema_amender_test.go b/session/schema_amender_test.go index ca05f4a74dbff..eda0e5e387e05 100644 --- a/session/schema_amender_test.go +++ b/session/schema_amender_test.go @@ -29,7 +29,6 @@ import ( "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/tikv" - "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" @@ -426,7 +425,7 @@ func (s *testSchemaAmenderSuite) TestAmendCollectAndGenMutations(c *C) { } c.Assert(err, IsNil) } - curVer, err := se.store.CurrentVersion(oracle.GlobalTxnScope) + curVer, err := se.store.CurrentVersion(kv.GlobalTxnScope) c.Assert(err, IsNil) se.sessionVars.TxnCtx.SetForUpdateTS(curVer.Ver + 1) mutationVals, err := txn.BatchGet(ctx, checkKeys) diff --git a/session/session.go b/session/session.go index 13df91510f61a..fe6fef23bf019 100644 --- a/session/session.go +++ b/session/session.go @@ -41,6 +41,10 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" + "github.com/pingcap/tidb/util/topsql" + "github.com/pingcap/tipb/go-binlog" + "go.uber.org/zap" + "github.com/pingcap/tidb/bindinfo" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl" @@ -58,6 +62,7 @@ import ( "github.com/pingcap/tidb/plugin" "github.com/pingcap/tidb/privilege" "github.com/pingcap/tidb/privilege/privileges" + txninfo "github.com/pingcap/tidb/session/txninfo" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/binloginfo" "github.com/pingcap/tidb/sessionctx/stmtctx" @@ -65,8 +70,6 @@ import ( "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/statistics/handle" "github.com/pingcap/tidb/store/tikv" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" - "github.com/pingcap/tidb/store/tikv/oracle" tikvutil "github.com/pingcap/tidb/store/tikv/util" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/telemetry" @@ -81,8 +84,6 @@ import ( "github.com/pingcap/tidb/util/sli" "github.com/pingcap/tidb/util/sqlexec" "github.com/pingcap/tidb/util/timeutil" - "github.com/pingcap/tipb/go-binlog" - "go.uber.org/zap" ) var ( @@ -100,6 +101,8 @@ var ( sessionExecuteParseDurationInternal = metrics.SessionExecuteParseDuration.WithLabelValues(metrics.LblInternal) sessionExecuteParseDurationGeneral = metrics.SessionExecuteParseDuration.WithLabelValues(metrics.LblGeneral) + telemetryCTEUsage = metrics.TelemetrySQLCTECnt + tiKVGCAutoConcurrency = "tikv_gc_auto_concurrency" ) @@ -145,6 +148,8 @@ type Session interface { Auth(user *auth.UserIdentity, auth []byte, salt []byte) bool AuthWithoutVerification(user *auth.UserIdentity) bool ShowProcess() *util.ProcessInfo + // Return the information of the txn current running + TxnInfo() *txninfo.TxnInfo // PrepareTxnCtx is exported for test. PrepareTxnCtx(context.Context) // FieldList returns fields list of a table. @@ -183,7 +188,7 @@ func (h *StmtHistory) Count() int { type session struct { // processInfo is used by ShowProcess(), and should be modified atomically. processInfo atomic.Value - txn TxnState + txn LazyTxn mu struct { sync.RWMutex @@ -406,7 +411,7 @@ func (s *session) StoreIndexUsage(tblID int64, idxID int64, rowsSelected int64) // FieldList returns fields list of a table. func (s *session) FieldList(tableName string) ([]*ast.ResultField, error) { - is := infoschema.GetInfoSchema(s) + is := s.GetInfoSchema().(infoschema.InfoSchema) dbName := model.NewCIStr(s.GetSessionVars().CurrentDB) tName := model.NewCIStr(tableName) pm := privilege.GetPrivilegeManager(s) @@ -442,6 +447,18 @@ func (s *session) FieldList(tableName string) ([]*ast.ResultField, error) { return fields, nil } +func (s *session) TxnInfo() *txninfo.TxnInfo { + txnInfo := s.txn.Info() + if txnInfo == nil { + return nil + } + processInfo := s.ShowProcess() + txnInfo.ConnectionID = processInfo.ID + txnInfo.Username = processInfo.User + txnInfo.CurrentDB = processInfo.DB + return txnInfo +} + func (s *session) doCommit(ctx context.Context) error { if !s.txn.Valid() { return nil @@ -457,6 +474,9 @@ func (s *session) doCommit(ctx context.Context) error { if err != nil { return err } + if err = s.removeTempTableFromBuffer(); err != nil { + return err + } // mockCommitError and mockGetTSErrorInRetry use to test PR #8743. failpoint.Inject("mockCommitError", func(val failpoint.Value) { @@ -480,58 +500,76 @@ func (s *session) doCommit(ctx context.Context) error { }, Client: s.sessionVars.BinlogClient, } - s.txn.SetOption(tikvstore.BinlogInfo, info) + s.txn.SetOption(kv.BinlogInfo, info) } } // Get the related table or partition IDs. relatedPhysicalTables := s.GetSessionVars().TxnCtx.TableDeltaMap + // Get accessed global temporary tables in the transaction. + temporaryTables := s.GetSessionVars().TxnCtx.GlobalTemporaryTables physicalTableIDs := make([]int64, 0, len(relatedPhysicalTables)) for id := range relatedPhysicalTables { + // Schema change on global temporary tables doesn't affect transactions. + if _, ok := temporaryTables[id]; ok { + continue + } physicalTableIDs = append(physicalTableIDs, id) } // Set this option for 2 phase commit to validate schema lease. - s.txn.SetOption(tikvstore.SchemaChecker, domain.NewSchemaChecker(domain.GetDomain(s), s.sessionVars.TxnCtx.SchemaVersion, physicalTableIDs)) - s.txn.SetOption(tikvstore.InfoSchema, s.sessionVars.TxnCtx.InfoSchema) - s.txn.SetOption(tikvstore.CommitHook, func(info string, _ error) { s.sessionVars.LastTxnInfo = info }) + s.txn.SetOption(kv.SchemaChecker, domain.NewSchemaChecker(domain.GetDomain(s), s.GetInfoSchema().SchemaMetaVersion(), physicalTableIDs)) + s.txn.SetOption(kv.InfoSchema, s.sessionVars.TxnCtx.InfoSchema) + s.txn.SetOption(kv.CommitHook, func(info string, _ error) { s.sessionVars.LastTxnInfo = info }) if s.GetSessionVars().EnableAmendPessimisticTxn { - s.txn.SetOption(tikvstore.SchemaAmender, NewSchemaAmenderForTikvTxn(s)) + s.txn.SetOption(kv.SchemaAmender, NewSchemaAmenderForTikvTxn(s)) } - s.txn.SetOption(tikvstore.EnableAsyncCommit, s.GetSessionVars().EnableAsyncCommit) - s.txn.SetOption(tikvstore.Enable1PC, s.GetSessionVars().Enable1PC) + s.txn.SetOption(kv.EnableAsyncCommit, s.GetSessionVars().EnableAsyncCommit) + s.txn.SetOption(kv.Enable1PC, s.GetSessionVars().Enable1PC) // priority of the sysvar is lower than `start transaction with causal consistency only` - if s.txn.GetOption(tikvstore.GuaranteeLinearizability) == nil { + if val := s.txn.GetOption(kv.GuaranteeLinearizability); val == nil || val.(bool) { // We needn't ask the TiKV client to guarantee linearizability for auto-commit transactions // because the property is naturally holds: // We guarantee the commitTS of any transaction must not exceed the next timestamp from the TSO. // An auto-commit transaction fetches its startTS from the TSO so its commitTS > its startTS > the commitTS // of any previously committed transactions. - s.txn.SetOption(tikvstore.GuaranteeLinearizability, + s.txn.SetOption(kv.GuaranteeLinearizability, s.GetSessionVars().TxnCtx.IsExplicit && s.GetSessionVars().GuaranteeLinearizability) } - // Filter out the temporary table key-values. - if tables := s.sessionVars.TxnCtx.GlobalTemporaryTables; tables != nil { - memBuffer := s.txn.GetMemBuffer() - for tid := range tables { - seekKey := tablecodec.EncodeTablePrefix(tid) - endKey := tablecodec.EncodeTablePrefix(tid + 1) - iter, err := memBuffer.Iter(seekKey, endKey) - if err != nil { + return s.txn.Commit(tikvutil.SetSessionID(ctx, s.GetSessionVars().ConnectionID)) +} + +// removeTempTableFromBuffer filters out the temporary table key-values. +func (s *session) removeTempTableFromBuffer() error { + tables := s.GetSessionVars().TxnCtx.GlobalTemporaryTables + if len(tables) == 0 { + return nil + } + memBuffer := s.txn.GetMemBuffer() + // Reset and new an empty stage buffer. + defer func() { + s.txn.cleanup() + }() + for tid := range tables { + seekKey := tablecodec.EncodeTablePrefix(tid) + endKey := tablecodec.EncodeTablePrefix(tid + 1) + iter, err := memBuffer.Iter(seekKey, endKey) + if err != nil { + return err + } + for iter.Valid() && iter.Key().HasPrefix(seekKey) { + if err = memBuffer.Delete(iter.Key()); err != nil { return err } - for iter.Valid() && iter.Key().HasPrefix(seekKey) { - if err = memBuffer.Delete(iter.Key()); err != nil { - return errors.Trace(err) - } - if err = iter.Next(); err != nil { - return errors.Trace(err) - } + s.txn.UpdateEntriesCountAndSize() + if err = iter.Next(); err != nil { + return err } } } - - return s.txn.Commit(tikvutil.SetSessionID(ctx, s.GetSessionVars().ConnectionID)) + // Flush to the root membuffer. + s.txn.flushStmtBuf() + return nil } // errIsNoisy is used to filter DUPLCATE KEY errors. @@ -573,7 +611,7 @@ func (s *session) doCommitWithRetry(ctx context.Context) error { // Finally t1 will have more data than t2, with no errors return to user! if s.isTxnRetryableError(err) && !s.sessionVars.BatchInsert && commitRetryLimit > 0 && !isPessimistic { logutil.Logger(ctx).Warn("sql", - zap.String("label", s.getSQLLabel()), + zap.String("label", s.GetSQLLabel()), zap.Error(err), zap.String("txn", s.txn.GoString())) // Transactions will retry 2 ~ commitRetryLimit times. @@ -583,7 +621,7 @@ func (s *session) doCommitWithRetry(ctx context.Context) error { err = s.retry(ctx, uint(maxRetryCount)) } else if !errIsNoisy(err) { logutil.Logger(ctx).Warn("can not retry txn", - zap.String("label", s.getSQLLabel()), + zap.String("label", s.GetSQLLabel()), zap.Error(err), zap.Bool("IsBatchInsert", s.sessionVars.BatchInsert), zap.Bool("IsPessimistic", isPessimistic), @@ -635,6 +673,7 @@ func (s *session) CommitTxn(ctx context.Context) error { } }) s.sessionVars.TxnCtx.Cleanup() + s.sessionVars.CleanupTxnReadTSIfUsed() return err } @@ -652,6 +691,7 @@ func (s *session) RollbackTxn(ctx context.Context) { } s.txn.changeToInvalid() s.sessionVars.TxnCtx.Cleanup() + s.sessionVars.CleanupTxnReadTSIfUsed() s.sessionVars.SetInTxn(false) } @@ -696,7 +736,7 @@ const sqlLogMaxLen = 1024 // SchemaChangedWithoutRetry is used for testing. var SchemaChangedWithoutRetry uint32 -func (s *session) getSQLLabel() string { +func (s *session) GetSQLLabel() string { if s.sessionVars.InRestrictedSQL { return metrics.LblInternal } @@ -756,7 +796,7 @@ func (s *session) retry(ctx context.Context, maxCnt uint) (err error) { var schemaVersion int64 sessVars := s.GetSessionVars() orgStartTS := sessVars.TxnCtx.StartTS - label := s.getSQLLabel() + label := s.GetSQLLabel() for { s.PrepareTxnCtx(ctx) s.sessionVars.RetryInfo.ResetOffset() @@ -960,6 +1000,7 @@ func (s *session) replaceTableValue(ctx context.Context, tblName string, varName return err } _, _, err = s.ExecRestrictedStmt(ctx, stmt) + domain.GetDomain(s).NotifyUpdateSysVarCache(s) return err } @@ -980,16 +1021,27 @@ func (s *session) GetGlobalSysVar(name string) (string, error) { // When running bootstrap or upgrade, we should not access global storage. return "", nil } - sysVar, err := s.getTableValue(context.TODO(), mysql.GlobalVariablesTable, name) + + sv := variable.GetSysVar(name) + if sv == nil { + // It might be a recently unregistered sysvar. We should return unknown + // since GetSysVar is the canonical version, but we can update the cache + // so the next request doesn't attempt to load this. + logutil.BgLogger().Info("sysvar does not exist. sysvar cache may be stale", zap.String("name", name)) + return "", variable.ErrUnknownSystemVar.GenWithStackByArgs(name) + } + + sysVar, err := domain.GetDomain(s).GetSysVarCache().GetGlobalVar(s, name) if err != nil { - if errResultIsEmpty.Equal(err) { - sv := variable.GetSysVar(name) - if sv != nil { - return sv.Value, nil - } - return "", variable.ErrUnknownSystemVar.GenWithStackByArgs(name) + // The sysvar exists, but there is no cache entry yet. + // This might be because the sysvar was only recently registered. + // In which case it is safe to return the default, but we can also + // update the cache for the future. + logutil.BgLogger().Info("sysvar not in cache yet. sysvar cache may be stale", zap.String("name", name)) + sysVar, err = s.getTableValue(context.TODO(), mysql.GlobalVariablesTable, name) + if err != nil { + return sv.Value, nil } - return "", err } // Fetch mysql.tidb values if required if s.varFromTiDBTable(name) { @@ -1034,12 +1086,7 @@ func (s *session) updateGlobalSysVar(sv *variable.SysVar, value string) error { return err } } - stmt, err := s.ParseWithParams(context.TODO(), "REPLACE %n.%n VALUES (%?, %?)", mysql.SystemDB, mysql.GlobalVariablesTable, sv.Name, value) - if err != nil { - return err - } - _, _, err = s.ExecRestrictedStmt(context.TODO(), stmt) - return err + return s.replaceTableValue(context.TODO(), mysql.GlobalVariablesTable, sv.Name, value) } // setTiDBTableValue handles tikv_* sysvars which need to update mysql.tidb @@ -1118,6 +1165,8 @@ func (s *session) getTiDBTableValue(name, val string) (string, error) { return validatedVal, nil } +var _ sqlexec.SQLParser = &session{} + func (s *session) ParseSQL(ctx context.Context, sql, charset, collation string) ([]ast.StmtNode, []error, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("session.ParseSQL", opentracing.ChildOf(span.Context())) @@ -1183,7 +1232,8 @@ func (s *session) SetProcessInfo(sql string, t time.Time, command byte, maxExecu if oldPi != nil && oldPi.Info == pi.Info { pi.Time = oldPi.Time } - _, pi.Digest = s.sessionVars.StmtCtx.SQLDigest() + _, digest := s.sessionVars.StmtCtx.SQLDigest() + pi.Digest = digest.String() // DO NOT reset the currentPlan to nil until this query finishes execution, otherwise reentrant calls // of SetProcessInfo would override Plan and PlanExplainRows to nil. if command == mysql.ComSleep { @@ -1339,6 +1389,13 @@ func (s *session) ParseWithParams(ctx context.Context, sql string, args ...inter for _, warn := range warns { s.sessionVars.StmtCtx.AppendWarning(util.SyntaxWarn(warn)) } + if variable.TopSQLEnabled() { + normalized, digest := parser.NormalizeDigest(sql) + if digest != nil { + // Fixme: reset/clean the label when internal sql execute finish. + topsql.AttachSQLInfo(ctx, normalized, digest, "", nil) + } + } return stmts[0], nil } @@ -1449,6 +1506,11 @@ func (s *session) ExecuteStmt(ctx context.Context, stmtNode ast.StmtNode) (sqlex if err := executor.ResetContextOfStmt(s, stmtNode); err != nil { return nil, err } + normalizedSQL, digest := s.sessionVars.StmtCtx.SQLDigest() + if variable.TopSQLEnabled() { + ctx = topsql.AttachSQLInfo(ctx, normalizedSQL, digest, "", nil) + } + if err := s.validateStatementReadOnlyInStaleness(stmtNode); err != nil { return nil, err } @@ -1456,6 +1518,8 @@ func (s *session) ExecuteStmt(ctx context.Context, stmtNode ast.StmtNode) (sqlex // Uncorrelated subqueries will execute once when building plan, so we reset process info before building plan. cmd32 := atomic.LoadUint32(&s.GetSessionVars().CommandValue) s.SetProcessInfo(stmtNode.Text(), time.Now(), byte(cmd32), 0) + s.txn.onStmtStart(digest.String()) + defer s.txn.onStmtEnd() // Transform abstract syntax tree to a physical plan(stored in executor.ExecStmt). compiler := executor.Compiler{Ctx: s} @@ -1480,12 +1544,12 @@ func (s *session) ExecuteStmt(ctx context.Context, stmtNode ast.StmtNode) (sqlex s.currentPlan = stmt.Plan // Execute the physical plan. - logStmt(stmt, s.sessionVars) + logStmt(stmt, s) recordSet, err := runStmt(ctx, s, stmt) if err != nil { if !kv.ErrKeyExists.Equal(err) { logutil.Logger(ctx).Warn("run statement failed", - zap.Int64("schemaVersion", s.sessionVars.TxnCtx.SchemaVersion), + zap.Int64("schemaVersion", s.GetInfoSchema().SchemaMetaVersion()), zap.Error(err), zap.String("session", s.String())) } @@ -1506,7 +1570,7 @@ func (s *session) ExecuteStmt(ctx context.Context, stmtNode ast.StmtNode) (sqlex func (s *session) validateStatementReadOnlyInStaleness(stmtNode ast.StmtNode) error { vars := s.GetSessionVars() - if !vars.TxnCtx.IsStaleness { + if !vars.TxnCtx.IsStaleness && vars.TxnReadTS.PeakTxnReadTS() == 0 { return nil } errMsg := "only support read-only statement during read-only staleness transactions" @@ -1587,6 +1651,7 @@ func runStmt(ctx context.Context, se *session, s sqlexec.Statement) (rs sqlexec. return nil, err } rs, err = s.Exec(ctx) + se.updateTelemetryMetric(s.(*executor.ExecStmt)) sessVars.TxnCtx.StatementCount++ if rs != nil { return &execStmtResult{ @@ -1632,8 +1697,40 @@ type execStmtResult struct { func (rs *execStmtResult) Close() error { se := rs.se - err := rs.RecordSet.Close() - return finishStmt(context.Background(), se, err, rs.sql) + if err := resetCTEStorageMap(se); err != nil { + return finishStmt(context.Background(), se, err, rs.sql) + } + if err := rs.RecordSet.Close(); err != nil { + return finishStmt(context.Background(), se, err, rs.sql) + } + return finishStmt(context.Background(), se, nil, rs.sql) +} + +func resetCTEStorageMap(se *session) error { + tmp := se.GetSessionVars().StmtCtx.CTEStorageMap + if tmp == nil { + // Close() is already called, so no need to reset. Such as TraceExec. + return nil + } + storageMap, ok := tmp.(map[int]*executor.CTEStorages) + if !ok { + return errors.New("type assertion for CTEStorageMap failed") + } + for _, v := range storageMap { + // No need to lock IterInTbl. + v.ResTbl.Lock() + defer v.ResTbl.Unlock() + err1 := v.ResTbl.DerefAndClose() + err2 := v.IterInTbl.DerefAndClose() + if err1 != nil { + return err1 + } + if err2 != nil { + return err2 + } + } + se.GetSessionVars().StmtCtx.CTEStorageMap = nil + return nil } // rollbackOnError makes sure the next statement starts a new transaction with the latest InfoSchema. @@ -1660,7 +1757,7 @@ func (s *session) PrepareStmt(sql string) (stmtID uint32, paramCount int, fields // So we have to call PrepareTxnCtx here. s.PrepareTxnCtx(ctx) s.PrepareTSFuture(ctx) - prepareExec := executor.NewPrepareExec(s, infoschema.GetInfoSchema(s), sql) + prepareExec := executor.NewPrepareExec(s, sql) err = prepareExec.Next(ctx, nil) if err != nil { return @@ -1688,7 +1785,7 @@ func (s *session) preparedStmtExec(ctx context.Context, } } sessionExecuteCompileDurationGeneral.Observe(time.Since(s.sessionVars.StartTime).Seconds()) - logQuery(st.OriginText(), s.sessionVars) + logQuery(st.OriginText(), s) return runStmt(ctx, s, st) } @@ -1701,7 +1798,7 @@ func (s *session) cachedPlanExec(ctx context.Context, if prepareStmt.ForUpdateRead { is = domain.GetDomain(s).InfoSchema() } else { - is = infoschema.GetInfoSchema(s) + is = s.GetInfoSchema().(infoschema.InfoSchema) } execAst := &ast.ExecuteStmt{ExecID: stmtID} if err := executor.ResetContextOfStmt(s, execAst); err != nil { @@ -1722,6 +1819,7 @@ func (s *session) cachedPlanExec(ctx context.Context, Ctx: s, OutputNames: execPlan.OutputNames(), PsStmt: prepareStmt, + Ti: &executor.TelemetryInfo{}, } compileDuration := time.Since(s.sessionVars.StartTime) sessionExecuteCompileDurationGeneral.Observe(compileDuration.Seconds()) @@ -1731,7 +1829,7 @@ func (s *session) cachedPlanExec(ctx context.Context, stmtCtx.OriginalSQL = stmt.Text stmtCtx.InitSQLDigest(prepareStmt.NormalizedSQL, prepareStmt.SQLDigest) stmtCtx.SetPlanDigest(prepareStmt.NormalizedPlan, prepareStmt.PlanDigest) - logQuery(stmt.GetTextToLog(), s.sessionVars) + logQuery(stmt.GetTextToLog(), s) if !s.isInternal() && config.GetGlobalConfig().EnableTelemetry { telemetry.CurrentExecuteCount.Inc() @@ -1777,11 +1875,11 @@ func (s *session) IsCachedExecOk(ctx context.Context, preparedStmt *plannercore. return false, nil } // check auto commit - if !s.GetSessionVars().IsAutocommit() { + if !plannercore.IsAutoCommitTxn(s) { return false, nil } // check schema version - is := infoschema.GetInfoSchema(s) + is := s.GetInfoSchema().(infoschema.InfoSchema) if prepared.SchemaVersion != is.SchemaMetaVersion() { prepared.CachedPlan = nil return false, nil @@ -1827,10 +1925,15 @@ func (s *session) ExecutePreparedStmt(ctx context.Context, stmtID uint32, args [ if err != nil { return nil, err } + s.txn.onStmtStart(preparedStmt.SQLDigest.String()) + var rs sqlexec.RecordSet if ok { - return s.cachedPlanExec(ctx, stmtID, preparedStmt, args) + rs, err = s.cachedPlanExec(ctx, stmtID, preparedStmt, args) + } else { + rs, err = s.preparedStmtExec(ctx, stmtID, preparedStmt, args) } - return s.preparedStmtExec(ctx, stmtID, preparedStmt, args) + s.txn.onStmtEnd() + return rs, err } func (s *session) DropPreparedStmt(stmtID uint32) error { @@ -1865,7 +1968,7 @@ func (s *session) Txn(active bool) (kv.Transaction, error) { } s.sessionVars.TxnCtx.StartTS = s.txn.StartTS() if s.sessionVars.TxnCtx.IsPessimistic { - s.txn.SetOption(tikvstore.Pessimistic, true) + s.txn.SetOption(kv.Pessimistic, true) } if !s.sessionVars.IsAutocommit() { s.sessionVars.SetInTxn(true) @@ -1873,7 +1976,7 @@ func (s *session) Txn(active bool) (kv.Transaction, error) { s.sessionVars.TxnCtx.CouldRetry = s.isTxnRetryable() s.txn.SetVars(s.sessionVars.KVVars) if s.sessionVars.GetReplicaRead().IsFollowerRead() { - s.txn.SetOption(tikvstore.ReplicaRead, tikvstore.ReplicaReadFollower) + s.txn.SetOption(kv.ReplicaRead, kv.ReplicaReadFollower) } } return &s.txn, nil @@ -1917,38 +2020,71 @@ func (s *session) isTxnRetryable() bool { } func (s *session) NewTxn(ctx context.Context) error { + if err := s.checkBeforeNewTxn(ctx); err != nil { + return err + } + txn, err := s.store.BeginWithOption(tikv.DefaultStartTSOption().SetTxnScope(s.sessionVars.CheckAndGetTxnScope())) + if err != nil { + return err + } + txn.SetVars(s.sessionVars.KVVars) + if s.GetSessionVars().GetReplicaRead().IsFollowerRead() { + txn.SetOption(kv.ReplicaRead, kv.ReplicaReadFollower) + } + s.txn.changeInvalidToValid(txn) + is := domain.GetDomain(s).InfoSchema() + s.sessionVars.TxnCtx = &variable.TransactionContext{ + InfoSchema: is, + CreateTime: time.Now(), + StartTS: txn.StartTS(), + ShardStep: int(s.sessionVars.ShardAllocateStep), + IsStaleness: false, + TxnScope: s.sessionVars.CheckAndGetTxnScope(), + } + return nil +} + +func (s *session) checkBeforeNewTxn(ctx context.Context) error { if s.txn.Valid() { - txnID := s.txn.StartTS() + txnStartTS := s.txn.StartTS() txnScope := s.GetSessionVars().TxnCtx.TxnScope err := s.CommitTxn(ctx) if err != nil { return err } - vars := s.GetSessionVars() - logutil.Logger(ctx).Info("NewTxn() inside a transaction auto commit", - zap.Int64("schemaVersion", vars.TxnCtx.SchemaVersion), - zap.Uint64("txnStartTS", txnID), + logutil.Logger(ctx).Info("Try to create a new txn inside a transaction auto commit", + zap.Int64("schemaVersion", s.GetInfoSchema().SchemaMetaVersion()), + zap.Uint64("txnStartTS", txnStartTS), zap.String("txnScope", txnScope)) } + return nil +} - txn, err := s.store.BeginWithOption(kv.TransactionOption{}.SetTxnScope(s.sessionVars.CheckAndGetTxnScope())) +// NewStaleTxnWithStartTS create a transaction with the given StartTS. +func (s *session) NewStaleTxnWithStartTS(ctx context.Context, startTS uint64) error { + if err := s.checkBeforeNewTxn(ctx); err != nil { + return err + } + txnScope := s.GetSessionVars().CheckAndGetTxnScope() + txn, err := s.store.BeginWithOption(tikv.DefaultStartTSOption().SetTxnScope(txnScope).SetStartTS(startTS)) if err != nil { return err } txn.SetVars(s.sessionVars.KVVars) - if s.GetSessionVars().GetReplicaRead().IsFollowerRead() { - txn.SetOption(tikvstore.ReplicaRead, tikvstore.ReplicaReadFollower) - } + txn.SetOption(kv.IsStalenessReadOnly, true) + txn.SetOption(kv.TxnScope, txnScope) s.txn.changeInvalidToValid(txn) - is := domain.GetDomain(s).InfoSchema() + is, err := domain.GetDomain(s).GetSnapshotInfoSchema(txn.StartTS()) + if err != nil { + return errors.Trace(err) + } s.sessionVars.TxnCtx = &variable.TransactionContext{ - InfoSchema: is, - SchemaVersion: is.SchemaMetaVersion(), - CreateTime: time.Now(), - StartTS: txn.StartTS(), - ShardStep: int(s.sessionVars.ShardAllocateStep), - IsStaleness: false, - TxnScope: s.sessionVars.CheckAndGetTxnScope(), + InfoSchema: is, + CreateTime: time.Now(), + StartTS: txn.StartTS(), + ShardStep: int(s.sessionVars.ShardAllocateStep), + IsStaleness: true, + TxnScope: txnScope, } return nil } @@ -2263,7 +2399,7 @@ func BootstrapSession(store kv.Storage) (*domain.Domain, error) { newCfg := *(config.GetGlobalConfig()) newCfg.MemQuotaQuery = newMemoryQuotaQuery config.StoreGlobalConfig(&newCfg) - variable.SetSysVar(variable.TIDBMemQuotaQuery, strconv.FormatInt(newCfg.MemQuotaQuery, 10)) + variable.SetSysVar(variable.TiDBMemQuotaQuery, strconv.FormatInt(newCfg.MemQuotaQuery, 10)) } newOOMAction, err := loadDefOOMAction(se) if err != nil { @@ -2294,41 +2430,54 @@ func BootstrapSession(store kv.Storage) (*domain.Domain, error) { } if !config.GetGlobalConfig().Security.SkipGrantTable { - err = dom.LoadPrivilegeLoop(se) + se4, err := createSession(store) + if err != nil { + return nil, err + } + err = dom.LoadPrivilegeLoop(se4) if err != nil { return nil, err } } + // Rebuild sysvar cache in a loop + se5, err := createSession(store) + if err != nil { + return nil, err + } + err = dom.LoadSysVarCacheLoop(se5) + if err != nil { + return nil, err + } + if len(cfg.Plugin.Load) > 0 { err := plugin.Init(context.Background(), plugin.Config{EtcdClient: dom.GetEtcdClient()}) if err != nil { return nil, err } } - - se4, err := createSession(store) + se6, err := createSession(store) if err != nil { return nil, err } - err = executor.LoadExprPushdownBlacklist(se4) + err = executor.LoadExprPushdownBlacklist(se6) if err != nil { return nil, err } - err = executor.LoadOptRuleBlacklist(se4) + err = executor.LoadOptRuleBlacklist(se6) if err != nil { return nil, err } - dom.TelemetryReportLoop(se4) - dom.TelemetryRotateSubWindowLoop(se4) + dom.TelemetryReportLoop(se6) + dom.TelemetryRotateSubWindowLoop(se6) - se5, err := createSession(store) + se7, err := createSession(store) if err != nil { return nil, err } - err = dom.UpdateTableStatsLoop(se5) + err = dom.UpdateTableStatsLoop(se7) if err != nil { return nil, err } @@ -2409,7 +2558,7 @@ func createSessionWithOpt(store kv.Storage, opt *Opt) (*session, error) { // CreateSessionWithDomain creates a new Session and binds it with a Domain. // We need this because when we start DDL in Domain, the DDL need a session // to change some system tables. But at that time, we have been already in -// a lock context, which cause we can't call createSesion directly. +// a lock context, which cause we can't call createSession directly. func CreateSessionWithDomain(store kv.Storage, dom *domain.Domain) (*session, error) { s := &session{ store: store, @@ -2482,130 +2631,6 @@ func finishBootstrap(store kv.Storage) { const quoteCommaQuote = "', '" -var builtinGlobalVariable = []string{ - variable.AutoCommit, - variable.SQLModeVar, - variable.MaxAllowedPacket, - variable.TimeZone, - variable.BlockEncryptionMode, - variable.WaitTimeout, - variable.InteractiveTimeout, - variable.MaxPreparedStmtCount, - variable.InitConnect, - variable.TxnIsolation, - variable.TxReadOnly, - variable.TransactionIsolation, - variable.TransactionReadOnly, - variable.NetBufferLength, - variable.QueryCacheType, - variable.QueryCacheSize, - variable.CharacterSetServer, - variable.AutoIncrementIncrement, - variable.AutoIncrementOffset, - variable.CollationServer, - variable.NetWriteTimeout, - variable.MaxExecutionTime, - variable.InnodbLockWaitTimeout, - variable.WindowingUseHighPrecision, - variable.SQLSelectLimit, - variable.DefaultWeekFormat, - - /* TiDB specific global variables: */ - variable.TiDBSkipASCIICheck, - variable.TiDBSkipUTF8Check, - variable.TiDBIndexJoinBatchSize, - variable.TiDBIndexLookupSize, - variable.TiDBIndexLookupConcurrency, - variable.TiDBIndexLookupJoinConcurrency, - variable.TiDBIndexSerialScanConcurrency, - variable.TiDBHashJoinConcurrency, - variable.TiDBProjectionConcurrency, - variable.TiDBHashAggPartialConcurrency, - variable.TiDBHashAggFinalConcurrency, - variable.TiDBWindowConcurrency, - variable.TiDBMergeJoinConcurrency, - variable.TiDBStreamAggConcurrency, - variable.TiDBExecutorConcurrency, - variable.TiDBBackoffLockFast, - variable.TiDBBackOffWeight, - variable.TiDBConstraintCheckInPlace, - variable.TiDBDDLReorgWorkerCount, - variable.TiDBDDLReorgBatchSize, - variable.TiDBDDLErrorCountLimit, - variable.TiDBOptInSubqToJoinAndAgg, - variable.TiDBOptPreferRangeScan, - variable.TiDBOptCorrelationThreshold, - variable.TiDBOptCorrelationExpFactor, - variable.TiDBOptCPUFactor, - variable.TiDBOptCopCPUFactor, - variable.TiDBOptNetworkFactor, - variable.TiDBOptScanFactor, - variable.TiDBOptDescScanFactor, - variable.TiDBOptMemoryFactor, - variable.TiDBOptDiskFactor, - variable.TiDBOptConcurrencyFactor, - variable.TiDBDistSQLScanConcurrency, - variable.TiDBInitChunkSize, - variable.TiDBMaxChunkSize, - variable.TiDBEnableCascadesPlanner, - variable.TiDBRetryLimit, - variable.TiDBDisableTxnAutoRetry, - variable.TiDBEnableWindowFunction, - variable.TiDBEnableStrictDoubleTypeCheck, - variable.TiDBEnableTablePartition, - variable.TiDBEnableVectorizedExpression, - variable.TiDBEnableFastAnalyze, - variable.TiDBExpensiveQueryTimeThreshold, - variable.TiDBEnableNoopFuncs, - variable.TiDBEnableIndexMerge, - variable.TiDBTxnMode, - variable.TiDBAllowBatchCop, - variable.TiDBAllowMPPExecution, - variable.TiDBOptBCJ, - variable.TiDBBCJThresholdSize, - variable.TiDBBCJThresholdCount, - variable.TiDBRowFormatVersion, - variable.TiDBEnableStmtSummary, - variable.TiDBStmtSummaryInternalQuery, - variable.TiDBStmtSummaryRefreshInterval, - variable.TiDBStmtSummaryHistorySize, - variable.TiDBStmtSummaryMaxStmtCount, - variable.TiDBStmtSummaryMaxSQLLength, - variable.TiDBMaxDeltaSchemaCount, - variable.TiDBCapturePlanBaseline, - variable.TiDBUsePlanBaselines, - variable.TiDBEvolvePlanBaselines, - variable.TiDBEnableExtendedStats, - variable.TiDBIsolationReadEngines, - variable.TiDBStoreLimit, - variable.TiDBAllowAutoRandExplicitInsert, - variable.TiDBEnableClusteredIndex, - variable.TiDBPartitionPruneMode, - variable.TiDBRedactLog, - variable.TiDBEnableTelemetry, - variable.TiDBShardAllocateStep, - variable.TiDBEnableChangeColumnType, - variable.TiDBEnableChangeMultiSchema, - variable.TiDBEnablePointGetCache, - variable.TiDBEnableAlterPlacement, - variable.TiDBEnableAmendPessimisticTxn, - variable.TiDBMemQuotaApplyCache, - variable.TiDBEnableParallelApply, - variable.TiDBMemoryUsageAlarmRatio, - variable.TiDBEnableRateLimitAction, - variable.TiDBEnableAsyncCommit, - variable.TiDBEnable1PC, - variable.TiDBGuaranteeLinearizability, - variable.TiDBAnalyzeVersion, - variable.TiDBEnableIndexMergeJoin, - variable.TiDBTrackAggregateMemoryUsage, - variable.TiDBMultiStatementMode, - variable.TiDBEnableExchangePartition, - variable.TiDBAllowFallbackToTiKV, - variable.TiDBEnableDynamicPrivileges, - variable.CTEMaxRecursionDepth, -} - // loadCommonGlobalVariablesIfNeeded loads and applies commonly used global variables for the session. func (s *session) loadCommonGlobalVariablesIfNeeded() error { vars := s.sessionVars @@ -2617,44 +2642,24 @@ func (s *session) loadCommonGlobalVariablesIfNeeded() error { return nil } - var err error - // Use GlobalVariableCache if TiDB just loaded global variables within 2 second ago. - // When a lot of connections connect to TiDB simultaneously, it can protect TiKV meta region from overload. - gvc := domain.GetDomain(s).GetGlobalVarsCache() - loadFunc := func() ([]chunk.Row, []*ast.ResultField, error) { - vars := append(make([]string, 0, len(builtinGlobalVariable)+len(variable.PluginVarNames)), builtinGlobalVariable...) - if len(variable.PluginVarNames) > 0 { - vars = append(vars, variable.PluginVarNames...) - } - - stmt, err := s.ParseWithParams(context.TODO(), "select HIGH_PRIORITY * from mysql.global_variables where variable_name in (%?) order by VARIABLE_NAME", vars) - if err != nil { - return nil, nil, errors.Trace(err) - } + vars.CommonGlobalLoaded = true - return s.ExecRestrictedStmt(context.TODO(), stmt) - } - rows, _, err := gvc.LoadGlobalVariables(loadFunc) + // Deep copy sessionvar cache + sessionCache, err := domain.GetDomain(s).GetSysVarCache().GetSessionCache(s) if err != nil { - logutil.BgLogger().Warn("failed to load global variables", - zap.Uint64("conn", s.sessionVars.ConnectionID), zap.Error(err)) return err } - vars.CommonGlobalLoaded = true - - for _, row := range rows { - varName := row.GetString(0) - varVal := row.GetString(1) - // `collation_server` is related to `character_set_server`, set `character_set_server` will also set `collation_server`. - // We have to make sure we set the `collation_server` with right value. - if _, ok := vars.GetSystemVar(varName); !ok || varName == variable.CollationServer { - err = vars.SetSystemVar(varName, varVal) + for varName, varVal := range sessionCache { + if _, ok := vars.GetSystemVar(varName); !ok { + err = vars.SetSystemVarWithRelaxedValidation(varName, varVal) if err != nil { + if variable.ErrUnknownSystemVar.Equal(err) { + continue // sessionCache is stale; sysvar has likely been unregistered + } return err } } } - // when client set Capability Flags CLIENT_INTERACTIVE, init wait_timeout with interactive_timeout if vars.ClientCapability&mysql.ClientInteractive > 0 { if varVal, ok := vars.GetSystemVar(variable.InteractiveTimeout); ok { @@ -2663,8 +2668,6 @@ func (s *session) loadCommonGlobalVariablesIfNeeded() error { } } } - - vars.CommonGlobalLoaded = true return nil } @@ -2678,11 +2681,10 @@ func (s *session) PrepareTxnCtx(ctx context.Context) { is := domain.GetDomain(s).InfoSchema() s.sessionVars.TxnCtx = &variable.TransactionContext{ - InfoSchema: is, - SchemaVersion: is.SchemaMetaVersion(), - CreateTime: time.Now(), - ShardStep: int(s.sessionVars.ShardAllocateStep), - TxnScope: s.GetSessionVars().CheckAndGetTxnScope(), + InfoSchema: is, + CreateTime: time.Now(), + ShardStep: int(s.sessionVars.ShardAllocateStep), + TxnScope: s.GetSessionVars().CheckAndGetTxnScope(), } if !s.sessionVars.IsAutocommit() || s.sessionVars.RetryInfo.Retrying { if s.sessionVars.TxnMode == ast.Pessimistic { @@ -2730,7 +2732,7 @@ func (s *session) InitTxnWithStartTS(startTS uint64) error { } // no need to get txn from txnFutureCh since txn should init with startTs - txn, err := s.store.BeginWithOption(kv.TransactionOption{}.SetTxnScope(s.GetSessionVars().CheckAndGetTxnScope()).SetStartTs(startTS)) + txn, err := s.store.BeginWithOption(tikv.DefaultStartTSOption().SetTxnScope(s.GetSessionVars().CheckAndGetTxnScope()).SetStartTS(startTS)) if err != nil { return err } @@ -2743,66 +2745,6 @@ func (s *session) InitTxnWithStartTS(startTS uint64) error { return nil } -// NewTxnWithStalenessOption create a transaction with Staleness option -func (s *session) NewTxnWithStalenessOption(ctx context.Context, option sessionctx.StalenessTxnOption) error { - if s.txn.Valid() { - txnID := s.txn.StartTS() - txnScope := s.txn.GetOption(tikvstore.TxnScope).(string) - err := s.CommitTxn(ctx) - if err != nil { - return err - } - vars := s.GetSessionVars() - logutil.Logger(ctx).Info("InitTxnWithExactStaleness() inside a transaction auto commit", - zap.Int64("schemaVersion", vars.TxnCtx.SchemaVersion), - zap.Uint64("txnStartTS", txnID), - zap.String("txnScope", txnScope)) - } - var txn kv.Transaction - var err error - txnScope := s.GetSessionVars().CheckAndGetTxnScope() - switch option.Mode { - case ast.TimestampBoundReadTimestamp: - txn, err = s.store.BeginWithOption(kv.TransactionOption{}.SetTxnScope(txnScope).SetStartTs(option.StartTS)) - if err != nil { - return err - } - case ast.TimestampBoundExactStaleness: - txn, err = s.store.BeginWithOption(kv.TransactionOption{}.SetTxnScope(txnScope).SetPrevSec(option.PrevSec)) - if err != nil { - return err - } - case ast.TimestampBoundMaxStaleness: - txn, err = s.store.BeginWithOption(kv.TransactionOption{}.SetTxnScope(txnScope).SetMaxPrevSec(option.PrevSec)) - if err != nil { - return err - } - case ast.TimestampBoundMinReadTimestamp: - txn, err = s.store.BeginWithOption(kv.TransactionOption{}.SetTxnScope(txnScope).SetMinStartTS(option.StartTS)) - if err != nil { - return err - } - default: - // For unsupported staleness txn cases, fallback to NewTxn - return s.NewTxn(ctx) - } - txn.SetVars(s.sessionVars.KVVars) - txn.SetOption(tikvstore.IsStalenessReadOnly, true) - txn.SetOption(tikvstore.TxnScope, txnScope) - s.txn.changeInvalidToValid(txn) - is := domain.GetDomain(s).InfoSchema() - s.sessionVars.TxnCtx = &variable.TransactionContext{ - InfoSchema: is, - SchemaVersion: is.SchemaMetaVersion(), - CreateTime: time.Now(), - StartTS: txn.StartTS(), - ShardStep: int(s.sessionVars.ShardAllocateStep), - IsStaleness: true, - TxnScope: txnScope, - } - return nil -} - // GetStore gets the store of session. func (s *session) GetStore() kv.Storage { return s.store @@ -2819,13 +2761,15 @@ func (s *session) ShowProcess() *util.ProcessInfo { // logStmt logs some crucial SQL including: CREATE USER/GRANT PRIVILEGE/CHANGE PASSWORD/DDL etc and normal SQL // if variable.ProcessGeneralLog is set. -func logStmt(execStmt *executor.ExecStmt, vars *variable.SessionVars) { +func logStmt(execStmt *executor.ExecStmt, s *session) { + vars := s.GetSessionVars() switch stmt := execStmt.StmtNode.(type) { case *ast.CreateUserStmt, *ast.DropUserStmt, *ast.AlterUserStmt, *ast.SetPwdStmt, *ast.GrantStmt, *ast.RevokeStmt, *ast.AlterTableStmt, *ast.CreateDatabaseStmt, *ast.CreateIndexStmt, *ast.CreateTableStmt, - *ast.DropDatabaseStmt, *ast.DropIndexStmt, *ast.DropTableStmt, *ast.RenameTableStmt, *ast.TruncateTableStmt: + *ast.DropDatabaseStmt, *ast.DropIndexStmt, *ast.DropTableStmt, *ast.RenameTableStmt, *ast.TruncateTableStmt, + *ast.RenameUserStmt: user := vars.User - schemaVersion := vars.TxnCtx.SchemaVersion + schemaVersion := s.GetInfoSchema().SchemaMetaVersion() if ss, ok := execStmt.StmtNode.(ast.SensitiveStmtNode); ok { logutil.BgLogger().Info("CRUCIAL OPERATION", zap.Uint64("conn", vars.ConnectionID), @@ -2841,11 +2785,12 @@ func logStmt(execStmt *executor.ExecStmt, vars *variable.SessionVars) { zap.Stringer("user", user)) } default: - logQuery(execStmt.GetTextToLog(), vars) + logQuery(execStmt.GetTextToLog(), s) } } -func logQuery(query string, vars *variable.SessionVars) { +func logQuery(query string, s *session) { + vars := s.GetSessionVars() if variable.ProcessGeneralLog.Load() && !vars.InRestrictedSQL { query = executor.QueryReplacer.Replace(query) if !vars.EnableRedactLog { @@ -2854,7 +2799,7 @@ func logQuery(query string, vars *variable.SessionVars) { logutil.BgLogger().Info("GENERAL_LOG", zap.Uint64("conn", vars.ConnectionID), zap.Stringer("user", vars.User), - zap.Int64("schemaVersion", vars.TxnCtx.SchemaVersion), + zap.Int64("schemaVersion", s.GetInfoSchema().SchemaMetaVersion()), zap.Uint64("txnStartTS", vars.TxnCtx.StartTS), zap.Uint64("forUpdateTS", vars.TxnCtx.GetForUpdateTS()), zap.Bool("isReadConsistency", vars.IsIsolation(ast.ReadCommitted)), @@ -2889,10 +2834,10 @@ func (s *session) checkPlacementPolicyBeforeCommit() error { // Get the txnScope of the transaction we're going to commit. txnScope := s.GetSessionVars().TxnCtx.TxnScope if txnScope == "" { - txnScope = oracle.GlobalTxnScope + txnScope = kv.GlobalTxnScope } - if txnScope != oracle.GlobalTxnScope { - is := infoschema.GetInfoSchema(s) + if txnScope != kv.GlobalTxnScope { + is := s.GetInfoSchema().(infoschema.InfoSchema) deltaMap := s.GetSessionVars().TxnCtx.TableDeltaMap for physicalTableID := range deltaMap { var tableName string @@ -2961,3 +2906,53 @@ func (s *session) SetPort(port string) { func (s *session) GetTxnWriteThroughputSLI() *sli.TxnWriteThroughputSLI { return &s.txn.writeSLI } + +var _ telemetry.TemporaryTableFeatureChecker = &session{} + +// TemporaryTableExists is used by the telemetry package to avoid circle dependency. +func (s *session) TemporaryTableExists() bool { + is := domain.GetDomain(s).InfoSchema() + for _, dbInfo := range is.AllSchemas() { + for _, tbInfo := range is.SchemaTables(dbInfo.Name) { + if tbInfo.Meta().TempTableType != model.TempTableNone { + return true + } + } + } + return false +} + +// GetInfoSchema returns snapshotInfoSchema if snapshot schema is set. +// Transaction infoschema is returned if inside an explicit txn. +// Otherwise the latest infoschema is returned. +func (s *session) GetInfoSchema() sessionctx.InfoschemaMetaVersion { + vars := s.GetSessionVars() + if snap, ok := vars.SnapshotInfoschema.(infoschema.InfoSchema); ok { + logutil.BgLogger().Info("use snapshot schema", zap.Uint64("conn", vars.ConnectionID), zap.Int64("schemaVersion", snap.SchemaMetaVersion())) + return snap + } + if vars.TxnCtx != nil && vars.InTxn() { + if is, ok := vars.TxnCtx.InfoSchema.(infoschema.InfoSchema); ok { + return is + } + } + return domain.GetDomain(s).InfoSchema() +} + +func (s *session) updateTelemetryMetric(es *executor.ExecStmt) { + if es.Ti == nil { + return + } + if s.isInternal() { + return + } + + ti := es.Ti + if ti.UseRecursive { + telemetryCTEUsage.WithLabelValues("recurCTE").Inc() + } else if ti.UseNonRecursive { + telemetryCTEUsage.WithLabelValues("nonRecurCTE").Inc() + } else { + telemetryCTEUsage.WithLabelValues("notCTE").Inc() + } +} diff --git a/session/session_fail_test.go b/session/session_fail_test.go index 12f49e0ed1abf..3488592051b9f 100644 --- a/session/session_fail_test.go +++ b/session/session_fail_test.go @@ -15,11 +15,11 @@ package session_test import ( "context" - "sync/atomic" + "strings" . "github.com/pingcap/check" "github.com/pingcap/failpoint" - "github.com/pingcap/tidb/store/tikv" + "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/util/testkit" ) @@ -79,20 +79,19 @@ func (s *testSessionSerialSuite) TestGetTSFailDirtyStateInretry(c *C) { func (s *testSessionSerialSuite) TestKillFlagInBackoff(c *C) { // This test checks the `killed` flag is passed down to the backoffer through - // session.KVVars. It works by setting the `killed = 3` first, then using - // failpoint to run backoff() and check the vars.Killed using the Hook() function. + // session.KVVars. tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("create table kill_backoff (id int)") - var killValue uint32 - tk.Se.GetSessionVars().KVVars.Hook = func(name string, vars *tikv.Variables) { - killValue = atomic.LoadUint32(vars.Killed) - } - c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/tikvStoreSendReqResult", `return("callBackofferHook")`), IsNil) + // Inject 1 time timeout. If `Killed` is not successfully passed, it will retry and complete query. + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/tikvStoreSendReqResult", `return("timeout")->return("")`), IsNil) defer failpoint.Disable("github.com/pingcap/tidb/store/tikv/tikvStoreSendReqResult") // Set kill flag and check its passed to backoffer. - tk.Se.GetSessionVars().Killed = 3 - tk.MustQuery("select * from kill_backoff") - c.Assert(killValue, Equals, uint32(3)) + tk.Se.GetSessionVars().Killed = 1 + rs, err := tk.Exec("select * from kill_backoff") + c.Assert(err, IsNil) + _, err = session.ResultSetToStringSlice(context.TODO(), tk.Se, rs) + // `interrupted` is returned when `Killed` is set. + c.Assert(strings.Contains(err.Error(), "Query execution was interrupted"), IsTrue) } func (s *testSessionSerialSuite) TestClusterTableSendError(c *C) { diff --git a/session/session_test.go b/session/session_test.go index 84442a8a16956..61e90f6f4b73e 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -19,6 +19,7 @@ import ( "fmt" "os" "path" + "strconv" "strings" "sync" "sync/atomic" @@ -36,12 +37,14 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl/placement" "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta/autoid" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/privilege/privileges" "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/session/txninfo" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/binloginfo" "github.com/pingcap/tidb/sessionctx/variable" @@ -50,9 +53,8 @@ import ( "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/store/mockstore/mockcopr" "github.com/pingcap/tidb/store/tikv" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" + tikvmockstore "github.com/pingcap/tidb/store/tikv/mockstore" "github.com/pingcap/tidb/store/tikv/mockstore/cluster" - "github.com/pingcap/tidb/store/tikv/oracle" tikvutil "github.com/pingcap/tidb/store/tikv/util" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/tablecodec" @@ -67,7 +69,6 @@ import ( ) var ( - withTiKV = flag.Bool("with-tikv", false, "run tests with TiKV cluster started. (not use the mock server)") pdAddrs = flag.String("pd-addrs", "127.0.0.1:2379", "pd addrs") pdAddrChan chan string initPdAddrsOnce sync.Once @@ -83,6 +84,7 @@ var _ = SerialSuites(&testSessionSerialSuite{}) var _ = SerialSuites(&testBackupRestoreSuite{}) var _ = Suite(&testClusteredSuite{}) var _ = SerialSuites(&testClusteredSerialSuite{}) +var _ = SerialSuites(&testTxnStateSerialSuite{}) type testSessionSuiteBase struct { cluster cluster.Cluster @@ -182,7 +184,7 @@ func initPdAddrs() { func (s *testSessionSuiteBase) SetUpSuite(c *C) { testleak.BeforeTest() - if *withTiKV { + if *tikvmockstore.WithTiKV { initPdAddrs() s.pdAddr = <-pdAddrChan var d driver.TiKVDriver @@ -212,14 +214,13 @@ func (s *testSessionSuiteBase) SetUpSuite(c *C) { var err error s.dom, err = session.BootstrapSession(s.store) c.Assert(err, IsNil) - s.dom.GetGlobalVarsCache().Disable() } func (s *testSessionSuiteBase) TearDownSuite(c *C) { s.dom.Close() s.store.Close() testleak.AfterTest(c)() - if *withTiKV { + if *tikvmockstore.WithTiKV { pdAddrChan <- s.pdAddr } } @@ -638,7 +639,6 @@ func (s *testSessionSuite) TestGlobalVarAccessor(c *C) { c.Assert(v, Equals, varValue2) // For issue 10955, make sure the new session load `max_execution_time` into sessionVars. - s.dom.GetGlobalVarsCache().Disable() tk1.MustExec("set @@global.max_execution_time = 100") tk2 := testkit.NewTestKitWithInit(c, s.store) c.Assert(tk2.Se.GetSessionVars().MaxExecutionTime, Equals, uint64(100)) @@ -788,6 +788,50 @@ func (s *testSessionSuite) TestRetryUnion(c *C) { c.Assert(err, ErrorMatches, ".*can not retry select for update statement") } +func (s *testSessionSuite) TestRetryGlobalTempTable(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("drop table if exists normal_table") + tk.MustExec("create table normal_table(a int primary key, b int)") + defer tk.MustExec("drop table if exists normal_table") + tk.MustExec("drop table if exists temp_table") + tk.MustExec("create global temporary table temp_table(a int primary key, b int) on commit delete rows") + defer tk.MustExec("drop table if exists temp_table") + + // insert select + tk.MustExec("set tidb_disable_txn_auto_retry = 0") + tk.MustExec("insert normal_table value(100, 100)") + tk.MustExec("set @@autocommit = 0") + // used to make conflicts + tk.MustExec("update normal_table set b=b+1 where a=100") + tk.MustExec("insert temp_table value(1, 1)") + tk.MustExec("insert normal_table select * from temp_table") + c.Assert(session.GetHistory(tk.Se).Count(), Equals, 3) + + // try to conflict with tk + tk1 := testkit.NewTestKitWithInit(c, s.store) + tk1.MustExec("update normal_table set b=b+1 where a=100") + + // It will retry internally. + tk.MustExec("commit") + tk.MustQuery("select a, b from normal_table order by a").Check(testkit.Rows("1 1", "100 102")) + tk.MustQuery("select a, b from temp_table order by a").Check(testkit.Rows()) + + // update multi-tables + tk.MustExec("update normal_table set b=b+1 where a=100") + tk.MustExec("insert temp_table value(1, 2)") + // before update: normal_table=(1 1) (100 102), temp_table=(1 2) + tk.MustExec("update normal_table, temp_table set normal_table.b=temp_table.b where normal_table.a=temp_table.a") + c.Assert(session.GetHistory(tk.Se).Count(), Equals, 3) + + // try to conflict with tk + tk1.MustExec("update normal_table set b=b+1 where a=100") + + // It will retry internally. + tk.MustExec("commit") + tk.MustQuery("select a, b from normal_table order by a").Check(testkit.Rows("1 2", "100 104")) +} + func (s *testSessionSuite) TestRetryShow(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("set @@autocommit = 0") @@ -2033,7 +2077,7 @@ func (s *testSchemaSerialSuite) TestLoadSchemaFailed(c *C) { _, err = tk1.Exec("commit") c.Check(err, NotNil) - ver, err := s.store.CurrentVersion(oracle.GlobalTxnScope) + ver, err := s.store.CurrentVersion(kv.GlobalTxnScope) c.Assert(err, IsNil) c.Assert(ver, NotNil) @@ -2089,6 +2133,46 @@ func (s *testSchemaSerialSuite) TestSchemaCheckerSQL(c *C) { c.Assert(err, NotNil) } +func (s *testSchemaSerialSuite) TestSchemaCheckerTempTable(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk1 := testkit.NewTestKitWithInit(c, s.store) + + // create table + tk.MustExec(`drop table if exists normal_table`) + tk.MustExec(`create table normal_table (id int, c int);`) + defer tk.MustExec(`drop table if exists normal_table`) + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec(`drop table if exists temp_table`) + tk.MustExec(`create global temporary table temp_table (id int, c int) on commit delete rows;`) + defer tk.MustExec(`drop table if exists temp_table`) + + // The schema version is out of date in the first transaction, and the SQL can't be retried. + atomic.StoreUint32(&session.SchemaChangedWithoutRetry, 1) + defer func() { + atomic.StoreUint32(&session.SchemaChangedWithoutRetry, 0) + }() + + // It's fine to change the schema of temporary tables. + tk.MustExec(`begin;`) + tk1.MustExec(`alter table temp_table modify column c bigint;`) + tk.MustExec(`insert into temp_table values(3, 3);`) + tk.MustExec(`commit;`) + + // Truncate will modify table ID. + tk.MustExec(`begin;`) + tk1.MustExec(`truncate table temp_table;`) + tk.MustExec(`insert into temp_table values(3, 3);`) + tk.MustExec(`commit;`) + + // It reports error when also changing the schema of a normal table. + tk.MustExec(`begin;`) + tk1.MustExec(`alter table normal_table modify column c bigint;`) + tk.MustExec(`insert into temp_table values(3, 3);`) + tk.MustExec(`insert into normal_table values(3, 3);`) + _, err := tk.Exec(`commit;`) + c.Assert(terror.ErrorEqual(err, domain.ErrInfoSchemaChanged), IsTrue, Commentf("err %v", err)) +} + func (s *testSchemaSuite) TestPrepareStmtCommitWhenSchemaChanged(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk1 := testkit.NewTestKitWithInit(c, s.store) @@ -2574,8 +2658,6 @@ func (s *testSessionSuite) TestSetGlobalTZ(c *C) { tk.MustQuery("show variables like 'time_zone'").Check(testkit.Rows("time_zone +08:00")) - // Disable global variable cache, so load global session variable take effect immediate. - s.dom.GetGlobalVarsCache().Disable() tk1 := testkit.NewTestKitWithInit(c, s.store) tk1.MustQuery("show variables like 'time_zone'").Check(testkit.Rows("time_zone +00:00")) } @@ -2717,8 +2799,6 @@ func (s *testSessionSuite3) TestEnablePartition(c *C) { tk.MustExec("set tidb_enable_list_partition=on") tk.MustQuery("show variables like 'tidb_enable_list_partition'").Check(testkit.Rows("tidb_enable_list_partition ON")) - // Disable global variable cache, so load global session variable take effect immediate. - s.dom.GetGlobalVarsCache().Disable() tk1 := testkit.NewTestKitWithInit(c, s.store) tk1.MustQuery("show variables like 'tidb_enable_table_partition'").Check(testkit.Rows("tidb_enable_table_partition ON")) } @@ -2892,7 +2972,7 @@ func (s *testSessionSuite2) TestUpdatePrivilege(c *C) { _, err := tk1.Exec("update t2 set id = 666 where id = 1;") c.Assert(err, NotNil) - c.Assert(strings.Contains(err.Error(), "privilege check fail"), IsTrue) + c.Assert(strings.Contains(err.Error(), "privilege check"), IsTrue) // Cover a bug that t1 and t2 both require update privilege. // In fact, the privlege check for t1 should be update, and for t2 should be select. @@ -3062,11 +3142,11 @@ func (s *testSessionSuite2) TestReplicaRead(c *C) { tk := testkit.NewTestKit(c, s.store) tk.Se, err = session.CreateSession4Test(s.store) c.Assert(err, IsNil) - c.Assert(tk.Se.GetSessionVars().GetReplicaRead(), Equals, tikvstore.ReplicaReadLeader) + c.Assert(tk.Se.GetSessionVars().GetReplicaRead(), Equals, kv.ReplicaReadLeader) tk.MustExec("set @@tidb_replica_read = 'follower';") - c.Assert(tk.Se.GetSessionVars().GetReplicaRead(), Equals, tikvstore.ReplicaReadFollower) + c.Assert(tk.Se.GetSessionVars().GetReplicaRead(), Equals, kv.ReplicaReadFollower) tk.MustExec("set @@tidb_replica_read = 'leader';") - c.Assert(tk.Se.GetSessionVars().GetReplicaRead(), Equals, tikvstore.ReplicaReadLeader) + c.Assert(tk.Se.GetSessionVars().GetReplicaRead(), Equals, kv.ReplicaReadLeader) } func (s *testSessionSuite3) TestIsolationRead(c *C) { @@ -3151,12 +3231,12 @@ func (s *testSessionSuite2) TestStmtHints(c *C) { c.Assert(tk.Se.GetSessionVars().GetEnableCascadesPlanner(), IsTrue) // Test READ_CONSISTENT_REPLICA hint - tk.Se.GetSessionVars().SetReplicaRead(tikvstore.ReplicaReadLeader) + tk.Se.GetSessionVars().SetReplicaRead(kv.ReplicaReadLeader) tk.MustExec("select /*+ READ_CONSISTENT_REPLICA() */ 1;") - c.Assert(tk.Se.GetSessionVars().GetReplicaRead(), Equals, tikvstore.ReplicaReadFollower) + c.Assert(tk.Se.GetSessionVars().GetReplicaRead(), Equals, kv.ReplicaReadFollower) tk.MustExec("select /*+ READ_CONSISTENT_REPLICA(), READ_CONSISTENT_REPLICA() */ 1;") c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings(), HasLen, 1) - c.Assert(tk.Se.GetSessionVars().GetReplicaRead(), Equals, tikvstore.ReplicaReadFollower) + c.Assert(tk.Se.GetSessionVars().GetReplicaRead(), Equals, kv.ReplicaReadFollower) } func (s *testSessionSuite3) TestPessimisticLockOnPartition(c *C) { @@ -3261,26 +3341,26 @@ func (s *testSessionSerialSuite) TestSetTxnScope(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) // assert default value result := tk.MustQuery("select @@txn_scope;") - result.Check(testkit.Rows(oracle.GlobalTxnScope)) - c.Assert(tk.Se.GetSessionVars().CheckAndGetTxnScope(), Equals, oracle.GlobalTxnScope) + result.Check(testkit.Rows(kv.GlobalTxnScope)) + c.Assert(tk.Se.GetSessionVars().CheckAndGetTxnScope(), Equals, kv.GlobalTxnScope) // assert set sys variable tk.MustExec("set @@session.txn_scope = 'local';") result = tk.MustQuery("select @@txn_scope;") - result.Check(testkit.Rows(oracle.GlobalTxnScope)) - c.Assert(tk.Se.GetSessionVars().CheckAndGetTxnScope(), Equals, oracle.GlobalTxnScope) + result.Check(testkit.Rows(kv.GlobalTxnScope)) + c.Assert(tk.Se.GetSessionVars().CheckAndGetTxnScope(), Equals, kv.GlobalTxnScope) failpoint.Disable("github.com/pingcap/tidb/store/tikv/config/injectTxnScope") failpoint.Enable("github.com/pingcap/tidb/store/tikv/config/injectTxnScope", `return("bj")`) defer failpoint.Disable("github.com/pingcap/tidb/store/tikv/config/injectTxnScope") tk = testkit.NewTestKitWithInit(c, s.store) // assert default value result = tk.MustQuery("select @@txn_scope;") - result.Check(testkit.Rows(oracle.LocalTxnScope)) + result.Check(testkit.Rows(kv.LocalTxnScope)) c.Assert(tk.Se.GetSessionVars().CheckAndGetTxnScope(), Equals, "bj") // assert set sys variable tk.MustExec("set @@session.txn_scope = 'global';") result = tk.MustQuery("select @@txn_scope;") - result.Check(testkit.Rows(oracle.GlobalTxnScope)) - c.Assert(tk.Se.GetSessionVars().CheckAndGetTxnScope(), Equals, oracle.GlobalTxnScope) + result.Check(testkit.Rows(kv.GlobalTxnScope)) + c.Assert(tk.Se.GetSessionVars().CheckAndGetTxnScope(), Equals, kv.GlobalTxnScope) // assert set invalid txn_scope err := tk.ExecToErr("set @@txn_scope='foo'") @@ -3291,7 +3371,7 @@ func (s *testSessionSerialSuite) TestSetTxnScope(c *C) { func (s *testSessionSerialSuite) TestGlobalAndLocalTxn(c *C) { // Because the PD config of check_dev_2 test is not compatible with local/global txn yet, // so we will skip this test for now. - if *withTiKV { + if *tikvmockstore.WithTiKV { return } tk := testkit.NewTestKitWithInit(c, s.store) @@ -3317,7 +3397,7 @@ PARTITION BY RANGE (c) ( GroupID: groupID, Role: placement.Leader, Count: 1, - LabelConstraints: []placement.Constraint{ + Constraints: []placement.Constraint{ { Key: placement.DCLabelKey, Op: placement.In, @@ -3337,9 +3417,9 @@ PARTITION BY RANGE (c) ( setBundle("p1", "dc-2") // set txn_scope to global - tk.MustExec(fmt.Sprintf("set @@session.txn_scope = '%s';", oracle.GlobalTxnScope)) + tk.MustExec(fmt.Sprintf("set @@session.txn_scope = '%s';", kv.GlobalTxnScope)) result := tk.MustQuery("select @@txn_scope;") - result.Check(testkit.Rows(oracle.GlobalTxnScope)) + result.Check(testkit.Rows(kv.GlobalTxnScope)) // test global txn auto commit tk.MustExec("insert into t1 (c) values (1)") // write dc-1 with global scope @@ -3350,7 +3430,7 @@ PARTITION BY RANGE (c) ( tk.MustExec("begin") txn, err := tk.Se.Txn(true) c.Assert(err, IsNil) - c.Assert(tk.Se.GetSessionVars().TxnCtx.TxnScope, Equals, oracle.GlobalTxnScope) + c.Assert(tk.Se.GetSessionVars().TxnCtx.TxnScope, Equals, kv.GlobalTxnScope) c.Assert(txn.Valid(), IsTrue) tk.MustExec("insert into t1 (c) values (1)") // write dc-1 with global scope result = tk.MustQuery("select * from t1") // read dc-1 and dc-2 with global scope @@ -3364,7 +3444,7 @@ PARTITION BY RANGE (c) ( tk.MustExec("begin") txn, err = tk.Se.Txn(true) c.Assert(err, IsNil) - c.Assert(tk.Se.GetSessionVars().TxnCtx.TxnScope, Equals, oracle.GlobalTxnScope) + c.Assert(tk.Se.GetSessionVars().TxnCtx.TxnScope, Equals, kv.GlobalTxnScope) c.Assert(txn.Valid(), IsTrue) tk.MustExec("insert into t1 (c) values (101)") // write dc-2 with global scope result = tk.MustQuery("select * from t1") // read dc-1 and dc-2 with global scope @@ -3687,7 +3767,7 @@ func (s *testSessionSerialSuite) TestDoDDLJobQuit(c *C) { func (s *testBackupRestoreSuite) TestBackupAndRestore(c *C) { // only run BR SQL integration test with tikv store. - if *withTiKV { + if *tikvmockstore.WithTiKV { cfg := config.GetGlobalConfig() cfg.Store = "tikv" cfg.Path = s.pdAddr @@ -3902,208 +3982,6 @@ func (s *testSessionSerialSuite) TestIssue21943(c *C) { c.Assert(err.Error(), Equals, "[variable:1238]Variable 'last_plan_from_cache' is a read only variable") } -func (s *testSessionSuite) TestValidateReadOnlyInStalenessTransaction(c *C) { - c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer", "return(false)"), IsNil) - defer failpoint.Disable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer") - testcases := []struct { - name string - sql string - isValidate bool - }{ - { - name: "select statement", - sql: `select * from t;`, - isValidate: true, - }, - { - name: "explain statement", - sql: `explain insert into t (id) values (1);`, - isValidate: true, - }, - { - name: "explain analyze insert statement", - sql: `explain analyze insert into t (id) values (1);`, - isValidate: false, - }, - { - name: "explain analyze select statement", - sql: `explain analyze select * from t `, - isValidate: true, - }, - { - name: "execute insert statement", - sql: `EXECUTE stmt1;`, - isValidate: false, - }, - { - name: "execute select statement", - sql: `EXECUTE stmt2;`, - isValidate: true, - }, - { - name: "show statement", - sql: `show tables;`, - isValidate: true, - }, - { - name: "set union", - sql: `SELECT 1, 2 UNION SELECT 'a', 'b';`, - isValidate: true, - }, - { - name: "insert", - sql: `insert into t (id) values (1);`, - isValidate: false, - }, - { - name: "delete", - sql: `delete from t where id =1`, - isValidate: false, - }, - { - name: "update", - sql: "update t set id =2 where id =1", - isValidate: false, - }, - { - name: "point get", - sql: `select * from t where id = 1`, - isValidate: true, - }, - { - name: "batch point get", - sql: `select * from t where id in (1,2,3);`, - isValidate: true, - }, - { - name: "split table", - sql: `SPLIT TABLE t BETWEEN (0) AND (1000000000) REGIONS 16;`, - isValidate: true, - }, - { - name: "do statement", - sql: `DO SLEEP(1);`, - isValidate: true, - }, - { - name: "select for update", - sql: "select * from t where id = 1 for update", - isValidate: false, - }, - { - name: "select lock in share mode", - sql: "select * from t where id = 1 lock in share mode", - isValidate: true, - }, - { - name: "select for update union statement", - sql: "select * from t for update union select * from t;", - isValidate: false, - }, - { - name: "replace statement", - sql: "replace into t(id) values (1)", - isValidate: false, - }, - { - name: "load data statement", - sql: "LOAD DATA LOCAL INFILE '/mn/asa.csv' INTO TABLE t FIELDS TERMINATED BY x'2c' ENCLOSED BY b'100010' LINES TERMINATED BY '\r\n' IGNORE 1 LINES (id);", - isValidate: false, - }, - { - name: "update multi tables", - sql: "update t,t1 set t.id = 1,t1.id = 2 where t.1 = 2 and t1.id = 3;", - isValidate: false, - }, - { - name: "delete multi tables", - sql: "delete t from t1 where t.id = t1.id", - isValidate: false, - }, - { - name: "insert select", - sql: "insert into t select * from t1;", - isValidate: false, - }, - } - tk := testkit.NewTestKit(c, s.store) - tk.MustExec("use test") - tk.MustExec("create table t (id int);") - tk.MustExec("create table t1 (id int);") - tk.MustExec(`PREPARE stmt1 FROM 'insert into t(id) values (5);';`) - tk.MustExec(`PREPARE stmt2 FROM 'select * from t';`) - tk.MustExec(`set @@tidb_enable_noop_functions=1;`) - for _, testcase := range testcases { - c.Log(testcase.name) - tk.MustExec(`START TRANSACTION READ ONLY WITH TIMESTAMP BOUND READ TIMESTAMP '2020-09-06 00:00:00';`) - if testcase.isValidate { - _, err := tk.Exec(testcase.sql) - c.Assert(err, IsNil) - tk.MustExec("commit") - } else { - err := tk.ExecToErr(testcase.sql) - c.Assert(err, NotNil) - c.Assert(err.Error(), Matches, `.*only support read-only statement during read-only staleness transactions.*`) - } - } -} - -func (s *testSessionSerialSuite) TestSpecialSQLInStalenessTxn(c *C) { - c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer", "return(false)"), IsNil) - defer failpoint.Disable("github.com/pingcap/tidb/executor/mockStalenessTxnSchemaVer") - tk := testkit.NewTestKit(c, s.store) - tk.MustExec("use test") - testcases := []struct { - name string - sql string - sameSession bool - }{ - { - name: "ddl", - sql: "create table t (id int, b int,INDEX(b));", - sameSession: false, - }, - { - name: "set global session", - sql: `SET GLOBAL sql_mode = 'STRICT_TRANS_TABLES,NO_AUTO_CREATE_USER';`, - sameSession: true, - }, - { - name: "analyze table", - sql: "analyze table t", - sameSession: true, - }, - { - name: "session binding", - sql: "CREATE SESSION BINDING FOR SELECT * FROM t WHERE b = 123 USING SELECT * FROM t IGNORE INDEX (b) WHERE b = 123;", - sameSession: true, - }, - { - name: "global binding", - sql: "CREATE GLOBAL BINDING FOR SELECT * FROM t WHERE b = 123 USING SELECT * FROM t IGNORE INDEX (b) WHERE b = 123;", - sameSession: true, - }, - { - name: "grant statements", - sql: "GRANT ALL ON test.* TO 'newuser';", - sameSession: false, - }, - { - name: "revoke statements", - sql: "REVOKE ALL ON test.* FROM 'newuser';", - sameSession: false, - }, - } - tk.MustExec("CREATE USER 'newuser' IDENTIFIED BY 'mypassword';") - for _, testcase := range testcases { - comment := Commentf(testcase.name) - tk.MustExec(`START TRANSACTION READ ONLY WITH TIMESTAMP BOUND READ TIMESTAMP '2020-09-06 00:00:00';`) - c.Assert(tk.Se.GetSessionVars().TxnCtx.IsStaleness, Equals, true, comment) - tk.MustExec(testcase.sql) - c.Assert(tk.Se.GetSessionVars().TxnCtx.IsStaleness, Equals, testcase.sameSession, comment) - } -} - func (s *testSessionSerialSuite) TestRemovedSysVars(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) @@ -4283,6 +4161,7 @@ func (s *testSessionSerialSuite) TestParseWithParams(c *C) { func (s *testSessionSuite3) TestGlobalTemporaryTable(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("set tidb_enable_global_temporary_table=true") tk.MustExec("create global temporary table g_tmp (a int primary key, b int, c int, index i_b(b)) on commit delete rows") tk.MustExec("begin") tk.MustExec("insert into g_tmp values (3, 3, 3)") @@ -4296,8 +4175,419 @@ func (s *testSessionSuite3) TestGlobalTemporaryTable(c *C) { tk.MustQuery("select c from g_tmp where b = 3").Check(testkit.Rows("3")) // Cover point get. tk.MustQuery("select * from g_tmp where a = 3").Check(testkit.Rows("3 3 3")) + // Cover batch point get. + tk.MustQuery("select * from g_tmp where a in (2,3,4)").Check(testkit.Rows("3 3 3", "4 7 9")) tk.MustExec("commit") // The global temporary table data is discard after the transaction commit. tk.MustQuery("select * from g_tmp").Check(testkit.Rows()) } + +type testTxnStateSerialSuite struct { + testSessionSuiteBase +} + +func (s *testTxnStateSerialSuite) TestBasic(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create table t(a int);") + tk.MustExec("insert into t(a) values (1);") + info := tk.Se.TxnInfo() + c.Assert(info, IsNil) + + tk.MustExec("begin pessimistic;") + startTSStr := tk.MustQuery("select @@tidb_current_ts;").Rows()[0][0].(string) + startTS, err := strconv.ParseUint(startTSStr, 10, 64) + c.Assert(err, IsNil) + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/beforePessimisticLock", "pause"), IsNil) + ch := make(chan interface{}) + go func() { + tk.MustExec("select * from t for update;") + ch <- nil + }() + time.Sleep(100 * time.Millisecond) + info = tk.Se.TxnInfo() + _, expectedDigest := parser.NormalizeDigest("select * from t for update;") + c.Assert(info.CurrentSQLDigest, Equals, expectedDigest.String()) + c.Assert(info.State, Equals, txninfo.TxnLockWaiting) + c.Assert((*time.Time)(info.BlockStartTime), NotNil) + c.Assert(info.StartTS, Equals, startTS) + + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/beforePessimisticLock"), IsNil) + <-ch + + info = tk.Se.TxnInfo() + c.Assert(info.CurrentSQLDigest, Equals, "") + c.Assert(info.State, Equals, txninfo.TxnRunningNormal) + c.Assert((*time.Time)(info.BlockStartTime), IsNil) + c.Assert(info.StartTS, Equals, startTS) + _, beginDigest := parser.NormalizeDigest("begin pessimistic;") + _, selectTSDigest := parser.NormalizeDigest("select @@tidb_current_ts;") + c.Assert(info.AllSQLDigests, DeepEquals, []string{beginDigest.String(), selectTSDigest.String(), expectedDigest.String()}) + + // len and size will be covered in TestLenAndSize + c.Assert(info.ConnectionID, Equals, tk.Se.GetSessionVars().ConnectionID) + c.Assert(info.Username, Equals, "") + c.Assert(info.CurrentDB, Equals, "test") + c.Assert(info.StartTS, Equals, startTS) + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/beforePrewrite", "pause"), IsNil) + go func() { + tk.MustExec("commit;") + ch <- nil + }() + time.Sleep(100 * time.Millisecond) + _, commitDigest := parser.NormalizeDigest("commit;") + info = tk.Se.TxnInfo() + c.Assert(info.CurrentSQLDigest, Equals, commitDigest.String()) + c.Assert(info.State, Equals, txninfo.TxnCommitting) + c.Assert(info.AllSQLDigests, DeepEquals, []string{beginDigest.String(), selectTSDigest.String(), expectedDigest.String(), commitDigest.String()}) + + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/beforePrewrite"), IsNil) + <-ch + info = tk.Se.TxnInfo() + c.Assert(info, IsNil) + + // Test autocommit transaction + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/beforePrewrite", "pause"), IsNil) + go func() { + tk.MustExec("insert into t values (2)") + ch <- nil + }() + time.Sleep(100 * time.Millisecond) + info = tk.Se.TxnInfo() + _, expectedDigest = parser.NormalizeDigest("insert into t values (2)") + c.Assert(info.CurrentSQLDigest, Equals, expectedDigest.String()) + c.Assert(info.State, Equals, txninfo.TxnCommitting) + c.Assert((*time.Time)(info.BlockStartTime), IsNil) + c.Assert(info.StartTS, Greater, startTS) + c.Assert(len(info.AllSQLDigests), Equals, 1) + c.Assert(info.AllSQLDigests[0], Equals, expectedDigest.String()) + + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/beforePrewrite"), IsNil) + <-ch + info = tk.Se.TxnInfo() + c.Assert(info, IsNil) +} + +func (s *testTxnStateSerialSuite) TestEntriesCountAndSize(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create table t(a int);") + tk.MustExec("begin pessimistic;") + tk.MustExec("insert into t(a) values (1);") + info := tk.Se.TxnInfo() + c.Assert(info.EntriesCount, Equals, uint64(1)) + c.Assert(info.EntriesSize, Equals, uint64(29)) + tk.MustExec("insert into t(a) values (2);") + info = tk.Se.TxnInfo() + c.Assert(info.EntriesCount, Equals, uint64(2)) + c.Assert(info.EntriesSize, Equals, uint64(58)) + tk.MustExec("commit;") +} + +func (s *testTxnStateSerialSuite) TestBlocked(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk2 := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create table t(a int);") + tk.MustExec("insert into t(a) values (1);") + tk.MustExec("begin pessimistic;") + tk.MustExec("select * from t where a = 1 for update;") + go func() { + tk2.MustExec("begin pessimistic") + tk2.MustExec("select * from t where a = 1 for update;") + tk2.MustExec("commit;") + }() + time.Sleep(100 * time.Millisecond) + c.Assert(tk2.Se.TxnInfo().State, Equals, txninfo.TxnLockWaiting) + c.Assert(tk2.Se.TxnInfo().BlockStartTime, NotNil) + tk.MustExec("commit;") +} + +func (s *testTxnStateSerialSuite) TestCommitting(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk2 := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create table t(a int);") + tk.MustExec("insert into t(a) values (1), (2);") + tk.MustExec("begin pessimistic;") + tk.MustExec("select * from t where a = 1 for update;") + ch := make(chan struct{}) + go func() { + tk2.MustExec("begin pessimistic") + c.Assert(tk2.Se.TxnInfo(), NotNil) + tk2.MustExec("select * from t where a = 2 for update;") + c.Assert(failpoint.Enable("github.com/pingcap/tidb/session/mockSlowCommit", "sleep(200)"), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/session/mockSlowCommit"), IsNil) + }() + tk2.MustExec("commit;") + ch <- struct{}{} + }() + time.Sleep(100 * time.Millisecond) + c.Assert(tk2.Se.TxnInfo().State, Equals, txninfo.TxnCommitting) + tk.MustExec("commit;") + <-ch +} + +func (s *testTxnStateSerialSuite) TestRollbacking(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create table t(a int);") + tk.MustExec("insert into t(a) values (1), (2);") + ch := make(chan struct{}) + go func() { + tk.MustExec("begin pessimistic") + tk.MustExec("insert into t(a) values (3);") + failpoint.Enable("github.com/pingcap/tidb/session/mockSlowRollback", "sleep(200)") + defer failpoint.Disable("github.com/pingcap/tidb/session/mockSlowRollback") + tk.MustExec("rollback;") + ch <- struct{}{} + }() + time.Sleep(100 * time.Millisecond) + c.Assert(tk.Se.TxnInfo().State, Equals, txninfo.TxnRollingBack) + <-ch +} + +func (s *testTxnStateSerialSuite) TestTxnInfoWithPreparedStmt(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create table t(a int)") + tk.MustExec("prepare s1 from 'insert into t values (?)'") + tk.MustExec("set @v = 1") + + tk.MustExec("begin pessimistic") + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/beforePessimisticLock", "pause"), IsNil) + ch := make(chan interface{}) + go func() { + tk.MustExec("execute s1 using @v") + ch <- nil + }() + time.Sleep(100 * time.Millisecond) + info := tk.Se.TxnInfo() + _, expectDigest := parser.NormalizeDigest("insert into t values (?)") + c.Assert(info.CurrentSQLDigest, Equals, expectDigest.String()) + + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/beforePessimisticLock"), IsNil) + <-ch + info = tk.Se.TxnInfo() + c.Assert(info.CurrentSQLDigest, Equals, "") + _, beginDigest := parser.NormalizeDigest("begin pessimistic") + c.Assert(info.AllSQLDigests, DeepEquals, []string{beginDigest.String(), expectDigest.String()}) + + tk.MustExec("rollback") +} + +func (s *testTxnStateSerialSuite) TestTxnInfoWithScalarSubquery(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create table t (a int, b int)") + tk.MustExec("insert into t values (1, 10), (2, 1)") + + tk.MustExec("begin pessimistic") + _, beginDigest := parser.NormalizeDigest("begin pessimistic") + tk.MustExec("select * from t where a = (select b from t where a = 2)") + _, s1Digest := parser.NormalizeDigest("select * from t where a = (select b from t where a = 2)") + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/beforePessimisticLock", "pause"), IsNil) + ch := make(chan interface{}) + go func() { + tk.MustExec("update t set b = b + 1 where a = (select b from t where a = 2)") + ch <- nil + }() + _, s2Digest := parser.NormalizeDigest("update t set b = b + 1 where a = (select b from t where a = 1)") + time.Sleep(100 * time.Millisecond) + info := tk.Se.TxnInfo() + c.Assert(info.CurrentSQLDigest, Equals, s2Digest.String()) + c.Assert(info.AllSQLDigests, DeepEquals, []string{beginDigest.String(), s1Digest.String(), s2Digest.String()}) + + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/beforePessimisticLock"), IsNil) + <-ch + tk.MustExec("rollback") +} + +func (s *testTxnStateSerialSuite) TestTxnInfoWithPSProtocol(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create table t (a int primary key)") + + // Test autocommit transaction + + idInsert, _, _, err := tk.Se.PrepareStmt("insert into t values (?)") + c.Assert(err, IsNil) + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/beforePrewrite", "pause"), IsNil) + ch := make(chan interface{}) + go func() { + _, err := tk.Se.ExecutePreparedStmt(context.Background(), idInsert, types.MakeDatums(1)) + c.Assert(err, IsNil) + ch <- nil + }() + time.Sleep(100 * time.Millisecond) + _, digest := parser.NormalizeDigest("insert into t values (1)") + info := tk.Se.TxnInfo() + c.Assert(info, NotNil) + c.Assert(info.StartTS, Greater, uint64(0)) + c.Assert(info.State, Equals, txninfo.TxnCommitting) + c.Assert(info.CurrentSQLDigest, Equals, digest.String()) + c.Assert(info.AllSQLDigests, DeepEquals, []string{digest.String()}) + + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/beforePrewrite"), IsNil) + <-ch + info = tk.Se.TxnInfo() + c.Assert(info, IsNil) + + // Test non-autocommit transaction + + id1, _, _, err := tk.Se.PrepareStmt("select * from t where a = ?") + c.Assert(err, IsNil) + _, digest1 := parser.NormalizeDigest("select * from t where a = ?") + id2, _, _, err := tk.Se.PrepareStmt("update t set a = a + 1 where a = ?") + c.Assert(err, IsNil) + _, digest2 := parser.NormalizeDigest("update t set a = a + 1 where a = ?") + + tk.MustExec("begin pessimistic") + + _, err = tk.Se.ExecutePreparedStmt(context.Background(), id1, types.MakeDatums(1)) + c.Assert(err, IsNil) + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/beforePessimisticLock", "pause"), IsNil) + go func() { + _, err := tk.Se.ExecutePreparedStmt(context.Background(), id2, types.MakeDatums(1)) + c.Assert(err, IsNil) + ch <- nil + }() + time.Sleep(100 * time.Millisecond) + info = tk.Se.TxnInfo() + c.Assert(info.StartTS, Greater, uint64(0)) + c.Assert(info.CurrentSQLDigest, Equals, digest2.String()) + c.Assert(info.State, Equals, txninfo.TxnLockWaiting) + c.Assert((*time.Time)(info.BlockStartTime), NotNil) + _, beginDigest := parser.NormalizeDigest("begin pessimistic") + c.Assert(info.AllSQLDigests, DeepEquals, []string{beginDigest.String(), digest1.String(), digest2.String()}) + + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/beforePessimisticLock"), IsNil) + <-ch + tk.MustExec("rollback") +} + +func (s *testSessionSuite) TestReadDMLBatchSize(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("set global tidb_dml_batch_size=1000") + se, err := session.CreateSession(s.store) + c.Assert(err, IsNil) + // `select 1` to load the global variables. + _, _ = se.Execute(context.TODO(), "select 1") + c.Assert(se.GetSessionVars().DMLBatchSize, Equals, 1000) +} + +func (s *testSessionSuite) TestInTxnPSProtoPointGet(c *C) { + ctx := context.Background() + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("create table t1(c1 int primary key, c2 int, c3 int)") + tk.MustExec("insert into t1 values(1, 10, 100)") + + // Generate the ps statement and make the prepared plan cached for point get. + id, _, _, err := tk.Se.PrepareStmt("select c1, c2 from t1 where c1 = ?") + c.Assert(err, IsNil) + idForUpdate, _, _, err := tk.Se.PrepareStmt("select c1, c2 from t1 where c1 = ? for update") + c.Assert(err, IsNil) + params := []types.Datum{types.NewDatum(1)} + rs, err := tk.Se.ExecutePreparedStmt(ctx, id, params) + c.Assert(err, IsNil) + tk.ResultSetToResult(rs, Commentf("%v", rs)).Check(testkit.Rows("1 10")) + rs, err = tk.Se.ExecutePreparedStmt(ctx, idForUpdate, params) + c.Assert(err, IsNil) + tk.ResultSetToResult(rs, Commentf("%v", rs)).Check(testkit.Rows("1 10")) + + // Query again the cached plan will be used. + rs, err = tk.Se.ExecutePreparedStmt(ctx, id, params) + c.Assert(err, IsNil) + tk.ResultSetToResult(rs, Commentf("%v", rs)).Check(testkit.Rows("1 10")) + rs, err = tk.Se.ExecutePreparedStmt(ctx, idForUpdate, params) + c.Assert(err, IsNil) + tk.ResultSetToResult(rs, Commentf("%v", rs)).Check(testkit.Rows("1 10")) + + // Start a transaction, now the in txn flag will be added to the session vars. + _, err = tk.Se.Execute(ctx, "start transaction") + c.Assert(err, IsNil) + rs, err = tk.Se.ExecutePreparedStmt(ctx, id, params) + c.Assert(err, IsNil) + tk.ResultSetToResult(rs, Commentf("%v", rs)).Check(testkit.Rows("1 10")) + txn, err := tk.Se.Txn(false) + c.Assert(err, IsNil) + c.Assert(txn.Valid(), IsTrue) + rs, err = tk.Se.ExecutePreparedStmt(ctx, idForUpdate, params) + c.Assert(err, IsNil) + tk.ResultSetToResult(rs, Commentf("%v", rs)).Check(testkit.Rows("1 10")) + txn, err = tk.Se.Txn(false) + c.Assert(err, IsNil) + c.Assert(txn.Valid(), IsTrue) + _, err = tk.Se.Execute(ctx, "update t1 set c2 = c2 + 1") + c.Assert(err, IsNil) + // Check the read result after in-transaction update. + rs, err = tk.Se.ExecutePreparedStmt(ctx, id, params) + c.Assert(err, IsNil) + tk.ResultSetToResult(rs, Commentf("%v", rs)).Check(testkit.Rows("1 11")) + rs, err = tk.Se.ExecutePreparedStmt(ctx, idForUpdate, params) + c.Assert(err, IsNil) + tk.ResultSetToResult(rs, Commentf("%v", rs)).Check(testkit.Rows("1 11")) + txn, err = tk.Se.Txn(false) + c.Assert(err, IsNil) + c.Assert(txn.Valid(), IsTrue) + tk.MustExec("commit") +} + +func (s *testSessionSuite) TestTMPTableSize(c *C) { + // Test the @@tmp_table_size system variable. + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("set tidb_enable_global_temporary_table=on") + tk.MustExec("create global temporary table t (c1 int, c2 varchar(512)) on commit delete rows") + + tk.MustQuery("select @@global.tmp_table_size").Check(testkit.Rows(strconv.Itoa(variable.DefTMPTableSize))) + c.Assert(tk.Se.GetSessionVars().TMPTableSize, Equals, int64(variable.DefTMPTableSize)) + + // Min value 1024, so the result is change to 1024, with a warning. + tk.MustExec("set @@global.tmp_table_size = 123") + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1292 Truncated incorrect tmp_table_size value: '123'")) + + // Change the session scope value. + tk.MustExec("set @@session.tmp_table_size = 2097152") + c.Assert(tk.Se.GetSessionVars().TMPTableSize, Equals, int64(2097152)) + + // Check in another sessin, change session scope value does not affect the global scope. + tk1 := testkit.NewTestKit(c, s.store) + tk1.MustQuery("select @@global.tmp_table_size").Check(testkit.Rows("1024")) + + // The value is now 1024, check the error when table size exceed it. + tk.MustExec("set @@session.tmp_table_size = 1024") + tk.MustExec("begin") + tk.MustExec("insert into t values (1, repeat('x', 512))") + tk.MustExec("insert into t values (1, repeat('x', 512))") + tk.MustGetErrCode("insert into t values (1, repeat('x', 512))", errno.ErrRecordFileFull) +} + +func (s *testSessionSuite) TestTiDBEnableGlobalTemporaryTable(c *C) { + // Test the @@tidb_enable_global_temporary_table system variable. + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + // variable 'tidb_enable_global_temporary_table' should not be seen when show variables + tk.MustQuery("show variables like 'tidb_enable_global_temporary_table'").Check(testkit.Rows()) + tk.MustQuery("show global variables like 'tidb_enable_global_temporary_table'").Check(testkit.Rows()) + + // variable 'tidb_enable_global_temporary_table' is turned off by default + tk.MustQuery("select @@global.tidb_enable_global_temporary_table").Check(testkit.Rows("0")) + tk.MustQuery("select @@tidb_enable_global_temporary_table").Check(testkit.Rows("0")) + c.Assert(tk.Se.GetSessionVars().EnableGlobalTemporaryTable, IsFalse) + + // cannot create global temporary table when 'tidb_enable_global_temporary_table' is off + tk.MustGetErrMsg( + "create global temporary table temp_test(id int primary key auto_increment) on commit delete rows", + "global temporary table is experimental and it is switched off by tidb_enable_global_temporary_table", + ) + tk.MustQuery("show tables like 'temp_test'").Check(testkit.Rows()) + + // you can create global temporary table when 'tidb_enable_global_temporary_table' is on + tk.MustExec("set tidb_enable_global_temporary_table=on") + tk.MustQuery("select @@tidb_enable_global_temporary_table").Check(testkit.Rows("1")) + c.Assert(tk.Se.GetSessionVars().EnableGlobalTemporaryTable, IsTrue) + tk.MustExec("create global temporary table temp_test(id int primary key auto_increment) on commit delete rows") + tk.MustQuery("show tables like 'temp_test'").Check(testkit.Rows("temp_test")) +} diff --git a/session/tidb.go b/session/tidb.go index 85732b457f7a6..583c5074e6805 100644 --- a/session/tidb.go +++ b/session/tidb.go @@ -239,7 +239,7 @@ func autoCommitAfterStmt(ctx context.Context, se *session, meetsErr error, sql s sessVars := se.sessionVars if meetsErr != nil { if !sessVars.InTxn() { - logutil.BgLogger().Info("rollbackTxn for ddl/autocommit failed") + logutil.BgLogger().Info("rollbackTxn called due to ddl/autocommit failure") se.RollbackTxn(ctx) recordAbortTxnDuration(sessVars) } else if se.txn.Valid() && se.txn.IsPessimistic() && executor.ErrDeadlock.Equal(meetsErr) { diff --git a/session/tidb_test.go b/session/tidb_test.go index 615f388ff1e2e..e351fa533a158 100644 --- a/session/tidb_test.go +++ b/session/tidb_test.go @@ -30,7 +30,6 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/store/mockstore" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/logutil" @@ -211,7 +210,7 @@ func (s *testMainSuite) TestKeysNeedLock(c *C) { for _, tt := range tests { c.Assert(keyNeedToLock(tt.key, tt.val, 0), Equals, tt.need) } - flag := tikvstore.KeyFlags(1) + flag := kv.KeyFlags(1) c.Assert(flag.HasPresumeKeyNotExists(), IsTrue) c.Assert(keyNeedToLock(indexKey, deleteVal, flag), IsTrue) } diff --git a/session/txn.go b/session/txn.go index eed4698f60a65..bb00265044ddf 100644 --- a/session/txn.go +++ b/session/txn.go @@ -20,6 +20,8 @@ import ( "runtime/trace" "strings" "sync/atomic" + "time" + "unsafe" "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" @@ -28,9 +30,10 @@ import ( "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/session/txninfo" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/binloginfo" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" + "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/logutil" @@ -39,12 +42,12 @@ import ( "go.uber.org/zap" ) -// TxnState wraps kv.Transaction to provide a new kv.Transaction. +// LazyTxn wraps kv.Transaction to provide a new kv.Transaction. // 1. It holds all statement related modification in the buffer before flush to the txn, // so if execute statement meets error, the txn won't be made dirty. // 2. It's a lazy transaction, that means it's a txnFuture before StartTS() is really need. -type TxnState struct { - // States of a TxnState should be one of the followings: +type LazyTxn struct { + // States of a LazyTxn should be one of the followings: // Invalid: kv.Transaction == nil && txnFuture == nil // Pending: kv.Transaction == nil && txnFuture != nil // Valid: kv.Transaction != nil && txnFuture == nil @@ -55,23 +58,34 @@ type TxnState struct { stagingHandle kv.StagingHandle mutations map[int64]*binlog.TableMutation writeSLI sli.TxnWriteThroughputSLI + + // following atomic fields are used for filling TxnInfo + // we need these fields because kv.Transaction provides no thread safety promise + // but we hope getting TxnInfo is a thread safe op + + // txnInfo provides information about the transaction in a thread-safe way. To atomically replace the struct, + // it's stored as an unsafe.Pointer. + txnInfo unsafe.Pointer } // GetTableInfo returns the cached index name. -func (txn *TxnState) GetTableInfo(id int64) *model.TableInfo { +func (txn *LazyTxn) GetTableInfo(id int64) *model.TableInfo { return txn.Transaction.GetTableInfo(id) } // CacheTableInfo caches the index name. -func (txn *TxnState) CacheTableInfo(id int64, info *model.TableInfo) { +func (txn *LazyTxn) CacheTableInfo(id int64, info *model.TableInfo) { txn.Transaction.CacheTableInfo(id, info) } -func (txn *TxnState) init() { +func (txn *LazyTxn) init() { txn.mutations = make(map[int64]*binlog.TableMutation) + txn.storeTxnInfo(&txninfo.TxnInfo{ + State: txninfo.TxnRunningNormal, + }) } -func (txn *TxnState) initStmtBuf() { +func (txn *LazyTxn) initStmtBuf() { if txn.Transaction == nil { return } @@ -81,14 +95,14 @@ func (txn *TxnState) initStmtBuf() { } // countHint is estimated count of mutations. -func (txn *TxnState) countHint() int { +func (txn *LazyTxn) countHint() int { if txn.stagingHandle == kv.InvalidStagingHandle { return 0 } return txn.Transaction.GetMemBuffer().Len() - txn.initCnt } -func (txn *TxnState) flushStmtBuf() { +func (txn *LazyTxn) flushStmtBuf() { if txn.stagingHandle == kv.InvalidStagingHandle { return } @@ -97,17 +111,48 @@ func (txn *TxnState) flushStmtBuf() { txn.initCnt = buf.Len() } -func (txn *TxnState) cleanupStmtBuf() { +func (txn *LazyTxn) cleanupStmtBuf() { if txn.stagingHandle == kv.InvalidStagingHandle { return } buf := txn.Transaction.GetMemBuffer() buf.Cleanup(txn.stagingHandle) txn.initCnt = buf.Len() + + txnInfo := txn.getTxnInfo() + atomic.StoreUint64(&txnInfo.EntriesCount, uint64(txn.Transaction.Len())) + atomic.StoreUint64(&txnInfo.EntriesSize, uint64(txn.Transaction.Size())) +} + +func (txn *LazyTxn) storeTxnInfo(info *txninfo.TxnInfo) { + atomic.StorePointer(&txn.txnInfo, unsafe.Pointer(info)) +} + +func (txn *LazyTxn) recreateTxnInfo( + startTS uint64, + state txninfo.TxnRunningState, + entriesCount, + entriesSize uint64, + currentSQLDigest string, + allSQLDigests []string, +) { + info := &txninfo.TxnInfo{ + StartTS: startTS, + State: state, + EntriesCount: entriesCount, + EntriesSize: entriesSize, + CurrentSQLDigest: currentSQLDigest, + AllSQLDigests: allSQLDigests, + } + txn.storeTxnInfo(info) +} + +func (txn *LazyTxn) getTxnInfo() *txninfo.TxnInfo { + return (*txninfo.TxnInfo)(atomic.LoadPointer(&txn.txnInfo)) } // Size implements the MemBuffer interface. -func (txn *TxnState) Size() int { +func (txn *LazyTxn) Size() int { if txn.Transaction == nil { return 0 } @@ -115,19 +160,19 @@ func (txn *TxnState) Size() int { } // Valid implements the kv.Transaction interface. -func (txn *TxnState) Valid() bool { +func (txn *LazyTxn) Valid() bool { return txn.Transaction != nil && txn.Transaction.Valid() } -func (txn *TxnState) pending() bool { +func (txn *LazyTxn) pending() bool { return txn.Transaction == nil && txn.txnFuture != nil } -func (txn *TxnState) validOrPending() bool { +func (txn *LazyTxn) validOrPending() bool { return txn.txnFuture != nil || txn.Valid() } -func (txn *TxnState) String() string { +func (txn *LazyTxn) String() string { if txn.Transaction != nil { return txn.Transaction.String() } @@ -138,7 +183,7 @@ func (txn *TxnState) String() string { } // GoString implements the "%#v" format for fmt.Printf. -func (txn *TxnState) GoString() string { +func (txn *LazyTxn) GoString() string { var s strings.Builder s.WriteString("Txn{") if txn.pending() { @@ -157,18 +202,25 @@ func (txn *TxnState) GoString() string { return s.String() } -func (txn *TxnState) changeInvalidToValid(kvTxn kv.Transaction) { +func (txn *LazyTxn) changeInvalidToValid(kvTxn kv.Transaction) { txn.Transaction = kvTxn txn.initStmtBuf() + txn.recreateTxnInfo( + kvTxn.StartTS(), + txninfo.TxnRunningNormal, + uint64(txn.Transaction.Len()), + uint64(txn.Transaction.Size()), + "", + nil) txn.txnFuture = nil } -func (txn *TxnState) changeInvalidToPending(future *txnFuture) { +func (txn *LazyTxn) changeInvalidToPending(future *txnFuture) { txn.Transaction = nil txn.txnFuture = future } -func (txn *TxnState) changePendingToValid(ctx context.Context) error { +func (txn *LazyTxn) changePendingToValid(ctx context.Context) error { if txn.txnFuture == nil { return errors.New("transaction future is not set") } @@ -184,16 +236,56 @@ func (txn *TxnState) changePendingToValid(ctx context.Context) error { } txn.Transaction = t txn.initStmtBuf() + + // The txnInfo may already recorded the first statement (usually "begin") when it's pending, so keep them. + txnInfo := txn.getTxnInfo() + txn.recreateTxnInfo( + t.StartTS(), + txninfo.TxnRunningNormal, + uint64(txn.Transaction.Len()), + uint64(txn.Transaction.Size()), + txnInfo.CurrentSQLDigest, + txnInfo.AllSQLDigests) return nil } -func (txn *TxnState) changeToInvalid() { +func (txn *LazyTxn) changeToInvalid() { if txn.stagingHandle != kv.InvalidStagingHandle { txn.Transaction.GetMemBuffer().Cleanup(txn.stagingHandle) } txn.stagingHandle = kv.InvalidStagingHandle txn.Transaction = nil txn.txnFuture = nil + + txn.recreateTxnInfo( + 0, + txninfo.TxnRunningNormal, + 0, + 0, + "", + nil) +} + +func (txn *LazyTxn) onStmtStart(currentSQLDigest string) { + if len(currentSQLDigest) == 0 { + return + } + + info := txn.getTxnInfo().ShallowClone() + info.CurrentSQLDigest = currentSQLDigest + // Keeps at most 50 history sqls to avoid consuming too much memory. + const maxTransactionStmtHistory int = 50 + if len(info.AllSQLDigests) < maxTransactionStmtHistory { + info.AllSQLDigests = append(info.AllSQLDigests, currentSQLDigest) + } + + txn.storeTxnInfo(info) +} + +func (txn *LazyTxn) onStmtEnd() { + info := txn.getTxnInfo().ShallowClone() + info.CurrentSQLDigest = "" + txn.storeTxnInfo(info) } var hasMockAutoIncIDRetry = int64(0) @@ -223,7 +315,7 @@ func ResetMockAutoRandIDRetryCount(failTimes int64) { } // Commit overrides the Transaction interface. -func (txn *TxnState) Commit(ctx context.Context) error { +func (txn *LazyTxn) Commit(ctx context.Context) error { defer txn.reset() if len(txn.mutations) != 0 || txn.countHint() != 0 { logutil.BgLogger().Error("the code should never run here", @@ -233,6 +325,10 @@ func (txn *TxnState) Commit(ctx context.Context) error { return errors.Trace(kv.ErrInvalidTxn) } + atomic.StoreInt32(&txn.getTxnInfo().State, txninfo.TxnCommitting) + + failpoint.Inject("mockSlowCommit", func(_ failpoint.Value) {}) + // mockCommitError8942 is used for PR #8942. failpoint.Inject("mockCommitError8942", func(val failpoint.Value) { if val.(bool) { @@ -259,17 +355,34 @@ func (txn *TxnState) Commit(ctx context.Context) error { } // Rollback overrides the Transaction interface. -func (txn *TxnState) Rollback() error { +func (txn *LazyTxn) Rollback() error { defer txn.reset() + atomic.StoreInt32(&txn.getTxnInfo().State, txninfo.TxnRollingBack) + // mockSlowRollback is used to mock a rollback which takes a long time + failpoint.Inject("mockSlowRollback", func(_ failpoint.Value) {}) return txn.Transaction.Rollback() } -func (txn *TxnState) reset() { +// LockKeys Wrap the inner transaction's `LockKeys` to record the status +func (txn *LazyTxn) LockKeys(ctx context.Context, lockCtx *kv.LockCtx, keys ...kv.Key) error { + txnInfo := txn.getTxnInfo() + originState := atomic.SwapInt32(&txnInfo.State, txninfo.TxnLockWaiting) + t := time.Now() + atomic.StorePointer(&txnInfo.BlockStartTime, unsafe.Pointer(&t)) + err := txn.Transaction.LockKeys(ctx, lockCtx, keys...) + atomic.StorePointer(&txnInfo.BlockStartTime, unsafe.Pointer(nil)) + atomic.StoreInt32(&txnInfo.State, originState) + atomic.StoreUint64(&txnInfo.EntriesCount, uint64(txn.Transaction.Len())) + atomic.StoreUint64(&txnInfo.EntriesSize, uint64(txn.Transaction.Size())) + return err +} + +func (txn *LazyTxn) reset() { txn.cleanup() txn.changeToInvalid() } -func (txn *TxnState) cleanup() { +func (txn *LazyTxn) cleanup() { txn.cleanupStmtBuf() txn.initStmtBuf() for key := range txn.mutations { @@ -278,13 +391,13 @@ func (txn *TxnState) cleanup() { } // KeysNeedToLock returns the keys need to be locked. -func (txn *TxnState) KeysNeedToLock() ([]kv.Key, error) { +func (txn *LazyTxn) KeysNeedToLock() ([]kv.Key, error) { if txn.stagingHandle == kv.InvalidStagingHandle { return nil, nil } keys := make([]kv.Key, 0, txn.countHint()) buf := txn.Transaction.GetMemBuffer() - buf.InspectStage(txn.stagingHandle, func(k kv.Key, flags tikvstore.KeyFlags, v []byte) { + buf.InspectStage(txn.stagingHandle, func(k kv.Key, flags kv.KeyFlags, v []byte) { if !keyNeedToLock(k, v, flags) { return } @@ -293,7 +406,7 @@ func (txn *TxnState) KeysNeedToLock() ([]kv.Key, error) { return keys, nil } -func keyNeedToLock(k, v []byte, flags tikvstore.KeyFlags) bool { +func keyNeedToLock(k, v []byte, flags kv.KeyFlags) bool { isTableKey := bytes.HasPrefix(k, tablecodec.TablePrefix()) if !isTableKey { // meta key always need to lock. @@ -316,6 +429,27 @@ func keyNeedToLock(k, v []byte, flags tikvstore.KeyFlags) bool { return !isNonUniqueIndex } +// Info dump the TxnState to Datum for displaying in `TIDB_TRX` +// This function is supposed to be thread safe +func (txn *LazyTxn) Info() *txninfo.TxnInfo { + info := txn.getTxnInfo().ShallowClone() + if info.StartTS == 0 { + return nil + } + return info +} + +// UpdateEntriesCountAndSize updates the EntriesCount and EntriesSize +// Note this function is not thread safe, because +// txn.Transaction can be changed during this function's execution if running parallel. +func (txn *LazyTxn) UpdateEntriesCountAndSize() { + if txn.Valid() { + txnInfo := txn.getTxnInfo() + atomic.StoreUint64(&txnInfo.EntriesCount, uint64(txn.Transaction.Len())) + atomic.StoreUint64(&txnInfo.EntriesSize, uint64(txn.Transaction.Size())) + } +} + func getBinlogMutation(ctx sessionctx.Context, tableID int64) *binlog.TableMutation { bin := binloginfo.GetPrewriteValue(ctx, true) for i := range bin.Mutations { @@ -353,14 +487,14 @@ type txnFuture struct { func (tf *txnFuture) wait() (kv.Transaction, error) { startTS, err := tf.future.Wait() if err == nil { - return tf.store.BeginWithOption(kv.TransactionOption{}.SetTxnScope(tf.txnScope).SetStartTs(startTS)) + return tf.store.BeginWithOption(tikv.DefaultStartTSOption().SetTxnScope(tf.txnScope).SetStartTS(startTS)) } else if config.GetGlobalConfig().Store == "unistore" { return nil, err } logutil.BgLogger().Warn("wait tso failed", zap.Error(err)) // It would retry get timestamp. - return tf.store.BeginWithOption(kv.TransactionOption{}.SetTxnScope(tf.txnScope)) + return tf.store.BeginWithOption(tikv.DefaultStartTSOption().SetTxnScope(tf.txnScope)) } func (s *session) getTxnFuture(ctx context.Context) *txnFuture { diff --git a/session/txninfo/txn_info.go b/session/txninfo/txn_info.go new file mode 100644 index 0000000000000..e7f5afc319c6b --- /dev/null +++ b/session/txninfo/txn_info.go @@ -0,0 +1,138 @@ +// Copyright 2021 PingCAP, Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package txninfo + +import ( + "strings" + "sync/atomic" + "time" + "unsafe" + + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/store/tikv/oracle" + "github.com/pingcap/tidb/types" +) + +// TxnRunningState is the current state of a transaction +type TxnRunningState = int32 + +const ( + // TxnRunningNormal means the transaction is running normally + TxnRunningNormal TxnRunningState = iota + // TxnLockWaiting means the transaction is blocked on a lock + TxnLockWaiting + // TxnCommitting means the transaction is (at least trying to) committing + TxnCommitting + // TxnRollingBack means the transaction is rolling back + TxnRollingBack +) + +// TxnRunningStateStrs is the names of the TxnRunningStates +var TxnRunningStateStrs = []string{ + "Normal", "LockWaiting", "Committing", "RollingBack", +} + +// TxnInfo is information about a running transaction +// This is supposed to be the datasource of `TIDB_TRX` in infoschema +type TxnInfo struct { + // The following fields are immutable and can be safely read across threads. + + StartTS uint64 + // Digest of SQL currently running + CurrentSQLDigest string + // Digests of all SQLs executed in the transaction. + AllSQLDigests []string + + // The following fields are mutable and needs to be read or written by atomic operations. But since only the + // transaction's thread can modify its value, it's ok for the transaction's thread to read it without atomic + // operations. + + // Current execution state of the transaction. + State TxnRunningState + // Last trying to block start time. Invalid if State is not TxnLockWaiting. It's an unsafe pointer to time.Time or nil. + BlockStartTime unsafe.Pointer + // How many entries are in MemDB + EntriesCount uint64 + // MemDB used memory + EntriesSize uint64 + + // The following fields will be filled in `session` instead of `LazyTxn` + + // Which session this transaction belongs to + ConnectionID uint64 + // The user who open this session + Username string + // The schema this transaction works on + CurrentDB string +} + +// ShallowClone shallow clones the TxnInfo. It's safe to call concurrently with the transaction. +// Note that this function doesn't do deep copy and some fields of the result may be unsafe to write. Use it at your own +// risk. +func (info *TxnInfo) ShallowClone() *TxnInfo { + return &TxnInfo{ + StartTS: info.StartTS, + CurrentSQLDigest: info.CurrentSQLDigest, + AllSQLDigests: info.AllSQLDigests, + State: atomic.LoadInt32(&info.State), + BlockStartTime: atomic.LoadPointer(&info.BlockStartTime), + EntriesCount: atomic.LoadUint64(&info.EntriesCount), + EntriesSize: atomic.LoadUint64(&info.EntriesSize), + ConnectionID: info.ConnectionID, + Username: info.Username, + CurrentDB: info.CurrentDB, + } +} + +// ToDatum Converts the `TxnInfo` to `Datum` to show in the `TIDB_TRX` table. +func (info *TxnInfo) ToDatum() []types.Datum { + humanReadableStartTime := time.Unix(0, oracle.ExtractPhysical(info.StartTS)*1e6) + + var currentDigest interface{} + if len(info.CurrentSQLDigest) != 0 { + currentDigest = info.CurrentSQLDigest + } + + var blockStartTime interface{} + if t := (*time.Time)(atomic.LoadPointer(&info.BlockStartTime)); t == nil { + blockStartTime = nil + } else { + blockStartTime = types.NewTime(types.FromGoTime(*t), mysql.TypeTimestamp, types.MaxFsp) + } + + e, err := types.ParseEnumValue(TxnRunningStateStrs, uint64(info.State+1)) + if err != nil { + panic("this should never happen") + } + + allSQLs := "[" + strings.Join(info.AllSQLDigests, ", ") + "]" + + state := types.NewMysqlEnumDatum(e) + + datums := types.MakeDatums( + info.StartTS, + types.NewTime(types.FromGoTime(humanReadableStartTime), mysql.TypeTimestamp, types.MaxFsp), + currentDigest, + ) + datums = append(datums, state) + datums = append(datums, types.MakeDatums( + blockStartTime, + info.EntriesCount, + info.EntriesSize, + info.ConnectionID, + info.Username, + info.CurrentDB, + allSQLs)...) + return datums +} diff --git a/sessionctx/binloginfo/binloginfo.go b/sessionctx/binloginfo/binloginfo.go index 044e6cdc11df9..a2d1047e93c65 100644 --- a/sessionctx/binloginfo/binloginfo.go +++ b/sessionctx/binloginfo/binloginfo.go @@ -29,7 +29,6 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/sessionctx" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" driver "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tipb/go-binlog" @@ -101,7 +100,7 @@ func GetPrewriteValue(ctx sessionctx.Context, createIfNotExists bool) *binlog.Pr vars := ctx.GetSessionVars() v, ok := vars.TxnCtx.Binlog.(*binlog.PrewriteValue) if !ok && createIfNotExists { - schemaVer := ctx.GetSessionVars().TxnCtx.SchemaVersion + schemaVer := ctx.GetInfoSchema().SchemaMetaVersion() v = &binlog.PrewriteValue{SchemaVersion: schemaVer} vars.TxnCtx.Binlog = v } @@ -155,17 +154,15 @@ func WaitBinlogRecover(timeout time.Duration) error { defer ticker.Stop() start := time.Now() for { - select { - case <-ticker.C: - if atomic.LoadInt32(&skippedCommitterCounter) == 0 { - logutil.BgLogger().Warn("[binloginfo] binlog recovered") - return nil - } - if time.Since(start) > timeout { - logutil.BgLogger().Warn("[binloginfo] waiting for binlog recovering timed out", - zap.Duration("duration", timeout)) - return errors.New("timeout") - } + <-ticker.C + if atomic.LoadInt32(&skippedCommitterCounter) == 0 { + logutil.BgLogger().Warn("[binloginfo] binlog recovered") + return nil + } + if time.Since(start) > timeout { + logutil.BgLogger().Warn("[binloginfo] waiting for binlog recovering timed out", + zap.Duration("duration", timeout)) + return errors.New("timeout") } } } @@ -295,7 +292,7 @@ func SetDDLBinlog(client *pumpcli.PumpsClient, txn kv.Transaction, jobID int64, }, Client: client, } - txn.SetOption(tikvstore.BinlogInfo, info) + txn.SetOption(kv.BinlogInfo, info) } const specialPrefix = `/*T! ` diff --git a/sessionctx/binloginfo/binloginfo_test.go b/sessionctx/binloginfo/binloginfo_test.go index 2dfca57d73f4c..6a586615bd8a0 100644 --- a/sessionctx/binloginfo/binloginfo_test.go +++ b/sessionctx/binloginfo/binloginfo_test.go @@ -698,3 +698,37 @@ func testGetTableByName(c *C, ctx sessionctx.Context, db, table string) table.Ta c.Assert(err, IsNil) return tbl } + +func (s *testBinlogSuite) TestTempTableBinlog(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.Se.GetSessionVars().BinlogClient = s.client + tk.MustExec("begin") + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("drop table if exists temp_table") + ddlQuery := "create global temporary table temp_table(id int) on commit delete rows" + tk.MustExec(ddlQuery) + ok := mustGetDDLBinlog(s, ddlQuery, c) + c.Assert(ok, IsTrue) + + tk.MustExec("insert temp_table value(1)") + tk.MustExec("update temp_table set id=id+1") + tk.MustExec("commit") + prewriteVal := getLatestBinlogPrewriteValue(c, s.pump) + c.Assert(len(prewriteVal.Mutations), Equals, 0) + + tk.MustExec("begin") + tk.MustExec("delete from temp_table") + tk.MustExec("commit") + prewriteVal = getLatestBinlogPrewriteValue(c, s.pump) + c.Assert(len(prewriteVal.Mutations), Equals, 0) + + ddlQuery = "truncate table temp_table" + tk.MustExec(ddlQuery) + ok = mustGetDDLBinlog(s, ddlQuery, c) + c.Assert(ok, IsTrue) + + ddlQuery = "drop table if exists temp_table" + tk.MustExec(ddlQuery) + ok = mustGetDDLBinlog(s, ddlQuery, c) + c.Assert(ok, IsTrue) +} diff --git a/sessionctx/context.go b/sessionctx/context.go index 2aeda663a038d..bd568c7e85f4b 100644 --- a/sessionctx/context.go +++ b/sessionctx/context.go @@ -17,7 +17,6 @@ import ( "context" "fmt" - "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/owner" @@ -28,12 +27,22 @@ import ( "github.com/pingcap/tipb/go-binlog" ) +// InfoschemaMetaVersion is a workaround. Due to circular dependency, +// can not return the complete interface. But SchemaMetaVersion is widely used for logging. +// So we give a convenience for that. +// FIXME: remove this interface +type InfoschemaMetaVersion interface { + SchemaMetaVersion() int64 +} + // Context is an interface for transaction and executive args environment. type Context interface { // NewTxn creates a new transaction for further execution. // If old transaction is valid, it is committed first. // It's used in BEGIN statement and DDL statements to commit old transaction. NewTxn(context.Context) error + // NewStaleTxnWithStartTS initializes a staleness transaction with the given StartTS. + NewStaleTxnWithStartTS(ctx context.Context, startTS uint64) error // Txn returns the current transaction which is created before executing a statement. // The returned kv.Transaction is not nil, but it maybe pending or invalid. @@ -56,6 +65,8 @@ type Context interface { // ClearValue clears the value associated with this context for key. ClearValue(key fmt.Stringer) + GetInfoSchema() InfoschemaMetaVersion + GetSessionVars() *variable.SessionVars GetSessionManager() util.SessionManager @@ -73,9 +84,6 @@ type Context interface { // It should be called right before we builds an executor. InitTxnWithStartTS(startTS uint64) error - // NewTxnWithStalenessOption initializes a transaction with StalenessTxnOption - NewTxnWithStalenessOption(ctx context.Context, option StalenessTxnOption) error - // GetStore returns the store of session. GetStore() kv.Storage @@ -141,10 +149,3 @@ const ( // LastExecuteDDL is the key for whether the session execute a ddl command last time. LastExecuteDDL basicCtxType = 3 ) - -// StalenessTxnOption represents available options for the InitTxnWithStaleness -type StalenessTxnOption struct { - Mode ast.TimestampBoundMode - PrevSec uint64 - StartTS uint64 -} diff --git a/sessionctx/stmtctx/stmtctx.go b/sessionctx/stmtctx/stmtctx.go index 8df0001427173..23fc0f52664f6 100644 --- a/sessionctx/stmtctx/stmtctx.go +++ b/sessionctx/stmtctx/stmtctx.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tidb/util/disk" "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/memory" + "github.com/pingcap/tidb/util/resourcegrouptag" atomic2 "go.uber.org/atomic" "go.uber.org/zap" ) @@ -140,18 +141,16 @@ type StatementContext struct { RuntimeStatsColl *execdetails.RuntimeStatsColl TableIDs []int64 IndexNames []string - nowTs time.Time // use this variable for now/current_timestamp calculation/cache for one stmt - stmtTimeCached bool StmtType string OriginalSQL string digestMemo struct { sync.Once normalized string - digest string + digest *parser.Digest } // planNormalized use for cache the normalized plan, avoid duplicate builds. planNormalized string - planDigest string + planDigest *parser.Digest encodedPlan string planHint string planHintSet bool @@ -164,6 +163,14 @@ type StatementContext struct { TblInfo2UnionScan map[*model.TableInfo]bool TaskID uint64 // unique ID for an execution of a statement TaskMapBakTS uint64 // counter for + + // stmtCache is used to store some statement-related values. + stmtCache map[StmtCacheKey]interface{} + // resourceGroupTag cache for the current statement resource group tag. + resourceGroupTag atomic.Value + // Map to store all CTE storages of current SQL. + // Will clean up at the end of the execution. + CTEStorageMap interface{} } // StmtHints are SessionVars related sql hints. @@ -195,24 +202,40 @@ func (sh *StmtHints) TaskMapNeedBackUp() bool { return sh.ForceNthPlan != -1 } -// GetNowTsCached getter for nowTs, if not set get now time and cache it -func (sc *StatementContext) GetNowTsCached() time.Time { - if !sc.stmtTimeCached { - now := time.Now() - sc.nowTs = now - sc.stmtTimeCached = true +// StmtCacheKey represents the key type in the StmtCache. +type StmtCacheKey int + +const ( + // StmtNowTsCacheKey is a variable for now/current_timestamp calculation/cache of one stmt. + StmtNowTsCacheKey StmtCacheKey = iota + // StmtSafeTSCacheKey is a variable for safeTS calculation/cache of one stmt. + StmtSafeTSCacheKey +) + +// GetOrStoreStmtCache gets the cached value of the given key if it exists, otherwise stores the value. +func (sc *StatementContext) GetOrStoreStmtCache(key StmtCacheKey, value interface{}) interface{} { + if sc.stmtCache == nil { + sc.stmtCache = make(map[StmtCacheKey]interface{}) + } + if _, ok := sc.stmtCache[key]; !ok { + sc.stmtCache[key] = value } - return sc.nowTs + return sc.stmtCache[key] +} + +// ResetInStmtCache resets the cache of given key. +func (sc *StatementContext) ResetInStmtCache(key StmtCacheKey) { + delete(sc.stmtCache, key) } -// ResetNowTs resetter for nowTs, clear cached time flag -func (sc *StatementContext) ResetNowTs() { - sc.stmtTimeCached = false +// ResetStmtCache resets all cached values. +func (sc *StatementContext) ResetStmtCache() { + sc.stmtCache = make(map[StmtCacheKey]interface{}) } // SQLDigest gets normalized and digest for provided sql. // it will cache result after first calling. -func (sc *StatementContext) SQLDigest() (normalized, sqlDigest string) { +func (sc *StatementContext) SQLDigest() (normalized string, sqlDigest *parser.Digest) { sc.digestMemo.Do(func() { sc.digestMemo.normalized, sc.digestMemo.digest = parser.NormalizeDigest(sc.OriginalSQL) }) @@ -220,20 +243,37 @@ func (sc *StatementContext) SQLDigest() (normalized, sqlDigest string) { } // InitSQLDigest sets the normalized and digest for sql. -func (sc *StatementContext) InitSQLDigest(normalized, digest string) { +func (sc *StatementContext) InitSQLDigest(normalized string, digest *parser.Digest) { sc.digestMemo.Do(func() { sc.digestMemo.normalized, sc.digestMemo.digest = normalized, digest }) } // GetPlanDigest gets the normalized plan and plan digest. -func (sc *StatementContext) GetPlanDigest() (normalized, planDigest string) { +func (sc *StatementContext) GetPlanDigest() (normalized string, planDigest *parser.Digest) { return sc.planNormalized, sc.planDigest } +// GetResourceGroupTag gets the resource group of the statement. +func (sc *StatementContext) GetResourceGroupTag() []byte { + tag, _ := sc.resourceGroupTag.Load().([]byte) + if len(tag) > 0 { + return tag + } + normalized, sqlDigest := sc.SQLDigest() + if len(normalized) == 0 { + return nil + } + tag = resourcegrouptag.EncodeResourceGroupTag(sqlDigest, sc.planDigest) + sc.resourceGroupTag.Store(tag) + return tag +} + // SetPlanDigest sets the normalized plan and plan digest. -func (sc *StatementContext) SetPlanDigest(normalized, planDigest string) { - sc.planNormalized, sc.planDigest = normalized, planDigest +func (sc *StatementContext) SetPlanDigest(normalized string, planDigest *parser.Digest) { + if planDigest != nil { + sc.planNormalized, sc.planDigest = normalized, planDigest + } } // GetEncodedPlan gets the encoded plan, it is used to avoid repeated encode. diff --git a/sessionctx/variable/noop.go b/sessionctx/variable/noop.go index a3382fc1159d1..fe3af2d33605d 100644 --- a/sessionctx/variable/noop.go +++ b/sessionctx/variable/noop.go @@ -38,7 +38,7 @@ var noopSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: SuperReadOnly, Value: Off, Type: TypeBool, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkReadOnly(vars, normalizedValue, originalValue, scope, false) }}, - {Scope: ScopeGlobal, Name: serverReadOnly, Value: Off, Type: TypeBool, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal, Name: ReadOnly, Value: Off, Type: TypeBool, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkReadOnly(vars, normalizedValue, originalValue, scope, false) }}, {Scope: ScopeGlobal, Name: ConnectTimeout, Value: "10", Type: TypeUnsigned, MinValue: 2, MaxValue: secondsPerYear, AutoConvertOutOfRange: true}, @@ -76,14 +76,13 @@ var noopSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: "log_backward_compatible_user_definitions", Value: ""}, {Scope: ScopeNone, Name: "lc_messages_dir", Value: "/usr/local/mysql-5.6.25-osx10.8-x86_64/share/"}, {Scope: ScopeGlobal, Name: "ft_boolean_syntax", Value: "+ -><()~*:\"\"&|"}, - {Scope: ScopeGlobal, Name: TableDefinitionCache, Value: "-1", Type: TypeUnsigned, MinValue: 400, MaxValue: 524288, AutoConvertOutOfRange: true}, + {Scope: ScopeGlobal, Name: TableDefinitionCache, Value: "2000", Type: TypeUnsigned, MinValue: 400, MaxValue: 524288, AutoConvertOutOfRange: true}, {Scope: ScopeNone, Name: SkipNameResolve, Value: Off, Type: TypeBool}, {Scope: ScopeNone, Name: "performance_schema_max_file_handles", Value: "32768"}, {Scope: ScopeSession, Name: "transaction_allow_batching", Value: ""}, {Scope: ScopeNone, Name: "performance_schema_max_statement_classes", Value: "168"}, {Scope: ScopeGlobal, Name: "server_id", Value: "0"}, {Scope: ScopeGlobal, Name: "innodb_flushing_avg_loops", Value: "30"}, - {Scope: ScopeGlobal | ScopeSession, Name: TmpTableSize, Value: "16777216", Type: TypeUnsigned, MinValue: 1024, MaxValue: math.MaxUint64, AutoConvertOutOfRange: true, IsHintUpdatable: true}, {Scope: ScopeGlobal, Name: "innodb_max_purge_lag", Value: "0"}, {Scope: ScopeGlobal | ScopeSession, Name: "preload_buffer_size", Value: "32768"}, {Scope: ScopeGlobal, Name: CheckProxyUsers, Value: Off, Type: TypeBool}, @@ -141,7 +140,7 @@ var noopSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: "innodb_file_format_max", Value: "Antelope"}, {Scope: ScopeGlobal | ScopeSession, Name: "debug", Value: ""}, {Scope: ScopeGlobal, Name: "log_warnings", Value: "1"}, - {Scope: ScopeGlobal | ScopeSession, Name: InnodbStrictMode, Value: "1", Type: TypeBool, AutoConvertNegativeBool: true}, + {Scope: ScopeGlobal | ScopeSession, Name: InnodbStrictMode, Value: On, Type: TypeBool, AutoConvertNegativeBool: true}, {Scope: ScopeGlobal, Name: "innodb_rollback_segments", Value: "128"}, {Scope: ScopeGlobal | ScopeSession, Name: "join_buffer_size", Value: "262144", IsHintUpdatable: true}, {Scope: ScopeNone, Name: "innodb_mirrored_log_groups", Value: "1"}, @@ -156,7 +155,7 @@ var noopSysVars = []*SysVar{ {Scope: ScopeNone, Name: "myisam_mmap_size", Value: "18446744073709551615"}, {Scope: ScopeNone, Name: "innodb_buffer_pool_instances", Value: "8"}, {Scope: ScopeGlobal | ScopeSession, Name: "max_length_for_sort_data", Value: "1024", IsHintUpdatable: true}, - {Scope: ScopeNone, Name: "character_set_system", Value: "utf8"}, + {Scope: ScopeNone, Name: CharacterSetSystem, Value: "utf8"}, {Scope: ScopeGlobal, Name: InnodbOptimizeFullTextOnly, Value: "0"}, {Scope: ScopeNone, Name: "character_sets_dir", Value: "/usr/local/mysql-5.6.25-osx10.8-x86_64/share/charsets/"}, {Scope: ScopeGlobal | ScopeSession, Name: QueryCacheType, Value: Off, Type: TypeEnum, PossibleValues: []string{Off, On, "DEMAND"}}, @@ -205,8 +204,6 @@ var noopSysVars = []*SysVar{ {Scope: ScopeGlobal | ScopeSession, Name: "sort_buffer_size", Value: "262144", IsHintUpdatable: true}, {Scope: ScopeGlobal, Name: "innodb_flush_neighbors", Value: "1"}, {Scope: ScopeNone, Name: "innodb_use_sys_malloc", Value: "1"}, - {Scope: ScopeSession, Name: PluginLoad, Value: ""}, - {Scope: ScopeSession, Name: PluginDir, Value: "/data/deploy/plugin"}, {Scope: ScopeNone, Name: "performance_schema_max_socket_classes", Value: "10"}, {Scope: ScopeNone, Name: "performance_schema_max_stage_classes", Value: "150"}, {Scope: ScopeGlobal, Name: "innodb_purge_batch_size", Value: "300"}, @@ -312,7 +309,7 @@ var noopSysVars = []*SysVar{ {Scope: ScopeNone, Name: "datetime_format", Value: "%Y-%m-%d %H:%i:%s"}, {Scope: ScopeGlobal, Name: "log_syslog", Value: ""}, {Scope: ScopeGlobal | ScopeSession, Name: "transaction_alloc_block_size", Value: "8192"}, - {Scope: ScopeGlobal, Name: "innodb_large_prefix", Type: TypeBool, Value: Off}, + {Scope: ScopeGlobal, Name: "innodb_large_prefix", Type: TypeBool, Value: On}, {Scope: ScopeNone, Name: "performance_schema_max_cond_classes", Value: "80"}, {Scope: ScopeGlobal, Name: "innodb_io_capacity", Value: "200"}, {Scope: ScopeGlobal, Name: "max_binlog_cache_size", Value: "18446744073709547520"}, @@ -475,7 +472,7 @@ var noopSysVars = []*SysVar{ // for compatibility purpose, we should leave them alone. // TODO: Follow the Terminology Updates of MySQL after their changes arrived. // https://mysqlhighavailability.com/mysql-terminology-updates/ - {Scope: ScopeSession, Name: PseudoSlaveMode, Value: "", Type: TypeInt}, + {Scope: ScopeSession, Name: PseudoSlaveMode, Value: Off, Type: TypeBool}, {Scope: ScopeGlobal, Name: "slave_pending_jobs_size_max", Value: "16777216"}, {Scope: ScopeGlobal, Name: "slave_transaction_retries", Value: "10"}, {Scope: ScopeGlobal, Name: "slave_checkpoint_period", Value: "300"}, @@ -493,7 +490,7 @@ var noopSysVars = []*SysVar{ {Scope: ScopeNone, Name: "slave_load_tmpdir", Value: "/var/tmp/"}, {Scope: ScopeGlobal, Name: "slave_parallel_type", Value: ""}, {Scope: ScopeGlobal, Name: "slave_parallel_workers", Value: "0"}, - {Scope: ScopeGlobal, Name: "rpl_semi_sync_master_timeout", Value: "10000", Type: TypeInt}, + {Scope: ScopeGlobal, Name: "rpl_semi_sync_master_timeout", Value: "10000", Type: TypeInt, MaxValue: math.MaxInt64}, {Scope: ScopeNone, Name: "slave_skip_errors", Value: Off}, {Scope: ScopeGlobal, Name: "sql_slave_skip_counter", Value: "0"}, {Scope: ScopeGlobal, Name: "rpl_semi_sync_slave_enabled", Value: Off, Type: TypeBool}, diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 1554f3c429d65..772882153a134 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -28,10 +28,10 @@ import ( "sync/atomic" "time" - "github.com/klauspost/cpuid" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/auth" + "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" pumpcli "github.com/pingcap/tidb-tools/tidb-binlog/pump_client" @@ -48,6 +48,7 @@ import ( "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/rowcodec" "github.com/pingcap/tidb/util/stringutil" + "github.com/pingcap/tidb/util/tableutil" "github.com/pingcap/tidb/util/timeutil" "github.com/twmb/murmur3" atomic2 "go.uber.org/atomic" @@ -127,13 +128,12 @@ func (r *retryInfoAutoIDs) getCurrent() (int64, bool) { // TransactionContext is used to store variables that has transaction scope. type TransactionContext struct { - forUpdateTS uint64 - stmtFuture oracle.Future - Binlog interface{} - InfoSchema interface{} - History interface{} - SchemaVersion int64 - StartTS uint64 + forUpdateTS uint64 + stmtFuture oracle.Future + Binlog interface{} + InfoSchema interface{} + History interface{} + StartTS uint64 // ShardStep indicates the max size of continuous rowid shard in one transaction. ShardStep int @@ -173,7 +173,9 @@ type TransactionContext struct { // TableDeltaMap lock to prevent potential data race tdmLock sync.Mutex - GlobalTemporaryTables map[int64]struct{} + // GlobalTemporaryTables is used to store transaction-specific information for global temporary tables. + // It can also be stored in sessionCtx with local temporary tables, but it's easier to clean this data after transaction ends. + GlobalTemporaryTables map[int64]tableutil.TempTable } // GetShard returns the shard prefix for the next `count` rowids. @@ -460,6 +462,9 @@ type SessionVars struct { // SnapshotTS is used for reading history data. For simplicity, SnapshotTS only supports distsql request. SnapshotTS uint64 + // TxnReadTS is used for staleness transaction, it provides next staleness transaction startTS. + TxnReadTS *TxnReadTS + // SnapshotInfoschema is used with SnapshotTS, when the schema version at snapshotTS less than current schema // version, we load an old version schema for query. SnapshotInfoschema interface{} @@ -481,6 +486,15 @@ type SessionVars struct { // AllowBCJ means allow broadcast join. AllowBCJ bool + + // AllowCartesianBCJ means allow broadcast CARTESIAN join, 0 means not allow, 1 means allow broadcast CARTESIAN join + // but the table size should under the broadcast threshold, 2 means allow broadcast CARTESIAN join even if the table + // size exceeds the broadcast threshold + AllowCartesianBCJ int + + // MPPOuterJoinFixedBuildSide means in MPP plan, always use right(left) table as build side for left(right) out join + MPPOuterJoinFixedBuildSide bool + // AllowDistinctAggPushDown can be set true to allow agg with distinct push down to tikv/tiflash. AllowDistinctAggPushDown bool @@ -491,11 +505,12 @@ type SessionVars struct { AllowWriteRowID bool // AllowBatchCop means if we should send batch coprocessor to TiFlash. Default value is 1, means to use batch cop in case of aggregation and join. - // If value is set to 2 , which means to force to send batch cop for any query. Value is set to 0 means never use batch cop. + // Value set to 2 means to force to send batch cop for any query. Value set to 0 means never use batch cop. AllowBatchCop int - // AllowMPPExecution will prefer using mpp way to execute a query. - AllowMPPExecution bool + // AllowMPPExecution means if we should use mpp way to execute query. Default value is "ON", means to be determined by the optimizer. + // Value set to "ENFORCE" means to use mpp whenever possible. Value set to means never use mpp. + allowMPPExecution string // TiDBAllowAutoRandExplicitInsert indicates whether explicit insertion on auto_random column is allowed. AllowAutoRandExplicitInsert bool @@ -520,14 +535,14 @@ type SessionVars struct { CopCPUFactor float64 // CopTiFlashConcurrencyFactor is the concurrency number of computation in tiflash coprocessor. CopTiFlashConcurrencyFactor float64 - // NetworkFactor is the network cost of transferring 1 byte data. - NetworkFactor float64 + // networkFactor is the network cost of transferring 1 byte data. + networkFactor float64 // ScanFactor is the IO cost of scanning 1 byte data on TiKV and TiFlash. - ScanFactor float64 - // DescScanFactor is the IO cost of scanning 1 byte data on TiKV and TiFlash in desc order. - DescScanFactor float64 - // SeekFactor is the IO cost of seeking the start value of a range in TiKV or TiFlash. - SeekFactor float64 + scanFactor float64 + // descScanFactor is the IO cost of scanning 1 byte data on TiKV and TiFlash in desc order. + descScanFactor float64 + // seekFactor is the IO cost of seeking the start value of a range in TiKV or TiFlash. + seekFactor float64 // MemoryFactor is the memory cost of storing one tuple. MemoryFactor float64 // DiskFactor is the IO cost of reading/writing one byte to temporary disk. @@ -590,6 +605,9 @@ type SessionVars struct { // EnableWindowFunction enables the window function. EnableWindowFunction bool + // EnablePipelinedWindowExec enables executing window functions in a pipelined manner. + EnablePipelinedWindowExec bool + // EnableStrictDoubleTypeCheck enables table field double type check. EnableStrictDoubleTypeCheck bool @@ -599,9 +617,6 @@ type SessionVars struct { // DDLReorgPriority is the operation priority of adding indices. DDLReorgPriority int - // EnableChangeColumnType is used to control whether to enable the change column type. - EnableChangeColumnType bool - // EnableChangeMultiSchema is used to control whether to enable the multi schema change. EnableChangeMultiSchema bool @@ -626,13 +641,6 @@ type SessionVars struct { writeStmtBufs WriteStmtBufs - // L2CacheSize indicates the size of CPU L2 cache, using byte as unit. - L2CacheSize int - - // EnableRadixJoin indicates whether to use radix hash join to execute - // HashJoin. - EnableRadixJoin bool - // ConstraintCheckInPlace indicates whether to check the constraint when the SQL executing. ConstraintCheckInPlace bool @@ -717,7 +725,7 @@ type SessionVars struct { enableIndexMerge bool // replicaRead is used for reading data from replicas, only follower is supported at this time. - replicaRead tikvstore.ReplicaReadType + replicaRead kv.ReplicaReadType // IsolationReadEngines is used to isolation read, tidb only read from the stores whose engine type is in the engines. IsolationReadEngines map[kv.StoreType]struct{} @@ -794,7 +802,7 @@ type SessionVars struct { PartitionPruneMode atomic2.String // TxnScope indicates the scope of the transactions. It should be `global` or equal to `dc-location` in configuration. - TxnScope oracle.TxnScope + TxnScope kv.TxnScopeVar // EnabledRateLimitAction indicates whether enabled ratelimit action during coprocessor EnabledRateLimitAction bool @@ -824,12 +832,18 @@ type SessionVars struct { // Now we only support TiFlash. AllowFallbackToTiKV map[kv.StoreType]struct{} - // EnableDynamicPrivileges indicates whether to permit experimental support for MySQL 8.0 compatible dynamic privileges. - EnableDynamicPrivileges bool - // CTEMaxRecursionDepth indicates The common table expression (CTE) maximum recursion depth. // see https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_cte_max_recursion_depth CTEMaxRecursionDepth int + + // The temporary table size threshold + // In MySQL, when a temporary table exceed this size, it spills to disk. + // In TiDB, as we do not support spill to disk for now, an error is reported. + // See https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_tmp_table_size + TMPTableSize int64 + + // EnableGlobalTemporaryTable indicates whether to enable global temporary table + EnableGlobalTemporaryTable bool } // AllocMPPTaskID allocates task id for mpp tasks. It will reset the task id if the query's @@ -844,15 +858,25 @@ func (s *SessionVars) AllocMPPTaskID(startTS uint64) int64 { return 1 } +// IsMPPAllowed returns whether mpp execution is allowed. +func (s *SessionVars) IsMPPAllowed() bool { + return s.allowMPPExecution != "OFF" +} + +// IsMPPEnforced returns whether mpp execution is enforced. +func (s *SessionVars) IsMPPEnforced() bool { + return s.allowMPPExecution == "ENFORCE" +} + // CheckAndGetTxnScope will return the transaction scope we should use in the current session. func (s *SessionVars) CheckAndGetTxnScope() string { if s.InRestrictedSQL { - return oracle.GlobalTxnScope + return kv.GlobalTxnScope } - if s.TxnScope.GetVarValue() == oracle.LocalTxnScope { + if s.TxnScope.GetVarValue() == kv.LocalTxnScope { return s.TxnScope.GetTxnScope() } - return oracle.GlobalTxnScope + return kv.GlobalTxnScope } // UseDynamicPartitionPrune indicates whether use new dynamic partition prune. @@ -959,6 +983,8 @@ func NewSessionVars() *SessionVars { StmtCtx: new(stmtctx.StatementContext), AllowAggPushDown: false, AllowBCJ: false, + AllowCartesianBCJ: DefOptCartesianBCJ, + MPPOuterJoinFixedBuildSide: DefOptMPPOuterJoinFixedBuildSide, BroadcastJoinThresholdSize: DefBroadcastJoinThresholdSize, BroadcastJoinThresholdCount: DefBroadcastJoinThresholdSize, OptimizerSelectivityLevel: DefTiDBOptimizerSelectivityLevel, @@ -972,16 +998,14 @@ func NewSessionVars() *SessionVars { CPUFactor: DefOptCPUFactor, CopCPUFactor: DefOptCopCPUFactor, CopTiFlashConcurrencyFactor: DefOptTiFlashConcurrencyFactor, - NetworkFactor: DefOptNetworkFactor, - ScanFactor: DefOptScanFactor, - DescScanFactor: DefOptDescScanFactor, - SeekFactor: DefOptSeekFactor, + networkFactor: DefOptNetworkFactor, + scanFactor: DefOptScanFactor, + descScanFactor: DefOptDescScanFactor, + seekFactor: DefOptSeekFactor, MemoryFactor: DefOptMemoryFactor, DiskFactor: DefOptDiskFactor, ConcurrencyFactor: DefOptConcurrencyFactor, - EnableRadixJoin: false, EnableVectorizedExpression: DefEnableVectorizedExpression, - L2CacheSize: cpuid.CPU.Cache.L2, CommandValue: uint32(mysql.ComSleep), TiDBOptJoinReorderThreshold: DefTiDBOptJoinReorderThreshold, SlowQueryFile: config.GetGlobalConfig().Log.SlowQueryFile, @@ -989,7 +1013,7 @@ func NewSessionVars() *SessionVars { WaitSplitRegionTimeout: DefWaitSplitRegionTimeout, enableIndexMerge: false, EnableNoopFuncs: DefTiDBEnableNoopFuncs, - replicaRead: tikvstore.ReplicaReadLeader, + replicaRead: kv.ReplicaReadLeader, AllowRemoveAutoInc: DefTiDBAllowRemoveAutoInc, UsePlanBaselines: DefTiDBUsePlanBaselines, EvolvePlanBaselines: DefTiDBEvolvePlanBaselines, @@ -1009,13 +1033,12 @@ func NewSessionVars() *SessionVars { EnableClusteredIndex: DefTiDBEnableClusteredIndex, EnableParallelApply: DefTiDBEnableParallelApply, ShardAllocateStep: DefTiDBShardAllocateStep, - EnableChangeColumnType: DefTiDBChangeColumnType, EnableChangeMultiSchema: DefTiDBChangeMultiSchema, EnablePointGetCache: DefTiDBPointGetCache, EnableAlterPlacement: DefTiDBEnableAlterPlacement, EnableAmendPessimisticTxn: DefTiDBEnableAmendPessimisticTxn, PartitionPruneMode: *atomic2.NewString(DefTiDBPartitionPruneMode), - TxnScope: oracle.GetTxnScope(), + TxnScope: kv.GetTxnScopeVar(), EnabledRateLimitAction: DefTiDBEnableRateLimitAction, EnableAsyncCommit: DefTiDBEnableAsyncCommit, Enable1PC: DefTiDBEnable1PC, @@ -1023,6 +1046,9 @@ func NewSessionVars() *SessionVars { AnalyzeVersion: DefTiDBAnalyzeVersion, EnableIndexMergeJoin: DefTiDBEnableIndexMergeJoin, AllowFallbackToTiKV: make(map[kv.StoreType]struct{}), + CTEMaxRecursionDepth: DefCTEMaxRecursionDepth, + TMPTableSize: DefTMPTableSize, + EnableGlobalTemporaryTable: DefTiDBEnableGlobalTemporaryTable, } vars.KVVars = tikvstore.NewVariables(&vars.Killed) vars.Concurrency = Concurrency{ @@ -1069,7 +1095,7 @@ func NewSessionVars() *SessionVars { terror.Log(vars.SetSystemVar(TiDBEnableStreaming, enableStreaming)) vars.AllowBatchCop = DefTiDBAllowBatchCop - vars.AllowMPPExecution = DefTiDBAllowMPPExecution + vars.allowMPPExecution = DefTiDBAllowMPPExecution var enableChunkRPC string if config.GetGlobalConfig().TiKVClient.EnableChunkRPC { @@ -1138,15 +1164,15 @@ func (s *SessionVars) SetEnableIndexMerge(val bool) { } // GetReplicaRead get ReplicaRead from sql hints and SessionVars.replicaRead. -func (s *SessionVars) GetReplicaRead() tikvstore.ReplicaReadType { +func (s *SessionVars) GetReplicaRead() kv.ReplicaReadType { if s.StmtCtx.HasReplicaReadHint { - return tikvstore.ReplicaReadType(s.StmtCtx.ReplicaRead) + return kv.ReplicaReadType(s.StmtCtx.ReplicaRead) } return s.replicaRead } // SetReplicaRead set SessionVars.replicaRead. -func (s *SessionVars) SetReplicaRead(val tikvstore.ReplicaReadType) { +func (s *SessionVars) SetReplicaRead(val kv.ReplicaReadType) { s.replicaRead = val } @@ -1228,7 +1254,7 @@ func (s *SessionVars) GetStatusFlag(flag uint16) bool { func (s *SessionVars) SetInTxn(val bool) { s.SetStatusFlag(mysql.ServerStatusInTrans, val) if val { - s.TxnCtx.IsExplicit = true + s.TxnCtx.IsExplicit = val } } @@ -1380,6 +1406,18 @@ func (s *SessionVars) SetSystemVar(name string, val string) error { return sv.SetSessionFromHook(s, val) } +// SetSystemVarWithRelaxedValidation sets the value of a system variable for session scope. +// Validation functions are called, but scope validation is skipped. +// Errors are not expected to be returned because this could cause upgrade issues. +func (s *SessionVars) SetSystemVarWithRelaxedValidation(name string, val string) error { + sv := GetSysVar(name) + if sv == nil { + return ErrUnknownSystemVar.GenWithStackByArgs(name) + } + val = sv.ValidateWithRelaxedValidation(s, val, ScopeSession) + return sv.SetSessionFromHook(s, val) +} + // GetReadableTxnMode returns the session variable TxnMode but rewrites it to "OPTIMISTIC" when it's empty. func (s *SessionVars) GetReadableTxnMode() string { txnMode := s.TxnMode @@ -1420,18 +1458,22 @@ func (s *SessionVars) LazyCheckKeyNotExists() bool { return s.PresumeKeyNotExists || (s.TxnCtx.IsPessimistic && !s.StmtCtx.DupKeyAsWarning) } -// SetLocalSystemVar sets values of the local variables which in "server" scope. -func SetLocalSystemVar(name string, val string) { - switch name { - case TiDBDDLReorgWorkerCount: - SetDDLReorgWorkerCounter(int32(tidbOptPositiveInt32(val, DefTiDBDDLReorgWorkerCount))) - case TiDBDDLReorgBatchSize: - SetDDLReorgBatchSize(int32(tidbOptPositiveInt32(val, DefTiDBDDLReorgBatchSize))) - case TiDBDDLErrorCountLimit: - SetDDLErrorCountLimit(tidbOptInt64(val, DefTiDBDDLErrorCountLimit)) - case TiDBRowFormatVersion: - SetDDLReorgRowFormat(tidbOptInt64(val, DefTiDBRowFormatV2)) +// GetTemporaryTable returns a TempTable by tableInfo. +func (s *SessionVars) GetTemporaryTable(tblInfo *model.TableInfo) tableutil.TempTable { + if tblInfo.TempTableType == model.TempTableGlobal { + if s.TxnCtx.GlobalTemporaryTables == nil { + s.TxnCtx.GlobalTemporaryTables = make(map[int64]tableutil.TempTable) + } + globalTempTables := s.TxnCtx.GlobalTemporaryTables + globalTempTable, ok := globalTempTables[tblInfo.ID] + if !ok { + globalTempTable = tableutil.TempTableFromMeta(tblInfo) + globalTempTables[tblInfo.ID] = globalTempTable + } + return globalTempTable } + // TODO: check local temporary tables + return nil } // special session variables. @@ -1444,22 +1486,7 @@ const ( TransactionIsolation = "transaction_isolation" TxnIsolationOneShot = "tx_isolation_one_shot" MaxExecutionTime = "max_execution_time" -) - -// these variables are useless for TiDB, but still need to validate their values for some compatible issues. -// TODO: some more variables need to be added here. -const ( - serverReadOnly = "read_only" -) - -var ( - // TxIsolationNames are the valid values of the variable "tx_isolation" or "transaction_isolation". - TxIsolationNames = map[string]struct{}{ - "READ-UNCOMMITTED": {}, - "READ-COMMITTED": {}, - "REPEATABLE-READ": {}, - "SERIALIZABLE": {}, - } + ReadOnly = "read_only" ) // TableDelta stands for the changed count for one table or partition. @@ -2049,3 +2076,98 @@ type QueryInfo struct { ForUpdateTS uint64 `json:"for_update_ts"` ErrMsg string `json:"error,omitempty"` } + +// TxnReadTS indicates the value and used situation for tx_read_ts +type TxnReadTS struct { + readTS uint64 + used bool +} + +// NewTxnReadTS creates TxnReadTS +func NewTxnReadTS(ts uint64) *TxnReadTS { + return &TxnReadTS{ + readTS: ts, + used: false, + } +} + +// UseTxnReadTS returns readTS, and mark used as true +func (t *TxnReadTS) UseTxnReadTS() uint64 { + if t == nil { + return 0 + } + t.used = true + return t.readTS +} + +// SetTxnReadTS update readTS, and refresh used +func (t *TxnReadTS) SetTxnReadTS(ts uint64) { + if t == nil { + return + } + t.used = false + t.readTS = ts +} + +// PeakTxnReadTS returns readTS +func (t *TxnReadTS) PeakTxnReadTS() uint64 { + if t == nil { + return 0 + } + return t.readTS +} + +// CleanupTxnReadTSIfUsed cleans txnReadTS if used +func (s *SessionVars) CleanupTxnReadTSIfUsed() { + if s.TxnReadTS == nil { + return + } + if s.TxnReadTS.used && s.TxnReadTS.readTS > 0 { + s.TxnReadTS = NewTxnReadTS(0) + s.SnapshotInfoschema = nil + } +} + +// GetNetworkFactor returns the session variable networkFactor +// returns 0 when tbl is a temporary table. +func (s *SessionVars) GetNetworkFactor(tbl *model.TableInfo) float64 { + if tbl != nil { + if tbl.TempTableType != model.TempTableNone { + return 0 + } + } + return s.networkFactor +} + +// GetScanFactor returns the session variable scanFactor +// returns 0 when tbl is a temporary table. +func (s *SessionVars) GetScanFactor(tbl *model.TableInfo) float64 { + if tbl != nil { + if tbl.TempTableType != model.TempTableNone { + return 0 + } + } + return s.scanFactor +} + +// GetDescScanFactor returns the session variable descScanFactor +// returns 0 when tbl is a temporary table. +func (s *SessionVars) GetDescScanFactor(tbl *model.TableInfo) float64 { + if tbl != nil { + if tbl.TempTableType != model.TempTableNone { + return 0 + } + } + return s.descScanFactor +} + +// GetSeekFactor returns the session variable seekFactor +// returns 0 when tbl is a temporary table. +func (s *SessionVars) GetSeekFactor(tbl *model.TableInfo) float64 { + if tbl != nil { + if tbl.TempTableType != model.TempTableNone { + return 0 + } + } + return s.seekFactor +} diff --git a/sessionctx/variable/session_test.go b/sessionctx/variable/session_test.go index ee9030056bf05..0ded1a362e9f5 100644 --- a/sessionctx/variable/session_test.go +++ b/sessionctx/variable/session_test.go @@ -46,13 +46,13 @@ func (*testSessionSuite) TestSetSystemVariable(c *C) { {variable.TimeZone, "xyz", true}, {variable.TiDBOptAggPushDown, "1", false}, {variable.TiDBOptDistinctAggPushDown, "1", false}, - {variable.TIDBMemQuotaQuery, "1024", false}, - {variable.TIDBMemQuotaHashJoin, "1024", false}, - {variable.TIDBMemQuotaMergeJoin, "1024", false}, - {variable.TIDBMemQuotaSort, "1024", false}, - {variable.TIDBMemQuotaTopn, "1024", false}, - {variable.TIDBMemQuotaIndexLookupReader, "1024", false}, - {variable.TIDBMemQuotaIndexLookupJoin, "1024", false}, + {variable.TiDBMemQuotaQuery, "1024", false}, + {variable.TiDBMemQuotaHashJoin, "1024", false}, + {variable.TiDBMemQuotaMergeJoin, "1024", false}, + {variable.TiDBMemQuotaSort, "1024", false}, + {variable.TiDBMemQuotaTopn, "1024", false}, + {variable.TiDBMemQuotaIndexLookupReader, "1024", false}, + {variable.TiDBMemQuotaIndexLookupJoin, "1024", false}, {variable.TiDBMemQuotaApplyCache, "1024", false}, {variable.TiDBEnableStmtSummary, "1", false}, } @@ -231,7 +231,7 @@ func (*testSessionSuite) TestSlowLogFormat(c *C) { logItems := &variable.SlowQueryLogItems{ TxnTS: txnTS, SQL: sql, - Digest: digest, + Digest: digest.String(), TimeTotal: costTime, TimeParse: time.Duration(10), TimeCompile: time.Duration(10), diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index c17238c9ae9c5..ff8650317dfa8 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -14,6 +14,7 @@ package variable import ( + "encoding/json" "fmt" "math" "strconv" @@ -29,10 +30,10 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/kv" tikvstore "github.com/pingcap/tidb/store/tikv/kv" - "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/collate" "github.com/pingcap/tidb/util/logutil" + "github.com/pingcap/tidb/util/stmtsummary" "github.com/pingcap/tidb/util/versioninfo" atomic2 "go.uber.org/atomic" ) @@ -120,6 +121,51 @@ type SysVar struct { // Aliases is a list of sysvars that should also be updated when this sysvar is updated. // Updating aliases calls the SET function of the aliases, but does not update their aliases (preventing SET recursion) Aliases []string + // GetSession is a getter function for session scope. + // It can be used by instance-scoped variables to overwrite the previously expected value. + GetSession func(*SessionVars) (string, error) + // GetGlobal is a getter function for global scope. + GetGlobal func(*SessionVars) (string, error) + // skipInit defines if the sysvar should be loaded into the session on init. + // This is only important to set for sysvars that include session scope, + // since global scoped sysvars are not-applicable. + skipInit bool + // IsNoop defines if the sysvar is a noop included for MySQL compatibility + IsNoop bool +} + +// GetGlobalFromHook calls the GetSession func if it exists. +func (sv *SysVar) GetGlobalFromHook(s *SessionVars) (string, error) { + // Call the Getter if there is one defined. + if sv.GetGlobal != nil { + return sv.GetGlobal(s) + } + if sv.HasNoneScope() { + return sv.Value, nil + } + return s.GlobalVarsAccessor.GetGlobalSysVar(sv.Name) +} + +// GetSessionFromHook calls the GetSession func if it exists. +func (sv *SysVar) GetSessionFromHook(s *SessionVars) (string, error) { + if sv.HasNoneScope() { + return sv.Value, nil + } + // Call the Getter if there is one defined. + if sv.GetSession != nil { + return sv.GetSession(s) + } + var ( + ok bool + val string + ) + if val, ok = s.stmtVars[sv.Name]; ok { + return val, nil + } + if val, ok = s.systems[sv.Name]; !ok { + return val, errors.New("sysvar has not yet loaded") + } + return val, nil } // SetSessionFromHook calls the SetSession func if it exists. @@ -171,6 +217,11 @@ func (sv *SysVar) SetGlobalFromHook(s *SessionVars, val string, skipAliases bool return nil } +// HasNoneScope returns true if the scope for the sysVar is None. +func (sv *SysVar) HasNoneScope() bool { + return sv.Scope == ScopeNone +} + // HasSessionScope returns true if the scope for the sysVar includes session. func (sv *SysVar) HasSessionScope() bool { return sv.Scope&ScopeSession != 0 @@ -183,6 +234,10 @@ func (sv *SysVar) HasGlobalScope() bool { // Validate checks if system variable satisfies specific restriction. func (sv *SysVar) Validate(vars *SessionVars, value string, scope ScopeFlag) (string, error) { + // Check that the scope is correct first. + if err := sv.validateScope(scope); err != nil { + return value, err + } // Normalize the value and apply validation based on type. // i.e. TypeBool converts 1/on/ON to ON. normalizedValue, err := sv.validateFromType(vars, value, scope) @@ -198,17 +253,6 @@ func (sv *SysVar) Validate(vars *SessionVars, value string, scope ScopeFlag) (st // validateFromType provides automatic validation based on the SysVar's type func (sv *SysVar) validateFromType(vars *SessionVars, value string, scope ScopeFlag) (string, error) { - // Check that the scope is correct and return the appropriate error message. - if sv.ReadOnly || sv.Scope == ScopeNone { - return value, ErrIncorrectScope.FastGenByArgs(sv.Name, "read only") - } - if scope == ScopeGlobal && !sv.HasGlobalScope() { - return value, errLocalVariable.FastGenByArgs(sv.Name) - } - if scope == ScopeSession && !sv.HasSessionScope() { - return value, errGlobalVariable.FastGenByArgs(sv.Name) - } - // The string "DEFAULT" is a special keyword in MySQL, which restores // the compiled sysvar value. In which case we can skip further validation. if strings.EqualFold(value, "DEFAULT") { @@ -240,6 +284,37 @@ func (sv *SysVar) validateFromType(vars *SessionVars, value string, scope ScopeF return value, nil // typeString } +func (sv *SysVar) validateScope(scope ScopeFlag) error { + if sv.ReadOnly || sv.Scope == ScopeNone { + return ErrIncorrectScope.FastGenByArgs(sv.Name, "read only") + } + if scope == ScopeGlobal && !sv.HasGlobalScope() { + return errLocalVariable.FastGenByArgs(sv.Name) + } + if scope == ScopeSession && !sv.HasSessionScope() { + return errGlobalVariable.FastGenByArgs(sv.Name) + } + return nil +} + +// ValidateWithRelaxedValidation normalizes values but can not return errors. +// Normalization+validation needs to be applied when reading values because older versions of TiDB +// may be less sophisticated in normalizing values. But errors should be caught and handled, +// because otherwise there will be upgrade issues. +func (sv *SysVar) ValidateWithRelaxedValidation(vars *SessionVars, value string, scope ScopeFlag) string { + normalizedValue, err := sv.validateFromType(vars, value, scope) + if err != nil { + return normalizedValue + } + if sv.Validation != nil { + normalizedValue, err = sv.Validation(vars, normalizedValue, value, scope) + if err != nil { + return normalizedValue + } + } + return normalizedValue +} + const ( localDayTimeFormat = "15:04" // FullDayTimeFormat is the full format of analyze start time and end time. @@ -443,6 +518,24 @@ func (sv *SysVar) GetNativeValType(val string) (types.Datum, byte, uint) { return types.NewStringDatum(val), mysql.TypeVarString, 0 } +// SkipInit returns true if when a new session is created we should "skip" copying +// an initial value to it (and call the SetSession func if it exists) +func (sv *SysVar) SkipInit() bool { + if sv.skipInit || sv.IsNoop { + return true + } + // These a special "Global-only" sysvars that for backward compatibility + // are currently cached in the session. Please don't add to this list. + switch sv.Name { + case TiDBEnableChangeMultiSchema, TiDBDDLReorgBatchSize, TiDBEnableAlterPlacement, + TiDBMaxDeltaSchemaCount, InitConnect, MaxPreparedStmtCount, + TiDBDDLReorgWorkerCount, TiDBDDLErrorCountLimit, TiDBRowFormatVersion, + TiDBEnableTelemetry, TiDBEnablePointGetCache: + return false + } + return !sv.HasSessionScope() +} + var sysVars map[string]*SysVar var sysVarsLock sync.RWMutex @@ -463,12 +556,21 @@ func UnregisterSysVar(name string) { sysVarsLock.Unlock() } +// Clone deep copies the sysvar struct to avoid a race +func (sv *SysVar) Clone() *SysVar { + dst := *sv + return &dst +} + // GetSysVar returns sys var info for name as key. func GetSysVar(name string) *SysVar { name = strings.ToLower(name) sysVarsLock.RLock() defer sysVarsLock.RUnlock() - return sysVars[name] + if sysVars[name] == nil { + return nil + } + return sysVars[name].Clone() } // SetSysVar sets a sysvar. This will not propagate to the cluster, so it should only be @@ -480,11 +582,15 @@ func SetSysVar(name string, value string) { sysVars[name].Value = value } -// GetSysVars returns the sysVars list under a RWLock +// GetSysVars deep copies the sysVars list under a RWLock func GetSysVars() map[string]*SysVar { sysVarsLock.RLock() defer sysVarsLock.RUnlock() - return sysVars + copy := make(map[string]*SysVar, len(sysVars)) + for name, sv := range sysVars { + copy[name] = sv.Clone() + } + return copy } // PluginVarNames is global plugin var names set. @@ -496,6 +602,7 @@ func init() { RegisterSysVar(v) } for _, v := range noopSysVars { + v.IsNoop = true RegisterSysVar(v) } } @@ -530,7 +637,7 @@ var defaultSysVars = []*SysVar{ s.SetStatusFlag(mysql.ServerStatusNoBackslashEscaped, sqlMode.HasNoBackslashEscapesMode()) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: MaxExecutionTime, Value: "0", Type: TypeUnsigned, MinValue: 0, MaxValue: math.MaxUint64, AutoConvertOutOfRange: true, IsHintUpdatable: true, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: MaxExecutionTime, Value: "0", Type: TypeUnsigned, MinValue: 0, MaxValue: math.MaxInt32, AutoConvertOutOfRange: true, IsHintUpdatable: true, SetSession: func(s *SessionVars, val string) error { timeoutMS := tidbOptPositiveInt32(val, 0) s.MaxExecutionTime = uint64(timeoutMS) return nil @@ -543,7 +650,7 @@ var defaultSysVars = []*SysVar{ } return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: SQLLogBin, Value: On, Type: TypeBool}, + {Scope: ScopeGlobal | ScopeSession, Name: SQLLogBin, Value: On, Type: TypeBool, skipInit: true}, {Scope: ScopeGlobal | ScopeSession, Name: TimeZone, Value: "SYSTEM", IsHintUpdatable: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { if strings.EqualFold(normalizedValue, "SYSTEM") { return "SYSTEM", nil @@ -559,7 +666,7 @@ var defaultSysVars = []*SysVar{ return nil }}, {Scope: ScopeNone, Name: SystemTimeZone, Value: "CST"}, - {Scope: ScopeGlobal | ScopeSession, Name: ForeignKeyChecks, Value: Off, Type: TypeBool, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: ForeignKeyChecks, Value: Off, Type: TypeBool, skipInit: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { if TiDBOptOn(normalizedValue) { // TiDB does not yet support foreign keys. // Return the original value in the warning, so that users are not confused. @@ -570,12 +677,12 @@ var defaultSysVars = []*SysVar{ } return normalizedValue, ErrWrongValueForVar.GenWithStackByArgs(ForeignKeyChecks, originalValue) }}, - {Scope: ScopeNone, Name: Hostname, Value: ServerHostname}, - {Scope: ScopeSession, Name: Timestamp, Value: ""}, - {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetFilesystem, Value: "binary", Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeNone, Name: Hostname, Value: DefHostname}, + {Scope: ScopeSession, Name: Timestamp, Value: "", skipInit: true}, + {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetFilesystem, Value: "binary", skipInit: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkCharacterSet(normalizedValue, CharacterSetFilesystem) }}, - {Scope: ScopeGlobal | ScopeSession, Name: CollationDatabase, Value: mysql.DefaultCollationName, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: CollationDatabase, Value: mysql.DefaultCollationName, skipInit: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkCollation(vars, normalizedValue, originalValue, scope) }, SetSession: func(s *SessionVars, val string) error { if coll, err := collate.GetCollationByName(val); err == nil { @@ -593,13 +700,13 @@ var defaultSysVars = []*SysVar{ s.AutoIncrementOffset = tidbOptPositiveInt32(val, DefAutoIncrementOffset) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetClient, Value: mysql.DefaultCharset, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetClient, Value: mysql.DefaultCharset, skipInit: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkCharacterSet(normalizedValue, CharacterSetClient) }}, {Scope: ScopeNone, Name: Port, Value: "4000", Type: TypeUnsigned, MinValue: 0, MaxValue: math.MaxUint16}, {Scope: ScopeNone, Name: LowerCaseTableNames, Value: "2"}, {Scope: ScopeNone, Name: LogBin, Value: Off, Type: TypeBool}, - {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetResults, Value: mysql.DefaultCharset, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetResults, Value: mysql.DefaultCharset, skipInit: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { if normalizedValue == "" { return normalizedValue, nil } @@ -615,7 +722,7 @@ var defaultSysVars = []*SysVar{ {Scope: ScopeGlobal | ScopeSession, Name: TransactionIsolation, Value: "REPEATABLE-READ", Type: TypeEnum, Aliases: []string{TxnIsolation}, PossibleValues: []string{"READ-UNCOMMITTED", "READ-COMMITTED", "REPEATABLE-READ", "SERIALIZABLE"}, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkIsolationLevel(vars, normalizedValue, originalValue, scope) }}, - {Scope: ScopeGlobal | ScopeSession, Name: CollationConnection, Value: mysql.DefaultCollationName, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: CollationConnection, Value: mysql.DefaultCollationName, skipInit: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkCollation(vars, normalizedValue, originalValue, scope) }, SetSession: func(s *SessionVars, val string) error { if coll, err := collate.GetCollationByName(val); err == nil { @@ -632,7 +739,7 @@ var defaultSysVars = []*SysVar{ } return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: CharsetDatabase, Value: mysql.DefaultCharset, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: CharsetDatabase, Value: mysql.DefaultCharset, skipInit: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkCharacterSet(normalizedValue, CharsetDatabase) }, SetSession: func(s *SessionVars, val string) error { if _, coll, err := charset.GetCharsetInfo(val); err == nil { @@ -649,7 +756,7 @@ var defaultSysVars = []*SysVar{ s.LockWaitTimeout = lockWaitSec * 1000 return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: GroupConcatMaxLen, Value: "1024", AutoConvertOutOfRange: true, IsHintUpdatable: true, Type: TypeUnsigned, MinValue: 4, MaxValue: math.MaxUint64, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: GroupConcatMaxLen, Value: "1024", AutoConvertOutOfRange: true, IsHintUpdatable: true, skipInit: true, Type: TypeUnsigned, MinValue: 4, MaxValue: math.MaxUint64, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { // https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_group_concat_max_len // Minimum Value 4 // Maximum Value (64-bit platforms) 18446744073709551615 @@ -665,7 +772,7 @@ var defaultSysVars = []*SysVar{ return normalizedValue, nil }}, {Scope: ScopeNone, Name: Socket, Value: "/tmp/myssock"}, - {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetConnection, Value: mysql.DefaultCharset, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetConnection, Value: mysql.DefaultCharset, skipInit: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkCharacterSet(normalizedValue, CharacterSetConnection) }, SetSession: func(s *SessionVars, val string) error { if _, coll, err := charset.GetCharsetInfo(val); err == nil { @@ -673,7 +780,7 @@ var defaultSysVars = []*SysVar{ } return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetServer, Value: mysql.DefaultCharset, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: CharacterSetServer, Value: mysql.DefaultCharset, skipInit: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkCharacterSet(normalizedValue, CharacterSetServer) }, SetSession: func(s *SessionVars, val string) error { if _, coll, err := charset.GetCharsetInfo(val); err == nil { @@ -682,15 +789,19 @@ var defaultSysVars = []*SysVar{ return nil }}, {Scope: ScopeGlobal | ScopeSession, Name: MaxAllowedPacket, Value: "67108864", Type: TypeUnsigned, MinValue: 1024, MaxValue: MaxOfMaxAllowedPacket, AutoConvertOutOfRange: true}, - {Scope: ScopeSession, Name: WarningCount, Value: "0", ReadOnly: true}, - {Scope: ScopeSession, Name: ErrorCount, Value: "0", ReadOnly: true}, + {Scope: ScopeSession, Name: WarningCount, Value: "0", ReadOnly: true, skipInit: true, GetSession: func(s *SessionVars) (string, error) { + return strconv.Itoa(s.SysWarningCount), nil + }}, + {Scope: ScopeSession, Name: ErrorCount, Value: "0", ReadOnly: true, skipInit: true, GetSession: func(s *SessionVars) (string, error) { + return strconv.Itoa(int(s.SysErrorCount)), nil + }}, {Scope: ScopeGlobal | ScopeSession, Name: WindowingUseHighPrecision, Value: On, Type: TypeBool, IsHintUpdatable: true, SetSession: func(s *SessionVars, val string) error { s.WindowingUseHighPrecision = TiDBOptOn(val) return nil }}, {Scope: ScopeNone, Name: "license", Value: "Apache License 2.0"}, {Scope: ScopeGlobal | ScopeSession, Name: BlockEncryptionMode, Value: "aes-128-ecb"}, - {Scope: ScopeSession, Name: "last_insert_id", Value: ""}, + {Scope: ScopeSession, Name: "last_insert_id", Value: "", skipInit: true}, {Scope: ScopeNone, Name: "have_ssl", Value: "DISABLED"}, {Scope: ScopeNone, Name: "have_openssl", Value: "DISABLED"}, {Scope: ScopeNone, Name: "ssl_ca", Value: ""}, @@ -699,24 +810,34 @@ var defaultSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: InitConnect, Value: ""}, /* TiDB specific variables */ - {Scope: ScopeSession, Name: TiDBTxnScope, Value: func() string { + {Scope: ScopeSession, Name: TiDBTxnScope, skipInit: true, Value: func() string { if isGlobal, _ := config.GetTxnScopeFromConfig(); isGlobal { - return oracle.GlobalTxnScope + return kv.GlobalTxnScope } - return oracle.LocalTxnScope + return kv.LocalTxnScope }(), SetSession: func(s *SessionVars, val string) error { switch val { - case oracle.GlobalTxnScope: - s.TxnScope = oracle.NewGlobalTxnScope() - case oracle.LocalTxnScope: - s.TxnScope = oracle.GetTxnScope() + case kv.GlobalTxnScope: + s.TxnScope = kv.NewGlobalTxnScopeVar() + case kv.LocalTxnScope: + s.TxnScope = kv.GetTxnScopeVar() default: return ErrWrongValueForVar.GenWithStack("@@txn_scope value should be global or local") } return nil + }, GetSession: func(s *SessionVars) (string, error) { + return s.TxnScope.GetVarValue(), nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBAllowMPPExecution, Type: TypeBool, Value: BoolToOnOff(DefTiDBAllowMPPExecution), SetSession: func(s *SessionVars, val string) error { - s.AllowMPPExecution = TiDBOptOn(val) + {Scope: ScopeSession, Name: TiDBTxnReadTS, Value: "", Hidden: true, SetSession: func(s *SessionVars, val string) error { + return setTxnReadTS(s, val) + }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + if vars.InTxn() { + return "", errors.New("as of timestamp can't be set in transaction") + } + return normalizedValue, nil + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBAllowMPPExecution, Value: On, Type: TypeEnum, PossibleValues: []string{"OFF", "ON", "ENFORCE"}, SetSession: func(s *SessionVars, val string) error { + s.allowMPPExecution = val return nil }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBBCJThresholdCount, Value: strconv.Itoa(DefBroadcastJoinThresholdCount), Type: TypeInt, MinValue: 0, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { @@ -727,14 +848,14 @@ var defaultSysVars = []*SysVar{ s.BroadcastJoinThresholdSize = tidbOptInt64(val, DefBroadcastJoinThresholdSize) return nil }}, - {Scope: ScopeSession, Name: TiDBSnapshot, Value: "", SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBSnapshot, Value: "", skipInit: true, SetSession: func(s *SessionVars, val string) error { err := setSnapshotTS(s, val) if err != nil { return err } return nil }}, - {Scope: ScopeSession, Name: TiDBOptAggPushDown, Value: BoolToOnOff(DefOptAggPushDown), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBOptAggPushDown, Value: BoolToOnOff(DefOptAggPushDown), Type: TypeBool, skipInit: true, SetSession: func(s *SessionVars, val string) error { s.AllowAggPushDown = TiDBOptOn(val) return nil }}, @@ -747,24 +868,32 @@ var defaultSysVars = []*SysVar{ s.AllowBCJ = TiDBOptOn(val) return nil }}, - {Scope: ScopeSession, Name: TiDBOptDistinctAggPushDown, Value: BoolToOnOff(config.GetGlobalConfig().Performance.DistinctAggPushDown), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBOptDistinctAggPushDown, Value: BoolToOnOff(config.GetGlobalConfig().Performance.DistinctAggPushDown), skipInit: true, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { s.AllowDistinctAggPushDown = TiDBOptOn(val) return nil }}, - {Scope: ScopeSession, Name: TiDBOptWriteRowID, Value: BoolToOnOff(DefOptWriteRowID), SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBOptWriteRowID, Value: BoolToOnOff(DefOptWriteRowID), skipInit: true, SetSession: func(s *SessionVars, val string) error { s.AllowWriteRowID = TiDBOptOn(val) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBBuildStatsConcurrency, Value: strconv.Itoa(DefBuildStatsConcurrency)}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBBuildStatsConcurrency, skipInit: true, Value: strconv.Itoa(DefBuildStatsConcurrency)}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptCartesianBCJ, Value: strconv.Itoa(DefOptCartesianBCJ), Type: TypeInt, MinValue: 0, MaxValue: 2, SetSession: func(s *SessionVars, val string) error { + s.AllowCartesianBCJ = tidbOptInt(val, DefOptCartesianBCJ) + return nil + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptMPPOuterJoinFixedBuildSide, Value: BoolToOnOff(DefOptMPPOuterJoinFixedBuildSide), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + s.MPPOuterJoinFixedBuildSide = TiDBOptOn(val) + return nil + }}, {Scope: ScopeGlobal, Name: TiDBAutoAnalyzeRatio, Value: strconv.FormatFloat(DefAutoAnalyzeRatio, 'f', -1, 64), Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64}, {Scope: ScopeGlobal, Name: TiDBAutoAnalyzeStartTime, Value: DefAutoAnalyzeStartTime, Type: TypeTime}, {Scope: ScopeGlobal, Name: TiDBAutoAnalyzeEndTime, Value: DefAutoAnalyzeEndTime, Type: TypeTime}, - {Scope: ScopeSession, Name: TiDBChecksumTableConcurrency, Value: strconv.Itoa(DefChecksumTableConcurrency)}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBExecutorConcurrency, Value: strconv.Itoa(DefExecutorConcurrency), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBChecksumTableConcurrency, skipInit: true, Value: strconv.Itoa(DefChecksumTableConcurrency)}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBExecutorConcurrency, Value: strconv.Itoa(DefExecutorConcurrency), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { s.ExecutorConcurrency = tidbOptPositiveInt32(val, DefExecutorConcurrency) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBDistSQLScanConcurrency, Value: strconv.Itoa(DefDistSQLScanConcurrency), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBDistSQLScanConcurrency, Value: strconv.Itoa(DefDistSQLScanConcurrency), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { s.distSQLScanConcurrency = tidbOptPositiveInt32(val, DefDistSQLScanConcurrency) return nil }}, @@ -780,7 +909,7 @@ var defaultSysVars = []*SysVar{ s.CorrelationThreshold = tidbOptFloat64(val, DefOptCorrelationThreshold) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptCorrelationExpFactor, Value: strconv.Itoa(DefOptCorrelationExpFactor), Type: TypeUnsigned, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptCorrelationExpFactor, Value: strconv.Itoa(DefOptCorrelationExpFactor), Type: TypeUnsigned, MinValue: 0, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { s.CorrelationExpFactor = int(tidbOptInt64(val, DefOptCorrelationExpFactor)) return nil }}, @@ -788,7 +917,7 @@ var defaultSysVars = []*SysVar{ s.CPUFactor = tidbOptFloat64(val, DefOptCPUFactor) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptTiFlashConcurrencyFactor, Value: strconv.FormatFloat(DefOptTiFlashConcurrencyFactor, 'f', -1, 64), Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptTiFlashConcurrencyFactor, Value: strconv.FormatFloat(DefOptTiFlashConcurrencyFactor, 'f', -1, 64), skipInit: true, Type: TypeFloat, MinValue: 1, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { s.CopTiFlashConcurrencyFactor = tidbOptFloat64(val, DefOptTiFlashConcurrencyFactor) return nil }}, @@ -797,19 +926,19 @@ var defaultSysVars = []*SysVar{ return nil }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptNetworkFactor, Value: strconv.FormatFloat(DefOptNetworkFactor, 'f', -1, 64), Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.NetworkFactor = tidbOptFloat64(val, DefOptNetworkFactor) + s.networkFactor = tidbOptFloat64(val, DefOptNetworkFactor) return nil }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptScanFactor, Value: strconv.FormatFloat(DefOptScanFactor, 'f', -1, 64), Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.ScanFactor = tidbOptFloat64(val, DefOptScanFactor) + s.scanFactor = tidbOptFloat64(val, DefOptScanFactor) return nil }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptDescScanFactor, Value: strconv.FormatFloat(DefOptDescScanFactor, 'f', -1, 64), Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.DescScanFactor = tidbOptFloat64(val, DefOptDescScanFactor) + s.descScanFactor = tidbOptFloat64(val, DefOptDescScanFactor) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptSeekFactor, Value: strconv.FormatFloat(DefOptSeekFactor, 'f', -1, 64), Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.SeekFactor = tidbOptFloat64(val, DefOptSeekFactor) + {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptSeekFactor, Value: strconv.FormatFloat(DefOptSeekFactor, 'f', -1, 64), skipInit: true, Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { + s.seekFactor = tidbOptFloat64(val, DefOptSeekFactor) return nil }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptMemoryFactor, Value: strconv.FormatFloat(DefOptMemoryFactor, 'f', -1, 64), Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { @@ -824,29 +953,29 @@ var defaultSysVars = []*SysVar{ s.ConcurrencyFactor = tidbOptFloat64(val, DefOptConcurrencyFactor) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBIndexJoinBatchSize, Value: strconv.Itoa(DefIndexJoinBatchSize), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBIndexJoinBatchSize, Value: strconv.Itoa(DefIndexJoinBatchSize), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { s.IndexJoinBatchSize = tidbOptPositiveInt32(val, DefIndexJoinBatchSize) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBIndexLookupSize, Value: strconv.Itoa(DefIndexLookupSize), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBIndexLookupSize, Value: strconv.Itoa(DefIndexLookupSize), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { s.IndexLookupSize = tidbOptPositiveInt32(val, DefIndexLookupSize) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBIndexLookupConcurrency, Value: strconv.Itoa(DefIndexLookupConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt64, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBIndexLookupConcurrency, Value: strconv.Itoa(DefIndexLookupConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt32, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { s.indexLookupConcurrency = tidbOptPositiveInt32(val, ConcurrencyUnset) return nil }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { appendDeprecationWarning(vars, TiDBIndexLookupConcurrency, TiDBExecutorConcurrency) return normalizedValue, nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBIndexLookupJoinConcurrency, Value: strconv.Itoa(DefIndexLookupJoinConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt64, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBIndexLookupJoinConcurrency, Value: strconv.Itoa(DefIndexLookupJoinConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt32, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { s.indexLookupJoinConcurrency = tidbOptPositiveInt32(val, ConcurrencyUnset) return nil }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { appendDeprecationWarning(vars, TiDBIndexLookupJoinConcurrency, TiDBExecutorConcurrency) return normalizedValue, nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBIndexSerialScanConcurrency, Value: strconv.Itoa(DefIndexSerialScanConcurrency), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBIndexSerialScanConcurrency, Value: strconv.Itoa(DefIndexSerialScanConcurrency), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { s.indexSerialScanConcurrency = tidbOptPositiveInt32(val, DefIndexSerialScanConcurrency) return nil }}, @@ -858,26 +987,36 @@ var defaultSysVars = []*SysVar{ s.SkipASCIICheck = TiDBOptOn(val) return nil }}, - {Scope: ScopeSession, Name: TiDBBatchInsert, Value: BoolToOnOff(DefBatchInsert), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBBatchInsert, Value: BoolToOnOff(DefBatchInsert), Type: TypeBool, skipInit: true, SetSession: func(s *SessionVars, val string) error { s.BatchInsert = TiDBOptOn(val) return nil }}, - {Scope: ScopeSession, Name: TiDBBatchDelete, Value: BoolToOnOff(DefBatchDelete), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBBatchDelete, Value: BoolToOnOff(DefBatchDelete), Type: TypeBool, skipInit: true, SetSession: func(s *SessionVars, val string) error { s.BatchDelete = TiDBOptOn(val) return nil }}, - {Scope: ScopeSession, Name: TiDBBatchCommit, Value: BoolToOnOff(DefBatchCommit), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBBatchCommit, Value: BoolToOnOff(DefBatchCommit), Type: TypeBool, skipInit: true, SetSession: func(s *SessionVars, val string) error { s.BatchCommit = TiDBOptOn(val) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBDMLBatchSize, Value: strconv.Itoa(DefDMLBatchSize), Type: TypeUnsigned, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.DMLBatchSize = int(tidbOptInt64(val, DefOptCorrelationExpFactor)) + {Scope: ScopeGlobal | ScopeSession, Name: TiDBDMLBatchSize, Value: strconv.Itoa(DefDMLBatchSize), Type: TypeUnsigned, MinValue: 0, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { + s.DMLBatchSize = int(tidbOptInt64(val, DefDMLBatchSize)) return nil }}, - {Scope: ScopeSession, Name: TiDBCurrentTS, Value: strconv.Itoa(DefCurretTS), ReadOnly: true}, - {Scope: ScopeSession, Name: TiDBLastTxnInfo, Value: strconv.Itoa(DefCurretTS), ReadOnly: true}, - {Scope: ScopeSession, Name: TiDBLastQueryInfo, Value: strconv.Itoa(DefCurretTS), ReadOnly: true}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBMaxChunkSize, Value: strconv.Itoa(DefMaxChunkSize), Type: TypeUnsigned, MinValue: maxChunkSizeLowerBound, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBCurrentTS, Value: strconv.Itoa(DefCurretTS), ReadOnly: true, skipInit: true, GetSession: func(s *SessionVars) (string, error) { + return fmt.Sprintf("%d", s.TxnCtx.StartTS), nil + }}, + {Scope: ScopeSession, Name: TiDBLastTxnInfo, Value: strconv.Itoa(DefCurretTS), ReadOnly: true, skipInit: true, GetSession: func(s *SessionVars) (string, error) { + return s.LastTxnInfo, nil + }}, + {Scope: ScopeSession, Name: TiDBLastQueryInfo, Value: strconv.Itoa(DefCurretTS), ReadOnly: true, skipInit: true, GetSession: func(s *SessionVars) (string, error) { + info, err := json.Marshal(s.LastQueryInfo) + if err != nil { + return "", err + } + return string(info), nil + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBMaxChunkSize, Value: strconv.Itoa(DefMaxChunkSize), Type: TypeUnsigned, MinValue: maxChunkSizeLowerBound, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { s.MaxChunkSize = tidbOptPositiveInt32(val, DefMaxChunkSize) return nil }}, @@ -902,61 +1041,61 @@ var defaultSysVars = []*SysVar{ s.SetEnableIndexMerge(TiDBOptOn(val)) return nil }}, - {Scope: ScopeSession, Name: TIDBMemQuotaQuery, Value: strconv.FormatInt(config.GetGlobalConfig().MemQuotaQuery, 10), Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBMemQuotaQuery, Value: strconv.FormatInt(config.GetGlobalConfig().MemQuotaQuery, 10), skipInit: true, Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { s.MemQuotaQuery = tidbOptInt64(val, config.GetGlobalConfig().MemQuotaQuery) return nil }}, - {Scope: ScopeSession, Name: TIDBMemQuotaHashJoin, Value: strconv.FormatInt(DefTiDBMemQuotaHashJoin, 10), Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBMemQuotaHashJoin, Value: strconv.FormatInt(DefTiDBMemQuotaHashJoin, 10), skipInit: true, Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { s.MemQuotaHashJoin = tidbOptInt64(val, DefTiDBMemQuotaHashJoin) return nil }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { - appendDeprecationWarning(vars, TIDBMemQuotaHashJoin, TIDBMemQuotaQuery) + appendDeprecationWarning(vars, TiDBMemQuotaHashJoin, TiDBMemQuotaQuery) return normalizedValue, nil }}, - {Scope: ScopeSession, Name: TIDBMemQuotaMergeJoin, Value: strconv.FormatInt(DefTiDBMemQuotaMergeJoin, 10), Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBMemQuotaMergeJoin, Value: strconv.FormatInt(DefTiDBMemQuotaMergeJoin, 10), skipInit: true, Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { s.MemQuotaMergeJoin = tidbOptInt64(val, DefTiDBMemQuotaMergeJoin) return nil }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { - appendDeprecationWarning(vars, TIDBMemQuotaMergeJoin, TIDBMemQuotaQuery) + appendDeprecationWarning(vars, TiDBMemQuotaMergeJoin, TiDBMemQuotaQuery) return normalizedValue, nil }}, - {Scope: ScopeSession, Name: TIDBMemQuotaSort, Value: strconv.FormatInt(DefTiDBMemQuotaSort, 10), Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBMemQuotaSort, Value: strconv.FormatInt(DefTiDBMemQuotaSort, 10), skipInit: true, Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { s.MemQuotaSort = tidbOptInt64(val, DefTiDBMemQuotaSort) return nil }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { - appendDeprecationWarning(vars, TIDBMemQuotaSort, TIDBMemQuotaQuery) + appendDeprecationWarning(vars, TiDBMemQuotaSort, TiDBMemQuotaQuery) return normalizedValue, nil }}, - {Scope: ScopeSession, Name: TIDBMemQuotaTopn, Value: strconv.FormatInt(DefTiDBMemQuotaTopn, 10), Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBMemQuotaTopn, Value: strconv.FormatInt(DefTiDBMemQuotaTopn, 10), skipInit: true, Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { s.MemQuotaTopn = tidbOptInt64(val, DefTiDBMemQuotaTopn) return nil }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { - appendDeprecationWarning(vars, TIDBMemQuotaTopn, TIDBMemQuotaQuery) + appendDeprecationWarning(vars, TiDBMemQuotaTopn, TiDBMemQuotaQuery) return normalizedValue, nil }}, - {Scope: ScopeSession, Name: TIDBMemQuotaIndexLookupReader, Value: strconv.FormatInt(DefTiDBMemQuotaIndexLookupReader, 10), Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBMemQuotaIndexLookupReader, Value: strconv.FormatInt(DefTiDBMemQuotaIndexLookupReader, 10), skipInit: true, Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { s.MemQuotaIndexLookupReader = tidbOptInt64(val, DefTiDBMemQuotaIndexLookupReader) return nil }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { - appendDeprecationWarning(vars, TIDBMemQuotaIndexLookupReader, TIDBMemQuotaQuery) + appendDeprecationWarning(vars, TiDBMemQuotaIndexLookupReader, TiDBMemQuotaQuery) return normalizedValue, nil }}, - {Scope: ScopeSession, Name: TIDBMemQuotaIndexLookupJoin, Value: strconv.FormatInt(DefTiDBMemQuotaIndexLookupJoin, 10), Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBMemQuotaIndexLookupJoin, Value: strconv.FormatInt(DefTiDBMemQuotaIndexLookupJoin, 10), skipInit: true, Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { s.MemQuotaIndexLookupJoin = tidbOptInt64(val, DefTiDBMemQuotaIndexLookupJoin) return nil }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { - appendDeprecationWarning(vars, TIDBMemQuotaIndexLookupJoin, TIDBMemQuotaQuery) + appendDeprecationWarning(vars, TiDBMemQuotaIndexLookupJoin, TiDBMemQuotaQuery) return normalizedValue, nil }}, - {Scope: ScopeSession, Name: TiDBEnableStreaming, Value: Off, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBEnableStreaming, Value: Off, Type: TypeBool, skipInit: true, SetSession: func(s *SessionVars, val string) error { s.EnableStreaming = TiDBOptOn(val) return nil }}, - {Scope: ScopeSession, Name: TiDBEnableChunkRPC, Value: On, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBEnableChunkRPC, Value: On, Type: TypeBool, skipInit: true, SetSession: func(s *SessionVars, val string) error { s.EnableChunkRPC = TiDBOptOn(val) return nil }}, - {Scope: ScopeSession, Name: TxnIsolationOneShot, Value: "", Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeSession, Name: TxnIsolationOneShot, Value: "", skipInit: true, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { return checkIsolationLevel(vars, normalizedValue, originalValue, scope) }, SetSession: func(s *SessionVars, val string) error { s.txnIsolationLevelOneShot.state = oneShotSet @@ -967,46 +1106,46 @@ var defaultSysVars = []*SysVar{ s.EnableTablePartition = val return nil }}, - {Scope: ScopeSession, Name: TiDBEnableListTablePartition, Value: Off, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBEnableListTablePartition, Value: Off, Type: TypeBool, skipInit: true, SetSession: func(s *SessionVars, val string) error { s.EnableListTablePartition = TiDBOptOn(val) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBHashJoinConcurrency, Value: strconv.Itoa(DefTiDBHashJoinConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt64, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBHashJoinConcurrency, Value: strconv.Itoa(DefTiDBHashJoinConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt32, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { s.hashJoinConcurrency = tidbOptPositiveInt32(val, ConcurrencyUnset) return nil }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { appendDeprecationWarning(vars, TiDBHashJoinConcurrency, TiDBExecutorConcurrency) return normalizedValue, nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBProjectionConcurrency, Value: strconv.Itoa(DefTiDBProjectionConcurrency), Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBProjectionConcurrency, Value: strconv.Itoa(DefTiDBProjectionConcurrency), Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { s.projectionConcurrency = tidbOptPositiveInt32(val, ConcurrencyUnset) return nil }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { appendDeprecationWarning(vars, TiDBProjectionConcurrency, TiDBExecutorConcurrency) return normalizedValue, nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBHashAggPartialConcurrency, Value: strconv.Itoa(DefTiDBHashAggPartialConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt64, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBHashAggPartialConcurrency, Value: strconv.Itoa(DefTiDBHashAggPartialConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt32, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { s.hashAggPartialConcurrency = tidbOptPositiveInt32(val, ConcurrencyUnset) return nil }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { appendDeprecationWarning(vars, TiDBHashAggPartialConcurrency, TiDBExecutorConcurrency) return normalizedValue, nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBHashAggFinalConcurrency, Value: strconv.Itoa(DefTiDBHashAggFinalConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt64, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBHashAggFinalConcurrency, Value: strconv.Itoa(DefTiDBHashAggFinalConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt32, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { s.hashAggFinalConcurrency = tidbOptPositiveInt32(val, ConcurrencyUnset) return nil }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { appendDeprecationWarning(vars, TiDBHashAggFinalConcurrency, TiDBExecutorConcurrency) return normalizedValue, nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBWindowConcurrency, Value: strconv.Itoa(DefTiDBWindowConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt64, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBWindowConcurrency, Value: strconv.Itoa(DefTiDBWindowConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt32, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { s.windowConcurrency = tidbOptPositiveInt32(val, ConcurrencyUnset) return nil }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { appendDeprecationWarning(vars, TiDBWindowConcurrency, TiDBExecutorConcurrency) return normalizedValue, nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBMergeJoinConcurrency, Value: strconv.Itoa(DefTiDBMergeJoinConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt64, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBMergeJoinConcurrency, Value: strconv.Itoa(DefTiDBMergeJoinConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt32, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { s.mergeJoinConcurrency = tidbOptPositiveInt32(val, ConcurrencyUnset) return nil }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { @@ -1014,7 +1153,7 @@ var defaultSysVars = []*SysVar{ return normalizedValue, nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBStreamAggConcurrency, Value: strconv.Itoa(DefTiDBStreamAggConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt64, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBStreamAggConcurrency, Value: strconv.Itoa(DefTiDBStreamAggConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt32, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error { s.streamAggConcurrency = tidbOptPositiveInt32(val, ConcurrencyUnset) return nil }, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { @@ -1025,15 +1164,15 @@ var defaultSysVars = []*SysVar{ s.EnableParallelApply = TiDBOptOn(val) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBMemQuotaApplyCache, Value: strconv.Itoa(DefTiDBMemQuotaApplyCache), SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBMemQuotaApplyCache, Value: strconv.Itoa(DefTiDBMemQuotaApplyCache), Type: TypeUnsigned, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { s.MemQuotaApplyCache = tidbOptInt64(val, DefTiDBMemQuotaApplyCache) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBBackoffLockFast, Value: strconv.Itoa(tikvstore.DefBackoffLockFast), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBBackoffLockFast, Value: strconv.Itoa(tikvstore.DefBackoffLockFast), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { s.KVVars.BackoffLockFast = tidbOptPositiveInt32(val, tikvstore.DefBackoffLockFast) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBBackOffWeight, Value: strconv.Itoa(tikvstore.DefBackOffWeight), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBBackOffWeight, Value: strconv.Itoa(tikvstore.DefBackOffWeight), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { s.KVVars.BackOffWeight = tidbOptPositiveInt32(val, tikvstore.DefBackOffWeight) return nil }}, @@ -1060,9 +1199,10 @@ var defaultSysVars = []*SysVar{ } else if formatVersion == DefTiDBRowFormatV2 { s.RowEncoder.Enable = true } + SetDDLReorgRowFormat(tidbOptInt64(val, DefTiDBRowFormatV2)) return nil }}, - {Scope: ScopeSession, Name: TiDBOptimizerSelectivityLevel, Value: strconv.Itoa(DefTiDBOptimizerSelectivityLevel), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBOptimizerSelectivityLevel, Value: strconv.Itoa(DefTiDBOptimizerSelectivityLevel), skipInit: true, Type: TypeUnsigned, MinValue: 0, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { s.OptimizerSelectivityLevel = tidbOptPositiveInt32(val, DefTiDBOptimizerSelectivityLevel) return nil }}, @@ -1070,6 +1210,10 @@ var defaultSysVars = []*SysVar{ s.EnableWindowFunction = TiDBOptOn(val) return nil }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnablePipelinedWindowFunction, Value: BoolToOnOff(DefEnablePipelinedWindowFunction), Hidden: true, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + s.EnablePipelinedWindowExec = TiDBOptOn(val) + return nil + }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableStrictDoubleTypeCheck, Value: BoolToOnOff(DefEnableStrictDoubleTypeCheck), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { s.EnableStrictDoubleTypeCheck = TiDBOptOn(val) return nil @@ -1082,7 +1226,7 @@ var defaultSysVars = []*SysVar{ s.EnableFastAnalyze = TiDBOptOn(val) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBSkipIsolationLevelCheck, Value: BoolToOnOff(DefTiDBSkipIsolationLevelCheck), Type: TypeBool}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBSkipIsolationLevelCheck, skipInit: true, Value: BoolToOnOff(DefTiDBSkipIsolationLevelCheck), Type: TypeBool}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableRateLimitAction, Value: BoolToOnOff(DefTiDBEnableRateLimitAction), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { s.EnabledRateLimitAction = TiDBOptOn(val) return nil @@ -1121,27 +1265,50 @@ var defaultSysVars = []*SysVar{ return nil }}, /* The following variable is defined as session scope but is actually server scope. */ - {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableDynamicPrivileges, Value: Off, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { - s.EnableDynamicPrivileges = TiDBOptOn(val) - return nil - }}, - {Scope: ScopeSession, Name: TiDBGeneralLog, Value: BoolToOnOff(DefTiDBGeneralLog), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableDynamicPrivileges, Value: On, Type: TypeBool, Hidden: true}, + {Scope: ScopeSession, Name: TiDBGeneralLog, Value: BoolToOnOff(DefTiDBGeneralLog), Type: TypeBool, skipInit: true, SetSession: func(s *SessionVars, val string) error { ProcessGeneralLog.Store(TiDBOptOn(val)) return nil + }, GetSession: func(s *SessionVars) (string, error) { + return BoolToOnOff(ProcessGeneralLog.Load()), nil }}, - {Scope: ScopeSession, Name: TiDBPProfSQLCPU, Value: strconv.Itoa(DefTiDBPProfSQLCPU), Type: TypeInt, MinValue: 0, MaxValue: 1, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBPProfSQLCPU, Value: strconv.Itoa(DefTiDBPProfSQLCPU), Type: TypeInt, skipInit: true, MinValue: 0, MaxValue: 1, SetSession: func(s *SessionVars, val string) error { EnablePProfSQLCPU.Store(uint32(tidbOptPositiveInt32(val, DefTiDBPProfSQLCPU)) > 0) return nil + }, GetSession: func(s *SessionVars) (string, error) { + val := "0" + if EnablePProfSQLCPU.Load() { + val = "1" + } + return val, nil }}, - {Scope: ScopeSession, Name: TiDBDDLSlowOprThreshold, Value: strconv.Itoa(DefTiDBDDLSlowOprThreshold), SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBDDLSlowOprThreshold, Value: strconv.Itoa(DefTiDBDDLSlowOprThreshold), skipInit: true, SetSession: func(s *SessionVars, val string) error { atomic.StoreUint32(&DDLSlowOprThreshold, uint32(tidbOptPositiveInt32(val, DefTiDBDDLSlowOprThreshold))) return nil + }, GetSession: func(s *SessionVars) (string, error) { + return strconv.FormatUint(uint64(atomic.LoadUint32(&DDLSlowOprThreshold)), 10), nil + }}, + {Scope: ScopeSession, Name: TiDBConfig, Value: "", ReadOnly: true, skipInit: true, GetSession: func(s *SessionVars) (string, error) { + conf := config.GetGlobalConfig() + j, err := json.MarshalIndent(conf, "", "\t") + if err != nil { + return "", err + } + return config.HideConfig(string(j)), nil + }}, + {Scope: ScopeGlobal, Name: TiDBDDLReorgWorkerCount, Value: strconv.Itoa(DefTiDBDDLReorgWorkerCount), Type: TypeUnsigned, MinValue: 1, MaxValue: uint64(maxDDLReorgWorkerCount), SetSession: func(s *SessionVars, val string) error { + SetDDLReorgWorkerCounter(int32(tidbOptPositiveInt32(val, DefTiDBDDLReorgWorkerCount))) + return nil + }}, + {Scope: ScopeGlobal, Name: TiDBDDLReorgBatchSize, Value: strconv.Itoa(DefTiDBDDLReorgBatchSize), Type: TypeUnsigned, MinValue: int64(MinDDLReorgBatchSize), MaxValue: uint64(MaxDDLReorgBatchSize), AutoConvertOutOfRange: true, SetSession: func(s *SessionVars, val string) error { + SetDDLReorgBatchSize(int32(tidbOptPositiveInt32(val, DefTiDBDDLReorgBatchSize))) + return nil }}, - {Scope: ScopeSession, Name: TiDBConfig, Value: "", ReadOnly: true}, - {Scope: ScopeGlobal, Name: TiDBDDLReorgWorkerCount, Value: strconv.Itoa(DefTiDBDDLReorgWorkerCount), Type: TypeUnsigned, MinValue: 1, MaxValue: uint64(maxDDLReorgWorkerCount)}, - {Scope: ScopeGlobal, Name: TiDBDDLReorgBatchSize, Value: strconv.Itoa(DefTiDBDDLReorgBatchSize), Type: TypeUnsigned, MinValue: int64(MinDDLReorgBatchSize), MaxValue: uint64(MaxDDLReorgBatchSize), AutoConvertOutOfRange: true}, - {Scope: ScopeGlobal, Name: TiDBDDLErrorCountLimit, Value: strconv.Itoa(DefTiDBDDLErrorCountLimit), Type: TypeUnsigned, MinValue: 0, MaxValue: uint64(math.MaxInt64), AutoConvertOutOfRange: true}, - {Scope: ScopeSession, Name: TiDBDDLReorgPriority, Value: "PRIORITY_LOW", SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal, Name: TiDBDDLErrorCountLimit, Value: strconv.Itoa(DefTiDBDDLErrorCountLimit), Type: TypeUnsigned, MinValue: 0, MaxValue: math.MaxInt64, AutoConvertOutOfRange: true, SetSession: func(s *SessionVars, val string) error { + SetDDLErrorCountLimit(tidbOptInt64(val, DefTiDBDDLErrorCountLimit)) + return nil + }}, + {Scope: ScopeSession, Name: TiDBDDLReorgPriority, Value: "PRIORITY_LOW", skipInit: true, SetSession: func(s *SessionVars, val string) error { s.setDDLReorgPriority(val) return nil }}, @@ -1150,10 +1317,6 @@ var defaultSysVars = []*SysVar{ SetMaxDeltaSchemaCount(tidbOptInt64(val, DefTiDBMaxDeltaSchemaCount)) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableChangeColumnType, Value: BoolToOnOff(DefTiDBChangeColumnType), Hidden: true, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { - s.EnableChangeColumnType = TiDBOptOn(val) - return nil - }}, {Scope: ScopeGlobal, Name: TiDBEnableChangeMultiSchema, Value: BoolToOnOff(DefTiDBChangeMultiSchema), Hidden: true, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { s.EnableChangeMultiSchema = TiDBOptOn(val) return nil @@ -1166,42 +1329,44 @@ var defaultSysVars = []*SysVar{ s.EnableAlterPlacement = TiDBOptOn(val) return nil }}, - {Scope: ScopeSession, Name: TiDBForcePriority, Value: mysql.Priority2Str[DefTiDBForcePriority], SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBForcePriority, skipInit: true, Value: mysql.Priority2Str[DefTiDBForcePriority], SetSession: func(s *SessionVars, val string) error { atomic.StoreInt32(&ForcePriority, int32(mysql.Str2Priority(val))) return nil + }, GetSession: func(s *SessionVars) (string, error) { + return mysql.Priority2Str[mysql.PriorityEnum(atomic.LoadInt32(&ForcePriority))], nil }}, - {Scope: ScopeSession, Name: TiDBEnableRadixJoin, Value: BoolToOnOff(DefTiDBUseRadixJoin), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { - s.EnableRadixJoin = TiDBOptOn(val) - return nil - }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptJoinReorderThreshold, Value: strconv.Itoa(DefTiDBOptJoinReorderThreshold), Type: TypeUnsigned, MinValue: 0, MaxValue: 63, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptJoinReorderThreshold, Value: strconv.Itoa(DefTiDBOptJoinReorderThreshold), skipInit: true, Type: TypeUnsigned, MinValue: 0, MaxValue: 63, SetSession: func(s *SessionVars, val string) error { s.TiDBOptJoinReorderThreshold = tidbOptPositiveInt32(val, DefTiDBOptJoinReorderThreshold) return nil }}, - {Scope: ScopeSession, Name: TiDBSlowQueryFile, Value: "", SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBSlowQueryFile, Value: "", skipInit: true, SetSession: func(s *SessionVars, val string) error { s.SlowQueryFile = val return nil }}, {Scope: ScopeGlobal, Name: TiDBScatterRegion, Value: BoolToOnOff(DefTiDBScatterRegion), Type: TypeBool}, - {Scope: ScopeSession, Name: TiDBWaitSplitRegionFinish, Value: BoolToOnOff(DefTiDBWaitSplitRegionFinish), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBWaitSplitRegionFinish, Value: BoolToOnOff(DefTiDBWaitSplitRegionFinish), skipInit: true, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { s.WaitSplitRegionFinish = TiDBOptOn(val) return nil }}, - {Scope: ScopeSession, Name: TiDBWaitSplitRegionTimeout, Value: strconv.Itoa(DefWaitSplitRegionTimeout), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBWaitSplitRegionTimeout, Value: strconv.Itoa(DefWaitSplitRegionTimeout), skipInit: true, Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { s.WaitSplitRegionTimeout = uint64(tidbOptPositiveInt32(val, DefWaitSplitRegionTimeout)) return nil }}, - {Scope: ScopeSession, Name: TiDBLowResolutionTSO, Value: Off, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBLowResolutionTSO, Value: Off, Type: TypeBool, skipInit: true, SetSession: func(s *SessionVars, val string) error { s.LowResolutionTSO = TiDBOptOn(val) return nil }}, - {Scope: ScopeSession, Name: TiDBExpensiveQueryTimeThreshold, Value: strconv.Itoa(DefTiDBExpensiveQueryTimeThreshold), Type: TypeUnsigned, MinValue: int64(MinExpensiveQueryTimeThreshold), MaxValue: uint64(math.MaxInt64), AutoConvertOutOfRange: true, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBExpensiveQueryTimeThreshold, Value: strconv.Itoa(DefTiDBExpensiveQueryTimeThreshold), Type: TypeUnsigned, MinValue: int64(MinExpensiveQueryTimeThreshold), MaxValue: math.MaxInt32, AutoConvertOutOfRange: true, SetSession: func(s *SessionVars, val string) error { atomic.StoreUint64(&ExpensiveQueryTimeThreshold, uint64(tidbOptPositiveInt32(val, DefTiDBExpensiveQueryTimeThreshold))) return nil + }, GetSession: func(s *SessionVars) (string, error) { + return fmt.Sprintf("%d", atomic.LoadUint64(&ExpensiveQueryTimeThreshold)), nil }}, - {Scope: ScopeSession, Name: TiDBMemoryUsageAlarmRatio, Value: strconv.FormatFloat(config.GetGlobalConfig().Performance.MemoryUsageAlarmRatio, 'f', -1, 64), Type: TypeFloat, MinValue: 0.0, MaxValue: 1.0, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBMemoryUsageAlarmRatio, Value: strconv.FormatFloat(config.GetGlobalConfig().Performance.MemoryUsageAlarmRatio, 'f', -1, 64), Type: TypeFloat, MinValue: 0.0, MaxValue: 1.0, skipInit: true, SetSession: func(s *SessionVars, val string) error { MemoryUsageAlarmRatio.Store(tidbOptFloat64(val, 0.8)) return nil + }, GetSession: func(s *SessionVars) (string, error) { + return fmt.Sprintf("%g", MemoryUsageAlarmRatio.Load()), nil }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableNoopFuncs, Value: BoolToOnOff(DefTiDBEnableNoopFuncs), Type: TypeBool, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { @@ -1210,7 +1375,7 @@ var defaultSysVars = []*SysVar{ // To prevent this strange position, prevent setting to OFF when any of these sysVars are ON of the same scope. if normalizedValue == Off { - for _, potentialIncompatibleSysVar := range []string{TxReadOnly, TransactionReadOnly, OfflineMode, SuperReadOnly, serverReadOnly} { + for _, potentialIncompatibleSysVar := range []string{TxReadOnly, TransactionReadOnly, OfflineMode, SuperReadOnly, ReadOnly} { val, _ := vars.GetSystemVar(potentialIncompatibleSysVar) // session scope if scope == ScopeGlobal { // global scope var err error @@ -1229,27 +1394,59 @@ var defaultSysVars = []*SysVar{ s.EnableNoopFuncs = TiDBOptOn(val) return nil }}, - {Scope: ScopeSession, Name: TiDBReplicaRead, Value: "leader", Type: TypeEnum, PossibleValues: []string{"leader", "follower", "leader-and-follower"}, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBReplicaRead, Value: "leader", Type: TypeEnum, PossibleValues: []string{"leader", "follower", "leader-and-follower"}, skipInit: true, SetSession: func(s *SessionVars, val string) error { if strings.EqualFold(val, "follower") { - s.SetReplicaRead(tikvstore.ReplicaReadFollower) + s.SetReplicaRead(kv.ReplicaReadFollower) } else if strings.EqualFold(val, "leader-and-follower") { - s.SetReplicaRead(tikvstore.ReplicaReadMixed) + s.SetReplicaRead(kv.ReplicaReadMixed) } else if strings.EqualFold(val, "leader") || len(val) == 0 { - s.SetReplicaRead(tikvstore.ReplicaReadLeader) + s.SetReplicaRead(kv.ReplicaReadLeader) } return nil }}, - {Scope: ScopeSession, Name: TiDBAllowRemoveAutoInc, Value: BoolToOnOff(DefTiDBAllowRemoveAutoInc), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBAllowRemoveAutoInc, Value: BoolToOnOff(DefTiDBAllowRemoveAutoInc), skipInit: true, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { s.AllowRemoveAutoInc = TiDBOptOn(val) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableStmtSummary, Value: BoolToOnOff(config.GetGlobalConfig().StmtSummary.Enable), Type: TypeBool, AllowEmpty: true}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBStmtSummaryInternalQuery, Value: BoolToOnOff(config.GetGlobalConfig().StmtSummary.EnableInternalQuery), Type: TypeBool, AllowEmpty: true}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBStmtSummaryRefreshInterval, Value: strconv.Itoa(config.GetGlobalConfig().StmtSummary.RefreshInterval), Type: TypeInt, MinValue: 1, MaxValue: uint64(math.MaxInt32), AllowEmpty: true}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBStmtSummaryHistorySize, Value: strconv.Itoa(config.GetGlobalConfig().StmtSummary.HistorySize), Type: TypeInt, MinValue: 0, MaxValue: uint64(math.MaxUint8), AllowEmpty: true}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBStmtSummaryMaxStmtCount, Value: strconv.FormatUint(uint64(config.GetGlobalConfig().StmtSummary.MaxStmtCount), 10), Type: TypeInt, MinValue: 1, MaxValue: uint64(math.MaxInt16), AllowEmpty: true}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBStmtSummaryMaxSQLLength, Value: strconv.FormatUint(uint64(config.GetGlobalConfig().StmtSummary.MaxSQLLength), 10), Type: TypeInt, MinValue: 0, MaxValue: uint64(math.MaxInt32), AllowEmpty: true}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBCapturePlanBaseline, Value: Off, Type: TypeBool, AllowEmptyAll: true}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableStmtSummary, Value: BoolToOnOff(config.GetGlobalConfig().StmtSummary.Enable), skipInit: true, Type: TypeBool, AllowEmpty: true, SetSession: func(s *SessionVars, val string) error { + return stmtsummary.StmtSummaryByDigestMap.SetEnabled(val, true) + }, SetGlobal: func(s *SessionVars, val string) error { + return stmtsummary.StmtSummaryByDigestMap.SetEnabled(val, false) + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBStmtSummaryInternalQuery, Value: BoolToOnOff(config.GetGlobalConfig().StmtSummary.EnableInternalQuery), skipInit: true, Type: TypeBool, AllowEmpty: true, SetSession: func(s *SessionVars, val string) error { + return stmtsummary.StmtSummaryByDigestMap.SetEnabledInternalQuery(val, true) + }, SetGlobal: func(s *SessionVars, val string) error { + return stmtsummary.StmtSummaryByDigestMap.SetEnabledInternalQuery(val, false) + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBStmtSummaryRefreshInterval, Value: strconv.Itoa(config.GetGlobalConfig().StmtSummary.RefreshInterval), skipInit: true, Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt32, AllowEmpty: true, SetSession: func(s *SessionVars, val string) error { + return stmtsummary.StmtSummaryByDigestMap.SetRefreshInterval(val, true) + }, SetGlobal: func(s *SessionVars, val string) error { + return stmtsummary.StmtSummaryByDigestMap.SetRefreshInterval(val, false) + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBStmtSummaryHistorySize, Value: strconv.Itoa(config.GetGlobalConfig().StmtSummary.HistorySize), skipInit: true, Type: TypeInt, MinValue: 0, MaxValue: math.MaxUint8, AllowEmpty: true, SetSession: func(s *SessionVars, val string) error { + return stmtsummary.StmtSummaryByDigestMap.SetHistorySize(val, true) + }, SetGlobal: func(s *SessionVars, val string) error { + return stmtsummary.StmtSummaryByDigestMap.SetHistorySize(val, false) + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBStmtSummaryMaxStmtCount, Value: strconv.FormatUint(uint64(config.GetGlobalConfig().StmtSummary.MaxStmtCount), 10), skipInit: true, Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt16, AllowEmpty: true, SetSession: func(s *SessionVars, val string) error { + return stmtsummary.StmtSummaryByDigestMap.SetMaxStmtCount(val, true) + }, SetGlobal: func(s *SessionVars, val string) error { + return stmtsummary.StmtSummaryByDigestMap.SetMaxStmtCount(val, false) + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBStmtSummaryMaxSQLLength, Value: strconv.FormatUint(uint64(config.GetGlobalConfig().StmtSummary.MaxSQLLength), 10), skipInit: true, Type: TypeInt, MinValue: 0, MaxValue: math.MaxInt32, AllowEmpty: true, SetSession: func(s *SessionVars, val string) error { + return stmtsummary.StmtSummaryByDigestMap.SetMaxSQLLength(val, true) + }, SetGlobal: func(s *SessionVars, val string) error { + return stmtsummary.StmtSummaryByDigestMap.SetMaxSQLLength(val, false) + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBCapturePlanBaseline, Value: Off, Type: TypeBool, AllowEmptyAll: true, skipInit: true, GetSession: func(s *SessionVars) (string, error) { + return CapturePlanBaseline.GetVal(), nil + }, SetSession: func(s *SessionVars, val string) error { + CapturePlanBaseline.Set(val, true) + return nil + }, SetGlobal: func(s *SessionVars, val string) error { + CapturePlanBaseline.Set(val, false) + return nil + }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBUsePlanBaselines, Value: BoolToOnOff(DefTiDBUsePlanBaselines), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { s.UsePlanBaselines = TiDBOptOn(val) return nil @@ -1265,7 +1462,7 @@ var defaultSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: TiDBEvolvePlanTaskMaxTime, Value: strconv.Itoa(DefTiDBEvolvePlanTaskMaxTime), Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64}, {Scope: ScopeGlobal, Name: TiDBEvolvePlanTaskStartTime, Value: DefTiDBEvolvePlanTaskStartTime, Type: TypeTime}, {Scope: ScopeGlobal, Name: TiDBEvolvePlanTaskEndTime, Value: DefTiDBEvolvePlanTaskEndTime, Type: TypeTime}, - {Scope: ScopeSession, Name: TiDBIsolationReadEngines, Value: strings.Join(config.GetGlobalConfig().IsolationRead.Engines, ", "), Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeSession, Name: TiDBIsolationReadEngines, Value: strings.Join(config.GetGlobalConfig().IsolationRead.Engines, ","), Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { engines := strings.Split(normalizedValue, ",") var formatVal string for i, engine := range engines { @@ -1299,51 +1496,65 @@ var defaultSysVars = []*SysVar{ } return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBStoreLimit, Value: strconv.FormatInt(atomic.LoadInt64(&config.GetGlobalConfig().TiKVClient.StoreLimit), 10), Type: TypeInt, MinValue: 0, MaxValue: uint64(math.MaxInt64), AutoConvertOutOfRange: true, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBStoreLimit, Value: strconv.FormatInt(atomic.LoadInt64(&config.GetGlobalConfig().TiKVClient.StoreLimit), 10), Type: TypeInt, MinValue: 0, MaxValue: math.MaxInt64, AutoConvertOutOfRange: true, SetSession: func(s *SessionVars, val string) error { tikvstore.StoreLimit.Store(tidbOptInt64(val, DefTiDBStoreLimit)) return nil }}, - {Scope: ScopeSession, Name: TiDBMetricSchemaStep, Value: strconv.Itoa(DefTiDBMetricSchemaStep), Type: TypeUnsigned, MinValue: 10, MaxValue: 60 * 60 * 60, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBMetricSchemaStep, Value: strconv.Itoa(DefTiDBMetricSchemaStep), Type: TypeUnsigned, skipInit: true, MinValue: 10, MaxValue: 60 * 60 * 60, SetSession: func(s *SessionVars, val string) error { s.MetricSchemaStep = tidbOptInt64(val, DefTiDBMetricSchemaStep) return nil }}, - {Scope: ScopeSession, Name: TiDBMetricSchemaRangeDuration, Value: strconv.Itoa(DefTiDBMetricSchemaRangeDuration), Type: TypeUnsigned, MinValue: 10, MaxValue: 60 * 60 * 60, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBMetricSchemaRangeDuration, Value: strconv.Itoa(DefTiDBMetricSchemaRangeDuration), skipInit: true, Type: TypeUnsigned, MinValue: 10, MaxValue: 60 * 60 * 60, SetSession: func(s *SessionVars, val string) error { s.MetricSchemaRangeDuration = tidbOptInt64(val, DefTiDBMetricSchemaRangeDuration) return nil }}, - {Scope: ScopeSession, Name: TiDBSlowLogThreshold, Value: strconv.Itoa(logutil.DefaultSlowThreshold), Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBSlowLogThreshold, Value: strconv.Itoa(logutil.DefaultSlowThreshold), skipInit: true, Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { atomic.StoreUint64(&config.GetGlobalConfig().Log.SlowThreshold, uint64(tidbOptInt64(val, logutil.DefaultSlowThreshold))) return nil + }, GetSession: func(s *SessionVars) (string, error) { + return strconv.FormatUint(atomic.LoadUint64(&config.GetGlobalConfig().Log.SlowThreshold), 10), nil }}, - {Scope: ScopeSession, Name: TiDBRecordPlanInSlowLog, Value: int32ToBoolStr(logutil.DefaultRecordPlanInSlowLog), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBRecordPlanInSlowLog, Value: int32ToBoolStr(logutil.DefaultRecordPlanInSlowLog), skipInit: true, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { atomic.StoreUint32(&config.GetGlobalConfig().Log.RecordPlanInSlowLog, uint32(tidbOptInt64(val, logutil.DefaultRecordPlanInSlowLog))) return nil + }, GetSession: func(s *SessionVars) (string, error) { + return strconv.FormatUint(uint64(atomic.LoadUint32(&config.GetGlobalConfig().Log.RecordPlanInSlowLog)), 10), nil }}, - {Scope: ScopeSession, Name: TiDBEnableSlowLog, Value: BoolToOnOff(logutil.DefaultTiDBEnableSlowLog), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBEnableSlowLog, Value: BoolToOnOff(logutil.DefaultTiDBEnableSlowLog), Type: TypeBool, skipInit: true, SetSession: func(s *SessionVars, val string) error { config.GetGlobalConfig().Log.EnableSlowLog = TiDBOptOn(val) return nil + }, GetSession: func(s *SessionVars) (string, error) { + return BoolToOnOff(config.GetGlobalConfig().Log.EnableSlowLog), nil }}, - {Scope: ScopeSession, Name: TiDBQueryLogMaxLen, Value: strconv.Itoa(logutil.DefaultQueryLogMaxLen), Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBQueryLogMaxLen, Value: strconv.Itoa(logutil.DefaultQueryLogMaxLen), Type: TypeInt, MinValue: -1, MaxValue: math.MaxInt64, skipInit: true, SetSession: func(s *SessionVars, val string) error { atomic.StoreUint64(&config.GetGlobalConfig().Log.QueryLogMaxLen, uint64(tidbOptInt64(val, logutil.DefaultQueryLogMaxLen))) return nil + }, GetSession: func(s *SessionVars) (string, error) { + return strconv.FormatUint(atomic.LoadUint64(&config.GetGlobalConfig().Log.QueryLogMaxLen), 10), nil }}, {Scope: ScopeGlobal | ScopeSession, Name: CTEMaxRecursionDepth, Value: strconv.Itoa(DefCTEMaxRecursionDepth), Type: TypeInt, MinValue: 0, MaxValue: 4294967295, AutoConvertOutOfRange: true, SetSession: func(s *SessionVars, val string) error { s.CTEMaxRecursionDepth = tidbOptInt(val, DefCTEMaxRecursionDepth) return nil }}, - {Scope: ScopeSession, Name: TiDBCheckMb4ValueInUTF8, Value: BoolToOnOff(config.GetGlobalConfig().CheckMb4ValueInUTF8), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBCheckMb4ValueInUTF8, Value: BoolToOnOff(config.GetGlobalConfig().CheckMb4ValueInUTF8), skipInit: true, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { config.GetGlobalConfig().CheckMb4ValueInUTF8 = TiDBOptOn(val) return nil + }, GetSession: func(s *SessionVars) (string, error) { + return BoolToOnOff(config.GetGlobalConfig().CheckMb4ValueInUTF8), nil }}, - {Scope: ScopeSession, Name: TiDBFoundInPlanCache, Value: BoolToOnOff(DefTiDBFoundInPlanCache), Type: TypeBool, ReadOnly: true, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBFoundInPlanCache, Value: BoolToOnOff(DefTiDBFoundInPlanCache), Type: TypeBool, ReadOnly: true, skipInit: true, SetSession: func(s *SessionVars, val string) error { s.FoundInPlanCache = TiDBOptOn(val) return nil + }, GetSession: func(s *SessionVars) (string, error) { + return BoolToOnOff(s.PrevFoundInPlanCache), nil }}, - {Scope: ScopeSession, Name: TiDBFoundInBinding, Value: BoolToOnOff(DefTiDBFoundInBinding), Type: TypeBool, ReadOnly: true, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBFoundInBinding, Value: BoolToOnOff(DefTiDBFoundInBinding), Type: TypeBool, ReadOnly: true, skipInit: true, SetSession: func(s *SessionVars, val string) error { s.FoundInBinding = TiDBOptOn(val) return nil + }, GetSession: func(s *SessionVars) (string, error) { + return BoolToOnOff(s.PrevFoundInBinding), nil }}, - {Scope: ScopeSession, Name: TiDBEnableCollectExecutionInfo, Value: BoolToOnOff(DefTiDBEnableCollectExecutionInfo), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeSession, Name: TiDBEnableCollectExecutionInfo, Value: BoolToOnOff(DefTiDBEnableCollectExecutionInfo), skipInit: true, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { oldConfig := config.GetGlobalConfig() newValue := TiDBOptOn(val) if oldConfig.EnableCollectExecutionInfo != newValue { @@ -1352,6 +1563,8 @@ var defaultSysVars = []*SysVar{ config.StoreGlobalConfig(&newConfig) } return nil + }, GetSession: func(s *SessionVars) (string, error) { + return BoolToOnOff(config.GetGlobalConfig().EnableCollectExecutionInfo), nil }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBAllowAutoRandExplicitInsert, Value: BoolToOnOff(DefTiDBAllowAutoRandExplicitInsert), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { s.AllowAutoRandExplicitInsert = TiDBOptOn(val) @@ -1366,7 +1579,7 @@ var defaultSysVars = []*SysVar{ s.EnableClusteredIndex = TiDBOptEnableClustered(val) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBPartitionPruneMode, Value: string(Static), Hidden: true, Type: TypeStr, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBPartitionPruneMode, Value: DefTiDBPartitionPruneMode, Hidden: true, Type: TypeStr, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { mode := PartitionPruneMode(normalizedValue).Update() if !mode.Valid() { return normalizedValue, ErrWrongTypeForVar.GenWithStackByArgs(TiDBPartitionPruneMode) @@ -1376,16 +1589,18 @@ var defaultSysVars = []*SysVar{ s.PartitionPruneMode.Store(strings.ToLower(strings.TrimSpace(val))) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBSlowLogMasking, Value: BoolToOnOff(DefTiDBRedactLog), Aliases: []string{TiDBRedactLog}, Type: TypeBool, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBSlowLogMasking, Value: BoolToOnOff(DefTiDBRedactLog), Aliases: []string{TiDBRedactLog}, skipInit: true, Type: TypeBool, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { appendDeprecationWarning(vars, TiDBSlowLogMasking, TiDBRedactLog) return normalizedValue, nil + }, GetSession: func(s *SessionVars) (string, error) { + return s.systems[TiDBRedactLog], nil }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBRedactLog, Value: BoolToOnOff(DefTiDBRedactLog), Aliases: []string{TiDBSlowLogMasking}, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { s.EnableRedactLog = TiDBOptOn(val) errors.RedactLogEnabled.Store(s.EnableRedactLog) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBShardAllocateStep, Value: strconv.Itoa(DefTiDBShardAllocateStep), Type: TypeInt, MinValue: 1, MaxValue: uint64(math.MaxInt64), AutoConvertOutOfRange: true, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBShardAllocateStep, Value: strconv.Itoa(DefTiDBShardAllocateStep), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt64, AutoConvertOutOfRange: true, SetSession: func(s *SessionVars, val string) error { s.ShardAllocateStep = tidbOptInt64(val, DefTiDBShardAllocateStep) return nil }}, @@ -1406,8 +1621,8 @@ var defaultSysVars = []*SysVar{ s.GuaranteeLinearizability = TiDBOptOn(val) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBAnalyzeVersion, Value: strconv.Itoa(DefTiDBAnalyzeVersion), Hidden: true, Type: TypeInt, MinValue: 1, MaxValue: 3, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { - if normalizedValue == "2" && FeedbackProbability.Load() > 0 { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBAnalyzeVersion, Value: strconv.Itoa(DefTiDBAnalyzeVersion), Hidden: false, Type: TypeInt, MinValue: 1, MaxValue: 2, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + if normalizedValue == "2" && FeedbackProbability != nil && FeedbackProbability.Load() > 0 { var original string var err error if scope == ScopeGlobal { @@ -1443,6 +1658,12 @@ var defaultSysVars = []*SysVar{ return nil }}, {Scope: ScopeNone, Name: TiDBEnableEnhancedSecurity, Value: Off, Type: TypeBool}, + {Scope: ScopeSession, Name: PluginLoad, Value: "", GetSession: func(s *SessionVars) (string, error) { + return config.GetGlobalConfig().Plugin.Dir, nil + }}, + {Scope: ScopeSession, Name: PluginDir, Value: "/data/deploy/plugin", GetSession: func(s *SessionVars) (string, error) { + return config.GetGlobalConfig().Plugin.Load, nil + }}, /* tikv gc metrics */ {Scope: ScopeGlobal, Name: TiDBGCEnable, Value: On, Type: TypeBool}, @@ -1450,6 +1671,89 @@ var defaultSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: TiDBGCLifetime, Value: "10m0s", Type: TypeDuration, MinValue: int64(time.Minute * 10), MaxValue: math.MaxInt64}, {Scope: ScopeGlobal, Name: TiDBGCConcurrency, Value: "-1", Type: TypeInt, MinValue: 1, MaxValue: 128, AllowAutoValue: true}, {Scope: ScopeGlobal, Name: TiDBGCScanLockMode, Value: "PHYSICAL", Type: TypeEnum, PossibleValues: []string{"PHYSICAL", "LEGACY"}}, + {Scope: ScopeGlobal, Name: TiDBGCScanLockMode, Value: "LEGACY", Type: TypeEnum, PossibleValues: []string{"PHYSICAL", "LEGACY"}}, + + // See https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_tmp_table_size + {Scope: ScopeGlobal | ScopeSession, Name: TMPTableSize, Value: strconv.Itoa(DefTMPTableSize), Type: TypeUnsigned, MinValue: 1024, MaxValue: math.MaxInt64, AutoConvertOutOfRange: true, IsHintUpdatable: true, AllowEmpty: true, SetSession: func(s *SessionVars, val string) error { + s.TMPTableSize = tidbOptInt64(val, DefTMPTableSize) + return nil + }}, + // variable for top SQL feature. + {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableTopSQL, Value: BoolToOnOff(DefTiDBTopSQLEnable), Type: TypeBool, AllowEmpty: true, GetSession: func(s *SessionVars) (string, error) { + return BoolToOnOff(TopSQLVariable.Enable.Load()), nil + }, SetSession: func(vars *SessionVars, s string) error { + TopSQLVariable.Enable.Store(TiDBOptOn(s)) + return nil + }, SetGlobal: func(vars *SessionVars, s string) error { + TopSQLVariable.Enable.Store(TiDBOptOn(s)) + return nil + }}, + // TODO(crazycs520): Add validation + {Scope: ScopeGlobal | ScopeSession, Name: TiDBTopSQLAgentAddress, Value: DefTiDBTopSQLAgentAddress, Type: TypeStr, AllowEmpty: true, GetSession: func(s *SessionVars) (string, error) { + return TopSQLVariable.AgentAddress.Load(), nil + }, SetSession: func(vars *SessionVars, s string) error { + TopSQLVariable.AgentAddress.Store(s) + return nil + }, SetGlobal: func(vars *SessionVars, s string) error { + TopSQLVariable.AgentAddress.Store(s) + return nil + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBTopSQLPrecisionSeconds, Value: strconv.Itoa(DefTiDBTopSQLPrecisionSeconds), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt64, AllowEmpty: true, GetSession: func(s *SessionVars) (string, error) { + return strconv.FormatInt(TopSQLVariable.PrecisionSeconds.Load(), 10), nil + }, SetSession: func(vars *SessionVars, s string) error { + val, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return err + } + TopSQLVariable.PrecisionSeconds.Store(val) + return nil + }, SetGlobal: func(vars *SessionVars, s string) error { + val, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return err + } + TopSQLVariable.PrecisionSeconds.Store(val) + return nil + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBTopSQLMaxStatementCount, Value: strconv.Itoa(DefTiDBTopSQLMaxStatementCount), Type: TypeInt, MinValue: 0, MaxValue: 5000, AllowEmpty: true, GetSession: func(s *SessionVars) (string, error) { + return strconv.FormatInt(TopSQLVariable.MaxStatementCount.Load(), 10), nil + }, SetSession: func(vars *SessionVars, s string) error { + val, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return err + } + TopSQLVariable.MaxStatementCount.Store(val) + return nil + }, SetGlobal: func(vars *SessionVars, s string) error { + val, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return err + } + TopSQLVariable.MaxStatementCount.Store(val) + return nil + }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBTopSQLReportIntervalSeconds, Value: strconv.Itoa(DefTiDBTopSQLReportIntervalSeconds), Type: TypeInt, MinValue: 1, MaxValue: 1 * 60 * 60, AllowEmpty: true, GetSession: func(s *SessionVars) (string, error) { + return strconv.FormatInt(TopSQLVariable.ReportIntervalSeconds.Load(), 10), nil + }, SetSession: func(vars *SessionVars, s string) error { + val, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return err + } + TopSQLVariable.ReportIntervalSeconds.Store(val) + return nil + }, SetGlobal: func(vars *SessionVars, s string) error { + val, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return err + } + TopSQLVariable.ReportIntervalSeconds.Store(val) + return nil + }}, + + {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableGlobalTemporaryTable, Value: BoolToOnOff(DefTiDBEnableGlobalTemporaryTable), Hidden: true, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + s.EnableGlobalTemporaryTable = TiDBOptOn(val) + return nil + }}, } // FeedbackProbability points to the FeedbackProbability in statistics package. @@ -1556,8 +1860,8 @@ const ( MaxConnectErrors = "max_connect_errors" // TableDefinitionCache is the name for 'table_definition_cache' system variable. TableDefinitionCache = "table_definition_cache" - // TmpTableSize is the name for 'tmp_table_size' system variable. - TmpTableSize = "tmp_table_size" + // TMPTableSize is the name for 'tmp_table_size' system variable. + TMPTableSize = "tmp_table_size" // Timestamp is the name for 'timestamp' system variable. Timestamp = "timestamp" // ConnectTimeout is the name for 'connect_timeout' system variable. diff --git a/sessionctx/variable/sysvar_test.go b/sessionctx/variable/sysvar_test.go index cc765854a5c0b..58f6ac803c563 100644 --- a/sessionctx/variable/sysvar_test.go +++ b/sessionctx/variable/sysvar_test.go @@ -223,14 +223,22 @@ func (*testSysVarSuite) TestScope(c *C) { sv := SysVar{Scope: ScopeGlobal | ScopeSession, Name: "mynewsysvar", Value: On, Type: TypeEnum, PossibleValues: []string{"OFF", "ON", "AUTO"}} c.Assert(sv.HasSessionScope(), IsTrue) c.Assert(sv.HasGlobalScope(), IsTrue) + c.Assert(sv.HasNoneScope(), IsFalse) sv = SysVar{Scope: ScopeGlobal, Name: "mynewsysvar", Value: On, Type: TypeEnum, PossibleValues: []string{"OFF", "ON", "AUTO"}} c.Assert(sv.HasSessionScope(), IsFalse) c.Assert(sv.HasGlobalScope(), IsTrue) + c.Assert(sv.HasNoneScope(), IsFalse) + + sv = SysVar{Scope: ScopeSession, Name: "mynewsysvar", Value: On, Type: TypeEnum, PossibleValues: []string{"OFF", "ON", "AUTO"}} + c.Assert(sv.HasSessionScope(), IsTrue) + c.Assert(sv.HasGlobalScope(), IsFalse) + c.Assert(sv.HasNoneScope(), IsFalse) sv = SysVar{Scope: ScopeNone, Name: "mynewsysvar", Value: On, Type: TypeEnum, PossibleValues: []string{"OFF", "ON", "AUTO"}} c.Assert(sv.HasSessionScope(), IsFalse) c.Assert(sv.HasGlobalScope(), IsFalse) + c.Assert(sv.HasNoneScope(), IsTrue) } func (*testSysVarSuite) TestBuiltInCase(c *C) { @@ -426,49 +434,61 @@ func (*testSysVarSuite) TestReadOnlyNoop(c *C) { } } -func (*testSysVarSuite) TestGetScopeNoneSystemVar(c *C) { - val, ok, err := GetScopeNoneSystemVar(Port) - c.Assert(err, IsNil) - c.Assert(ok, IsTrue) - c.Assert(val, Equals, "4000") +func (*testSysVarSuite) TestSkipInit(c *C) { + sv := SysVar{Scope: ScopeGlobal, Name: "skipinit1", Value: On, Type: TypeBool} + c.Assert(sv.SkipInit(), IsTrue) - val, ok, err = GetScopeNoneSystemVar("nonsensevar") - c.Assert(err.Error(), Equals, "[variable:1193]Unknown system variable 'nonsensevar'") - c.Assert(ok, IsFalse) - c.Assert(val, Equals, "") + sv = SysVar{Scope: ScopeGlobal | ScopeSession, Name: "skipinit1", Value: On, Type: TypeBool} + c.Assert(sv.SkipInit(), IsFalse) - val, ok, err = GetScopeNoneSystemVar(CharacterSetClient) - c.Assert(err, IsNil) - c.Assert(ok, IsFalse) - c.Assert(val, Equals, "") + sv = SysVar{Scope: ScopeSession, Name: "skipinit1", Value: On, Type: TypeBool} + c.Assert(sv.SkipInit(), IsFalse) + + sv = SysVar{Scope: ScopeSession, Name: "skipinit1", Value: On, Type: TypeBool, skipInit: true} + c.Assert(sv.SkipInit(), IsTrue) +} + +// IsNoop is used by the documentation to auto-generate docs for real sysvars. +func (*testSysVarSuite) TestIsNoop(c *C) { + sv := GetSysVar(TiDBMultiStatementMode) + c.Assert(sv.IsNoop, IsFalse) + + sv = GetSysVar(InnodbLockWaitTimeout) + c.Assert(sv.IsNoop, IsFalse) + + sv = GetSysVar(InnodbFastShutdown) + c.Assert(sv.IsNoop, IsTrue) + + sv = GetSysVar(ReadOnly) + c.Assert(sv.IsNoop, IsTrue) } func (*testSysVarSuite) TestInstanceScopedVars(c *C) { - // This tests instance scoped variables through GetSessionSystemVar(). + // This tests instance scoped variables through GetSessionOrGlobalSystemVar(). // Eventually these should be changed to use getters so that the switch // statement in GetSessionOnlySysVars can be removed. vars := NewSessionVars() - val, err := GetSessionSystemVar(vars, TiDBCurrentTS) + val, err := GetSessionOrGlobalSystemVar(vars, TiDBCurrentTS) c.Assert(err, IsNil) c.Assert(val, Equals, fmt.Sprintf("%d", vars.TxnCtx.StartTS)) - val, err = GetSessionSystemVar(vars, TiDBLastTxnInfo) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBLastTxnInfo) c.Assert(err, IsNil) c.Assert(val, Equals, vars.LastTxnInfo) - val, err = GetSessionSystemVar(vars, TiDBLastQueryInfo) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBLastQueryInfo) c.Assert(err, IsNil) info, err := json.Marshal(vars.LastQueryInfo) c.Assert(err, IsNil) c.Assert(val, Equals, string(info)) - val, err = GetSessionSystemVar(vars, TiDBGeneralLog) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBGeneralLog) c.Assert(err, IsNil) c.Assert(val, Equals, BoolToOnOff(ProcessGeneralLog.Load())) - val, err = GetSessionSystemVar(vars, TiDBPProfSQLCPU) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBPProfSQLCPU) c.Assert(err, IsNil) expected := "0" if EnablePProfSQLCPU.Load() { @@ -476,74 +496,126 @@ func (*testSysVarSuite) TestInstanceScopedVars(c *C) { } c.Assert(val, Equals, expected) - val, err = GetSessionSystemVar(vars, TiDBExpensiveQueryTimeThreshold) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBExpensiveQueryTimeThreshold) c.Assert(err, IsNil) c.Assert(val, Equals, fmt.Sprintf("%d", atomic.LoadUint64(&ExpensiveQueryTimeThreshold))) - val, err = GetSessionSystemVar(vars, TiDBMemoryUsageAlarmRatio) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBMemoryUsageAlarmRatio) c.Assert(err, IsNil) c.Assert(val, Equals, fmt.Sprintf("%g", MemoryUsageAlarmRatio.Load())) - val, err = GetSessionSystemVar(vars, TiDBConfig) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBConfig) c.Assert(err, IsNil) conf := config.GetGlobalConfig() j, err := json.MarshalIndent(conf, "", "\t") c.Assert(err, IsNil) c.Assert(val, Equals, config.HideConfig(string(j))) - val, err = GetSessionSystemVar(vars, TiDBForcePriority) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBForcePriority) c.Assert(err, IsNil) c.Assert(val, Equals, mysql.Priority2Str[mysql.PriorityEnum(atomic.LoadInt32(&ForcePriority))]) - val, err = GetSessionSystemVar(vars, TiDBDDLSlowOprThreshold) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBDDLSlowOprThreshold) c.Assert(err, IsNil) c.Assert(val, Equals, strconv.FormatUint(uint64(atomic.LoadUint32(&DDLSlowOprThreshold)), 10)) - val, err = GetSessionSystemVar(vars, PluginDir) + val, err = GetSessionOrGlobalSystemVar(vars, PluginDir) c.Assert(err, IsNil) c.Assert(val, Equals, config.GetGlobalConfig().Plugin.Dir) - val, err = GetSessionSystemVar(vars, PluginLoad) + val, err = GetSessionOrGlobalSystemVar(vars, PluginLoad) c.Assert(err, IsNil) c.Assert(val, Equals, config.GetGlobalConfig().Plugin.Load) - val, err = GetSessionSystemVar(vars, TiDBSlowLogThreshold) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBSlowLogThreshold) c.Assert(err, IsNil) c.Assert(val, Equals, strconv.FormatUint(atomic.LoadUint64(&config.GetGlobalConfig().Log.SlowThreshold), 10)) - val, err = GetSessionSystemVar(vars, TiDBRecordPlanInSlowLog) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBRecordPlanInSlowLog) c.Assert(err, IsNil) c.Assert(val, Equals, strconv.FormatUint(uint64(atomic.LoadUint32(&config.GetGlobalConfig().Log.RecordPlanInSlowLog)), 10)) - val, err = GetSessionSystemVar(vars, TiDBEnableSlowLog) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBEnableSlowLog) c.Assert(err, IsNil) c.Assert(val, Equals, BoolToOnOff(config.GetGlobalConfig().Log.EnableSlowLog)) - val, err = GetSessionSystemVar(vars, TiDBQueryLogMaxLen) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBQueryLogMaxLen) c.Assert(err, IsNil) c.Assert(val, Equals, strconv.FormatUint(atomic.LoadUint64(&config.GetGlobalConfig().Log.QueryLogMaxLen), 10)) - val, err = GetSessionSystemVar(vars, TiDBCheckMb4ValueInUTF8) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBCheckMb4ValueInUTF8) c.Assert(err, IsNil) c.Assert(val, Equals, BoolToOnOff(config.GetGlobalConfig().CheckMb4ValueInUTF8)) - val, err = GetSessionSystemVar(vars, TiDBCapturePlanBaseline) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBCapturePlanBaseline) c.Assert(err, IsNil) c.Assert(val, Equals, CapturePlanBaseline.GetVal()) - val, err = GetSessionSystemVar(vars, TiDBFoundInPlanCache) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBFoundInPlanCache) c.Assert(err, IsNil) c.Assert(val, Equals, BoolToOnOff(vars.PrevFoundInPlanCache)) - val, err = GetSessionSystemVar(vars, TiDBFoundInBinding) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBFoundInBinding) c.Assert(err, IsNil) c.Assert(val, Equals, BoolToOnOff(vars.PrevFoundInBinding)) - val, err = GetSessionSystemVar(vars, TiDBEnableCollectExecutionInfo) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBEnableCollectExecutionInfo) c.Assert(err, IsNil) c.Assert(val, Equals, BoolToOnOff(config.GetGlobalConfig().EnableCollectExecutionInfo)) - val, err = GetSessionSystemVar(vars, TiDBTxnScope) + val, err = GetSessionOrGlobalSystemVar(vars, TiDBTxnScope) c.Assert(err, IsNil) c.Assert(val, Equals, vars.TxnScope.GetVarValue()) } + +// Calling GetSysVars/GetSysVar needs to return a deep copy, otherwise there will be data races. +// This is a bit unfortunate, since the only time the race occurs is in the testsuite (Enabling/Disabling SEM) and +// during startup (setting the .Value of ScopeNone variables). In future it might also be able +// to fix this by delaying the LoadSysVarCacheLoop start time until after the server is fully initialized. +func (*testSysVarSuite) TestDeepCopyGetSysVars(c *C) { + // Check GetSysVar + sv := SysVar{Scope: ScopeGlobal | ScopeSession, Name: "datarace", Value: On, Type: TypeBool} + RegisterSysVar(&sv) + svcopy := GetSysVar("datarace") + svcopy.Name = "datarace2" + c.Assert(sv.Name, Equals, "datarace") + c.Assert(GetSysVar("datarace").Name, Equals, "datarace") + UnregisterSysVar("datarace") + + // Check GetSysVars + sv = SysVar{Scope: ScopeGlobal | ScopeSession, Name: "datarace", Value: On, Type: TypeBool} + RegisterSysVar(&sv) + for name, svcopy := range GetSysVars() { + if name == "datarace" { + svcopy.Name = "datarace2" + } + } + c.Assert(sv.Name, Equals, "datarace") + c.Assert(GetSysVar("datarace").Name, Equals, "datarace") + UnregisterSysVar("datarace") +} + +// Test that sysvars defaults are logically valid. i.e. +// the default itself must validate without error provided the scope and read-only is correct. +// The default values should also be normalized for consistency. +func (*testSysVarSuite) TestDefaultValuesAreSettable(c *C) { + vars := NewSessionVars() + for _, sv := range GetSysVars() { + if sv.HasSessionScope() && !sv.ReadOnly { + val, err := sv.Validate(vars, sv.Value, ScopeSession) + c.Assert(sv.Value, Equals, val) + c.Assert(err, IsNil) + } + + if sv.HasGlobalScope() && !sv.ReadOnly { + if sv.Name == TiDBEnableNoopFuncs { + // TODO: this requires access to the global var accessor, + // which is not available in this test. + continue + } + val, err := sv.Validate(vars, sv.Value, ScopeGlobal) + c.Assert(sv.Value, Equals, val) + c.Assert(err, IsNil) + } + } +} diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 30d52ac54f386..2a5fd6360cacc 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -15,7 +15,6 @@ package variable import ( "math" - "os" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/config" @@ -47,7 +46,14 @@ const ( // tidb_opt_agg_push_down is used to enable/disable the optimizer rule of aggregation push down. TiDBOptAggPushDown = "tidb_opt_agg_push_down" + // TiDBOptBCJ is used to enable/disable broadcast join in MPP mode TiDBOptBCJ = "tidb_opt_broadcast_join" + + // TiDBOptCartesianBCJ is used to disable/enable broadcast cartesian join in MPP mode + TiDBOptCartesianBCJ = "tidb_opt_broadcast_cartesian_join" + + TiDBOptMPPOuterJoinFixedBuildSide = "tidb_opt_mpp_outer_join_fixed_build_side" + // tidb_opt_distinct_agg_push_down is used to decide whether agg with distinct should be pushed to tikv/tiflash. TiDBOptDistinctAggPushDown = "tidb_opt_distinct_agg_push_down" @@ -107,15 +113,15 @@ const ( // The following session variables controls the memory quota during query execution. // "tidb_mem_quota_query": control the memory quota of a query. - TIDBMemQuotaQuery = "tidb_mem_quota_query" // Bytes. + TiDBMemQuotaQuery = "tidb_mem_quota_query" // Bytes. TiDBMemQuotaApplyCache = "tidb_mem_quota_apply_cache" - // TODO: remove them below sometime, it should have only one Quota(TIDBMemQuotaQuery). - TIDBMemQuotaHashJoin = "tidb_mem_quota_hashjoin" // Bytes. - TIDBMemQuotaMergeJoin = "tidb_mem_quota_mergejoin" // Bytes. - TIDBMemQuotaSort = "tidb_mem_quota_sort" // Bytes. - TIDBMemQuotaTopn = "tidb_mem_quota_topn" // Bytes. - TIDBMemQuotaIndexLookupReader = "tidb_mem_quota_indexlookupreader" // Bytes. - TIDBMemQuotaIndexLookupJoin = "tidb_mem_quota_indexlookupjoin" // Bytes. + // TODO: remove them below sometime, it should have only one Quota(TiDBMemQuotaQuery). + TiDBMemQuotaHashJoin = "tidb_mem_quota_hashjoin" // Bytes. + TiDBMemQuotaMergeJoin = "tidb_mem_quota_mergejoin" // Bytes. + TiDBMemQuotaSort = "tidb_mem_quota_sort" // Bytes. + TiDBMemQuotaTopn = "tidb_mem_quota_topn" // Bytes. + TiDBMemQuotaIndexLookupReader = "tidb_mem_quota_indexlookupreader" // Bytes. + TiDBMemQuotaIndexLookupJoin = "tidb_mem_quota_indexlookupjoin" // Bytes. // tidb_general_log is used to log every query in the server in info level. TiDBGeneralLog = "tidb_general_log" @@ -204,6 +210,9 @@ const ( // TiDBTxnScope indicates whether using global transactions or local transactions. TiDBTxnScope = "txn_scope" + + // TiDBTxnReadTS indicates the next transaction should be staleness transaction and provide the startTS + TiDBTxnReadTS = "tx_read_ts" ) // TiDB system variable names that both in session and global scope. @@ -216,7 +225,6 @@ const ( // A distsql scan task can be a table scan or a index scan, which may be distributed to many TiKV nodes. // Higher concurrency may reduce latency, but with the cost of higher memory usage and system performance impact. // If the query has a LIMIT clause, high concurrency makes the system do much more work than needed. - // tidb_distsql_scan_concurrency is deprecated, use tidb_executor_concurrency instead. TiDBDistSQLScanConcurrency = "tidb_distsql_scan_concurrency" // tidb_opt_insubquery_to_join_and_agg is used to enable/disable the optimizer rule of rewriting IN subquery. @@ -291,6 +299,8 @@ const ( // The default value is 0 TiDBAllowBatchCop = "tidb_allow_batch_cop" + // TiDBAllowMPPExecution means if we should use mpp way to execute query. Default value is 1 (or 'ON'), means to be determined by the optimizer. + // Value set to 2 (or 'ENFORCE') which means to use mpp whenever possible. Value set to 2 (or 'OFF') means never use mpp. TiDBAllowMPPExecution = "tidb_allow_mpp" // TiDBInitChunkSize is used to control the init chunk size during query execution. @@ -364,9 +374,6 @@ const ( // It can be: PRIORITY_LOW, PRIORITY_NORMAL, PRIORITY_HIGH TiDBDDLReorgPriority = "tidb_ddl_reorg_priority" - // TiDBEnableChangeColumnType is used to control whether to enable the change column type. - TiDBEnableChangeColumnType = "tidb_enable_change_column_type" - // TiDBEnableChangeMultiSchema is used to control whether to enable the change multi schema. TiDBEnableChangeMultiSchema = "tidb_enable_change_multi_schema" @@ -393,10 +400,6 @@ const ( // It can be "NO_PRIORITY", "LOW_PRIORITY", "HIGH_PRIORITY", "DELAYED" TiDBForcePriority = "tidb_force_priority" - // tidb_enable_radix_join indicates to use radix hash join algorithm to execute - // HashJoin. - TiDBEnableRadixJoin = "tidb_enable_radix_join" - // tidb_constraint_check_in_place indicates to check the constraint when the SQL executing. // It could hurt the performance of bulking insert when it is ON. TiDBConstraintCheckInPlace = "tidb_constraint_check_in_place" @@ -404,13 +407,16 @@ const ( // tidb_enable_window_function is used to control whether to enable the window function. TiDBEnableWindowFunction = "tidb_enable_window_function" + // tidb_enable_pipelined_window_function is used to control whether to use pipelined window function, it only works when tidb_enable_window_function = true. + TiDBEnablePipelinedWindowFunction = "tidb_enable_pipelined_window_function" + // tidb_enable_strict_double_type_check is used to control table field double type syntax check. TiDBEnableStrictDoubleTypeCheck = "tidb_enable_strict_double_type_check" // tidb_enable_vectorized_expression is used to control whether to enable the vectorized expression evaluation. TiDBEnableVectorizedExpression = "tidb_enable_vectorized_expression" - // TIDBOptJoinReorderThreshold defines the threshold less than which + // TiDBOptJoinReorderThreshold defines the threshold less than which // we'll choose a rather time consuming algorithm to calculate the join order. TiDBOptJoinReorderThreshold = "tidb_opt_join_reorder_threshold" @@ -532,6 +538,23 @@ const ( // TiDBEnableDynamicPrivileges enables MySQL 8.0 compatible dynamic privileges (experimental). TiDBEnableDynamicPrivileges = "tidb_enable_dynamic_privileges" + + // TiDBEnableTopSQL indicates whether the top SQL is enabled. + TiDBEnableTopSQL = "tidb_enable_top_sql" + + // TiDBTopSQLAgentAddress indicates the top SQL agent address. + TiDBTopSQLAgentAddress = "tidb_top_sql_agent_address" + + // TiDBTopSQLPrecisionSeconds indicates the top SQL precision seconds. + TiDBTopSQLPrecisionSeconds = "tidb_top_sql_precision_seconds" + + // TiDBTopSQLMaxStatementCount indicates the max number of statements been collected. + TiDBTopSQLMaxStatementCount = "tidb_top_sql_max_statement_count" + + // TiDBTopSQLReportIntervalSeconds indicates the top SQL report interval seconds. + TiDBTopSQLReportIntervalSeconds = "tidb_top_sql_report_interval_seconds" + // TiDBEnableGlobalTemporaryTable indicates whether to enable global temporary table + TiDBEnableGlobalTemporaryTable = "tidb_enable_global_temporary_table" ) // TiDB vars that have only global scope @@ -571,6 +594,8 @@ const ( DefSkipASCIICheck = false DefOptAggPushDown = false DefOptBCJ = false + DefOptCartesianBCJ = 1 + DefOptMPPOuterJoinFixedBuildSide = false DefOptWriteRowID = false DefOptCorrelationThreshold = 0.9 DefOptCorrelationExpFactor = 1 @@ -614,7 +639,7 @@ const ( DefBroadcastJoinThresholdCount = 10 * 1024 DefTiDBOptimizerSelectivityLevel = 0 DefTiDBAllowBatchCop = 1 - DefTiDBAllowMPPExecution = true + DefTiDBAllowMPPExecution = "ON" DefTiDBTxnMode = "" DefTiDBRowFormatV1 = 1 DefTiDBRowFormatV2 = 2 @@ -622,7 +647,6 @@ const ( DefTiDBDDLReorgBatchSize = 256 DefTiDBDDLErrorCountLimit = 512 DefTiDBMaxDeltaSchemaCount = 1024 - DefTiDBChangeColumnType = false DefTiDBChangeMultiSchema = false DefTiDBPointGetCache = false DefTiDBEnableAlterPlacement = false @@ -632,8 +656,8 @@ const ( DefTiDBMergeJoinConcurrency = 1 // disable optimization by default DefTiDBStreamAggConcurrency = 1 DefTiDBForcePriority = mysql.NoPriority - DefTiDBUseRadixJoin = false DefEnableWindowFunction = true + DefEnablePipelinedWindowFunction = true DefEnableStrictDoubleTypeCheck = true DefEnableVectorizedExpression = true DefTiDBOptJoinReorderThreshold = 0 @@ -665,16 +689,23 @@ const ( DefTiDBEnableTelemetry = true DefTiDBEnableParallelApply = false DefTiDBEnableAmendPessimisticTxn = false - DefTiDBPartitionPruneMode = "static" + DefTiDBPartitionPruneMode = "dynamic" DefTiDBEnableRateLimitAction = true DefTiDBEnableAsyncCommit = false DefTiDBEnable1PC = false DefTiDBGuaranteeLinearizability = true - DefTiDBAnalyzeVersion = 1 + DefTiDBAnalyzeVersion = 2 DefTiDBEnableIndexMergeJoin = false DefTiDBTrackAggregateMemoryUsage = true DefTiDBEnableExchangePartition = false DefCTEMaxRecursionDepth = 1000 + DefTiDBTopSQLEnable = false + DefTiDBTopSQLAgentAddress = "" + DefTiDBTopSQLPrecisionSeconds = 1 + DefTiDBTopSQLMaxStatementCount = 200 + DefTiDBTopSQLReportIntervalSeconds = 60 + DefTiDBEnableGlobalTemporaryTable = false + DefTMPTableSize = 16777216 ) // Process global variables. @@ -693,11 +724,36 @@ var ( // DDLSlowOprThreshold is the threshold for ddl slow operations, uint is millisecond. DDLSlowOprThreshold uint32 = DefTiDBDDLSlowOprThreshold ForcePriority = int32(DefTiDBForcePriority) - ServerHostname, _ = os.Hostname() MaxOfMaxAllowedPacket uint64 = 1073741824 ExpensiveQueryTimeThreshold uint64 = DefTiDBExpensiveQueryTimeThreshold MinExpensiveQueryTimeThreshold uint64 = 10 // 10s CapturePlanBaseline = serverGlobalVariable{globalVal: Off} DefExecutorConcurrency = 5 MemoryUsageAlarmRatio = atomic.NewFloat64(config.GetGlobalConfig().Performance.MemoryUsageAlarmRatio) + TopSQLVariable = TopSQL{ + Enable: atomic.NewBool(DefTiDBTopSQLEnable), + AgentAddress: atomic.NewString(DefTiDBTopSQLAgentAddress), + PrecisionSeconds: atomic.NewInt64(DefTiDBTopSQLPrecisionSeconds), + MaxStatementCount: atomic.NewInt64(DefTiDBTopSQLMaxStatementCount), + ReportIntervalSeconds: atomic.NewInt64(DefTiDBTopSQLReportIntervalSeconds), + } ) + +// TopSQL is the variable for control top sql feature. +type TopSQL struct { + // Enable top-sql or not. + Enable *atomic.Bool + // AgentAddress indicate the collect agent address. + AgentAddress *atomic.String + // The refresh interval of top-sql. + PrecisionSeconds *atomic.Int64 + // The maximum number of statements kept in memory. + MaxStatementCount *atomic.Int64 + // The report data interval of top-sql. + ReportIntervalSeconds *atomic.Int64 +} + +// TopSQLEnabled uses to check whether enabled the top SQL feature. +func TopSQLEnabled() bool { + return TopSQLVariable.Enable.Load() && TopSQLVariable.AgentAddress.Load() != "" +} diff --git a/sessionctx/variable/varsutil.go b/sessionctx/variable/varsutil.go index d43a45b1cdbc7..4447606508beb 100644 --- a/sessionctx/variable/varsutil.go +++ b/sessionctx/variable/varsutil.go @@ -14,8 +14,6 @@ package variable import ( - "encoding/json" - "fmt" "strconv" "strings" "sync" @@ -25,7 +23,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/parser/charset" "github.com/pingcap/parser/mysql" - "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/collate" "github.com/pingcap/tidb/util/timeutil" @@ -161,131 +159,47 @@ func checkIsolationLevel(vars *SessionVars, normalizedValue string, originalValu return normalizedValue, nil } -// GetSessionSystemVar gets a system variable. +// GetSessionOrGlobalSystemVar gets a system variable. // If it is a session only variable, use the default value defined in code. // Returns error if there is no such variable. -func GetSessionSystemVar(s *SessionVars, key string) (string, error) { - key = strings.ToLower(key) - gVal, ok, err := GetSessionOnlySysVars(s, key) - if err != nil || ok { - return gVal, err - } - gVal, err = s.GlobalVarsAccessor.GetGlobalSysVar(key) - if err != nil { - return "", err - } - s.systems[key] = gVal - return gVal, nil -} - -// GetSessionOnlySysVars get the default value defined in code for session only variable. -// The return bool value indicates whether it's a session only variable. -func GetSessionOnlySysVars(s *SessionVars, key string) (string, bool, error) { - sysVar := GetSysVar(key) - if sysVar == nil { - return "", false, ErrUnknownSystemVar.GenWithStackByArgs(key) - } - // For virtual system variables: - switch sysVar.Name { - case TiDBCurrentTS: - return fmt.Sprintf("%d", s.TxnCtx.StartTS), true, nil - case TiDBLastTxnInfo: - return s.LastTxnInfo, true, nil - case TiDBLastQueryInfo: - info, err := json.Marshal(s.LastQueryInfo) - if err != nil { - return "", true, err - } - return string(info), true, nil - case TiDBGeneralLog: - return BoolToOnOff(ProcessGeneralLog.Load()), true, nil - case TiDBPProfSQLCPU: - val := "0" - if EnablePProfSQLCPU.Load() { - val = "1" +func GetSessionOrGlobalSystemVar(s *SessionVars, name string) (string, error) { + sv := GetSysVar(name) + if sv == nil { + return "", ErrUnknownSystemVar.GenWithStackByArgs(name) + } + if sv.HasNoneScope() { + return sv.Value, nil + } + if sv.HasSessionScope() { + // Populate the value to s.systems if it is not there already. + // in future should be already loaded on session init + if sv.GetSession != nil { + // shortcut to the getter, we won't use the value + return sv.GetSessionFromHook(s) } - return val, true, nil - case TiDBExpensiveQueryTimeThreshold: - return fmt.Sprintf("%d", atomic.LoadUint64(&ExpensiveQueryTimeThreshold)), true, nil - case TiDBMemoryUsageAlarmRatio: - return fmt.Sprintf("%g", MemoryUsageAlarmRatio.Load()), true, nil - case TiDBConfig: - conf := config.GetGlobalConfig() - j, err := json.MarshalIndent(conf, "", "\t") - if err != nil { - return "", false, err + if _, ok := s.systems[sv.Name]; !ok { + if sv.HasGlobalScope() { + if val, err := s.GlobalVarsAccessor.GetGlobalSysVar(sv.Name); err == nil { + s.systems[sv.Name] = val + } + } else { + s.systems[sv.Name] = sv.Value // no global scope, use default + } } - return config.HideConfig(string(j)), true, nil - case TiDBForcePriority: - return mysql.Priority2Str[mysql.PriorityEnum(atomic.LoadInt32(&ForcePriority))], true, nil - case TiDBDDLSlowOprThreshold: - return strconv.FormatUint(uint64(atomic.LoadUint32(&DDLSlowOprThreshold)), 10), true, nil - case PluginDir: - return config.GetGlobalConfig().Plugin.Dir, true, nil - case PluginLoad: - return config.GetGlobalConfig().Plugin.Load, true, nil - case TiDBSlowLogThreshold: - return strconv.FormatUint(atomic.LoadUint64(&config.GetGlobalConfig().Log.SlowThreshold), 10), true, nil - case TiDBRecordPlanInSlowLog: - return strconv.FormatUint(uint64(atomic.LoadUint32(&config.GetGlobalConfig().Log.RecordPlanInSlowLog)), 10), true, nil - case TiDBEnableSlowLog: - return BoolToOnOff(config.GetGlobalConfig().Log.EnableSlowLog), true, nil - case TiDBQueryLogMaxLen: - return strconv.FormatUint(atomic.LoadUint64(&config.GetGlobalConfig().Log.QueryLogMaxLen), 10), true, nil - case TiDBCheckMb4ValueInUTF8: - return BoolToOnOff(config.GetGlobalConfig().CheckMb4ValueInUTF8), true, nil - case TiDBCapturePlanBaseline: - return CapturePlanBaseline.GetVal(), true, nil - case TiDBFoundInPlanCache: - return BoolToOnOff(s.PrevFoundInPlanCache), true, nil - case TiDBFoundInBinding: - return BoolToOnOff(s.PrevFoundInBinding), true, nil - case TiDBEnableCollectExecutionInfo: - return BoolToOnOff(config.GetGlobalConfig().EnableCollectExecutionInfo), true, nil - case TiDBTxnScope: - return s.TxnScope.GetVarValue(), true, nil - } - sVal, ok := s.GetSystemVar(key) - if ok { - return sVal, true, nil - } - if !sysVar.HasGlobalScope() { - // None-Global variable can use pre-defined default value. - return sysVar.Value, true, nil - } - return "", false, nil -} - -// GetGlobalSystemVar gets a global system variable. -func GetGlobalSystemVar(s *SessionVars, key string) (string, error) { - key = strings.ToLower(key) - gVal, ok, err := GetScopeNoneSystemVar(key) - if err != nil || ok { - return gVal, err - } - gVal, err = s.GlobalVarsAccessor.GetGlobalSysVar(key) - if err != nil { - return "", err + return sv.GetSessionFromHook(s) } - return gVal, nil + return sv.GetGlobalFromHook(s) } -// GetScopeNoneSystemVar checks the validation of `key`, -// and return the default value if its scope is `ScopeNone`. -func GetScopeNoneSystemVar(key string) (string, bool, error) { - sysVar := GetSysVar(key) - if sysVar == nil { - return "", false, ErrUnknownSystemVar.GenWithStackByArgs(key) - } - if sysVar.Scope == ScopeNone { - return sysVar.Value, true, nil +// GetGlobalSystemVar gets a global system variable. +func GetGlobalSystemVar(s *SessionVars, name string) (string, error) { + sv := GetSysVar(name) + if sv == nil { + return "", ErrUnknownSystemVar.GenWithStackByArgs(name) } - return "", false, nil + return sv.GetGlobalFromHook(s) } -// epochShiftBits is used to reserve logical part of the timestamp. -const epochShiftBits = 18 - // SetSessionSystemVar sets system variable and updates SessionVars states. func SetSessionSystemVar(vars *SessionVars, name string, value string) error { sysVar := GetSysVar(name) @@ -313,25 +227,6 @@ func SetStmtVar(vars *SessionVars, name string, value string) error { return vars.SetStmtVar(name, sVal) } -// ValidateGetSystemVar checks if system variable exists and validates its scope when get system variable. -func ValidateGetSystemVar(name string, isGlobal bool) error { - sysVar := GetSysVar(name) - if sysVar == nil { - return ErrUnknownSystemVar.GenWithStackByArgs(name) - } - switch sysVar.Scope { - case ScopeGlobal: - if !isGlobal { - return ErrIncorrectScope.GenWithStackByArgs(name, "GLOBAL") - } - case ScopeSession: - if isGlobal { - return ErrIncorrectScope.GenWithStackByArgs(name, "SESSION") - } - } - return nil -} - const ( // initChunkSizeUpperBound indicates upper bound value of tidb_init_chunk_size. initChunkSizeUpperBound = 32 @@ -478,14 +373,29 @@ func setSnapshotTS(s *SessionVars, sVal string) error { } t1, err := t.GoTime(s.TimeZone) - s.SnapshotTS = GoTimeToTS(t1) + s.SnapshotTS = oracle.GoTimeToTS(t1) + // tx_read_ts should be mutual exclusive with tidb_snapshot + s.TxnReadTS = NewTxnReadTS(0) return err } -// GoTimeToTS converts a Go time to uint64 timestamp. -func GoTimeToTS(t time.Time) uint64 { - ts := (t.UnixNano() / int64(time.Millisecond)) << epochShiftBits - return uint64(ts) +func setTxnReadTS(s *SessionVars, sVal string) error { + if sVal == "" { + s.TxnReadTS = NewTxnReadTS(0) + return nil + } + t, err := types.ParseTime(s.StmtCtx, sVal, mysql.TypeTimestamp, types.MaxFsp) + if err != nil { + return err + } + t1, err := t.GoTime(s.TimeZone) + if err != nil { + return err + } + s.TxnReadTS = NewTxnReadTS(oracle.GoTimeToTS(t1)) + // tx_read_ts should be mutual exclusive with tidb_snapshot + s.SnapshotTS = 0 + return err } // serverGlobalVariable is used to handle variables that acts in server and global scope. diff --git a/sessionctx/variable/varsutil_test.go b/sessionctx/variable/varsutil_test.go index 5757eaeeac403..41f379a9d6353 100644 --- a/sessionctx/variable/varsutil_test.go +++ b/sessionctx/variable/varsutil_test.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/util/testleak" ) @@ -95,7 +95,6 @@ func (s *testVarsutilSuite) TestNewSessionVars(c *C) { c.Assert(vars.MemQuotaIndexLookupReader, Equals, int64(DefTiDBMemQuotaIndexLookupReader)) c.Assert(vars.MemQuotaIndexLookupJoin, Equals, int64(DefTiDBMemQuotaIndexLookupJoin)) c.Assert(vars.MemQuotaApplyCache, Equals, int64(DefTiDBMemQuotaApplyCache)) - c.Assert(vars.EnableRadixJoin, Equals, DefTiDBUseRadixJoin) c.Assert(vars.AllowWriteRowID, Equals, DefOptWriteRowID) c.Assert(vars.TiDBOptJoinReorderThreshold, Equals, DefTiDBOptJoinReorderThreshold) c.Assert(vars.EnableFastAnalyze, Equals, DefTiDBUseFastAnalyze) @@ -103,8 +102,9 @@ func (s *testVarsutilSuite) TestNewSessionVars(c *C) { c.Assert(vars.FoundInBinding, Equals, DefTiDBFoundInBinding) c.Assert(vars.AllowAutoRandExplicitInsert, Equals, DefTiDBAllowAutoRandExplicitInsert) c.Assert(vars.ShardAllocateStep, Equals, int64(DefTiDBShardAllocateStep)) - c.Assert(vars.EnableChangeColumnType, Equals, DefTiDBChangeColumnType) c.Assert(vars.AnalyzeVersion, Equals, DefTiDBAnalyzeVersion) + c.Assert(vars.CTEMaxRecursionDepth, Equals, DefCTEMaxRecursionDepth) + c.Assert(vars.TMPTableSize, Equals, int64(DefTMPTableSize)) assertFieldsGreaterThanZero(c, reflect.ValueOf(vars.MemQuota)) assertFieldsGreaterThanZero(c, reflect.ValueOf(vars.BatchSize)) @@ -124,7 +124,7 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { err := SetSessionSystemVar(v, "autocommit", "1") c.Assert(err, IsNil) - val, err := GetSessionSystemVar(v, "autocommit") + val, err := GetSessionOrGlobalSystemVar(v, "autocommit") c.Assert(err, IsNil) c.Assert(val, Equals, "ON") c.Assert(SetSessionSystemVar(v, "autocommit", ""), NotNil) @@ -132,20 +132,20 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { // 0 converts to OFF err = SetSessionSystemVar(v, "foreign_key_checks", "0") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, "foreign_key_checks") + val, err = GetSessionOrGlobalSystemVar(v, "foreign_key_checks") c.Assert(err, IsNil) c.Assert(val, Equals, "OFF") // 1/ON is not supported (generates a warning and sets to OFF) err = SetSessionSystemVar(v, "foreign_key_checks", "1") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, "foreign_key_checks") + val, err = GetSessionOrGlobalSystemVar(v, "foreign_key_checks") c.Assert(err, IsNil) c.Assert(val, Equals, "OFF") err = SetSessionSystemVar(v, "sql_mode", "strict_trans_tables") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, "sql_mode") + val, err = GetSessionOrGlobalSystemVar(v, "sql_mode") c.Assert(err, IsNil) c.Assert(val, Equals, "STRICT_TRANS_TABLES") c.Assert(v.StrictSQLMode, IsTrue) @@ -253,7 +253,7 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { // Test case for TiDBConfig session variable. err = SetSessionSystemVar(v, TiDBConfig, "abc") c.Assert(terror.ErrorEqual(err, ErrIncorrectScope), IsTrue) - val, err = GetSessionSystemVar(v, TiDBConfig) + val, err = GetSessionOrGlobalSystemVar(v, TiDBConfig) c.Assert(err, IsNil) bVal, err := json.MarshalIndent(config.GetGlobalConfig(), "", "\t") c.Assert(err, IsNil) @@ -261,13 +261,13 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { err = SetSessionSystemVar(v, TiDBEnableStreaming, "1") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBEnableStreaming) + val, err = GetSessionOrGlobalSystemVar(v, TiDBEnableStreaming) c.Assert(err, IsNil) c.Assert(val, Equals, "ON") c.Assert(v.EnableStreaming, Equals, true) err = SetSessionSystemVar(v, TiDBEnableStreaming, "0") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBEnableStreaming) + val, err = GetSessionOrGlobalSystemVar(v, TiDBEnableStreaming) c.Assert(err, IsNil) c.Assert(val, Equals, "OFF") c.Assert(v.EnableStreaming, Equals, false) @@ -282,7 +282,7 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { err = SetSessionSystemVar(v, TiDBRetryLimit, "3") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBRetryLimit) + val, err = GetSessionOrGlobalSystemVar(v, TiDBRetryLimit) c.Assert(err, IsNil) c.Assert(val, Equals, "3") c.Assert(v.RetryLimit, Equals, int64(3)) @@ -290,7 +290,7 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { c.Assert(v.EnableTablePartition, Equals, "") err = SetSessionSystemVar(v, TiDBEnableTablePartition, "on") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBEnableTablePartition) + val, err = GetSessionOrGlobalSystemVar(v, TiDBEnableTablePartition) c.Assert(err, IsNil) c.Assert(val, Equals, "ON") c.Assert(v.EnableTablePartition, Equals, "ON") @@ -298,7 +298,7 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { c.Assert(v.EnableListTablePartition, Equals, false) err = SetSessionSystemVar(v, TiDBEnableListTablePartition, "on") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBEnableListTablePartition) + val, err = GetSessionOrGlobalSystemVar(v, TiDBEnableListTablePartition) c.Assert(err, IsNil) c.Assert(val, Equals, "ON") c.Assert(v.EnableListTablePartition, Equals, true) @@ -306,33 +306,33 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { c.Assert(v.TiDBOptJoinReorderThreshold, Equals, DefTiDBOptJoinReorderThreshold) err = SetSessionSystemVar(v, TiDBOptJoinReorderThreshold, "5") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBOptJoinReorderThreshold) + val, err = GetSessionOrGlobalSystemVar(v, TiDBOptJoinReorderThreshold) c.Assert(err, IsNil) c.Assert(val, Equals, "5") c.Assert(v.TiDBOptJoinReorderThreshold, Equals, 5) err = SetSessionSystemVar(v, TiDBCheckMb4ValueInUTF8, "1") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBCheckMb4ValueInUTF8) + val, err = GetSessionOrGlobalSystemVar(v, TiDBCheckMb4ValueInUTF8) c.Assert(err, IsNil) c.Assert(val, Equals, "ON") c.Assert(config.GetGlobalConfig().CheckMb4ValueInUTF8, Equals, true) err = SetSessionSystemVar(v, TiDBCheckMb4ValueInUTF8, "0") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBCheckMb4ValueInUTF8) + val, err = GetSessionOrGlobalSystemVar(v, TiDBCheckMb4ValueInUTF8) c.Assert(err, IsNil) c.Assert(val, Equals, "OFF") c.Assert(config.GetGlobalConfig().CheckMb4ValueInUTF8, Equals, false) err = SetSessionSystemVar(v, TiDBLowResolutionTSO, "1") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBLowResolutionTSO) + val, err = GetSessionOrGlobalSystemVar(v, TiDBLowResolutionTSO) c.Assert(err, IsNil) c.Assert(val, Equals, "ON") c.Assert(v.LowResolutionTSO, Equals, true) err = SetSessionSystemVar(v, TiDBLowResolutionTSO, "0") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBLowResolutionTSO) + val, err = GetSessionOrGlobalSystemVar(v, TiDBLowResolutionTSO) c.Assert(err, IsNil) c.Assert(val, Equals, "OFF") c.Assert(v.LowResolutionTSO, Equals, false) @@ -340,7 +340,7 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { c.Assert(v.CorrelationThreshold, Equals, 0.9) err = SetSessionSystemVar(v, TiDBOptCorrelationThreshold, "0") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBOptCorrelationThreshold) + val, err = GetSessionOrGlobalSystemVar(v, TiDBOptCorrelationThreshold) c.Assert(err, IsNil) c.Assert(val, Equals, "0") c.Assert(v.CorrelationThreshold, Equals, float64(0)) @@ -348,7 +348,7 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { c.Assert(v.CPUFactor, Equals, 3.0) err = SetSessionSystemVar(v, TiDBOptCPUFactor, "5.0") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBOptCPUFactor) + val, err = GetSessionOrGlobalSystemVar(v, TiDBOptCPUFactor) c.Assert(err, IsNil) c.Assert(val, Equals, "5.0") c.Assert(v.CPUFactor, Equals, 5.0) @@ -356,7 +356,7 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { c.Assert(v.CopCPUFactor, Equals, 3.0) err = SetSessionSystemVar(v, TiDBOptCopCPUFactor, "5.0") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBOptCopCPUFactor) + val, err = GetSessionOrGlobalSystemVar(v, TiDBOptCopCPUFactor) c.Assert(err, IsNil) c.Assert(val, Equals, "5.0") c.Assert(v.CopCPUFactor, Equals, 5.0) @@ -364,47 +364,47 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { c.Assert(v.CopTiFlashConcurrencyFactor, Equals, 24.0) err = SetSessionSystemVar(v, TiDBOptTiFlashConcurrencyFactor, "5.0") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBOptTiFlashConcurrencyFactor) + val, err = GetSessionOrGlobalSystemVar(v, TiDBOptTiFlashConcurrencyFactor) c.Assert(err, IsNil) c.Assert(val, Equals, "5.0") c.Assert(v.CopCPUFactor, Equals, 5.0) - c.Assert(v.NetworkFactor, Equals, 1.0) + c.Assert(v.GetNetworkFactor(nil), Equals, 1.0) err = SetSessionSystemVar(v, TiDBOptNetworkFactor, "3.0") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBOptNetworkFactor) + val, err = GetSessionOrGlobalSystemVar(v, TiDBOptNetworkFactor) c.Assert(err, IsNil) c.Assert(val, Equals, "3.0") - c.Assert(v.NetworkFactor, Equals, 3.0) + c.Assert(v.GetNetworkFactor(nil), Equals, 3.0) - c.Assert(v.ScanFactor, Equals, 1.5) + c.Assert(v.GetScanFactor(nil), Equals, 1.5) err = SetSessionSystemVar(v, TiDBOptScanFactor, "3.0") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBOptScanFactor) + val, err = GetSessionOrGlobalSystemVar(v, TiDBOptScanFactor) c.Assert(err, IsNil) c.Assert(val, Equals, "3.0") - c.Assert(v.ScanFactor, Equals, 3.0) + c.Assert(v.GetScanFactor(nil), Equals, 3.0) - c.Assert(v.DescScanFactor, Equals, 3.0) + c.Assert(v.GetDescScanFactor(nil), Equals, 3.0) err = SetSessionSystemVar(v, TiDBOptDescScanFactor, "5.0") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBOptDescScanFactor) + val, err = GetSessionOrGlobalSystemVar(v, TiDBOptDescScanFactor) c.Assert(err, IsNil) c.Assert(val, Equals, "5.0") - c.Assert(v.DescScanFactor, Equals, 5.0) + c.Assert(v.GetDescScanFactor(nil), Equals, 5.0) - c.Assert(v.SeekFactor, Equals, 20.0) + c.Assert(v.GetSeekFactor(nil), Equals, 20.0) err = SetSessionSystemVar(v, TiDBOptSeekFactor, "50.0") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBOptSeekFactor) + val, err = GetSessionOrGlobalSystemVar(v, TiDBOptSeekFactor) c.Assert(err, IsNil) c.Assert(val, Equals, "50.0") - c.Assert(v.SeekFactor, Equals, 50.0) + c.Assert(v.GetSeekFactor(nil), Equals, 50.0) c.Assert(v.MemoryFactor, Equals, 0.001) err = SetSessionSystemVar(v, TiDBOptMemoryFactor, "1.0") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBOptMemoryFactor) + val, err = GetSessionOrGlobalSystemVar(v, TiDBOptMemoryFactor) c.Assert(err, IsNil) c.Assert(val, Equals, "1.0") c.Assert(v.MemoryFactor, Equals, 1.0) @@ -412,7 +412,7 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { c.Assert(v.DiskFactor, Equals, 1.5) err = SetSessionSystemVar(v, TiDBOptDiskFactor, "1.1") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBOptDiskFactor) + val, err = GetSessionOrGlobalSystemVar(v, TiDBOptDiskFactor) c.Assert(err, IsNil) c.Assert(val, Equals, "1.1") c.Assert(v.DiskFactor, Equals, 1.1) @@ -420,57 +420,57 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { c.Assert(v.ConcurrencyFactor, Equals, 3.0) err = SetSessionSystemVar(v, TiDBOptConcurrencyFactor, "5.0") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBOptConcurrencyFactor) + val, err = GetSessionOrGlobalSystemVar(v, TiDBOptConcurrencyFactor) c.Assert(err, IsNil) c.Assert(val, Equals, "5.0") c.Assert(v.ConcurrencyFactor, Equals, 5.0) err = SetSessionSystemVar(v, TiDBReplicaRead, "follower") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBReplicaRead) + val, err = GetSessionOrGlobalSystemVar(v, TiDBReplicaRead) c.Assert(err, IsNil) c.Assert(val, Equals, "follower") - c.Assert(v.GetReplicaRead(), Equals, tikvstore.ReplicaReadFollower) + c.Assert(v.GetReplicaRead(), Equals, kv.ReplicaReadFollower) err = SetSessionSystemVar(v, TiDBReplicaRead, "leader") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBReplicaRead) + val, err = GetSessionOrGlobalSystemVar(v, TiDBReplicaRead) c.Assert(err, IsNil) c.Assert(val, Equals, "leader") - c.Assert(v.GetReplicaRead(), Equals, tikvstore.ReplicaReadLeader) + c.Assert(v.GetReplicaRead(), Equals, kv.ReplicaReadLeader) err = SetSessionSystemVar(v, TiDBReplicaRead, "leader-and-follower") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBReplicaRead) + val, err = GetSessionOrGlobalSystemVar(v, TiDBReplicaRead) c.Assert(err, IsNil) c.Assert(val, Equals, "leader-and-follower") - c.Assert(v.GetReplicaRead(), Equals, tikvstore.ReplicaReadMixed) + c.Assert(v.GetReplicaRead(), Equals, kv.ReplicaReadMixed) err = SetSessionSystemVar(v, TiDBEnableStmtSummary, "ON") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBEnableStmtSummary) + val, err = GetSessionOrGlobalSystemVar(v, TiDBEnableStmtSummary) c.Assert(err, IsNil) c.Assert(val, Equals, "ON") err = SetSessionSystemVar(v, TiDBRedactLog, "ON") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBRedactLog) + val, err = GetSessionOrGlobalSystemVar(v, TiDBRedactLog) c.Assert(err, IsNil) c.Assert(val, Equals, "ON") err = SetSessionSystemVar(v, TiDBStmtSummaryRefreshInterval, "10") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBStmtSummaryRefreshInterval) + val, err = GetSessionOrGlobalSystemVar(v, TiDBStmtSummaryRefreshInterval) c.Assert(err, IsNil) c.Assert(val, Equals, "10") err = SetSessionSystemVar(v, TiDBStmtSummaryHistorySize, "10") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBStmtSummaryHistorySize) + val, err = GetSessionOrGlobalSystemVar(v, TiDBStmtSummaryHistorySize) c.Assert(err, IsNil) c.Assert(val, Equals, "10") err = SetSessionSystemVar(v, TiDBStmtSummaryMaxStmtCount, "10") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBStmtSummaryMaxStmtCount) + val, err = GetSessionOrGlobalSystemVar(v, TiDBStmtSummaryMaxStmtCount) c.Assert(err, IsNil) c.Assert(val, Equals, "10") err = SetSessionSystemVar(v, TiDBStmtSummaryMaxStmtCount, "a") @@ -478,7 +478,7 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { err = SetSessionSystemVar(v, TiDBStmtSummaryMaxSQLLength, "10") c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBStmtSummaryMaxSQLLength) + val, err = GetSessionOrGlobalSystemVar(v, TiDBStmtSummaryMaxSQLLength) c.Assert(err, IsNil) c.Assert(val, Equals, "10") err = SetSessionSystemVar(v, TiDBStmtSummaryMaxSQLLength, "a") @@ -490,13 +490,6 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) { err = SetSessionSystemVar(v, TiDBFoundInBinding, "1") c.Assert(err, ErrorMatches, ".*]Variable 'last_plan_from_binding' is a read only variable") - err = SetSessionSystemVar(v, TiDBEnableChangeColumnType, "ON") - c.Assert(err, IsNil) - val, err = GetSessionSystemVar(v, TiDBEnableChangeColumnType) - c.Assert(err, IsNil) - c.Assert(val, Equals, "ON") - c.Assert(v.systems[TiDBEnableChangeColumnType], Equals, "ON") - err = SetSessionSystemVar(v, "UnknownVariable", "on") c.Assert(err, ErrorMatches, ".*]Unknown system variable 'UnknownVariable'") diff --git a/statistics/builder.go b/statistics/builder.go index 81c2f52dc84a9..d3d66294650b2 100644 --- a/statistics/builder.go +++ b/statistics/builder.go @@ -15,6 +15,7 @@ package statistics import ( "bytes" + "math" "github.com/pingcap/errors" "github.com/pingcap/tidb/sessionctx" @@ -42,7 +43,7 @@ func NewSortedBuilder(sc *stmtctx.StatementContext, numBuckets, id int64, tp *ty numBuckets: numBuckets, valuesPerBucket: 1, hist: NewHistogram(id, 0, 0, 0, tp, int(numBuckets), 0), - needBucketNDV: statsVer == Version2, + needBucketNDV: statsVer >= Version2, } } @@ -210,15 +211,32 @@ func BuildColumn(ctx sessionctx.Context, numBuckets, id int64, collector *Sample return BuildColumnHist(ctx, numBuckets, id, collector, tp, collector.Count, collector.FMSketch.NDV(), collector.NullCount) } -// BuildColumnHistAndTopN build a histogram and TopN for a column from samples. -func BuildColumnHistAndTopN(ctx sessionctx.Context, numBuckets, numTopN int, id int64, collector *SampleCollector, tp *types.FieldType) (*Histogram, *TopN, error) { +// BuildHistAndTopN build a histogram and TopN for a column or an index from samples. +func BuildHistAndTopN( + ctx sessionctx.Context, + numBuckets, numTopN int, + id int64, + collector *SampleCollector, + tp *types.FieldType, + isColumn bool, +) (*Histogram, *TopN, error) { + var getComparedBytes func(datum types.Datum) ([]byte, error) + if isColumn { + getComparedBytes = func(datum types.Datum) ([]byte, error) { + return codec.EncodeKey(ctx.GetSessionVars().StmtCtx, nil, datum) + } + } else { + getComparedBytes = func(datum types.Datum) ([]byte, error) { + return datum.GetBytes(), nil + } + } count := collector.Count ndv := collector.FMSketch.NDV() nullCount := collector.NullCount if ndv > count { ndv = count } - if count == 0 || len(collector.Samples) == 0 { + if count == 0 || len(collector.Samples) == 0 || ndv == 0 { return NewHistogram(id, ndv, nullCount, 0, tp, 0, collector.TotalSize), nil, nil } sc := ctx.GetSessionVars().StmtCtx @@ -237,7 +255,7 @@ func BuildColumnHistAndTopN(ctx sessionctx.Context, numBuckets, numTopN int, id // the topNList is always sorted by count from more to less topNList := make([]TopNMeta, 0, numTopN) - cur, err := codec.EncodeKey(ctx.GetSessionVars().StmtCtx, nil, samples[0].Value) + cur, err := getComparedBytes(samples[0].Value) if err != nil { return nil, nil, errors.Trace(err) } @@ -246,9 +264,11 @@ func BuildColumnHistAndTopN(ctx sessionctx.Context, numBuckets, numTopN int, id // Iterate through the samples for i := int64(0); i < sampleNum; i++ { - corrXYSum += float64(i) * float64(samples[i].Ordinal) + if isColumn { + corrXYSum += float64(i) * float64(samples[i].Ordinal) + } - sampleBytes, err := codec.EncodeKey(ctx.GetSessionVars().StmtCtx, nil, samples[i].Value) + sampleBytes, err := getComparedBytes(samples[i].Value) if err != nil { return nil, nil, errors.Trace(err) } @@ -286,7 +306,9 @@ func BuildColumnHistAndTopN(ctx sessionctx.Context, numBuckets, numTopN int, id } // Calc the correlation of the column between the handle column. - hg.Correlation = calcCorrelation(sampleNum, corrXYSum) + if isColumn { + hg.Correlation = calcCorrelation(sampleNum, corrXYSum) + } // Handle the counting for the last value. Basically equal to the case 2 above. // now topn is empty: append the "current" count directly @@ -308,9 +330,11 @@ func BuildColumnHistAndTopN(ctx sessionctx.Context, numBuckets, numTopN int, id } } + topNList = pruneTopNItem(topNList, ndv, nullCount, sampleNum, count) + // Step2: exclude topn from samples for i := int64(0); i < int64(len(samples)); i++ { - sampleBytes, err := codec.EncodeKey(ctx.GetSessionVars().StmtCtx, nil, samples[i].Value) + sampleBytes, err := getComparedBytes(samples[i].Value) if err != nil { return nil, nil, errors.Trace(err) } @@ -345,3 +369,55 @@ func BuildColumnHistAndTopN(ctx sessionctx.Context, numBuckets, numTopN int, id return hg, topn, nil } + +// pruneTopNItem tries to prune the least common values in the top-n list if it is not significantly more common than the values not in the list. +// We assume that the ones not in the top-n list's selectivity is 1/remained_ndv which is the internal implementation of EqualRowCount +func pruneTopNItem(topns []TopNMeta, ndv, nullCount, sampleRows, totalRows int64) []TopNMeta { + // If the sampleRows holds all rows. We just return the top-n directly. + if sampleRows == totalRows || totalRows <= 1 { + return topns + } + // Sum the occurrence except the least common one from the top-n list. To check whether the lest common one is worth + // storing later. + sumCount := uint64(0) + for i := 0; i < len(topns)-1; i++ { + sumCount += topns[i].Count + } + topNNum := len(topns) + for topNNum > 0 { + // Selectivity for the ones not in the top-n list. + // (1 - things in top-n list - null) / remained ndv. + selectivity := 1.0 - float64(sumCount)/float64(sampleRows) - float64(nullCount)/float64(totalRows) + if selectivity < 0.0 { + selectivity = 0 + } + if selectivity > 1 { + selectivity = 1 + } + otherNDV := float64(ndv) - float64(topNNum) + if otherNDV > 1 { + selectivity /= otherNDV + } + N := float64(totalRows) + n := float64(sampleRows) + K := N * float64(topns[topNNum-1].Count) / n + // Since we are sampling without replacement. The distribution would be a hypergeometric distribution. + // Thus the variance is the following formula. + variance := n * K * (N - K) * (N - n) / (N * N * (N - 1)) + stddev := math.Sqrt(variance) + // We choose the bound that plus two stddev of the sample frequency, plus an additional 0.5 for the continuity correction. + // Note: + // The mean + 2 * stddev is known as Wald confidence interval, plus 0.5 would be continuity-corrected Wald interval + if float64(topns[topNNum-1].Count) > selectivity*n+2*stddev+0.5 { + // If the current one is worth storing, the latter ones too. So we just break here. + break + } + // Current one is not worth storing, remove it and subtract it from sumCount, go to next one. + topNNum-- + if topNNum == 0 { + break + } + sumCount -= topns[topNNum-1].Count + } + return topns[:topNNum] +} diff --git a/statistics/cmsketch.go b/statistics/cmsketch.go index f682a1507a4bf..07d90434a6cc7 100644 --- a/statistics/cmsketch.go +++ b/statistics/cmsketch.go @@ -386,9 +386,13 @@ func CMSketchToProto(c *CMSketch, topn *TopN) *tipb.CMSketch { // CMSketchAndTopNFromProto converts CMSketch and TopN from its protobuf representation. func CMSketchAndTopNFromProto(protoSketch *tipb.CMSketch) (*CMSketch, *TopN) { - if protoSketch == nil || len(protoSketch.Rows) == 0 { + if protoSketch == nil { return nil, nil } + retTopN := TopNFromProto(protoSketch.TopN) + if len(protoSketch.Rows) == 0 { + return nil, retTopN + } c := NewCMSketch(int32(len(protoSketch.Rows)), int32(len(protoSketch.Rows[0].Counters))) for i, row := range protoSketch.Rows { c.count = 0 @@ -398,14 +402,14 @@ func CMSketchAndTopNFromProto(protoSketch *tipb.CMSketch) (*CMSketch, *TopN) { } } c.defaultValue = protoSketch.DefaultValue - if len(protoSketch.TopN) == 0 { - return c, nil - } - return c, TopNFromProto(protoSketch.TopN) + return c, retTopN } // TopNFromProto converts TopN from its protobuf representation. func TopNFromProto(protoTopN []*tipb.CMSketchTopN) *TopN { + if len(protoTopN) == 0 { + return nil + } topN := NewTopN(32) for _, e := range protoTopN { d := make([]byte, len(e.Data)) @@ -517,6 +521,23 @@ func (c *TopN) String() string { return builder.String() } +// Num returns the ndv of the TopN. +// TopN is declared directly in Histogram. So the Len is occupied by the Histogram. We use Num instead. +func (c *TopN) Num() int { + if c == nil { + return 0 + } + return len(c.TopN) +} + +// outOfRange checks whether the the given value falls back in [TopN.LowestOne, TopN.HighestOne]. +func (c *TopN) outOfRange(val []byte) bool { + if c == nil || len(c.TopN) == 0 { + return true + } + return bytes.Compare(c.TopN[0].Encoded, val) > 0 || bytes.Compare(val, c.TopN[c.Num()-1].Encoded) > 0 +} + // DecodedString returns the value with decoded result. func (c *TopN) DecodedString(ctx sessionctx.Context, colTypes []byte) (string, error) { builder := &strings.Builder{} @@ -818,14 +839,11 @@ func MergeTopN(topNs []*TopN, n uint32) (*TopN, []TopNMeta) { } func checkEmptyTopNs(topNs []*TopN) bool { - totCnt := uint64(0) + count := uint64(0) for _, topN := range topNs { - totCnt += topN.TotalCount() - } - if totCnt == 0 { - return true + count += topN.TotalCount() } - return false + return count == 0 } func getMergedTopNFromSortedSlice(sorted []TopNMeta, n uint32) (*TopN, []TopNMeta) { diff --git a/statistics/feedback.go b/statistics/feedback.go index 89aeab32152b3..5b69163e21b2b 100644 --- a/statistics/feedback.go +++ b/statistics/feedback.go @@ -107,7 +107,6 @@ func (m *QueryFeedbackMap) Append(q *QueryFeedback) { Tp: q.Tp, } m.append(k, []*QueryFeedback{q}) - return } // MaxQueryFeedbackCount is the max number of feedbacks that are cached in memory. @@ -136,7 +135,6 @@ func (m *QueryFeedbackMap) Merge(r *QueryFeedbackMap) { break } } - return } var ( diff --git a/statistics/handle/bootstrap.go b/statistics/handle/bootstrap.go index 5fd48e096ca3f..99189221a0444 100644 --- a/statistics/handle/bootstrap.go +++ b/statistics/handle/bootstrap.go @@ -131,7 +131,7 @@ func (h *Handle) initStatsHistograms4Chunk(is infoschema.InfoSchema, cache *stat var topnCount int64 // If this is stats of the Version2, we need to consider the topn's count as well. // See the comments of Version2 for more details. - if statsVer == statistics.Version2 { + if statsVer >= statistics.Version2 { var err error topnCount, err = h.initTopNCountSum(tblID, id) if err != nil { @@ -185,7 +185,7 @@ func (h *Handle) initStatsTopN4Chunk(cache *statsCache, iter *chunk.Iterator4Chu continue } idx, ok := table.Indices[row.GetInt64(1)] - if !ok || idx.CMSketch == nil { + if !ok || (idx.CMSketch == nil && idx.StatsVer <= statistics.Version1) { continue } if idx.TopN == nil { diff --git a/statistics/handle/ddl.go b/statistics/handle/ddl.go index 168c8a07daad1..6f7c37556a36c 100644 --- a/statistics/handle/ddl.go +++ b/statistics/handle/ddl.go @@ -74,7 +74,7 @@ var analyzeOptionDefault = map[ast.AnalyzeOptionType]uint64{ func (h *Handle) updateGlobalStats(tblInfo *model.TableInfo) error { // We need to merge the partition-level stats to global-stats when we drop table partition in dynamic mode. tableID := tblInfo.ID - is := infoschema.GetInfoSchema(h.mu.ctx) + is := h.mu.ctx.GetInfoSchema().(infoschema.InfoSchema) globalStats, err := h.TableStatsFromStorage(tblInfo, tableID, true, 0) if err != nil { return err @@ -112,7 +112,8 @@ func (h *Handle) updateGlobalStats(tblInfo *model.TableInfo) error { } for i := 0; i < newColGlobalStats.Num; i++ { hg, cms, topN, fms := newColGlobalStats.Hg[i], newColGlobalStats.Cms[i], newColGlobalStats.TopN[i], newColGlobalStats.Fms[i] - err = h.SaveStatsToStorage(tableID, newColGlobalStats.Count, 0, hg, cms, topN, fms, 2, 1) + // fms for global stats doesn't need to dump to kv. + err = h.SaveStatsToStorage(tableID, newColGlobalStats.Count, 0, hg, cms, topN, fms, 2, 1, false) if err != nil { return err } @@ -141,7 +142,8 @@ func (h *Handle) updateGlobalStats(tblInfo *model.TableInfo) error { } for i := 0; i < newIndexGlobalStats.Num; i++ { hg, cms, topN, fms := newIndexGlobalStats.Hg[i], newIndexGlobalStats.Cms[i], newIndexGlobalStats.TopN[i], newIndexGlobalStats.Fms[i] - err = h.SaveStatsToStorage(tableID, newIndexGlobalStats.Count, 1, hg, cms, topN, fms, 2, 1) + // fms for global stats doesn't need to dump to kv. + err = h.SaveStatsToStorage(tableID, newIndexGlobalStats.Count, 1, hg, cms, topN, fms, 2, 1, false) if err != nil { return err } diff --git a/statistics/handle/ddl_test.go b/statistics/handle/ddl_test.go index c62e80d372766..6bdd897270b8d 100644 --- a/statistics/handle/ddl_test.go +++ b/statistics/handle/ddl_test.go @@ -179,6 +179,8 @@ func (s *testStatsSuite) TestDDLHistogram(c *C) { rs := testKit.MustQuery("select count(*) from mysql.stats_histograms where table_id = ? and hist_id = 1 and is_index =1", tableInfo.ID) rs.Check(testkit.Rows("1")) rs = testKit.MustQuery("select count(*) from mysql.stats_buckets where table_id = ? and hist_id = 1 and is_index = 1", tableInfo.ID) + rs.Check(testkit.Rows("0")) + rs = testKit.MustQuery("select count(*) from mysql.stats_top_n where table_id = ? and hist_id = 1 and is_index = 1", tableInfo.ID) rs.Check(testkit.Rows("2")) } diff --git a/statistics/handle/dump.go b/statistics/handle/dump.go index 1f20855742a76..36971076b4644 100644 --- a/statistics/handle/dump.go +++ b/statistics/handle/dump.go @@ -230,13 +230,15 @@ func (h *Handle) loadStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, } for _, col := range tbl.Columns { - err = h.SaveStatsToStorage(tbl.PhysicalID, tbl.Count, 0, &col.Histogram, col.CMSketch, col.TopN, col.FMSketch, int(col.StatsVer), 1) + // loadStatsFromJSON doesn't support partition table now. + err = h.SaveStatsToStorage(tbl.PhysicalID, tbl.Count, 0, &col.Histogram, col.CMSketch, col.TopN, col.FMSketch, int(col.StatsVer), 1, false) if err != nil { return errors.Trace(err) } } for _, idx := range tbl.Indices { - err = h.SaveStatsToStorage(tbl.PhysicalID, tbl.Count, 1, &idx.Histogram, idx.CMSketch, idx.TopN, nil, int(idx.StatsVer), 1) + // loadStatsFromJSON doesn't support partition table now. + err = h.SaveStatsToStorage(tbl.PhysicalID, tbl.Count, 1, &idx.Histogram, idx.CMSketch, idx.TopN, nil, int(idx.StatsVer), 1, false) if err != nil { return errors.Trace(err) } diff --git a/statistics/handle/dump_test.go b/statistics/handle/dump_test.go index ef02403c48ac5..f0c7c2fb09110 100644 --- a/statistics/handle/dump_test.go +++ b/statistics/handle/dump_test.go @@ -215,7 +215,7 @@ func (s *testStatsSuite) TestDumpCMSketchWithTopN(c *C) { cms, _, _, _ := statistics.NewCMSketchAndTopN(5, 2048, fakeData, 20, 100) stat := h.GetTableStats(tableInfo) - err = h.SaveStatsToStorage(tableInfo.ID, 1, 0, &stat.Columns[tableInfo.Columns[0].ID].Histogram, cms, nil, nil, statistics.Version2, 1) + err = h.SaveStatsToStorage(tableInfo.ID, 1, 0, &stat.Columns[tableInfo.Columns[0].ID].Histogram, cms, nil, nil, statistics.Version2, 1, false) c.Assert(err, IsNil) c.Assert(h.Update(is), IsNil) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 7036242b4e3a7..4279d2bd9b18c 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -17,6 +17,7 @@ import ( "context" "encoding/json" "fmt" + "runtime/pprof" "sort" "strconv" "sync" @@ -121,6 +122,10 @@ func (h *Handle) withRestrictedSQLExecutor(ctx context.Context, fn func(context. func (h *Handle) execRestrictedSQL(ctx context.Context, sql string, params ...interface{}) ([]chunk.Row, []*ast.ResultField, error) { return h.withRestrictedSQLExecutor(ctx, func(ctx context.Context, exec sqlexec.RestrictedSQLExecutor) ([]chunk.Row, []*ast.ResultField, error) { + if variable.TopSQLEnabled() { + // Restore the goroutine label by using the original ctx after execution is finished. + defer pprof.SetGoroutineLabels(ctx) + } stmt, err := exec.ParseWithParams(ctx, sql, params...) if err != nil { return nil, nil, errors.Trace(err) @@ -554,7 +559,6 @@ func (sc statsCache) initMemoryUsage() { sum += tb.MemoryUsage() } sc.memUsage = sum - return } // update updates the statistics table cache using copy on write. @@ -952,12 +956,12 @@ func (h *Handle) extendedStatsFromStorage(reader *statsReader, table *statistics } // SaveStatsToStorage saves the stats to storage. -func (h *Handle) SaveStatsToStorage(tableID int64, count int64, isIndex int, hg *statistics.Histogram, cms *statistics.CMSketch, topN *statistics.TopN, fms *statistics.FMSketch, statsVersion int, isAnalyzed int64) (err error) { +func (h *Handle) SaveStatsToStorage(tableID int64, count int64, isIndex int, hg *statistics.Histogram, cms *statistics.CMSketch, topN *statistics.TopN, fms *statistics.FMSketch, statsVersion int, isAnalyzed int64, needDumpFMS bool) (err error) { h.mu.Lock() defer h.mu.Unlock() ctx := context.TODO() exec := h.mu.ctx.(sqlexec.SQLExecutor) - _, err = exec.ExecuteInternal(ctx, "begin") + _, err = exec.ExecuteInternal(ctx, "begin pessimistic") if err != nil { return errors.Trace(err) } @@ -1001,7 +1005,7 @@ func (h *Handle) SaveStatsToStorage(tableID int64, count int64, isIndex int, hg if _, err := exec.ExecuteInternal(ctx, "delete from mysql.stats_fm_sketch where table_id = %? and is_index = %? and hist_id = %?", tableID, isIndex, hg.ID); err != nil { return err } - if fmSketch != nil { + if fmSketch != nil && needDumpFMS { if _, err = exec.ExecuteInternal(ctx, "insert into mysql.stats_fm_sketch (table_id, is_index, hist_id, value) values (%?, %?, %?, %?)", tableID, isIndex, hg.ID, fmSketch); err != nil { return err } @@ -1119,7 +1123,7 @@ func (h *Handle) columnCountFromStorage(reader *statsReader, tableID, colID, sta if err != nil { return 0, errors.Trace(err) } - if statsVer == statistics.Version2 { + if statsVer >= statistics.Version2 { // Before stats ver 2, histogram represents all data in this column. // In stats ver 2, histogram + TopN represent all data in this column. // So we need to add TopN total count here. diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 3bb95e8219478..3a2ee051680ea 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -37,6 +37,7 @@ import ( "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/collate" + "github.com/pingcap/tidb/util/ranger" "github.com/pingcap/tidb/util/testkit" ) @@ -279,7 +280,14 @@ func (s *testStatsSuite) TestColumnIDs(c *C) { tableInfo := tbl.Meta() statsTbl := do.StatsHandle().GetTableStats(tableInfo) sc := new(stmtctx.StatementContext) - count := statsTbl.ColumnLessRowCount(sc, types.NewDatum(2), tableInfo.Columns[0].ID) + ran := &ranger.Range{ + LowVal: []types.Datum{types.MinNotNullDatum()}, + HighVal: []types.Datum{types.NewIntDatum(2)}, + LowExclude: false, + HighExclude: true, + } + count, err := statsTbl.GetRowCountByColumnRanges(sc, tableInfo.Columns[0].ID, []*ranger.Range{ran}) + c.Assert(err, IsNil) c.Assert(count, Equals, float64(1)) // Drop a column and the offset changed, @@ -293,7 +301,8 @@ func (s *testStatsSuite) TestColumnIDs(c *C) { tableInfo = tbl.Meta() statsTbl = do.StatsHandle().GetTableStats(tableInfo) // At that time, we should get c2's stats instead of c1's. - count = statsTbl.ColumnLessRowCount(sc, types.NewDatum(2), tableInfo.Columns[0].ID) + count, err = statsTbl.GetRowCountByColumnRanges(sc, tableInfo.Columns[0].ID, []*ranger.Range{ran}) + c.Assert(err, IsNil) c.Assert(count, Equals, 0.0) } @@ -620,13 +629,22 @@ func (s *testStatsSuite) TestCorrelation(c *C) { testKit := testkit.NewTestKit(c, s.store) testKit.MustExec("use test") testKit.MustExec("create table t(c1 int primary key, c2 int)") + testKit.MustExec("select * from t where c1 > 10 and c2 > 10") testKit.MustExec("insert into t values(1,1),(3,12),(4,20),(2,7),(5,21)") + testKit.MustExec("set @@session.tidb_analyze_version=1") testKit.MustExec("analyze table t") result := testKit.MustQuery("show stats_histograms where Table_name = 't'").Sort() c.Assert(len(result.Rows()), Equals, 2) c.Assert(result.Rows()[0][9], Equals, "0") c.Assert(result.Rows()[1][9], Equals, "1") + testKit.MustExec("set @@session.tidb_analyze_version=2") + testKit.MustExec("analyze table t") + result = testKit.MustQuery("show stats_histograms where Table_name = 't'").Sort() + c.Assert(len(result.Rows()), Equals, 2) + c.Assert(result.Rows()[0][9], Equals, "1") + c.Assert(result.Rows()[1][9], Equals, "1") testKit.MustExec("insert into t values(8,18)") + testKit.MustExec("set @@session.tidb_analyze_version=1") testKit.MustExec("analyze table t") result = testKit.MustQuery("show stats_histograms where Table_name = 't'").Sort() c.Assert(len(result.Rows()), Equals, 2) @@ -636,20 +654,27 @@ func (s *testStatsSuite) TestCorrelation(c *C) { testKit.MustExec("analyze table t") result = testKit.MustQuery("show stats_histograms where Table_name = 't'").Sort() c.Assert(len(result.Rows()), Equals, 2) - c.Assert(result.Rows()[0][9], Equals, "0") + c.Assert(result.Rows()[0][9], Equals, "1") c.Assert(result.Rows()[1][9], Equals, "0.8285714285714286") testKit.MustExec("truncate table t") - testKit.MustExec("set @@session.tidb_analyze_version=1") result = testKit.MustQuery("show stats_histograms where Table_name = 't'").Sort() c.Assert(len(result.Rows()), Equals, 0) testKit.MustExec("insert into t values(1,21),(3,12),(4,7),(2,20),(5,1)") + testKit.MustExec("set @@session.tidb_analyze_version=1") testKit.MustExec("analyze table t") result = testKit.MustQuery("show stats_histograms where Table_name = 't'").Sort() c.Assert(len(result.Rows()), Equals, 2) c.Assert(result.Rows()[0][9], Equals, "0") c.Assert(result.Rows()[1][9], Equals, "-1") + testKit.MustExec("set @@session.tidb_analyze_version=2") + testKit.MustExec("analyze table t") + result = testKit.MustQuery("show stats_histograms where Table_name = 't'").Sort() + c.Assert(len(result.Rows()), Equals, 2) + c.Assert(result.Rows()[0][9], Equals, "1") + c.Assert(result.Rows()[1][9], Equals, "-1") testKit.MustExec("insert into t values(8,4)") + testKit.MustExec("set @@session.tidb_analyze_version=1") testKit.MustExec("analyze table t") result = testKit.MustQuery("show stats_histograms where Table_name = 't'").Sort() c.Assert(len(result.Rows()), Equals, 2) @@ -659,21 +684,28 @@ func (s *testStatsSuite) TestCorrelation(c *C) { testKit.MustExec("analyze table t") result = testKit.MustQuery("show stats_histograms where Table_name = 't'").Sort() c.Assert(len(result.Rows()), Equals, 2) - c.Assert(result.Rows()[0][9], Equals, "0") + c.Assert(result.Rows()[0][9], Equals, "1") c.Assert(result.Rows()[1][9], Equals, "-0.9428571428571428") testKit.MustExec("truncate table t") - testKit.MustExec("set @@session.tidb_analyze_version=1") testKit.MustExec("insert into t values (1,1),(2,1),(3,1),(4,1),(5,1),(6,1),(7,1),(8,1),(9,1),(10,1),(11,1),(12,1),(13,1),(14,1),(15,1),(16,1),(17,1),(18,1),(19,1),(20,2),(21,2),(22,2),(23,2),(24,2),(25,2)") + testKit.MustExec("set @@session.tidb_analyze_version=1") testKit.MustExec("analyze table t") result = testKit.MustQuery("show stats_histograms where Table_name = 't'").Sort() c.Assert(len(result.Rows()), Equals, 2) c.Assert(result.Rows()[0][9], Equals, "0") c.Assert(result.Rows()[1][9], Equals, "1") + testKit.MustExec("set @@session.tidb_analyze_version=2") + testKit.MustExec("analyze table t") + result = testKit.MustQuery("show stats_histograms where Table_name = 't'").Sort() + c.Assert(len(result.Rows()), Equals, 2) + c.Assert(result.Rows()[0][9], Equals, "1") + c.Assert(result.Rows()[1][9], Equals, "1") testKit.MustExec("drop table t") testKit.MustExec("create table t(c1 int, c2 int)") testKit.MustExec("insert into t values(1,1),(2,7),(3,12),(4,20),(5,21),(8,18)") + testKit.MustExec("set @@session.tidb_analyze_version=1") testKit.MustExec("analyze table t") result = testKit.MustQuery("show stats_histograms where Table_name = 't'").Sort() c.Assert(len(result.Rows()), Equals, 2) @@ -688,6 +720,13 @@ func (s *testStatsSuite) TestCorrelation(c *C) { testKit.MustExec("truncate table t") testKit.MustExec("insert into t values(1,1),(2,7),(3,12),(8,18),(4,20),(5,21)") + testKit.MustExec("set @@session.tidb_analyze_version=1") + testKit.MustExec("analyze table t") + result = testKit.MustQuery("show stats_histograms where Table_name = 't'").Sort() + c.Assert(len(result.Rows()), Equals, 2) + c.Assert(result.Rows()[0][9], Equals, "0.8285714285714286") + c.Assert(result.Rows()[1][9], Equals, "1") + testKit.MustExec("set @@session.tidb_analyze_version=2") testKit.MustExec("analyze table t") result = testKit.MustQuery("show stats_histograms where Table_name = 't'").Sort() c.Assert(len(result.Rows()), Equals, 2) @@ -697,6 +736,7 @@ func (s *testStatsSuite) TestCorrelation(c *C) { testKit.MustExec("drop table t") testKit.MustExec("create table t(c1 int primary key, c2 int, c3 int, key idx_c2(c2))") testKit.MustExec("insert into t values(1,1,1),(2,2,2),(3,3,3)") + testKit.MustExec("set @@session.tidb_analyze_version=1") testKit.MustExec("analyze table t") result = testKit.MustQuery("show stats_histograms where Table_name = 't' and Is_index = 0").Sort() c.Assert(len(result.Rows()), Equals, 3) @@ -706,6 +746,16 @@ func (s *testStatsSuite) TestCorrelation(c *C) { result = testKit.MustQuery("show stats_histograms where Table_name = 't' and Is_index = 1").Sort() c.Assert(len(result.Rows()), Equals, 1) c.Assert(result.Rows()[0][9], Equals, "0") + testKit.MustExec("set @@tidb_analyze_version=2") + testKit.MustExec("analyze table t") + result = testKit.MustQuery("show stats_histograms where Table_name = 't' and Is_index = 0").Sort() + c.Assert(len(result.Rows()), Equals, 3) + c.Assert(result.Rows()[0][9], Equals, "1") + c.Assert(result.Rows()[1][9], Equals, "1") + c.Assert(result.Rows()[2][9], Equals, "1") + result = testKit.MustQuery("show stats_histograms where Table_name = 't' and Is_index = 1").Sort() + c.Assert(len(result.Rows()), Equals, 1) + c.Assert(result.Rows()[0][9], Equals, "0") } func (s *testStatsSuite) TestShowGlobalStats(c *C) { @@ -796,17 +846,22 @@ func (s *testStatsSuite) TestBuildGlobalLevelStats(c *C) { c.Assert(len(result.Rows()), Equals, 20) } -func (s *testStatsSuite) prepareForGlobalStatsWithOpts(c *C, tk *testkit.TestKit) { - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec(` create table t (a int, key(a)) partition by range (a) ` + +func (s *testStatsSuite) prepareForGlobalStatsWithOpts(c *C, tk *testkit.TestKit, tblName, dbName string) { + tk.MustExec("create database if not exists " + dbName) + tk.MustExec("use " + dbName) + tk.MustExec("drop table if exists " + tblName) + tk.MustExec(` create table ` + tblName + ` (a int, key(a)) partition by range (a) ` + `(partition p0 values less than (100000), partition p1 values less than (200000))`) - buf1 := bytes.NewBufferString("insert into t values (0)") - buf2 := bytes.NewBufferString("insert into t values (100000)") + buf1 := bytes.NewBufferString("insert into " + tblName + " values (0)") + buf2 := bytes.NewBufferString("insert into " + tblName + " values (100000)") for i := 0; i < 5000; i += 3 { buf1.WriteString(fmt.Sprintf(", (%v)", i)) buf2.WriteString(fmt.Sprintf(", (%v)", 100000+i)) } + for i := 0; i < 1000; i++ { + buf1.WriteString(fmt.Sprintf(", (%v)", 0)) + buf2.WriteString(fmt.Sprintf(", (%v)", 100000)) + } tk.MustExec(buf1.String()) tk.MustExec(buf2.String()) tk.MustExec("set @@tidb_analyze_version=2") @@ -827,9 +882,10 @@ func (s *testStatsSuite) checkForGlobalStatsWithOpts(c *C, tk *testkit.TestKit, } func (s *testStatsSuite) TestAnalyzeGlobalStatsWithOpts(c *C) { + c.Skip("unstable, skip race test") defer cleanEnv(c, s.store, s.do) tk := testkit.NewTestKit(c, s.store) - s.prepareForGlobalStatsWithOpts(c, tk) + s.prepareForGlobalStatsWithOpts(c, tk, "test_gstats_opt", "test_gstats_opt") type opt struct { topn int @@ -848,7 +904,7 @@ func (s *testStatsSuite) TestAnalyzeGlobalStatsWithOpts(c *C) { {77, 47000, true}, } for _, ca := range cases { - sql := fmt.Sprintf("analyze table t with %v topn, %v buckets", ca.topn, ca.buckets) + sql := fmt.Sprintf("analyze table test_gstats_opt with %v topn, %v buckets", ca.topn, ca.buckets) if !ca.err { tk.MustExec(sql) s.checkForGlobalStatsWithOpts(c, tk, "global", ca.topn, ca.buckets) @@ -862,27 +918,28 @@ func (s *testStatsSuite) TestAnalyzeGlobalStatsWithOpts(c *C) { } func (s *testStatsSuite) TestAnalyzeGlobalStatsWithOpts2(c *C) { + c.Skip("unstable, skip race test") defer cleanEnv(c, s.store, s.do) tk := testkit.NewTestKit(c, s.store) - s.prepareForGlobalStatsWithOpts(c, tk) + s.prepareForGlobalStatsWithOpts(c, tk, "test_gstats_opt2", "test_gstats_opt2") - tk.MustExec("analyze table t with 20 topn, 50 buckets") - s.checkForGlobalStatsWithOpts(c, tk, "global", 20, 50) - s.checkForGlobalStatsWithOpts(c, tk, "p0", 20, 50) - s.checkForGlobalStatsWithOpts(c, tk, "p1", 20, 50) + tk.MustExec("analyze table test_gstats_opt2 with 20 topn, 50 buckets, 1000 samples") + s.checkForGlobalStatsWithOpts(c, tk, "global", 2, 50) + s.checkForGlobalStatsWithOpts(c, tk, "p0", 1, 50) + s.checkForGlobalStatsWithOpts(c, tk, "p1", 1, 50) // analyze a partition to let its options be different with others' - tk.MustExec("analyze table t partition p0 with 10 topn, 20 buckets") + tk.MustExec("analyze table test_gstats_opt2 partition p0 with 10 topn, 20 buckets") s.checkForGlobalStatsWithOpts(c, tk, "global", 10, 20) // use new options s.checkForGlobalStatsWithOpts(c, tk, "p0", 10, 20) - s.checkForGlobalStatsWithOpts(c, tk, "p1", 20, 50) + s.checkForGlobalStatsWithOpts(c, tk, "p1", 1, 50) - tk.MustExec("analyze table t partition p1 with 100 topn, 200 buckets") + tk.MustExec("analyze table test_gstats_opt2 partition p1 with 100 topn, 200 buckets") s.checkForGlobalStatsWithOpts(c, tk, "global", 100, 200) s.checkForGlobalStatsWithOpts(c, tk, "p0", 10, 20) s.checkForGlobalStatsWithOpts(c, tk, "p1", 100, 200) - tk.MustExec("analyze table t partition p0") // default options + tk.MustExec("analyze table test_gstats_opt2 partition p0 with 20 topn") // change back to 20 topn s.checkForGlobalStatsWithOpts(c, tk, "global", 20, 256) s.checkForGlobalStatsWithOpts(c, tk, "p0", 20, 256) s.checkForGlobalStatsWithOpts(c, tk, "p1", 100, 200) @@ -986,7 +1043,7 @@ partition by range (a) ( tk.MustQuery("select distinct_count, null_count, tot_col_size, correlation=0 from mysql.stats_histograms where is_index=0 order by table_id asc").Check( testkit.Rows("15 1 17 1", "6 1 7 0", "9 0 10 0")) tk.MustQuery("select distinct_count, null_count, tot_col_size, correlation=0 from mysql.stats_histograms where is_index=1 order by table_id asc").Check( - testkit.Rows("15 1 0 1", "6 1 0 1", "9 0 0 1")) + testkit.Rows("15 1 0 1", "6 1 6 1", "9 0 10 1")) tk.MustQuery("show stats_buckets where is_index=0").Check( // db table partition col is_idx bucket_id count repeats lower upper ndv @@ -999,10 +1056,10 @@ partition by range (a) ( tk.MustQuery("show stats_buckets where is_index=1").Check( testkit.Rows("test t global a 1 0 7 2 1 6 0", "test t global a 1 1 17 2 6 19 0", - "test t p0 a 1 0 4 1 1 4 4", - "test t p0 a 1 1 7 2 5 6 2", - "test t p1 a 1 0 8 1 11 18 8", - "test t p1 a 1 1 10 2 19 19 1")) + "test t p0 a 1 0 4 1 1 4 0", + "test t p0 a 1 1 7 2 5 6 0", + "test t p1 a 1 0 6 1 11 16 0", + "test t p1 a 1 1 10 2 17 19 0")) } func (s *testStatsSuite) TestGlobalStatsData2(c *C) { @@ -1056,12 +1113,12 @@ func (s *testStatsSuite) TestGlobalStatsData2(c *C) { tk.MustQuery("show stats_buckets where is_index=1").Check(testkit.Rows( // db, tbl, part, col, isIdx, bucketID, count, repeat, lower, upper, ndv - "test tint global c 1 0 5 2 1 4 0", // 4 is popped from p0.TopN, so g.ndv = p0.ndv+1 - "test tint global c 1 1 12 2 4 17 0", - "test tint p0 c 1 0 3 0 1 4 3", - "test tint p0 c 1 1 3 0 5 5 0", - "test tint p1 c 1 0 5 0 11 16 5", - "test tint p1 c 1 1 5 0 17 17 0")) + "test tint global c 1 0 5 2 1 4 0", // 4 is popped from p0.TopN, so g.ndv = p0.ndv+1 + "test tint global c 1 1 12 2 17 17 0", // same with the column's + "test tint p0 c 1 0 2 1 1 2 0", + "test tint p0 c 1 1 3 1 3 3 0", + "test tint p1 c 1 0 3 1 11 13 0", + "test tint p1 c 1 1 5 1 14 15 0")) tk.MustQuery("select distinct_count, null_count from mysql.stats_histograms where is_index=1 order by table_id asc").Check( testkit.Rows("12 1", // global, g = p0 + p1 @@ -1119,11 +1176,11 @@ func (s *testStatsSuite) TestGlobalStatsData2(c *C) { tk.MustQuery("show stats_buckets where table_name='tdouble' and is_index=1 and column_name='c'").Check(testkit.Rows( // db, tbl, part, col, isIdx, bucketID, count, repeat, lower, upper, ndv "test tdouble global c 1 0 5 2 1 4 0", // 4 is popped from p0.TopN, so g.ndv = p0.ndv+1 - "test tdouble global c 1 1 12 2 4 17 0", - "test tdouble p0 c 1 0 3 0 1 4 3", - "test tdouble p0 c 1 1 3 0 5 5 0", - "test tdouble p1 c 1 0 5 0 11 16 5", - "test tdouble p1 c 1 1 5 0 17 17 0")) + "test tdouble global c 1 1 12 2 17 17 0", + "test tdouble p0 c 1 0 2 1 1 2 0", + "test tdouble p0 c 1 1 3 1 3 3 0", + "test tdouble p1 c 1 0 3 1 11 13 0", + "test tdouble p1 c 1 1 5 1 14 15 0")) rs = tk.MustQuery("show stats_histograms where table_name='tdouble' and column_name='c' and is_index=1").Rows() c.Assert(rs[0][6].(string), Equals, "12") // g.ndv = p0 + p1 @@ -1184,11 +1241,11 @@ func (s *testStatsSuite) TestGlobalStatsData2(c *C) { tk.MustQuery("show stats_buckets where table_name='tdecimal' and is_index=1 and column_name='c'").Check(testkit.Rows( // db, tbl, part, col, isIdx, bucketID, count, repeat, lower, upper, ndv "test tdecimal global c 1 0 5 2 1.00 4.00 0", // 4 is popped from p0.TopN, so g.ndv = p0.ndv+1 - "test tdecimal global c 1 1 12 2 4.00 17.00 0", - "test tdecimal p0 c 1 0 3 0 1.00 4.00 3", - "test tdecimal p0 c 1 1 3 0 5.00 5.00 0", - "test tdecimal p1 c 1 0 5 0 11.00 16.00 5", - "test tdecimal p1 c 1 1 5 0 17.00 17.00 0")) + "test tdecimal global c 1 1 12 2 17.00 17.00 0", + "test tdecimal p0 c 1 0 2 1 1.00 2.00 0", + "test tdecimal p0 c 1 1 3 1 3.00 3.00 0", + "test tdecimal p1 c 1 0 3 1 11.00 13.00 0", + "test tdecimal p1 c 1 1 5 1 14.00 15.00 0")) rs = tk.MustQuery("show stats_histograms where table_name='tdecimal' and column_name='c' and is_index=1").Rows() c.Assert(rs[0][6].(string), Equals, "12") // g.ndv = p0 + p1 @@ -1249,11 +1306,11 @@ func (s *testStatsSuite) TestGlobalStatsData2(c *C) { tk.MustQuery("show stats_buckets where table_name='tdatetime' and is_index=1 and column_name='c'").Check(testkit.Rows( // db, tbl, part, col, isIdx, bucketID, count, repeat, lower, upper, ndv "test tdatetime global c 1 0 5 2 2000-01-01 00:00:00 2000-01-04 00:00:00 0", // 4 is popped from p0.TopN, so g.ndv = p0.ndv+1 - "test tdatetime global c 1 1 12 2 2000-01-04 00:00:00 2000-01-17 00:00:00 0", - "test tdatetime p0 c 1 0 3 0 2000-01-01 00:00:00 2000-01-04 00:00:00 3", - "test tdatetime p0 c 1 1 3 0 2000-01-05 00:00:00 2000-01-05 00:00:00 0", - "test tdatetime p1 c 1 0 5 0 2000-01-11 00:00:00 2000-01-16 00:00:00 5", - "test tdatetime p1 c 1 1 5 0 2000-01-17 00:00:00 2000-01-17 00:00:00 0")) + "test tdatetime global c 1 1 12 2 2000-01-17 00:00:00 2000-01-17 00:00:00 0", + "test tdatetime p0 c 1 0 2 1 2000-01-01 00:00:00 2000-01-02 00:00:00 0", + "test tdatetime p0 c 1 1 3 1 2000-01-03 00:00:00 2000-01-03 00:00:00 0", + "test tdatetime p1 c 1 0 3 1 2000-01-11 00:00:00 2000-01-13 00:00:00 0", + "test tdatetime p1 c 1 1 5 1 2000-01-14 00:00:00 2000-01-15 00:00:00 0")) rs = tk.MustQuery("show stats_histograms where table_name='tdatetime' and column_name='c' and is_index=1").Rows() c.Assert(rs[0][6].(string), Equals, "12") // g.ndv = p0 + p1 @@ -1314,11 +1371,11 @@ func (s *testStatsSuite) TestGlobalStatsData2(c *C) { tk.MustQuery("show stats_buckets where table_name='tstring' and is_index=1 and column_name='c'").Check(testkit.Rows( // db, tbl, part, col, isIdx, bucketID, count, repeat, lower, upper, ndv "test tstring global c 1 0 5 2 a1 a4 0", // 4 is popped from p0.TopN, so g.ndv = p0.ndv+1 - "test tstring global c 1 1 12 2 a4 b17 0", - "test tstring p0 c 1 0 3 0 a1 a4 3", - "test tstring p0 c 1 1 3 0 a5 a5 0", - "test tstring p1 c 1 0 5 0 b11 b16 5", - "test tstring p1 c 1 1 5 0 b17 b17 0")) + "test tstring global c 1 1 12 2 b17 b17 0", + "test tstring p0 c 1 0 2 1 a1 a2 0", + "test tstring p0 c 1 1 3 1 a3 a3 0", + "test tstring p1 c 1 0 3 1 b11 b13 0", + "test tstring p1 c 1 1 5 1 b14 b15 0")) rs = tk.MustQuery("show stats_histograms where table_name='tstring' and column_name='c' and is_index=1").Rows() c.Assert(rs[0][6].(string), Equals, "12") // g.ndv = p0 + p1 @@ -1358,12 +1415,12 @@ func (s *testStatsSuite) TestGlobalStatsData3(c *C) { "test tintint p1 a 1 (13, 2) 3")) tk.MustQuery("show stats_buckets where table_name='tintint' and is_index=1").Check(testkit.Rows( - "test tintint global a 1 0 6 2 (1, 1) (2, 3) 0", // (2, 3) is popped into it - "test tintint global a 1 1 11 2 (2, 3) (13, 1) 0", // (13, 1) is popped into it - "test tintint p0 a 1 0 4 1 (1, 1) (2, 2) 4", - "test tintint p0 a 1 1 4 0 (2, 3) (3, 1) 0", - "test tintint p1 a 1 0 3 0 (11, 1) (13, 1) 3", - "test tintint p1 a 1 1 3 0 (13, 2) (13, 2) 0")) + "test tintint global a 1 0 6 2 (1, 1) (2, 3) 0", // (2, 3) is popped into it + "test tintint global a 1 1 11 2 (13, 1) (13, 1) 0", // (13, 1) is popped into it + "test tintint p0 a 1 0 3 1 (1, 1) (2, 1) 0", + "test tintint p0 a 1 1 4 1 (2, 2) (2, 2) 0", + "test tintint p1 a 1 0 2 1 (11, 1) (12, 1) 0", + "test tintint p1 a 1 1 3 1 (12, 2) (12, 2) 0")) rs = tk.MustQuery("show stats_histograms where table_name='tintint' and is_index=1").Rows() c.Assert(rs[0][6].(string), Equals, "11") // g.ndv = p0.ndv + p1.ndv @@ -1392,12 +1449,12 @@ func (s *testStatsSuite) TestGlobalStatsData3(c *C) { "test tintstr p1 a 1 (13, 2) 3")) tk.MustQuery("show stats_buckets where table_name='tintstr' and is_index=1").Check(testkit.Rows( - "test tintstr global a 1 0 6 2 (1, 1) (2, 3) 0", // (2, 3) is popped into it - "test tintstr global a 1 1 11 2 (2, 3) (13, 1) 0", // (13, 1) is popped into it - "test tintstr p0 a 1 0 4 1 (1, 1) (2, 2) 4", - "test tintstr p0 a 1 1 4 0 (2, 3) (3, 1) 0", - "test tintstr p1 a 1 0 3 0 (11, 1) (13, 1) 3", - "test tintstr p1 a 1 1 3 0 (13, 2) (13, 2) 0")) + "test tintstr global a 1 0 6 2 (1, 1) (2, 3) 0", // (2, 3) is popped into it + "test tintstr global a 1 1 11 2 (13, 1) (13, 1) 0", // (13, 1) is popped into it + "test tintstr p0 a 1 0 3 1 (1, 1) (2, 1) 0", + "test tintstr p0 a 1 1 4 1 (2, 2) (2, 2) 0", + "test tintstr p1 a 1 0 2 1 (11, 1) (12, 1) 0", + "test tintstr p1 a 1 1 3 1 (12, 2) (12, 2) 0")) rs = tk.MustQuery("show stats_histograms where table_name='tintstr' and is_index=1").Rows() c.Assert(rs[0][6].(string), Equals, "11") // g.ndv = p0.ndv + p1.ndv @@ -1426,12 +1483,12 @@ func (s *testStatsSuite) TestGlobalStatsData3(c *C) { "test tintdouble p1 a 1 (13, 2) 3")) tk.MustQuery("show stats_buckets where table_name='tintdouble' and is_index=1").Check(testkit.Rows( - "test tintdouble global a 1 0 6 2 (1, 1) (2, 3) 0", // (2, 3) is popped into it - "test tintdouble global a 1 1 11 2 (2, 3) (13, 1) 0", // (13, 1) is popped into it - "test tintdouble p0 a 1 0 4 1 (1, 1) (2, 2) 4", - "test tintdouble p0 a 1 1 4 0 (2, 3) (3, 1) 0", - "test tintdouble p1 a 1 0 3 0 (11, 1) (13, 1) 3", - "test tintdouble p1 a 1 1 3 0 (13, 2) (13, 2) 0")) + "test tintdouble global a 1 0 6 2 (1, 1) (2, 3) 0", // (2, 3) is popped into it + "test tintdouble global a 1 1 11 2 (13, 1) (13, 1) 0", // (13, 1) is popped into it + "test tintdouble p0 a 1 0 3 1 (1, 1) (2, 1) 0", + "test tintdouble p0 a 1 1 4 1 (2, 2) (2, 2) 0", + "test tintdouble p1 a 1 0 2 1 (11, 1) (12, 1) 0", + "test tintdouble p1 a 1 1 3 1 (12, 2) (12, 2) 0")) rs = tk.MustQuery("show stats_histograms where table_name='tintdouble' and is_index=1").Rows() c.Assert(rs[0][6].(string), Equals, "11") // g.ndv = p0.ndv + p1.ndv @@ -1460,12 +1517,12 @@ func (s *testStatsSuite) TestGlobalStatsData3(c *C) { "test tdoubledecimal p1 a 1 (13, 2.00) 3")) tk.MustQuery("show stats_buckets where table_name='tdoubledecimal' and is_index=1").Check(testkit.Rows( - "test tdoubledecimal global a 1 0 6 2 (1, 1.00) (2, 3.00) 0", // (2, 3) is popped into it - "test tdoubledecimal global a 1 1 11 2 (2, 3.00) (13, 1.00) 0", // (13, 1) is popped into it - "test tdoubledecimal p0 a 1 0 4 1 (1, 1.00) (2, 2.00) 4", - "test tdoubledecimal p0 a 1 1 4 0 (2, 3.00) (3, 1.00) 0", - "test tdoubledecimal p1 a 1 0 3 0 (11, 1.00) (13, 1.00) 3", - "test tdoubledecimal p1 a 1 1 3 0 (13, 2.00) (13, 2.00) 0")) + "test tdoubledecimal global a 1 0 6 2 (1, 1.00) (2, 3.00) 0", // (2, 3) is popped into it + "test tdoubledecimal global a 1 1 11 2 (13, 1.00) (13, 1.00) 0", // (13, 1) is popped into it + "test tdoubledecimal p0 a 1 0 3 1 (1, 1.00) (2, 1.00) 0", + "test tdoubledecimal p0 a 1 1 4 1 (2, 2.00) (2, 2.00) 0", + "test tdoubledecimal p1 a 1 0 2 1 (11, 1.00) (12, 1.00) 0", + "test tdoubledecimal p1 a 1 1 3 1 (12, 2.00) (12, 2.00) 0")) rs = tk.MustQuery("show stats_histograms where table_name='tdoubledecimal' and is_index=1").Rows() c.Assert(rs[0][6].(string), Equals, "11") // g.ndv = p0.ndv + p1.ndv @@ -1494,12 +1551,12 @@ func (s *testStatsSuite) TestGlobalStatsData3(c *C) { "test tstrdt p1 a 1 (13, 2000-01-02 00:00:00) 3")) tk.MustQuery("show stats_buckets where table_name='tstrdt' and is_index=1").Check(testkit.Rows( - "test tstrdt global a 1 0 6 2 (1, 2000-01-01 00:00:00) (2, 2000-01-03 00:00:00) 0", // (2, 3) is popped into it - "test tstrdt global a 1 1 11 2 (2, 2000-01-03 00:00:00) (13, 2000-01-01 00:00:00) 0", // (13, 1) is popped into it - "test tstrdt p0 a 1 0 4 1 (1, 2000-01-01 00:00:00) (2, 2000-01-02 00:00:00) 4", - "test tstrdt p0 a 1 1 4 0 (2, 2000-01-03 00:00:00) (3, 2000-01-01 00:00:00) 0", - "test tstrdt p1 a 1 0 3 0 (11, 2000-01-01 00:00:00) (13, 2000-01-01 00:00:00) 3", - "test tstrdt p1 a 1 1 3 0 (13, 2000-01-02 00:00:00) (13, 2000-01-02 00:00:00) 0")) + "test tstrdt global a 1 0 6 2 (1, 2000-01-01 00:00:00) (2, 2000-01-03 00:00:00) 0", // (2, 3) is popped into it + "test tstrdt global a 1 1 11 2 (13, 2000-01-01 00:00:00) (13, 2000-01-01 00:00:00) 0", // (13, 1) is popped into it + "test tstrdt p0 a 1 0 3 1 (1, 2000-01-01 00:00:00) (2, 2000-01-01 00:00:00) 0", + "test tstrdt p0 a 1 1 4 1 (2, 2000-01-02 00:00:00) (2, 2000-01-02 00:00:00) 0", + "test tstrdt p1 a 1 0 2 1 (11, 2000-01-01 00:00:00) (12, 2000-01-01 00:00:00) 0", + "test tstrdt p1 a 1 1 3 1 (12, 2000-01-02 00:00:00) (12, 2000-01-02 00:00:00) 0")) rs = tk.MustQuery("show stats_histograms where table_name='tstrdt' and is_index=1").Rows() c.Assert(rs[0][6].(string), Equals, "11") // g.ndv = p0.ndv + p1.ndv @@ -1846,6 +1903,12 @@ func (s *testStatsSuite) TestCorrelationStatsCompute(c *C) { c.Assert(statsTbl.ExtendedStats, NotNil) c.Assert(len(statsTbl.ExtendedStats.Stats), Equals, 0) + tk.MustExec("analyze table t") + tk.MustQuery("select type, column_ids, stats, status from mysql.stats_extended").Sort().Check(testkit.Rows( + "2 [1,2] 1.000000 1", + "2 [1,3] -1.000000 1", + )) + tk.MustExec("set @@session.tidb_analyze_version=2") tk.MustExec("analyze table t") tk.MustQuery("select type, column_ids, stats, status from mysql.stats_extended").Sort().Check(testkit.Rows( "2 [1,2] 1.000000 1", @@ -1875,12 +1938,26 @@ func (s *testStatsSuite) TestCorrelationStatsCompute(c *C) { // Check that table with NULLs won't cause panic tk.MustExec("delete from t") tk.MustExec("insert into t values(1,null,2), (2,null,null)") + tk.MustExec("set @@session.tidb_analyze_version=1") + tk.MustExec("analyze table t") + tk.MustQuery("select type, column_ids, stats, status from mysql.stats_extended").Sort().Check(testkit.Rows( + "2 [1,2] 0.000000 1", + "2 [1,3] 1.000000 1", + )) + tk.MustExec("set @@session.tidb_analyze_version=2") tk.MustExec("analyze table t") tk.MustQuery("select type, column_ids, stats, status from mysql.stats_extended").Sort().Check(testkit.Rows( "2 [1,2] 0.000000 1", "2 [1,3] 1.000000 1", )) tk.MustExec("insert into t values(3,3,3)") + tk.MustExec("set @@session.tidb_analyze_version=1") + tk.MustExec("analyze table t") + tk.MustQuery("select type, column_ids, stats, status from mysql.stats_extended").Sort().Check(testkit.Rows( + "2 [1,2] 1.000000 1", + "2 [1,3] 1.000000 1", + )) + tk.MustExec("set @@session.tidb_analyze_version=2") tk.MustExec("analyze table t") tk.MustQuery("select type, column_ids, stats, status from mysql.stats_extended").Sort().Check(testkit.Rows( "2 [1,2] 1.000000 1", @@ -1985,8 +2062,8 @@ func (s *testStatsSuite) TestAnalyzeWithDynamicPartitionPruneMode(c *C) { tk.MustExec("insert into t values (3)") tk.MustExec("analyze table t partition p0 index a with 1 topn, 2 buckets") rows = tk.MustQuery("show stats_buckets where partition_name = 'global' and is_index=1").Rows() - c.Assert(len(rows), Equals, 2) - c.Assert(rows[1][6], Equals, "6") + c.Assert(len(rows), Equals, 1) + c.Assert(rows[0][6], Equals, "6") } func (s *testStatsSuite) TestPartitionPruneModeSessionVariable(c *C) { @@ -2332,16 +2409,18 @@ func (s *testStatsSuite) TestDuplicateFMSketch(c *C) { defer cleanEnv(c, s.store, s.do) tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") - tk.MustExec("create table t(a int, b int, c int)") + tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") + defer tk.MustExec("set @@tidb_partition_prune_mode='static'") + tk.MustExec("create table t(a int, b int, c int) partition by hash(a) partitions 3") tk.MustExec("insert into t values (1, 1, 1)") tk.MustExec("analyze table t") - tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("3")) + tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("9")) tk.MustExec("analyze table t") - tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("3")) + tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("9")) - tk.MustExec("alter table t drop column a") + tk.MustExec("alter table t drop column b") c.Assert(s.do.StatsHandle().GCStats(s.do.InfoSchema(), time.Duration(0)), IsNil) - tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("2")) + tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("6")) } func (s *testStatsSuite) TestIndexFMSketch(c *C) { @@ -2349,31 +2428,33 @@ func (s *testStatsSuite) TestIndexFMSketch(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") - tk.MustExec("create table t(a int, b int, c int, index ia(a), index ibc(b, c))") + tk.MustExec("create table t(a int, b int, c int, index ia(a), index ibc(b, c)) partition by hash(a) partitions 3") tk.MustExec("insert into t values (1, 1, 1)") + tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") + defer tk.MustExec("set @@tidb_partition_prune_mode='static'") tk.MustExec("analyze table t index ia") - tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("1")) + tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("3")) tk.MustExec("analyze table t index ibc") - tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("2")) + tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("6")) tk.MustExec("analyze table t") - tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("5")) + tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("15")) tk.MustExec("drop table if exists t") c.Assert(s.do.StatsHandle().GCStats(s.do.InfoSchema(), 0), IsNil) // clustered index tk.MustExec("drop table if exists t") tk.MustExec("set @@tidb_enable_clustered_index=ON") - tk.MustExec("create table t (a datetime, b datetime, primary key (a))") + tk.MustExec("create table t (a datetime, b datetime, primary key (a)) partition by hash(year(a)) partitions 3") tk.MustExec("insert into t values ('2000-01-01', '2000-01-01')") tk.MustExec("analyze table t") - tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("2")) + tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("6")) tk.MustExec("drop table if exists t") c.Assert(s.do.StatsHandle().GCStats(s.do.InfoSchema(), 0), IsNil) // test NDV checkNDV := func(rows, ndv int) { tk.MustExec("analyze table t") - rs := tk.MustQuery(fmt.Sprintf("select value from mysql.stats_fm_sketch")).Rows() + rs := tk.MustQuery("select value from mysql.stats_fm_sketch").Rows() c.Assert(len(rs), Equals, rows) for i := range rs { fm, err := statistics.DecodeFMSketch([]byte(rs[i][0].(string))) @@ -2383,25 +2464,23 @@ func (s *testStatsSuite) TestIndexFMSketch(c *C) { } tk.MustExec("set @@tidb_enable_clustered_index=OFF") - tk.MustExec("create table t(a int, key(a))") + tk.MustExec("create table t(a int, key(a)) partition by hash(a) partitions 3") tk.MustExec("insert into t values (1), (2), (2), (3)") - checkNDV(2, 3) - tk.MustExec("insert into t values (4), (5)") - checkNDV(2, 5) + checkNDV(6, 1) + tk.MustExec("insert into t values (4), (5), (6)") + checkNDV(6, 2) tk.MustExec("insert into t values (2), (5)") - checkNDV(2, 5) + checkNDV(6, 2) tk.MustExec("drop table if exists t") c.Assert(s.do.StatsHandle().GCStats(s.do.InfoSchema(), 0), IsNil) // clustered index tk.MustExec("set @@tidb_enable_clustered_index=ON") - tk.MustExec("create table t (a datetime, b datetime, primary key (a))") - tk.MustExec("insert into t values ('2000-01-01', '2000-01-01')") - checkNDV(2, 1) - tk.MustExec("insert into t values ('2020-01-01', '2020-01-01')") - checkNDV(2, 2) - tk.MustExec("insert into t values ('1999-01-01', '1999-01-01'), ('1999-01-02', '1999-01-02'), ('1999-01-03', '1999-01-03')") - checkNDV(2, 5) + tk.MustExec("create table t (a datetime, b datetime, primary key (a)) partition by hash(year(a)) partitions 3") + tk.MustExec("insert into t values ('2000-01-01', '2001-01-01'), ('2001-01-01', '2001-01-01'), ('2002-01-01', '2001-01-01')") + checkNDV(6, 1) + tk.MustExec("insert into t values ('1999-01-01', '1998-01-01'), ('1997-01-02', '1999-01-02'), ('1998-01-03', '1999-01-03')") + checkNDV(6, 2) } func (s *testStatsSuite) TestShowExtendedStats4DropColumn(c *C) { @@ -2802,3 +2881,40 @@ func (s *testSerialStatsSuite) TestIssues24349(c *C) { "test t global b 0 1 10 1 4 4 0", )) } + +func (s *testStatsSuite) TestIssues24401(c *C) { + defer cleanEnv(c, s.store, s.do) + testKit := testkit.NewTestKit(c, s.store) + testKit.MustExec("use test") + + // normal table with static prune mode + testKit.MustExec("set @@tidb_partition_prune_mode='static'") + testKit.MustExec("create table t(a int, index(a))") + testKit.MustExec("insert into t values (1), (2), (3)") + testKit.MustExec("analyze table t") + testKit.MustQuery("select * from mysql.stats_fm_sketch").Check(testkit.Rows()) + + // partition table with static prune mode + testKit.MustExec("create table tp(a int, index(a)) partition by hash(a) partitions 3") + testKit.MustExec("insert into tp values (1), (2), (3)") + testKit.MustExec("analyze table tp") + testKit.MustQuery("select * from mysql.stats_fm_sketch").Check(testkit.Rows()) + + // normal table with dynamic prune mode + testKit.MustExec("set @@tidb_partition_prune_mode='dynamic'") + defer testKit.MustExec("set @@tidb_partition_prune_mode='static'") + testKit.MustExec("analyze table t") + testKit.MustQuery("select * from mysql.stats_fm_sketch").Check(testkit.Rows()) + + // partition table with dynamic prune mode + testKit.MustExec("analyze table tp") + rows := testKit.MustQuery("select * from mysql.stats_fm_sketch").Rows() + lenRows := len(rows) + c.Assert(lenRows, Equals, 6) + + // check fm-sketch won't increase infinitely + testKit.MustExec("insert into t values (10), (20), (30), (12), (23), (23), (4344)") + testKit.MustExec("analyze table tp") + rows = testKit.MustQuery("select * from mysql.stats_fm_sketch").Rows() + c.Assert(len(rows), Equals, lenRows) +} diff --git a/statistics/handle/update.go b/statistics/handle/update.go index c65f0885877f6..03df857ec4a5d 100644 --- a/statistics/handle/update.go +++ b/statistics/handle/update.go @@ -480,7 +480,7 @@ func (h *Handle) dumpTableStatCountToKV(id int64, delta variable.TableDelta) (up affectedRows := h.mu.ctx.GetSessionVars().StmtCtx.AffectedRows() // if it's a partitioned table and its global-stats exists, update its count and modify_count as well. - is := infoschema.GetInfoSchema(h.mu.ctx) + is := h.mu.ctx.GetInfoSchema().(infoschema.InfoSchema) if is == nil { return false, errors.New("cannot get the information schema") } @@ -750,11 +750,11 @@ func (h *Handle) handleSingleHistogramUpdate(is infoschema.InfoSchema, rows []ch return nil } var tbl *statistics.Table - if table.Meta().GetPartitionInfo() == nil || h.CurrentPruneMode() == variable.Dynamic { - tbl = h.GetTableStats(table.Meta()) - } else { - tbl = h.GetPartitionStats(table.Meta(), physicalTableID) + // feedback for partition is not ready + if table.Meta().GetPartitionInfo() != nil { + return nil } + tbl = h.GetTableStats(table.Meta()) var cms *statistics.CMSketch var hist *statistics.Histogram var topN *statistics.TopN @@ -822,7 +822,8 @@ func (h *Handle) deleteOutdatedFeedback(tableID, histID, isIndex int64) error { func (h *Handle) dumpStatsUpdateToKV(tableID, isIndex int64, q *statistics.QueryFeedback, hist *statistics.Histogram, cms *statistics.CMSketch, topN *statistics.TopN, fms *statistics.FMSketch, statsVersion int64) error { hist = statistics.UpdateHistogram(hist, q, int(statsVersion)) - err := h.SaveStatsToStorage(tableID, -1, int(isIndex), hist, cms, topN, fms, int(statsVersion), 0) + // feedback for partition is not ready. + err := h.SaveStatsToStorage(tableID, -1, int(isIndex), hist, cms, topN, fms, int(statsVersion), 0, false) metrics.UpdateStatsCounter.WithLabelValues(metrics.RetLabel(err)).Inc() return errors.Trace(err) } diff --git a/statistics/handle/update_test.go b/statistics/handle/update_test.go index b105738098f4b..fe8107fd8679a 100644 --- a/statistics/handle/update_test.go +++ b/statistics/handle/update_test.go @@ -504,7 +504,8 @@ func (s *testStatsSuite) TestAutoUpdate(c *C) { hg, ok := stats.Indices[tableInfo.Indices[0].ID] c.Assert(ok, IsTrue) c.Assert(hg.NDV, Equals, int64(3)) - c.Assert(hg.Len(), Equals, 3) + c.Assert(hg.Len(), Equals, 0) + c.Assert(hg.TopN.Num(), Equals, 3) }) } @@ -577,8 +578,8 @@ func (s *testSerialStatsSuite) TestAutoAnalyzeOnEmptyTable(c *C) { // test if it will be limited by the time range c.Assert(s.do.StatsHandle().HandleAutoAnalyze(s.do.InfoSchema()), IsFalse) - tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_start_time='00:00 +0000'")) - tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_end_time='23:59 +0000'")) + tk.MustExec("set global tidb_auto_analyze_start_time='00:00 +0000'") + tk.MustExec("set global tidb_auto_analyze_end_time='23:59 +0000'") c.Assert(s.do.StatsHandle().HandleAutoAnalyze(s.do.InfoSchema()), IsTrue) } @@ -1453,7 +1454,7 @@ func (s *testStatsSuite) TestNeedAnalyzeTable(c *C) { }{ // table was never analyzed and has reach the limit { - tbl: &statistics.Table{Version: oracle.EncodeTSO(oracle.GetPhysical(time.Now()))}, + tbl: &statistics.Table{Version: oracle.GoTimeToTS(time.Now())}, limit: 0, ratio: 0, start: "00:00 +0800", @@ -1464,7 +1465,7 @@ func (s *testStatsSuite) TestNeedAnalyzeTable(c *C) { }, // table was never analyzed but has not reach the limit { - tbl: &statistics.Table{Version: oracle.EncodeTSO(oracle.GetPhysical(time.Now()))}, + tbl: &statistics.Table{Version: oracle.GoTimeToTS(time.Now())}, limit: time.Hour, ratio: 0, start: "00:00 +0800", @@ -1758,6 +1759,7 @@ func (s *testStatsSuite) TestAbnormalIndexFeedback(c *C) { for i := 0; i < 20; i++ { testKit.MustExec(fmt.Sprintf("insert into t values (%d, %d)", i/5, i)) } + testKit.MustExec("set @@session.tidb_analyze_version = 1") testKit.MustExec("analyze table t with 3 buckets, 0 topn") testKit.MustExec("delete from t where a = 1") testKit.MustExec("delete from t where b > 10") @@ -1836,6 +1838,7 @@ func (s *testStatsSuite) TestFeedbackRanges(c *C) { err := h.HandleDDLEvent(<-h.DDLEventCh()) c.Assert(err, IsNil) c.Assert(h.DumpStatsDeltaToKV(handle.DumpAll), IsNil) + testKit.MustExec("set @@session.tidb_analyze_version=1") testKit.MustExec("analyze table t with 3 buckets") for i := 30; i < 40; i++ { testKit.MustExec(fmt.Sprintf("insert into t values (%d, %d)", i, i)) diff --git a/statistics/histogram.go b/statistics/histogram.go index 37a78fe1b92de..80d61b0c3bdf5 100644 --- a/statistics/histogram.go +++ b/statistics/histogram.go @@ -295,13 +295,10 @@ const ( Version1 = 1 // Version2 maintains the statistics in the following way. // Column stats: CM Sketch is not used. TopN and Histogram are built from samples. TopN + Histogram represent all data. - // Index stats: CM SKetch is not used. TopN and Histograms are built in TiKV using full data. NDV is also collected for each bucket in histogram. + // Index stats: CM SKetch is not used. TopN and Histograms are built from samples. // Then values covered by TopN is removed from Histogram. TopN + Histogram represent all data. + // Both Column and Index's NDVs are collected by full scan. Version2 = 2 - // Version3 is used for testing now. Once it finished, we will fallback the Version3 to Version2. - // The difference between Version2 and Version3 is that we construct the index's statistics based on sampling also. - // The data structure between them are then same. - Version3 = 3 ) // AnalyzeFlag is set when the statistics comes from analyze and has not been modified by feedback. @@ -506,20 +503,12 @@ func (hg *Histogram) BetweenRowCount(a, b types.Datum) float64 { } // BetweenRowCount estimates the row count for interval [l, r). -func (c *Column) BetweenRowCount(sc *stmtctx.StatementContext, l, r types.Datum) (float64, error) { +func (c *Column) BetweenRowCount(sc *stmtctx.StatementContext, l, r types.Datum, lowEncoded, highEncoded []byte) float64 { histBetweenCnt := c.Histogram.BetweenRowCount(l, r) if c.StatsVer <= Version1 { - return histBetweenCnt, nil - } - lBytes, err := codec.EncodeKey(sc, nil, l) - if err != nil { - return 0, errors.Trace(err) - } - rBytes, err := codec.EncodeKey(sc, nil, r) - if err != nil { - return 0, errors.Trace(err) + return histBetweenCnt } - return float64(c.TopN.BetweenCount(lBytes, rBytes)) + histBetweenCnt, nil + return float64(c.TopN.BetweenCount(lowEncoded, highEncoded)) + histBetweenCnt } // TotalRowCount returns the total count of this histogram. @@ -814,7 +803,7 @@ func MergeHistograms(sc *stmtctx.StatementContext, lh *Histogram, rh *Histogram, rAvg *= 2 } for i := 0; i < rh.Len(); i++ { - if statsVer == Version2 { + if statsVer >= Version2 { lh.AppendBucketWithNDV(rh.GetLower(i), rh.GetUpper(i), rh.Buckets[i].Count+lCount-offset, rh.Buckets[i].Repeat, rh.Buckets[i].NDV) continue } @@ -925,14 +914,14 @@ func (c *Column) String() string { // TotalRowCount returns the total count of this column. func (c *Column) TotalRowCount() float64 { - if c.StatsVer == Version2 { + if c.StatsVer >= Version2 { return c.Histogram.TotalRowCount() + float64(c.TopN.TotalCount()) } return c.Histogram.TotalRowCount() } func (c *Column) notNullCount() float64 { - if c.StatsVer == Version2 { + if c.StatsVer >= Version2 { return c.Histogram.notNullCount() + float64(c.TopN.TotalCount()) } return c.Histogram.notNullCount() @@ -978,7 +967,7 @@ func (c *Column) IsInvalid(sc *stmtctx.StatementContext, collPseudo bool) bool { return c.TotalRowCount() == 0 || (c.Histogram.NDV > 0 && c.notNullCount() == 0) } -func (c *Column) equalRowCount(sc *stmtctx.StatementContext, val types.Datum, modifyCount int64) (float64, error) { +func (c *Column) equalRowCount(sc *stmtctx.StatementContext, val types.Datum, encodedVal []byte, modifyCount int64) (float64, error) { if val.IsNull() { return float64(c.NullCount), nil } @@ -987,7 +976,7 @@ func (c *Column) equalRowCount(sc *stmtctx.StatementContext, val types.Datum, mo if c.Histogram.Bounds.NumRows() == 0 { return 0.0, nil } - if c.Histogram.NDV > 0 && c.outOfRange(val) { + if c.Histogram.NDV > 0 && c.outOfRange(val, encodedVal) { return outOfRangeEQSelectivity(c.Histogram.NDV, modifyCount, int64(c.TotalRowCount())) * c.TotalRowCount(), nil } if c.CMSketch != nil { @@ -996,14 +985,17 @@ func (c *Column) equalRowCount(sc *stmtctx.StatementContext, val types.Datum, mo } return c.Histogram.equalRowCount(val, false), nil } + // All the values are null. + if c.Histogram.Bounds.NumRows() == 0 && c.TopN.Num() == 0 { + return 0, nil + } + if c.Histogram.NDV+int64(c.TopN.Num()) > 0 && c.outOfRange(val, encodedVal) { + return outOfRangeEQSelectivity(c.Histogram.NDV, modifyCount, int64(c.TotalRowCount())) * c.TotalRowCount(), nil + } // Stats version == 2 // 1. try to find this value in TopN if c.TopN != nil { - valBytes, err := codec.EncodeKey(sc, nil, val) - if err != nil { - return 0, errors.Trace(err) - } - rowcount, ok := c.QueryTopN(valBytes) + rowcount, ok := c.QueryTopN(encodedVal) if ok { return float64(rowcount), nil } @@ -1054,6 +1046,14 @@ func (c *Column) GetColumnRowCount(sc *stmtctx.StatementContext, ranges []*range if err != nil { return 0, errors.Trace(err) } + lowEncoded, err := codec.EncodeKey(sc, nil, lowVal) + if err != nil { + return 0, err + } + highEncoded, err := codec.EncodeKey(sc, nil, highVal) + if err != nil { + return 0, err + } if cmp == 0 { // the point case. if !rg.LowExclude && !rg.HighExclude { @@ -1063,7 +1063,7 @@ func (c *Column) GetColumnRowCount(sc *stmtctx.StatementContext, ranges []*range continue } var cnt float64 - cnt, err = c.equalRowCount(sc, lowVal, modifyCount) + cnt, err = c.equalRowCount(sc, lowVal, lowEncoded, modifyCount) if err != nil { return 0, errors.Trace(err) } @@ -1075,7 +1075,7 @@ func (c *Column) GetColumnRowCount(sc *stmtctx.StatementContext, ranges []*range // The small range case. if rangeVals != nil { for _, val := range rangeVals { - cnt, err := c.equalRowCount(sc, val, modifyCount) + cnt, err := c.equalRowCount(sc, val, lowEncoded, modifyCount) if err != nil { return 0, err } @@ -1084,18 +1084,15 @@ func (c *Column) GetColumnRowCount(sc *stmtctx.StatementContext, ranges []*range continue } // The interval case. - cnt, err := c.BetweenRowCount(sc, lowVal, highVal) - if err != nil { - return 0, err - } - if (c.outOfRange(lowVal) && !lowVal.IsNull()) || c.outOfRange(highVal) { + cnt := c.BetweenRowCount(sc, lowVal, highVal, lowEncoded, highEncoded) + if (c.outOfRange(lowVal, lowEncoded) && !lowVal.IsNull()) || c.outOfRange(highVal, highEncoded) { cnt += outOfRangeEQSelectivity(outOfRangeBetweenRate, modifyCount, int64(c.TotalRowCount())) * c.TotalRowCount() } // `betweenRowCount` returns count for [l, h) range, we adjust cnt for boudaries here. // Note that, `cnt` does not include null values, we need specially handle cases // where null is the lower bound. if rg.LowExclude && !lowVal.IsNull() { - lowCnt, err := c.equalRowCount(sc, lowVal, modifyCount) + lowCnt, err := c.equalRowCount(sc, lowVal, lowEncoded, modifyCount) if err != nil { return 0, errors.Trace(err) } @@ -1105,7 +1102,7 @@ func (c *Column) GetColumnRowCount(sc *stmtctx.StatementContext, ranges []*range cnt += float64(c.NullCount) } if !rg.HighExclude { - highCnt, err := c.equalRowCount(sc, highVal, modifyCount) + highCnt, err := c.equalRowCount(sc, highVal, highEncoded, modifyCount) if err != nil { return 0, errors.Trace(err) } @@ -1121,6 +1118,15 @@ func (c *Column) GetColumnRowCount(sc *stmtctx.StatementContext, ranges []*range return rowCount, nil } +func (c *Column) outOfRange(val types.Datum, encodedVal []byte) bool { + outOfHist := c.Histogram.outOfRange(val) + if !outOfHist { + return false + } + // Already out of hist. + return c.TopN.outOfRange(encodedVal) +} + // Index represents an index histogram. type Index struct { Histogram @@ -1140,7 +1146,7 @@ func (idx *Index) String() string { // TotalRowCount returns the total count of this index. func (idx *Index) TotalRowCount() float64 { - if idx.StatsVer == Version2 { + if idx.StatsVer >= Version2 { return idx.Histogram.TotalRowCount() + float64(idx.TopN.TotalCount()) } return idx.Histogram.TotalRowCount() @@ -1177,7 +1183,7 @@ func (idx *Index) equalRowCount(b []byte, modifyCount int64) float64 { return float64(idx.QueryBytes(b)) } // If it's version2, query the top-n first. - if idx.StatsVer == Version2 { + if idx.StatsVer >= Version2 { count, found := idx.TopN.QueryTopN(b) if found { return float64(count) @@ -1245,7 +1251,7 @@ func (idx *Index) GetRowCount(sc *stmtctx.StatementContext, coll *HistColl, inde expBackoffSuccess := false // Due to the limitation of calcFraction and convertDatumToScalar, the histogram actually won't estimate anything. // If the first column's range is point. - if rangePosition := GetOrdinalOfRangeCond(sc, indexRange); rangePosition > 0 && idx.StatsVer == Version2 && coll != nil { + if rangePosition := GetOrdinalOfRangeCond(sc, indexRange); rangePosition > 0 && idx.StatsVer >= Version2 && coll != nil { var expBackoffSel float64 expBackoffSel, expBackoffSuccess, err = idx.expBackoffEstimation(sc, coll, indexRange) if err != nil { @@ -1504,7 +1510,15 @@ func (coll *HistColl) NewHistCollBySelectivity(sc *stmtctx.StatementContext, sta } func (idx *Index) outOfRange(val types.Datum) bool { - if idx.Histogram.Len() == 0 { + outOfTopN := idx.TopN.outOfRange(val.GetBytes()) + // The val is in TopN, return false. + if !outOfTopN { + return false + } + + histEmpty := idx.Histogram.Len() == 0 + // HistEmpty->Hist out of range. + if histEmpty { return true } withInLowBoundOrPrefixMatch := chunk.Compare(idx.Bounds.GetRow(0), 0, &val) <= 0 || diff --git a/statistics/integration_test.go b/statistics/integration_test.go index 9edef42f17698..20aea4b621e95 100644 --- a/statistics/integration_test.go +++ b/statistics/integration_test.go @@ -94,14 +94,16 @@ func (s *testIntegrationSuite) TestChangeVerTo2Behavior(c *C) { } tk.MustExec("set @@session.tidb_analyze_version = 1") tk.MustExec("analyze table t index idx") - tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 The analyze version from the session is not compatible with the existing statistics of the table. Use the existing version instead")) + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 The analyze version from the session is not compatible with the existing statistics of the table. Use the existing version instead", + "Warning 1105 The version 2 would collect all statistics not only the selected indexes")) c.Assert(h.Update(is), IsNil) statsTblT = h.GetTableStats(tblT.Meta()) for _, idx := range statsTblT.Indices { c.Assert(idx.StatsVer, Equals, int64(2)) } tk.MustExec("analyze table t index") - tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 The analyze version from the session is not compatible with the existing statistics of the table. Use the existing version instead")) + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 The analyze version from the session is not compatible with the existing statistics of the table. Use the existing version instead", + "Warning 1105 The version 2 would collect all statistics not only the selected indexes")) c.Assert(h.Update(is), IsNil) statsTblT = h.GetTableStats(tblT.Meta()) for _, idx := range statsTblT.Indices { @@ -165,13 +167,6 @@ func (s *testIntegrationSuite) TestFastAnalyzeOnVer2(c *C) { } } -func (s *testIntegrationSuite) TestHideAnalyzeVerOnShow(c *C) { - defer cleanEnv(c, s.store, s.do) - tk := testkit.NewTestKit(c, s.store) - // TODO: remove this test when the version2 is GA. - c.Assert(len(tk.MustQuery("show variables like '%analyze_version%'").Rows()), Equals, 0) -} - func (s *testIntegrationSuite) TestIncAnalyzeOnVer2(c *C) { defer cleanEnv(c, s.store, s.do) tk := testkit.NewTestKit(c, s.store) @@ -188,8 +183,8 @@ func (s *testIntegrationSuite) TestIncAnalyzeOnVer2(c *C) { tk.MustExec("analyze incremental table t index idx with 2 topn") // After analyze, there's two val in hist. tk.MustQuery("show stats_buckets where table_name = 't' and column_name = 'idx'").Check(testkit.Rows( - "test t idx 1 0 2 2 1 1 1", - "test t idx 1 1 3 0 2 4 1", + "test t idx 1 0 2 2 1 1 0", + "test t idx 1 1 3 1 3 3 0", )) // Two val in topn. tk.MustQuery("show stats_topn where table_name = 't' and column_name = 'idx'").Check(testkit.Rows( @@ -332,7 +327,7 @@ func (s *testIntegrationSuite) TestNULLOnFullSampling(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t;") - tk.MustExec("set @@session.tidb_analyze_version = 3;") + tk.MustExec("set @@session.tidb_analyze_version = 2;") tk.MustExec("create table t(a int, index idx(a))") tk.MustExec("insert into t values(1), (1), (1), (2), (2), (3), (4), (null), (null), (null)") var ( diff --git a/statistics/row_sampler.go b/statistics/row_sampler.go index f4f40945af697..fce358e45ae24 100644 --- a/statistics/row_sampler.go +++ b/statistics/row_sampler.go @@ -14,13 +14,11 @@ package statistics import ( - "bytes" "container/heap" "context" "math/rand" - "github.com/pingcap/errors" - "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" @@ -52,6 +50,7 @@ type RowSampleCollector struct { type RowSampleItem struct { Columns []types.Datum Weight int64 + Handle kv.Handle } // WeightedRowSampleHeap implements the Heap interface. @@ -299,151 +298,3 @@ func RowSamplesToProto(samples WeightedRowSampleHeap) []*tipb.RowSample { } return rows } - -// BuildHistAndTopNOnRowSample build a histogram and TopN for a column from samples. -func BuildHistAndTopNOnRowSample( - ctx sessionctx.Context, - numBuckets, numTopN int, - id int64, - collector *SampleCollector, - tp *types.FieldType, - isColumn bool, -) (*Histogram, *TopN, error) { - var getComparedBytes func(datum types.Datum) ([]byte, error) - if isColumn { - getComparedBytes = func(datum types.Datum) ([]byte, error) { - return codec.EncodeKey(ctx.GetSessionVars().StmtCtx, nil, datum) - } - } else { - getComparedBytes = func(datum types.Datum) ([]byte, error) { - return datum.GetBytes(), nil - } - } - count := collector.Count - ndv := collector.FMSketch.NDV() - nullCount := collector.NullCount - if ndv > count { - ndv = count - } - if count == 0 || len(collector.Samples) == 0 { - return NewHistogram(id, ndv, nullCount, 0, tp, 0, collector.TotalSize), nil, nil - } - sc := ctx.GetSessionVars().StmtCtx - samples := collector.Samples - samples, err := SortSampleItems(sc, samples) - if err != nil { - return nil, nil, err - } - hg := NewHistogram(id, ndv, nullCount, 0, tp, numBuckets, collector.TotalSize) - - sampleNum := int64(len(samples)) - // As we use samples to build the histogram, the bucket number and repeat should multiply a factor. - sampleFactor := float64(count) / float64(len(samples)) - - // Step1: collect topn from samples - - // the topNList is always sorted by count from more to less - topNList := make([]TopNMeta, 0, numTopN) - cur, err := getComparedBytes(samples[0].Value) - if err != nil { - return nil, nil, errors.Trace(err) - } - curCnt := float64(0) - - // Iterate through the samples - for i := int64(0); i < sampleNum; i++ { - - sampleBytes, err := getComparedBytes(samples[i].Value) - if err != nil { - return nil, nil, errors.Trace(err) - } - // case 1, this value is equal to the last one: current count++ - if bytes.Equal(cur, sampleBytes) { - curCnt += 1 - continue - } - // case 2, meet a different value: counting for the "current" is complete - // case 2-1, now topn is empty: append the "current" count directly - if len(topNList) == 0 { - topNList = append(topNList, TopNMeta{Encoded: cur, Count: uint64(curCnt)}) - cur, curCnt = sampleBytes, 1 - continue - } - // case 2-2, now topn is full, and the "current" count is less than the least count in the topn: no need to insert the "current" - if len(topNList) >= numTopN && uint64(curCnt) <= topNList[len(topNList)-1].Count { - cur, curCnt = sampleBytes, 1 - continue - } - // case 2-3, now topn is not full, or the "current" count is larger than the least count in the topn: need to find a slot to insert the "current" - j := len(topNList) - for ; j > 0; j-- { - if uint64(curCnt) < topNList[j-1].Count { - break - } - } - topNList = append(topNList, TopNMeta{}) - copy(topNList[j+1:], topNList[j:]) - topNList[j] = TopNMeta{Encoded: cur, Count: uint64(curCnt)} - if len(topNList) > numTopN { - topNList = topNList[:numTopN] - } - cur, curCnt = sampleBytes, 1 - } - - // Handle the counting for the last value. Basically equal to the case 2 above. - // now topn is empty: append the "current" count directly - if len(topNList) == 0 { - topNList = append(topNList, TopNMeta{Encoded: cur, Count: uint64(curCnt)}) - } else if len(topNList) < numTopN || uint64(curCnt) > topNList[len(topNList)-1].Count { - // now topn is not full, or the "current" count is larger than the least count in the topn: need to find a slot to insert the "current" - j := len(topNList) - for ; j > 0; j-- { - if uint64(curCnt) < topNList[j-1].Count { - break - } - } - topNList = append(topNList, TopNMeta{}) - copy(topNList[j+1:], topNList[j:]) - topNList[j] = TopNMeta{Encoded: cur, Count: uint64(curCnt)} - if len(topNList) > numTopN { - topNList = topNList[:numTopN] - } - } - - // Step2: exclude topn from samples - for i := int64(0); i < int64(len(samples)); i++ { - sampleBytes, err := getComparedBytes(samples[i].Value) - if err != nil { - return nil, nil, errors.Trace(err) - } - for j := 0; j < len(topNList); j++ { - if bytes.Equal(sampleBytes, topNList[j].Encoded) { - // find the same value in topn: need to skip over this value in samples - copy(samples[i:], samples[uint64(i)+topNList[j].Count:]) - samples = samples[:uint64(len(samples))-topNList[j].Count] - i-- - continue - } - } - } - - for i := 0; i < len(topNList); i++ { - topNList[i].Count *= uint64(sampleFactor) - } - topn := &TopN{TopN: topNList} - - if uint64(count) <= topn.TotalCount() || int(hg.NDV) <= len(topn.TopN) { - // TopN includes all sample data - return hg, topn, nil - } - - // Step3: build histogram with the rest samples - if len(samples) > 0 { - _, err = buildHist(sc, hg, samples, count-int64(topn.TotalCount()), ndv-int64(len(topn.TopN)), int64(numBuckets)) - if err != nil { - return nil, nil, err - } - } - - return hg, topn, nil -} diff --git a/statistics/sample_test.go b/statistics/sample_test.go index 61ec41b85870a..3624b233395ae 100644 --- a/statistics/sample_test.go +++ b/statistics/sample_test.go @@ -297,7 +297,7 @@ func (s *testSampleSuite) TestBuildStatsOnRowSample(c *C) { TotalSize: int64(len(data)) * 8, } tp := types.NewFieldType(mysql.TypeLonglong) - hist, topN, err := BuildHistAndTopNOnRowSample(ctx, 5, 4, 1, collector, tp, true) + hist, topN, err := BuildHistAndTopN(ctx, 5, 4, 1, collector, tp, true) c.Assert(err, IsNil, Commentf("%+v", err)) topNStr, err := topN.DecodedString(ctx, []byte{tp.Tp}) c.Assert(err, IsNil) diff --git a/statistics/selectivity_test.go b/statistics/selectivity_test.go index 32ed9fcb9650b..4b2b38ceed61d 100644 --- a/statistics/selectivity_test.go +++ b/statistics/selectivity_test.go @@ -237,7 +237,6 @@ func (s *testStatsSuite) TestSelectivity(c *C) { defer cleanEnv(c, s.store, s.do) testKit := testkit.NewTestKit(c, s.store) statsTbl := s.prepareSelectivity(testKit, c) - is := s.do.InfoSchema() longExpr := "0 < a and a = 1 " for i := 1; i < 64; i++ { @@ -294,9 +293,10 @@ func (s *testStatsSuite) TestSelectivity(c *C) { c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, tt.exprs)) c.Assert(stmts, HasLen, 1) - err = plannercore.Preprocess(sctx, stmts[0], is) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) c.Assert(err, IsNil, comment) - p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is) + p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], ret.InfoSchema) c.Assert(err, IsNil, Commentf("error %v, for building plan, expr %s", err, tt.exprs)) sel := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) @@ -406,6 +406,7 @@ func (s *testStatsSuite) TestEstimationForUnknownValues(c *C) { testKit.MustExec("use test") testKit.MustExec("drop table if exists t") testKit.MustExec("create table t(a int, b int, key idx(a, b))") + testKit.MustExec("set @@tidb_analyze_version=1") testKit.MustExec("analyze table t") for i := 0; i < 10; i++ { testKit.MustExec(fmt.Sprintf("insert into t values (%d, %d)", i, i)) @@ -542,7 +543,6 @@ func BenchmarkSelectivity(b *testing.B) { testKit := testkit.NewTestKit(c, s.store) statsTbl := s.prepareSelectivity(testKit, c) - is := s.do.InfoSchema() exprs := "a > 1 and b < 2 and c > 3 and d < 4 and e > 5" sql := "select * from t where " + exprs comment := Commentf("for %s", exprs) @@ -550,9 +550,10 @@ func BenchmarkSelectivity(b *testing.B) { stmts, err := session.Parse(sctx, sql) c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, exprs)) c.Assert(stmts, HasLen, 1) - err = plannercore.Preprocess(sctx, stmts[0], is) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) c.Assert(err, IsNil, comment) - p, _, err := plannercore.BuildLogicalPlan(context.Background(), sctx, stmts[0], is) + p, _, err := plannercore.BuildLogicalPlan(context.Background(), sctx, stmts[0], ret.InfoSchema) c.Assert(err, IsNil, Commentf("error %v, for building plan, expr %s", err, exprs)) file, err := os.Create("cpu.profile") @@ -641,6 +642,42 @@ func (s *testStatsSuite) TestStatsVer2(c *C) { } } +func (s *testStatsSuite) TestTopNOutOfHist(c *C) { + defer cleanEnv(c, s.store, s.do) + testKit := testkit.NewTestKit(c, s.store) + testKit.MustExec("use test") + testKit.MustExec("set tidb_analyze_version=2") + + testKit.MustExec("drop table if exists topn_before_hist") + testKit.MustExec("create table topn_before_hist(a int, index idx(a))") + testKit.MustExec("insert into topn_before_hist values(1), (1), (1), (1), (3), (3), (4), (5), (6)") + testKit.MustExec("analyze table topn_before_hist with 2 topn, 3 buckets") + + testKit.MustExec("create table topn_after_hist(a int, index idx(a))") + testKit.MustExec("insert into topn_after_hist values(2), (2), (3), (4), (5), (7), (7), (7), (7)") + testKit.MustExec("analyze table topn_after_hist with 2 topn, 3 buckets") + + testKit.MustExec("create table topn_before_hist_no_index(a int)") + testKit.MustExec("insert into topn_before_hist_no_index values(1), (1), (1), (1), (3), (3), (4), (5), (6)") + testKit.MustExec("analyze table topn_before_hist_no_index with 2 topn, 3 buckets") + + testKit.MustExec("create table topn_after_hist_no_index(a int)") + testKit.MustExec("insert into topn_after_hist_no_index values(2), (2), (3), (4), (5), (7), (7), (7), (7)") + testKit.MustExec("analyze table topn_after_hist_no_index with 2 topn, 3 buckets") + + var ( + input []string + output [][]string + ) + s.testData.GetTestCases(c, &input, &output) + for i := range input { + s.testData.OnRecord(func() { + output[i] = s.testData.ConvertRowsToStrings(testKit.MustQuery(input[i]).Rows()) + }) + testKit.MustQuery(input[i]).Check(testkit.Rows(output[i]...)) + } +} + func (s *testStatsSuite) TestColumnIndexNullEstimation(c *C) { defer cleanEnv(c, s.store, s.do) testKit := testkit.NewTestKit(c, s.store) @@ -724,6 +761,7 @@ func (s *testStatsSuite) TestCollationColumnEstimate(c *C) { tk.MustExec("drop table if exists t") tk.MustExec("create table t(a varchar(20) collate utf8mb4_general_ci)") tk.MustExec("insert into t values('aaa'), ('bbb'), ('AAA'), ('BBB')") + tk.MustExec("set @@session.tidb_analyze_version=1") h := s.do.StatsHandle() c.Assert(h.DumpStatsDeltaToKV(handle.DumpAll), IsNil) tk.MustExec("analyze table t") @@ -756,9 +794,8 @@ func (s *testStatsSuite) TestDNFCondSelectivity(c *C) { testKit.MustExec(`analyze table t`) ctx := context.Background() - is := s.do.InfoSchema() h := s.do.StatsHandle() - tb, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + tb, err := s.do.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) tblInfo := tb.Meta() statsTbl := h.GetTableStats(tblInfo) @@ -777,9 +814,10 @@ func (s *testStatsSuite) TestDNFCondSelectivity(c *C) { c.Assert(err, IsNil, Commentf("error %v, for sql %s", err, tt)) c.Assert(stmts, HasLen, 1) - err = plannercore.Preprocess(sctx, stmts[0], is) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) c.Assert(err, IsNil, Commentf("error %v, for sql %s", err, tt)) - p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is) + p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], ret.InfoSchema) c.Assert(err, IsNil, Commentf("error %v, for building plan, sql %s", err, tt)) sel := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) diff --git a/statistics/statistics_test.go b/statistics/statistics_test.go index e74e04f7596ac..2b9f9771ed12a 100644 --- a/statistics/statistics_test.go +++ b/statistics/statistics_test.go @@ -278,26 +278,29 @@ func (s *testStatisticsSuite) TestBuild(c *C) { count = col.lessRowCount(types.NewIntDatum(1)) c.Check(int(count), Equals, 5) - colv2, topnv2, err := BuildColumnHistAndTopN(ctx, int(bucketCount), topNCount, 2, collector, types.NewFieldType(mysql.TypeLonglong)) + colv2, topnv2, err := BuildHistAndTopN(ctx, int(bucketCount), topNCount, 2, collector, types.NewFieldType(mysql.TypeLonglong), true) c.Check(err, IsNil) c.Check(topnv2.TopN, NotNil) - expectedTopNCount := []uint64{9990, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30} + // The most common one's occurrence is 9990, the second most common one's occurrence is 30. + // The ndv of the histogram is 73344, the total count of it is 90010. 90010/73344 vs 30, it's not a bad estimate. + expectedTopNCount := []uint64{9990} + c.Assert(len(topnv2.TopN), Equals, len(expectedTopNCount)) for i, meta := range topnv2.TopN { c.Check(meta.Count, Equals, expectedTopNCount[i]) } - c.Check(colv2.Len(), Equals, 256) + c.Check(colv2.Len(), Equals, 251) count = colv2.lessRowCount(types.NewIntDatum(1000)) - c.Check(int(count), Equals, 325) + c.Check(int(count), Equals, 328) count = colv2.lessRowCount(types.NewIntDatum(2000)) - c.Check(int(count), Equals, 9430) + c.Check(int(count), Equals, 10007) count = colv2.greaterRowCount(types.NewIntDatum(2000)) - c.Check(int(count), Equals, 80008) + c.Check(int(count), Equals, 80001) count = colv2.lessRowCount(types.NewIntDatum(200000000)) - c.Check(int(count), Equals, 89440) + c.Check(int(count), Equals, 90010) count = colv2.greaterRowCount(types.NewIntDatum(200000000)) c.Check(count, Equals, 0.0) count = colv2.BetweenRowCount(types.NewIntDatum(3000), types.NewIntDatum(3500)) - c.Check(int(count), Equals, 4995) + c.Check(int(count), Equals, 5001) count = colv2.lessRowCount(types.NewIntDatum(1)) c.Check(int(count), Equals, 0) @@ -458,7 +461,7 @@ func (s *testStatisticsSuite) TestPseudoTable(c *C) { count, err := tbl.ColumnEqualRowCount(sc, types.NewIntDatum(1000), colInfo.ID) c.Assert(err, IsNil) c.Assert(int(count), Equals, 10) - count = tbl.ColumnBetweenRowCount(sc, types.NewIntDatum(1000), types.NewIntDatum(5000), colInfo.ID) + count, _ = tbl.ColumnBetweenRowCount(sc, types.NewIntDatum(1000), types.NewIntDatum(5000), colInfo.ID) c.Assert(int(count), Equals, 250) } diff --git a/statistics/table.go b/statistics/table.go index 7628e018e25a5..85807fbefc67f 100644 --- a/statistics/table.go +++ b/statistics/table.go @@ -277,19 +277,24 @@ func (t *Table) ColumnLessRowCount(sc *stmtctx.StatementContext, value types.Dat } // ColumnBetweenRowCount estimates the row count where column greater or equal to a and less than b. -func (t *Table) ColumnBetweenRowCount(sc *stmtctx.StatementContext, a, b types.Datum, colID int64) float64 { +func (t *Table) ColumnBetweenRowCount(sc *stmtctx.StatementContext, a, b types.Datum, colID int64) (float64, error) { c, ok := t.Columns[colID] if !ok || c.IsInvalid(sc, t.Pseudo) { - return float64(t.Count) / pseudoBetweenRate + return float64(t.Count) / pseudoBetweenRate, nil } - count, err := c.BetweenRowCount(sc, a, b) + aEncoded, err := codec.EncodeKey(sc, nil, a) if err != nil { - return 0 + return 0, err } + bEncoded, err := codec.EncodeKey(sc, nil, b) + if err != nil { + return 0, err + } + count := c.BetweenRowCount(sc, a, b, aEncoded, bEncoded) if a.IsNull() { count += float64(c.NullCount) } - return count * c.GetIncreaseFactor(t.Count) + return count * c.GetIncreaseFactor(t.Count), nil } // ColumnEqualRowCount estimates the row count where the column equals to value. @@ -298,7 +303,11 @@ func (t *Table) ColumnEqualRowCount(sc *stmtctx.StatementContext, value types.Da if !ok || c.IsInvalid(sc, t.Pseudo) { return float64(t.Count) / pseudoEqualRate, nil } - result, err := c.equalRowCount(sc, value, t.ModifyCount) + encodedVal, err := codec.EncodeKey(sc, nil, value) + if err != nil { + return 0, err + } + result, err := c.equalRowCount(sc, value, encodedVal, t.ModifyCount) result *= c.GetIncreaseFactor(t.Count) return result, errors.Trace(err) } diff --git a/statistics/testdata/stats_suite_in.json b/statistics/testdata/stats_suite_in.json index 6aef592ae4fa5..b20b6d8300433 100644 --- a/statistics/testdata/stats_suite_in.json +++ b/statistics/testdata/stats_suite_in.json @@ -66,6 +66,21 @@ "explain select * from ct2 where a=8 and b>=1 and b<=8" ] }, + { + "name": "TestTopNOutOfHist", + "cases": [ + "show stats_topn", + "show stats_buckets", + "explain select * from topn_before_hist where a = 1", + "explain select * from topn_before_hist where a = 2", + "explain select * from topn_after_hist where a = 7", + "explain select * from topn_after_hist where a = 6", + "explain select * from topn_after_hist_no_index where a = 7", + "explain select * from topn_after_hist_no_index where a = 6", + "explain select * from topn_before_hist_no_index where a = 1", + "explain select * from topn_before_hist_no_index where a = 2" + ] + }, { "name": "TestColumnIndexNullEstimation", "cases": [ diff --git a/statistics/testdata/stats_suite_out.json b/statistics/testdata/stats_suite_out.json index 80f4ad9e9e562..33babd9401b4d 100644 --- a/statistics/testdata/stats_suite_out.json +++ b/statistics/testdata/stats_suite_out.json @@ -89,71 +89,70 @@ "test tint b 0 1 6 1 6 8 0", "test tint c 0 0 3 1 3 5 0", "test tint c 0 1 6 1 6 8 0", - "test tint singular 1 0 2 1 1 4 2", - "test tint singular 1 1 6 1 5 8 4", - "test tint multi 1 0 2 1 (1, 1) (4, 4) 2", - "test tint multi 1 1 6 1 (5, 5) (8, 8) 4", + "test tint singular 1 0 3 1 3 5 0", + "test tint singular 1 1 6 1 6 8 0", + "test tint multi 1 0 3 1 (3, 3) (5, 5) 0", + "test tint multi 1 1 6 1 (6, 6) (8, 8) 0", "test tdouble a 0 0 3 1 3 5 0", "test tdouble a 0 1 6 1 6 8 0", "test tdouble b 0 0 3 1 3 5 0", "test tdouble b 0 1 6 1 6 8 0", "test tdouble c 0 0 3 1 3 5 0", "test tdouble c 0 1 6 1 6 8 0", - "test tdouble singular 1 0 2 1 1 4 2", - "test tdouble singular 1 1 6 1 5 8 4", - "test tdouble multi 1 0 2 1 (1, 1) (4, 4) 2", - "test tdouble multi 1 1 6 1 (5, 5) (8, 8) 4", + "test tdouble singular 1 0 3 1 3 5 0", + "test tdouble singular 1 1 6 1 6 8 0", + "test tdouble multi 1 0 3 1 (3, 3) (5, 5) 0", + "test tdouble multi 1 1 6 1 (6, 6) (8, 8) 0", "test tdecimal a 0 0 3 1 3.00000000000000000000 5.00000000000000000000 0", "test tdecimal a 0 1 6 1 6.00000000000000000000 8.00000000000000000000 0", "test tdecimal b 0 0 3 1 3.00000000000000000000 5.00000000000000000000 0", "test tdecimal b 0 1 6 1 6.00000000000000000000 8.00000000000000000000 0", "test tdecimal c 0 0 3 1 3.00000000000000000000 5.00000000000000000000 0", "test tdecimal c 0 1 6 1 6.00000000000000000000 8.00000000000000000000 0", - "test tdecimal singular 1 0 2 1 1.00000000000000000000 4.00000000000000000000 2", - "test tdecimal singular 1 1 6 1 5.00000000000000000000 8.00000000000000000000 4", - "test tdecimal multi 1 0 2 1 (1.00000000000000000000, 1.00000000000000000000) (4.00000000000000000000, 4.00000000000000000000) 2", - "test tdecimal multi 1 1 6 1 (5.00000000000000000000, 5.00000000000000000000) (8.00000000000000000000, 8.00000000000000000000) 4", + "test tdecimal singular 1 0 3 1 3.00000000000000000000 5.00000000000000000000 0", + "test tdecimal singular 1 1 6 1 6.00000000000000000000 8.00000000000000000000 0", + "test tdecimal multi 1 0 3 1 (3.00000000000000000000, 3.00000000000000000000) (5.00000000000000000000, 5.00000000000000000000) 0", + "test tdecimal multi 1 1 6 1 (6.00000000000000000000, 6.00000000000000000000) (8.00000000000000000000, 8.00000000000000000000) 0", "test tstring a 0 0 3 1 3 5 0", "test tstring a 0 1 6 1 6 8 0", "test tstring b 0 0 3 1 3 5 0", "test tstring b 0 1 6 1 6 8 0", "test tstring c 0 0 3 1 3 5 0", "test tstring c 0 1 6 1 6 8 0", - "test tstring singular 1 0 2 1 1 4 2", - "test tstring singular 1 1 6 1 5 8 4", - "test tstring multi 1 0 2 1 (1, 1) (4, 4) 2", - "test tstring multi 1 1 6 1 (5, 5) (8, 8) 4", + "test tstring singular 1 0 3 1 3 5 0", + "test tstring singular 1 1 6 1 6 8 0", + "test tstring multi 1 0 3 1 (3, 3) (5, 5) 0", + "test tstring multi 1 1 6 1 (6, 6) (8, 8) 0", "test tdatetime a 0 0 1 1 2001-01-03 00:00:00 2001-01-03 00:00:00 0", "test tdatetime a 0 1 2 1 2001-01-04 00:00:00 2001-01-04 00:00:00 0", "test tdatetime b 0 0 1 1 2001-01-03 00:00:00 2001-01-03 00:00:00 0", "test tdatetime b 0 1 2 1 2001-01-04 00:00:00 2001-01-04 00:00:00 0", "test tdatetime c 0 0 1 1 2001-01-03 00:00:00 2001-01-03 00:00:00 0", "test tdatetime c 0 1 2 1 2001-01-04 00:00:00 2001-01-04 00:00:00 0", - "test tdatetime singular 1 0 0 0 2001-01-01 00:00:00 2001-01-02 00:00:00 0", - "test tdatetime singular 1 1 2 1 2001-01-03 00:00:00 2001-01-04 00:00:00 2", - "test tdatetime multi 1 0 0 0 (2001-01-01 00:00:00, 2001-01-01 00:00:00) (2001-01-02 00:00:00, 2001-01-02 00:00:00) 0", - "test tdatetime multi 1 1 2 1 (2001-01-03 00:00:00, 2001-01-03 00:00:00) (2001-01-04 00:00:00, 2001-01-04 00:00:00) 2", + "test tdatetime singular 1 0 1 1 2001-01-03 00:00:00 2001-01-03 00:00:00 0", + "test tdatetime singular 1 1 2 1 2001-01-04 00:00:00 2001-01-04 00:00:00 0", + "test tdatetime multi 1 0 1 1 (2001-01-03 00:00:00, 2001-01-03 00:00:00) (2001-01-03 00:00:00, 2001-01-03 00:00:00) 0", + "test tdatetime multi 1 1 2 1 (2001-01-04 00:00:00, 2001-01-04 00:00:00) (2001-01-04 00:00:00, 2001-01-04 00:00:00) 0", "test tprefix a 0 0 2 1 333 444 0", "test tprefix a 0 1 4 1 555 666 0", "test tprefix b 0 0 2 1 333 444 0", "test tprefix b 0 1 4 1 555 666 0", - "test tprefix prefixa 1 0 0 0 11 22 0", - "test tprefix prefixa 1 1 2 1 33 44 2", - "test tprefix prefixa 1 2 4 1 55 66 2", + "test tprefix prefixa 1 0 2 1 33 44 0", + "test tprefix prefixa 1 1 4 1 55 66 0", "test ct1 a 0 0 3 1 3 5 0", "test ct1 a 0 1 6 1 6 8 0", "test ct1 pk 0 0 3 1 3 5 0", "test ct1 pk 0 1 6 1 6 8 0", - "test ct1 PRIMARY 1 0 2 1 1 4 2", - "test ct1 PRIMARY 1 1 6 1 5 8 4", + "test ct1 PRIMARY 1 0 3 1 3 5 0", + "test ct1 PRIMARY 1 1 6 1 6 8 0", "test ct2 a 0 0 3 1 3 5 0", "test ct2 a 0 1 6 1 6 8 0", "test ct2 b 0 0 3 1 3 5 0", "test ct2 b 0 1 6 1 6 8 0", "test ct2 c 0 0 3 1 3 5 0", "test ct2 c 0 1 6 1 6 8 0", - "test ct2 PRIMARY 1 0 2 1 (1, 1) (4, 4) 2", - "test ct2 PRIMARY 1 1 6 1 (5, 5) (8, 8) 4" + "test ct2 PRIMARY 1 0 3 1 (3, 3) (5, 5) 0", + "test ct2 PRIMARY 1 1 6 1 (6, 6) (8, 8) 0" ], [ "TableReader_7 1.00 root data:Selection_6", @@ -161,8 +160,8 @@ " └─TableFullScan_5 8.00 cop[tikv] table:tint keep order:false" ], [ - "TableReader_7 1.00 root data:Selection_6", - "└─Selection_6 1.00 cop[tikv] eq(test.tint.a, 4)", + "TableReader_7 0.75 root data:Selection_6", + "└─Selection_6 0.75 cop[tikv] eq(test.tint.a, 4)", " └─TableFullScan_5 8.00 cop[tikv] table:tint keep order:false" ], [ @@ -176,9 +175,9 @@ "└─TableRowIDScan_9(Probe) 1.00 cop[tikv] table:tdouble keep order:false" ], [ - "IndexLookUp_10 1.00 root ", - "├─IndexRangeScan_8(Build) 1.00 cop[tikv] table:tdouble, index:singular(a) range:[4,4], keep order:false", - "└─TableRowIDScan_9(Probe) 1.00 cop[tikv] table:tdouble keep order:false" + "IndexLookUp_10 0.75 root ", + "├─IndexRangeScan_8(Build) 0.75 cop[tikv] table:tdouble, index:singular(a) range:[4,4], keep order:false", + "└─TableRowIDScan_9(Probe) 0.75 cop[tikv] table:tdouble keep order:false" ], [ "IndexLookUp_10 1.00 root ", @@ -191,9 +190,9 @@ "└─TableRowIDScan_9(Probe) 1.00 cop[tikv] table:tdecimal keep order:false" ], [ - "IndexLookUp_10 1.00 root ", - "├─IndexRangeScan_8(Build) 1.00 cop[tikv] table:tdecimal, index:singular(a) range:[4.00000000000000000000,4.00000000000000000000], keep order:false", - "└─TableRowIDScan_9(Probe) 1.00 cop[tikv] table:tdecimal keep order:false" + "IndexLookUp_10 0.75 root ", + "├─IndexRangeScan_8(Build) 0.75 cop[tikv] table:tdecimal, index:singular(a) range:[4.00000000000000000000,4.00000000000000000000], keep order:false", + "└─TableRowIDScan_9(Probe) 0.75 cop[tikv] table:tdecimal keep order:false" ], [ "IndexLookUp_10 1.00 root ", @@ -206,8 +205,8 @@ " └─TableFullScan_5 8.00 cop[tikv] table:tstring keep order:false" ], [ - "TableReader_7 1.00 root data:Selection_6", - "└─Selection_6 1.00 cop[tikv] eq(test.tstring.a, \"4\")", + "TableReader_7 0.75 root data:Selection_6", + "└─Selection_6 0.75 cop[tikv] eq(test.tstring.a, \"4\")", " └─TableFullScan_5 8.00 cop[tikv] table:tstring keep order:false" ], [ @@ -252,8 +251,8 @@ " └─TableFullScan_5 8.00 cop[tikv] table:tint keep order:false" ], [ - "TableReader_7 1.00 root data:Selection_6", - "└─Selection_6 1.00 cop[tikv] eq(test.tint.b, 4), eq(test.tint.c, 4)", + "TableReader_7 0.75 root data:Selection_6", + "└─Selection_6 0.75 cop[tikv] eq(test.tint.b, 4), eq(test.tint.c, 4)", " └─TableFullScan_5 8.00 cop[tikv] table:tint keep order:false" ], [ @@ -267,9 +266,9 @@ "└─TableRowIDScan_9(Probe) 1.00 cop[tikv] table:tdouble keep order:false" ], [ - "IndexLookUp_10 1.00 root ", - "├─IndexRangeScan_8(Build) 1.00 cop[tikv] table:tdouble, index:multi(b, c) range:[4 4,4 4], keep order:false", - "└─TableRowIDScan_9(Probe) 1.00 cop[tikv] table:tdouble keep order:false" + "IndexLookUp_10 0.75 root ", + "├─IndexRangeScan_8(Build) 0.75 cop[tikv] table:tdouble, index:multi(b, c) range:[4 4,4 4], keep order:false", + "└─TableRowIDScan_9(Probe) 0.75 cop[tikv] table:tdouble keep order:false" ], [ "IndexLookUp_10 1.00 root ", @@ -282,9 +281,9 @@ "└─TableRowIDScan_9(Probe) 1.00 cop[tikv] table:tdecimal keep order:false" ], [ - "IndexLookUp_10 1.00 root ", - "├─IndexRangeScan_8(Build) 1.00 cop[tikv] table:tdecimal, index:multi(b, c) range:[4.00000000000000000000 4.00000000000000000000,4.00000000000000000000 4.00000000000000000000], keep order:false", - "└─TableRowIDScan_9(Probe) 1.00 cop[tikv] table:tdecimal keep order:false" + "IndexLookUp_10 0.75 root ", + "├─IndexRangeScan_8(Build) 0.75 cop[tikv] table:tdecimal, index:multi(b, c) range:[4.00000000000000000000 4.00000000000000000000,4.00000000000000000000 4.00000000000000000000], keep order:false", + "└─TableRowIDScan_9(Probe) 0.75 cop[tikv] table:tdecimal keep order:false" ], [ "IndexLookUp_10 1.00 root ", @@ -297,8 +296,8 @@ " └─TableFullScan_5 8.00 cop[tikv] table:tstring keep order:false" ], [ - "TableReader_7 1.00 root data:Selection_6", - "└─Selection_6 1.00 cop[tikv] eq(test.tstring.b, \"4\"), eq(test.tstring.c, \"4\")", + "TableReader_7 0.75 root data:Selection_6", + "└─Selection_6 0.75 cop[tikv] eq(test.tstring.b, \"4\"), eq(test.tstring.c, \"4\")", " └─TableFullScan_5 8.00 cop[tikv] table:tstring keep order:false" ], [ @@ -397,8 +396,8 @@ " └─TableFullScan_5 4.00 cop[tikv] table:tdatetime keep order:false" ], [ - "TableReader_6 4.00 root data:TableRangeScan_5", - "└─TableRangeScan_5 4.00 cop[tikv] table:ct1 range:[\"1\",\"4\"], keep order:false" + "TableReader_6 5.00 root data:TableRangeScan_5", + "└─TableRangeScan_5 5.00 cop[tikv] table:ct1 range:[\"1\",\"4\"], keep order:false" ], [ "TableReader_6 3.75 root data:TableRangeScan_5", @@ -422,6 +421,75 @@ ] ] }, + { + "Name": "TestTopNOutOfHist", + "Cases": [ + [ + "test topn_before_hist a 0 1 4", + "test topn_before_hist a 0 3 2", + "test topn_before_hist idx 1 1 4", + "test topn_before_hist idx 1 3 2", + "test topn_after_hist a 0 2 2", + "test topn_after_hist a 0 7 4", + "test topn_after_hist idx 1 2 2", + "test topn_after_hist idx 1 7 4", + "test topn_before_hist_no_index a 0 1 4", + "test topn_before_hist_no_index a 0 3 2", + "test topn_after_hist_no_index a 0 2 2", + "test topn_after_hist_no_index a 0 7 4" + ], + [ + "test topn_before_hist a 0 0 2 1 4 5 0", + "test topn_before_hist a 0 1 3 1 6 6 0", + "test topn_before_hist idx 1 0 2 1 4 5 0", + "test topn_before_hist idx 1 1 3 1 6 6 0", + "test topn_after_hist a 0 0 2 1 3 4 0", + "test topn_after_hist a 0 1 3 1 5 5 0", + "test topn_after_hist idx 1 0 2 1 3 4 0", + "test topn_after_hist idx 1 1 3 1 5 5 0", + "test topn_before_hist_no_index a 0 0 2 1 4 5 0", + "test topn_before_hist_no_index a 0 1 3 1 6 6 0", + "test topn_after_hist_no_index a 0 0 2 1 3 4 0", + "test topn_after_hist_no_index a 0 1 3 1 5 5 0" + ], + [ + "IndexReader_6 4.00 root index:IndexRangeScan_5", + "└─IndexRangeScan_5 4.00 cop[tikv] table:topn_before_hist, index:idx(a) range:[1,1], keep order:false" + ], + [ + "IndexReader_6 0.00 root index:IndexRangeScan_5", + "└─IndexRangeScan_5 0.00 cop[tikv] table:topn_before_hist, index:idx(a) range:[2,2], keep order:false" + ], + [ + "IndexReader_6 4.00 root index:IndexRangeScan_5", + "└─IndexRangeScan_5 4.00 cop[tikv] table:topn_after_hist, index:idx(a) range:[7,7], keep order:false" + ], + [ + "IndexReader_6 0.00 root index:IndexRangeScan_5", + "└─IndexRangeScan_5 0.00 cop[tikv] table:topn_after_hist, index:idx(a) range:[6,6], keep order:false" + ], + [ + "TableReader_7 4.00 root data:Selection_6", + "└─Selection_6 4.00 cop[tikv] eq(test.topn_after_hist_no_index.a, 7)", + " └─TableFullScan_5 9.00 cop[tikv] table:topn_after_hist_no_index keep order:false" + ], + [ + "TableReader_7 1.00 root data:Selection_6", + "└─Selection_6 1.00 cop[tikv] eq(test.topn_after_hist_no_index.a, 6)", + " └─TableFullScan_5 9.00 cop[tikv] table:topn_after_hist_no_index keep order:false" + ], + [ + "TableReader_7 4.00 root data:Selection_6", + "└─Selection_6 4.00 cop[tikv] eq(test.topn_before_hist_no_index.a, 1)", + " └─TableFullScan_5 9.00 cop[tikv] table:topn_before_hist_no_index keep order:false" + ], + [ + "TableReader_7 1.00 root data:Selection_6", + "└─Selection_6 1.00 cop[tikv] eq(test.topn_before_hist_no_index.a, 2)", + " └─TableFullScan_5 9.00 cop[tikv] table:topn_before_hist_no_index keep order:false" + ] + ] + }, { "Name": "TestColumnIndexNullEstimation", "Cases": [ @@ -539,7 +607,7 @@ }, { "SQL": "select * from t where a < 8 and (b > 10 or c < 3 or b > 4) and a > 2", - "Selectivity": 0.3125 + "Selectivity": 0 } ] } diff --git a/store/copr/batch_coprocessor.go b/store/copr/batch_coprocessor.go index 5506c9d497ac2..ade644f411897 100644 --- a/store/copr/batch_coprocessor.go +++ b/store/copr/batch_coprocessor.go @@ -16,25 +16,25 @@ package copr import ( "context" "io" + "math" + "strconv" "sync" "sync/atomic" "time" - "unsafe" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/coprocessor" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/log" "github.com/pingcap/tidb/kv" - txndriver "github.com/pingcap/tidb/store/driver/txn" + "github.com/pingcap/tidb/store/driver/backoff" + derr "github.com/pingcap/tidb/store/driver/error" "github.com/pingcap/tidb/store/tikv" - tikverr "github.com/pingcap/tidb/store/tikv/error" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" - "github.com/pingcap/tidb/store/tikv/util" - "github.com/pingcap/tidb/util/memory" "go.uber.org/zap" ) @@ -42,8 +42,9 @@ import ( type batchCopTask struct { storeAddr string cmdType tikvrpc.CmdType + ctx *tikv.RPCContext - copTasks []copTaskAndRPCContext + regionInfos []RegionInfo } type batchCopResponse struct { @@ -95,31 +96,174 @@ func (rs *batchCopResponse) RespTime() time.Duration { return rs.respTime } -type copTaskAndRPCContext struct { - task *copTask - ctx *tikv.RPCContext +// balanceBatchCopTask balance the regions between available stores, the basic rule is +// 1. the first region of each original batch cop task belongs to its original store because some +// meta data(like the rpc context) in batchCopTask is related to it +// 2. for the remaining regions: +// if there is only 1 available store, then put the region to the related store +// otherwise, use a greedy algorithm to put it into the store with highest weight +func balanceBatchCopTask(originalTasks []*batchCopTask) []*batchCopTask { + if len(originalTasks) <= 1 { + return originalTasks + } + storeTaskMap := make(map[uint64]*batchCopTask) + storeCandidateRegionMap := make(map[uint64]map[string]RegionInfo) + totalRegionCandidateNum := 0 + totalRemainingRegionNum := 0 + + for _, task := range originalTasks { + taskStoreID := task.regionInfos[0].AllStores[0] + batchTask := &batchCopTask{ + storeAddr: task.storeAddr, + cmdType: task.cmdType, + ctx: task.ctx, + regionInfos: []RegionInfo{task.regionInfos[0]}, + } + storeTaskMap[taskStoreID] = batchTask + } + + for _, task := range originalTasks { + taskStoreID := task.regionInfos[0].AllStores[0] + for index, ri := range task.regionInfos { + // for each region, figure out the valid store num + validStoreNum := 0 + if index == 0 { + continue + } + if len(ri.AllStores) <= 1 { + validStoreNum = 1 + } else { + for _, storeID := range ri.AllStores { + if _, ok := storeTaskMap[storeID]; ok { + validStoreNum++ + } + } + } + if validStoreNum == 1 { + // if only one store is valid, just put it to storeTaskMap + storeTaskMap[taskStoreID].regionInfos = append(storeTaskMap[taskStoreID].regionInfos, ri) + } else { + // if more than one store is valid, put the region + // to store candidate map + totalRegionCandidateNum += validStoreNum + totalRemainingRegionNum += 1 + taskKey := ri.Region.String() + for _, storeID := range ri.AllStores { + if _, validStore := storeTaskMap[storeID]; !validStore { + continue + } + if _, ok := storeCandidateRegionMap[storeID]; !ok { + candidateMap := make(map[string]RegionInfo) + storeCandidateRegionMap[storeID] = candidateMap + } + if _, duplicateRegion := storeCandidateRegionMap[storeID][taskKey]; duplicateRegion { + // duplicated region, should not happen, just give up balance + logutil.BgLogger().Warn("Meet duplicated region info during when trying to balance batch cop task, give up balancing") + return originalTasks + } + storeCandidateRegionMap[storeID][taskKey] = ri + } + } + } + } + if totalRemainingRegionNum == 0 { + return originalTasks + } + + avgStorePerRegion := float64(totalRegionCandidateNum) / float64(totalRemainingRegionNum) + findNextStore := func(candidateStores []uint64) uint64 { + store := uint64(math.MaxUint64) + weightedRegionNum := math.MaxFloat64 + if candidateStores != nil { + for _, storeID := range candidateStores { + if _, validStore := storeCandidateRegionMap[storeID]; !validStore { + continue + } + num := float64(len(storeCandidateRegionMap[storeID]))/avgStorePerRegion + float64(len(storeTaskMap[storeID].regionInfos)) + if num < weightedRegionNum { + store = storeID + weightedRegionNum = num + } + } + if store != uint64(math.MaxUint64) { + return store + } + } + for storeID := range storeTaskMap { + if _, validStore := storeCandidateRegionMap[storeID]; !validStore { + continue + } + num := float64(len(storeCandidateRegionMap[storeID]))/avgStorePerRegion + float64(len(storeTaskMap[storeID].regionInfos)) + if num < weightedRegionNum { + store = storeID + weightedRegionNum = num + } + } + return store + } + + store := findNextStore(nil) + for totalRemainingRegionNum > 0 { + if store == uint64(math.MaxUint64) { + break + } + var key string + var ri RegionInfo + for key, ri = range storeCandidateRegionMap[store] { + // get the first region + break + } + storeTaskMap[store].regionInfos = append(storeTaskMap[store].regionInfos, ri) + totalRemainingRegionNum-- + for _, id := range ri.AllStores { + if _, ok := storeCandidateRegionMap[id]; ok { + delete(storeCandidateRegionMap[id], key) + totalRegionCandidateNum-- + if len(storeCandidateRegionMap[id]) == 0 { + delete(storeCandidateRegionMap, id) + } + } + } + if totalRemainingRegionNum > 0 { + avgStorePerRegion = float64(totalRegionCandidateNum) / float64(totalRemainingRegionNum) + // it is not optimal because we only check the stores that affected by this region, in fact in order + // to find out the store with the lowest weightedRegionNum, all stores should be checked, but I think + // check only the affected stores is more simple and will get a good enough result + store = findNextStore(ri.AllStores) + } + } + if totalRemainingRegionNum > 0 { + logutil.BgLogger().Warn("Some regions are not used when trying to balance batch cop task, give up balancing") + return originalTasks + } + + var ret []*batchCopTask + for _, task := range storeTaskMap { + ret = append(ret, task) + } + return ret } -func buildBatchCopTasks(bo *backoffer, cache *tikv.RegionCache, ranges *tikv.KeyRanges, storeType kv.StoreType) ([]*batchCopTask, error) { +func buildBatchCopTasks(bo *Backoffer, cache *RegionCache, ranges *KeyRanges, storeType kv.StoreType) ([]*batchCopTask, error) { start := time.Now() const cmdType = tikvrpc.CmdBatchCop rangesLen := ranges.Len() for { + + locations, err := cache.SplitKeyRangesByLocations(bo, ranges) + if err != nil { + return nil, errors.Trace(err) + } var tasks []*copTask - appendTask := func(regionWithRangeInfo *tikv.KeyLocation, ranges *tikv.KeyRanges) { + for _, lo := range locations { tasks = append(tasks, &copTask{ - region: regionWithRangeInfo.Region, - ranges: ranges, + region: lo.Location.Region, + ranges: lo.Ranges, cmdType: cmdType, storeType: storeType, }) } - err := tikv.SplitKeyRanges(bo.TiKVBackoffer(), cache, ranges, appendTask) - if err != nil { - return nil, errors.Trace(err) - } - var batchTasks []*batchCopTask storeTaskMap := make(map[string]*batchCopTask) @@ -129,8 +273,10 @@ func buildBatchCopTasks(bo *backoffer, cache *tikv.RegionCache, ranges *tikv.Key if err != nil { return nil, errors.Trace(err) } - // If the region is not found in cache, it must be out - // of date and already be cleaned up. We should retry and generate new tasks. + // When rpcCtx is nil, it's not only attributed to the miss region, but also + // some TiFlash stores crash and can't be recovered. + // That is not an error that can be easily recovered, so we regard this error + // same as rpc error. if rpcCtx == nil { needRetry = true logutil.BgLogger().Info("retry for TiFlash peer with region missing", zap.Uint64("region id", task.region.GetID())) @@ -138,28 +284,48 @@ func buildBatchCopTasks(bo *backoffer, cache *tikv.RegionCache, ranges *tikv.Key // Then `splitRegion` will reloads these regions. continue } + allStores := cache.GetAllValidTiFlashStores(task.region, rpcCtx.Store) if batchCop, ok := storeTaskMap[rpcCtx.Addr]; ok { - batchCop.copTasks = append(batchCop.copTasks, copTaskAndRPCContext{task: task, ctx: rpcCtx}) + batchCop.regionInfos = append(batchCop.regionInfos, RegionInfo{Region: task.region, Meta: rpcCtx.Meta, Ranges: task.ranges, AllStores: allStores}) } else { batchTask := &batchCopTask{ - storeAddr: rpcCtx.Addr, - cmdType: cmdType, - copTasks: []copTaskAndRPCContext{{task, rpcCtx}}, + storeAddr: rpcCtx.Addr, + cmdType: cmdType, + ctx: rpcCtx, + regionInfos: []RegionInfo{{Region: task.region, Meta: rpcCtx.Meta, Ranges: task.ranges, AllStores: allStores}}, } storeTaskMap[rpcCtx.Addr] = batchTask } } if needRetry { - // Backoff once for each retry. - err = bo.Backoff(tikv.BoRegionMiss, errors.New("Cannot find region with TiFlash peer")) + // As mentioned above, nil rpcCtx is always attributed to failed stores. + // It's equal to long poll the store but get no response. Here we'd better use + // TiFlash error to trigger the TiKV fallback mechanism. + err = bo.Backoff(tikv.BoTiFlashRPC(), errors.New("Cannot find region with TiFlash peer")) if err != nil { return nil, errors.Trace(err) } continue } + for _, task := range storeTaskMap { batchTasks = append(batchTasks, task) } + if log.GetLevel() <= zap.DebugLevel { + msg := "Before region balance:" + for _, task := range batchTasks { + msg += " store " + task.storeAddr + ": " + strconv.Itoa(len(task.regionInfos)) + " regions," + } + logutil.BgLogger().Debug(msg) + } + batchTasks = balanceBatchCopTask(batchTasks) + if log.GetLevel() <= zap.DebugLevel { + msg := "After region balance:" + for _, task := range batchTasks { + msg += " store " + task.storeAddr + ": " + strconv.Itoa(len(task.regionInfos)) + " regions," + } + logutil.BgLogger().Debug(msg) + } if elapsed := time.Since(start); elapsed > time.Millisecond*500 { logutil.BgLogger().Warn("buildBatchCopTasks takes too much time", @@ -176,21 +342,19 @@ func (c *CopClient) sendBatch(ctx context.Context, req *kv.Request, vars *tikv.V if req.KeepOrder || req.Desc { return copErrorResponse{errors.New("batch coprocessor cannot prove keep order or desc property")} } - ctx = context.WithValue(ctx, tikv.TxnStartKey, req.StartTs) - bo := newBackofferWithVars(ctx, copBuildTaskMaxBackoff, vars) - ranges := toTiKVKeyRanges(req.KeyRanges) + ctx = context.WithValue(ctx, tikv.TxnStartKey(), req.StartTs) + bo := backoff.NewBackofferWithVars(ctx, copBuildTaskMaxBackoff, vars) + ranges := NewKeyRanges(req.KeyRanges) tasks, err := buildBatchCopTasks(bo, c.store.GetRegionCache(), ranges, req.StoreType) if err != nil { return copErrorResponse{err} } it := &batchCopIterator{ - store: c.store.kvStore, - req: req, - finishCh: make(chan struct{}), - vars: vars, - memTracker: req.MemTracker, - ClientHelper: tikv.NewClientHelper(c.store.kvStore.store, util.NewTSSet(5)), - rpcCancel: tikv.NewRPCanceller(), + store: c.store.kvStore, + req: req, + finishCh: make(chan struct{}), + vars: vars, + rpcCancel: tikv.NewRPCanceller(), } ctx = context.WithValue(ctx, tikv.RPCCancellerCtxKey{}, it.rpcCancel) it.tasks = tasks @@ -200,8 +364,6 @@ func (c *CopClient) sendBatch(ctx context.Context, req *kv.Request, vars *tikv.V } type batchCopIterator struct { - *tikv.ClientHelper - store *kvStore req *kv.Request finishCh chan struct{} @@ -213,8 +375,6 @@ type batchCopIterator struct { vars *tikv.Variables - memTracker *memory.Tracker - rpcCancel *tikv.RPCCanceller wg sync.WaitGroup @@ -228,7 +388,13 @@ func (b *batchCopIterator) run(ctx context.Context) { // We run workers for every batch cop. for _, task := range b.tasks { b.wg.Add(1) - bo := newBackofferWithVars(ctx, copNextMaxBackoff, b.vars) + boMaxSleep := copNextMaxBackoff + failpoint.Inject("ReduceCopNextMaxBackoff", func(value failpoint.Value) { + if value.(bool) { + boMaxSleep = 2 + } + }) + bo := backoff.NewBackofferWithVars(ctx, boMaxSleep, b.vars) go b.handleTask(ctx, bo, task) } b.wg.Wait() @@ -270,7 +436,7 @@ func (b *batchCopIterator) recvFromRespCh(ctx context.Context) (resp *batchCopRe return case <-ticker.C: if atomic.LoadUint32(b.vars.Killed) == 1 { - resp = &batchCopResponse{err: tikverr.ErrQueryInterrupted} + resp = &batchCopResponse{err: derr.ErrQueryInterrupted} ok = true return } @@ -298,7 +464,7 @@ func (b *batchCopIterator) Close() error { return nil } -func (b *batchCopIterator) handleTask(ctx context.Context, bo *backoffer, task *batchCopTask) { +func (b *batchCopIterator) handleTask(ctx context.Context, bo *Backoffer, task *batchCopTask) { tasks := []*batchCopTask{task} for idx := 0; idx < len(tasks); idx++ { ret, err := b.handleTaskOnce(ctx, bo, tasks[idx]) @@ -313,27 +479,29 @@ func (b *batchCopIterator) handleTask(ctx context.Context, bo *backoffer, task * } // Merge all ranges and request again. -func (b *batchCopIterator) retryBatchCopTask(ctx context.Context, bo *backoffer, batchTask *batchCopTask) ([]*batchCopTask, error) { - var ranges []tikvstore.KeyRange - for _, taskCtx := range batchTask.copTasks { - taskCtx.task.ranges.Do(func(ran *tikvstore.KeyRange) { +func (b *batchCopIterator) retryBatchCopTask(ctx context.Context, bo *Backoffer, batchTask *batchCopTask) ([]*batchCopTask, error) { + var ranges []kv.KeyRange + for _, ri := range batchTask.regionInfos { + ri.Ranges.Do(func(ran *kv.KeyRange) { ranges = append(ranges, *ran) }) } - return buildBatchCopTasks(bo, b.store.GetRegionCache(), tikv.NewKeyRanges(ranges), b.req.StoreType) + return buildBatchCopTasks(bo, b.store.GetRegionCache(), NewKeyRanges(ranges), b.req.StoreType) } -func (b *batchCopIterator) handleTaskOnce(ctx context.Context, bo *backoffer, task *batchCopTask) ([]*batchCopTask, error) { +const readTimeoutUltraLong = 3600 * time.Second // For requests that may scan many regions for tiflash. + +func (b *batchCopIterator) handleTaskOnce(ctx context.Context, bo *Backoffer, task *batchCopTask) ([]*batchCopTask, error) { sender := NewRegionBatchRequestSender(b.store.GetRegionCache(), b.store.GetTiKVClient()) - var regionInfos = make([]*coprocessor.RegionInfo, 0, len(task.copTasks)) - for _, task := range task.copTasks { + var regionInfos = make([]*coprocessor.RegionInfo, 0, len(task.regionInfos)) + for _, ri := range task.regionInfos { regionInfos = append(regionInfos, &coprocessor.RegionInfo{ - RegionId: task.task.region.GetID(), + RegionId: ri.Region.GetID(), RegionEpoch: &metapb.RegionEpoch{ - ConfVer: task.task.region.GetConfVer(), - Version: task.task.region.GetVer(), + ConfVer: ri.Region.GetConfVer(), + Version: ri.Region.GetVer(), }, - Ranges: task.task.ranges.ToPBRanges(), + Ranges: ri.Ranges.ToPBRanges(), }) } @@ -346,29 +514,31 @@ func (b *batchCopIterator) handleTaskOnce(ctx context.Context, bo *backoffer, ta } req := tikvrpc.NewRequest(task.cmdType, &copReq, kvrpcpb.Context{ - IsolationLevel: isolationLevelToPB(b.req.IsolationLevel), - Priority: priorityToPB(b.req.Priority), - NotFillCache: b.req.NotFillCache, - RecordTimeStat: true, - RecordScanStat: true, - TaskId: b.req.TaskID, + IsolationLevel: isolationLevelToPB(b.req.IsolationLevel), + Priority: priorityToPB(b.req.Priority), + NotFillCache: b.req.NotFillCache, + RecordTimeStat: true, + RecordScanStat: true, + TaskId: b.req.TaskID, + ResourceGroupTag: b.req.ResourceGroupTag, }) req.StoreTp = tikvrpc.TiFlash - logutil.BgLogger().Debug("send batch request to ", zap.String("req info", req.String()), zap.Int("cop task len", len(task.copTasks))) - resp, retry, cancel, err := sender.sendStreamReqToAddr(bo, task.copTasks, req, tikv.ReadTimeoutUltraLong) + logutil.BgLogger().Debug("send batch request to ", zap.String("req info", req.String()), zap.Int("cop task len", len(task.regionInfos))) + resp, retry, cancel, err := sender.SendReqToAddr(bo, task.ctx, task.regionInfos, req, readTimeoutUltraLong) // If there are store errors, we should retry for all regions. if retry { return b.retryBatchCopTask(ctx, bo, task) } if err != nil { + err = derr.ToTiDBErr(err) return nil, errors.Trace(err) } defer cancel() return nil, b.handleStreamedBatchCopResponse(ctx, bo, resp.Resp.(*tikvrpc.BatchCopStreamResponse), task) } -func (b *batchCopIterator) handleStreamedBatchCopResponse(ctx context.Context, bo *backoffer, response *tikvrpc.BatchCopStreamResponse, task *batchCopTask) (err error) { +func (b *batchCopIterator) handleStreamedBatchCopResponse(ctx context.Context, bo *Backoffer, response *tikvrpc.BatchCopStreamResponse, task *batchCopTask) (err error) { defer response.Close() resp := response.BatchResponse if resp == nil { @@ -386,7 +556,7 @@ func (b *batchCopIterator) handleStreamedBatchCopResponse(ctx context.Context, b return nil } - if err1 := bo.Backoff(tikv.BoTiKVRPC, errors.Errorf("recv stream response error: %v, task store addr: %s", err, task.storeAddr)); err1 != nil { + if err1 := bo.Backoff(tikv.BoTiKVRPC(), errors.Errorf("recv stream response error: %v, task store addr: %s", err, task.storeAddr)); err1 != nil { return errors.Trace(err) } @@ -396,12 +566,12 @@ func (b *batchCopIterator) handleStreamedBatchCopResponse(ctx context.Context, b } else { logutil.BgLogger().Info("stream unknown error", zap.Error(err)) } - return txndriver.ErrTiFlashServerTimeout + return derr.ErrTiFlashServerTimeout } } } -func (b *batchCopIterator) handleBatchCopResponse(bo *backoffer, response *coprocessor.BatchResponse, task *batchCopTask) (err error) { +func (b *batchCopIterator) handleBatchCopResponse(bo *Backoffer, response *coprocessor.BatchResponse, task *batchCopTask) (err error) { if otherErr := response.GetOtherError(); otherErr != "" { err = errors.Errorf("other error: %s", otherErr) logutil.BgLogger().Warn("other error", @@ -411,6 +581,20 @@ func (b *batchCopIterator) handleBatchCopResponse(bo *backoffer, response *copro return errors.Trace(err) } + if len(response.RetryRegions) > 0 { + logutil.BgLogger().Info("multiple regions are stale and need to be refreshed", zap.Int("region size", len(response.RetryRegions))) + for idx, retry := range response.RetryRegions { + id := tikv.NewRegionVerID(retry.Id, retry.RegionEpoch.ConfVer, retry.RegionEpoch.Version) + logutil.BgLogger().Info("invalid region because tiflash detected stale region", zap.String("region id", id.String())) + b.store.GetRegionCache().InvalidateCachedRegionWithReason(id, tikv.EpochNotMatch) + if idx >= 10 { + logutil.BgLogger().Info("stale regions are too many, so we omit the rest ones") + break + } + } + return + } + resp := batchCopResponse{ pbResp: response, detail: new(CopRuntimeStats), @@ -421,9 +605,8 @@ func (b *batchCopIterator) handleBatchCopResponse(bo *backoffer, response *copro resp.detail.BackoffSleep = make(map[string]time.Duration, len(backoffTimes)) resp.detail.BackoffTimes = make(map[string]int, len(backoffTimes)) for backoff := range backoffTimes { - backoffName := backoff.String() - resp.detail.BackoffTimes[backoffName] = backoffTimes[backoff] - resp.detail.BackoffSleep[backoffName] = time.Duration(bo.GetBackoffSleepMS()[backoff]) * time.Millisecond + resp.detail.BackoffTimes[backoff] = backoffTimes[backoff] + resp.detail.BackoffSleep[backoff] = time.Duration(bo.GetBackoffSleepMS()[backoff]) * time.Millisecond } resp.detail.CalleeAddress = task.storeAddr @@ -440,8 +623,3 @@ func (b *batchCopIterator) sendToRespCh(resp *batchCopResponse) (exit bool) { } return } - -func toTiKVKeyRanges(ranges []kv.KeyRange) *tikv.KeyRanges { - res := *(*[]tikvstore.KeyRange)(unsafe.Pointer(&ranges)) - return tikv.NewKeyRanges(res) -} diff --git a/store/copr/batch_request_sender.go b/store/copr/batch_request_sender.go index 139ee087ec290..b12ced3f65c12 100644 --- a/store/copr/batch_request_sender.go +++ b/store/copr/batch_request_sender.go @@ -15,33 +15,41 @@ package copr import ( "context" - "sync/atomic" "time" "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/tidb/store/tikv" tikverr "github.com/pingcap/tidb/store/tikv/error" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) +// RegionInfo contains region related information for batchCopTask +type RegionInfo struct { + Region tikv.RegionVerID + Meta *metapb.Region + Ranges *KeyRanges + AllStores []uint64 +} + // RegionBatchRequestSender sends BatchCop requests to TiFlash server by stream way. type RegionBatchRequestSender struct { *tikv.RegionRequestSender } // NewRegionBatchRequestSender creates a RegionBatchRequestSender object. -func NewRegionBatchRequestSender(cache *tikv.RegionCache, client tikv.Client) *RegionBatchRequestSender { +func NewRegionBatchRequestSender(cache *RegionCache, client tikv.Client) *RegionBatchRequestSender { return &RegionBatchRequestSender{ - RegionRequestSender: tikv.NewRegionRequestSender(cache, client), + RegionRequestSender: tikv.NewRegionRequestSender(cache.RegionCache, client), } } -func (ss *RegionBatchRequestSender) sendStreamReqToAddr(bo *backoffer, ctxs []copTaskAndRPCContext, req *tikvrpc.Request, timout time.Duration) (resp *tikvrpc.Response, retry bool, cancel func(), err error) { - // use the first ctx to send request, because every ctx has same address. +// SendReqToAddr send batch cop request +func (ss *RegionBatchRequestSender) SendReqToAddr(bo *Backoffer, rpcCtx *tikv.RPCContext, regionInfos []RegionInfo, req *tikvrpc.Request, timout time.Duration) (resp *tikvrpc.Response, retry bool, cancel func(), err error) { cancel = func() {} - rpcCtx := ctxs[0].ctx if e := tikvrpc.SetContext(req, rpcCtx.Meta, rpcCtx.Peer); e != nil { return nil, false, cancel, errors.Trace(e) } @@ -57,7 +65,7 @@ func (ss *RegionBatchRequestSender) sendStreamReqToAddr(bo *backoffer, ctxs []co if err != nil { cancel() ss.SetRPCError(err) - e := ss.onSendFail(bo, ctxs, err) + e := ss.onSendFailForBatchRegions(bo, rpcCtx, regionInfos, err) if e != nil { return nil, false, func() {}, errors.Trace(e) } @@ -67,30 +75,26 @@ func (ss *RegionBatchRequestSender) sendStreamReqToAddr(bo *backoffer, ctxs []co return } -func (ss *RegionBatchRequestSender) onSendFail(bo *backoffer, ctxs []copTaskAndRPCContext, err error) error { +func (ss *RegionBatchRequestSender) onSendFailForBatchRegions(bo *Backoffer, ctx *tikv.RPCContext, regionInfos []RegionInfo, err error) error { // If it failed because the context is cancelled by ourself, don't retry. if errors.Cause(err) == context.Canceled || status.Code(errors.Cause(err)) == codes.Canceled { return errors.Trace(err) - } else if atomic.LoadUint32(&tikv.ShuttingDown) > 0 { + } else if tikv.LoadShuttingDown() > 0 { return tikverr.ErrTiDBShuttingDown } - for _, failedCtx := range ctxs { - ctx := failedCtx.ctx - if ctx.Meta != nil { - // The reload region param is always true. Because that every time we try, we must - // re-build the range then re-create the batch sender. As a result, the len of "failStores" - // will change. If tiflash's replica is more than two, the "reload region" will always be false. - // Now that the batch cop and mpp has a relative low qps, it's reasonable to reload every time - // when meeting io error. - ss.GetRegionCache().OnSendFail(bo.TiKVBackoffer(), ctx, true, err) - } - } + // The reload region param is always true. Because that every time we try, we must + // re-build the range then re-create the batch sender. As a result, the len of "failStores" + // will change. If tiflash's replica is more than two, the "reload region" will always be false. + // Now that the batch cop and mpp has a relative low qps, it's reasonable to reload every time + // when meeting io error. + rc := RegionCache{ss.GetRegionCache()} + rc.OnSendFailForBatchRegions(bo, ctx.Store, regionInfos, true, err) // Retry on send request failure when it's not canceled. // When a store is not available, the leader of related region should be elected quickly. // TODO: the number of retry time should be limited:since region may be unavailable // when some unrecoverable disaster happened. - err = bo.Backoff(tikv.BoTiFlashRPC, errors.Errorf("send tikv request error: %v, ctxs: %v, try next peer later", err, ctxs)) + err = bo.Backoff(retry.BoTiFlashRPC, errors.Errorf("send request error: %v, ctx: %v, regionInfos: %v", err, ctx, regionInfos)) return errors.Trace(err) } diff --git a/store/copr/coprocessor.go b/store/copr/coprocessor.go index cb799edb16f70..49e6d1d281ced 100644 --- a/store/copr/coprocessor.go +++ b/store/copr/coprocessor.go @@ -35,9 +35,10 @@ import ( "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/kv" tidbmetrics "github.com/pingcap/tidb/metrics" - txndriver "github.com/pingcap/tidb/store/driver/txn" + "github.com/pingcap/tidb/store/driver/backoff" + derr "github.com/pingcap/tidb/store/driver/error" + "github.com/pingcap/tidb/store/driver/options" "github.com/pingcap/tidb/store/tikv" - tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" @@ -73,9 +74,9 @@ func (c *CopClient) Send(ctx context.Context, req *kv.Request, variables interfa logutil.BgLogger().Debug("send batch requests") return c.sendBatch(ctx, req, vars) } - ctx = context.WithValue(ctx, tikv.TxnStartKey, req.StartTs) - bo := newBackofferWithVars(ctx, copBuildTaskMaxBackoff, vars) - ranges := toTiKVKeyRanges(req.KeyRanges) + ctx = context.WithValue(ctx, tikv.TxnStartKey(), req.StartTs) + bo := backoff.NewBackofferWithVars(ctx, copBuildTaskMaxBackoff, vars) + ranges := NewKeyRanges(req.KeyRanges) tasks, err := buildCopTasks(bo, c.store.GetRegionCache(), ranges, req) if err != nil { return copErrorResponse{err} @@ -129,7 +130,7 @@ func (c *CopClient) Send(ctx context.Context, req *kv.Request, variables interfa // copTask contains a related Region and KeyRange for a kv.Request. type copTask struct { region tikv.RegionVerID - ranges *tikv.KeyRanges + ranges *KeyRanges respChan chan *copResponse storeAddr string @@ -145,7 +146,7 @@ func (r *copTask) String() string { // rangesPerTask limits the length of the ranges slice sent in one copTask. const rangesPerTask = 25000 -func buildCopTasks(bo *backoffer, cache *tikv.RegionCache, ranges *tikv.KeyRanges, req *kv.Request) ([]*copTask, error) { +func buildCopTasks(bo *Backoffer, cache *RegionCache, ranges *KeyRanges, req *kv.Request) ([]*copTask, error) { start := time.Now() cmdType := tikvrpc.CmdCop if req.Streaming { @@ -157,16 +158,22 @@ func buildCopTasks(bo *backoffer, cache *tikv.RegionCache, ranges *tikv.KeyRange } rangesLen := ranges.Len() + + locs, err := cache.SplitKeyRangesByLocations(bo, ranges) + if err != nil { + return nil, errors.Trace(err) + } + var tasks []*copTask - appendTask := func(regionWithRangeInfo *tikv.KeyLocation, ranges *tikv.KeyRanges) { + for _, loc := range locs { // TiKV will return gRPC error if the message is too large. So we need to limit the length of the ranges slice // to make sure the message can be sent successfully. - rLen := ranges.Len() + rLen := loc.Ranges.Len() for i := 0; i < rLen; { nextI := mathutil.Min(i+rangesPerTask, rLen) tasks = append(tasks, &copTask{ - region: regionWithRangeInfo.Region, - ranges: ranges.Slice(i, nextI), + region: loc.Location.Region, + ranges: loc.Ranges.Slice(i, nextI), // Channel buffer is 2 for handling region split. // In a common case, two region split tasks will not be blocked. respChan: make(chan *copResponse, 2), @@ -177,11 +184,6 @@ func buildCopTasks(bo *backoffer, cache *tikv.RegionCache, ranges *tikv.KeyRange } } - err := tikv.SplitKeyRanges(bo.TiKVBackoffer(), cache, ranges, appendTask) - if err != nil { - return nil, errors.Trace(err) - } - if req.Desc { reverseTasks(tasks) } @@ -195,7 +197,7 @@ func buildCopTasks(bo *backoffer, cache *tikv.RegionCache, ranges *tikv.KeyRange return tasks, nil } -func buildTiDBMemCopTasks(ranges *tikv.KeyRanges, req *kv.Request) ([]*copTask, error) { +func buildTiDBMemCopTasks(ranges *KeyRanges, req *kv.Request) ([]*copTask, error) { servers, err := infosync.GetAllServerInfo(context.Background()) if err != nil { return nil, err @@ -477,7 +479,7 @@ func (it *copIterator) recvFromRespCh(ctx context.Context, respCh <-chan *copRes return case <-ticker.C: if atomic.LoadUint32(it.vars.Killed) == 1 { - resp = &copResponse{err: tikverr.ErrQueryInterrupted} + resp = &copResponse{err: derr.ErrQueryInterrupted} ok = true return } @@ -606,12 +608,12 @@ func (it *copIterator) Next(ctx context.Context) (kv.ResultSubset, error) { // Associate each region with an independent backoffer. In this way, when multiple regions are // unavailable, TiDB can execute very quickly without blocking -func chooseBackoffer(ctx context.Context, backoffermap map[uint64]*backoffer, task *copTask, worker *copIteratorWorker) *backoffer { +func chooseBackoffer(ctx context.Context, backoffermap map[uint64]*Backoffer, task *copTask, worker *copIteratorWorker) *Backoffer { bo, ok := backoffermap[task.region.GetID()] if ok { return bo } - newbo := newBackofferWithVars(ctx, copNextMaxBackoff, worker.vars) + newbo := backoff.NewBackofferWithVars(ctx, copNextMaxBackoff, worker.vars) backoffermap[task.region.GetID()] = newbo return newbo } @@ -630,7 +632,7 @@ func (worker *copIteratorWorker) handleTask(ctx context.Context, task *copTask, } }() remainTasks := []*copTask{task} - backoffermap := make(map[uint64]*backoffer) + backoffermap := make(map[uint64]*Backoffer) for len(remainTasks) > 0 { curTask := remainTasks[0] bo := chooseBackoffer(ctx, backoffermap, curTask, worker) @@ -658,7 +660,7 @@ func (worker *copIteratorWorker) handleTask(ctx context.Context, task *copTask, // handleTaskOnce handles single copTask, successful results are send to channel. // If error happened, returns error. If region split or meet lock, returns the remain tasks. -func (worker *copIteratorWorker) handleTaskOnce(bo *backoffer, task *copTask, ch chan<- *copResponse) ([]*copTask, error) { +func (worker *copIteratorWorker) handleTaskOnce(bo *Backoffer, task *copTask, ch chan<- *copResponse) ([]*copTask, error) { failpoint.Inject("handleTaskOnceError", func(val failpoint.Value) { if val.(bool) { failpoint.Return(nil, errors.New("mock handleTaskOnce error")) @@ -697,28 +699,30 @@ func (worker *copIteratorWorker) handleTaskOnce(bo *backoffer, task *copTask, ch } } - req := tikvrpc.NewReplicaReadRequest(task.cmdType, &copReq, worker.req.ReplicaRead, &worker.replicaReadSeed, kvrpcpb.Context{ - IsolationLevel: isolationLevelToPB(worker.req.IsolationLevel), - Priority: priorityToPB(worker.req.Priority), - NotFillCache: worker.req.NotFillCache, - RecordTimeStat: true, - RecordScanStat: true, - TaskId: worker.req.TaskID, + req := tikvrpc.NewReplicaReadRequest(task.cmdType, &copReq, options.GetTiKVReplicaReadType(worker.req.ReplicaRead), &worker.replicaReadSeed, kvrpcpb.Context{ + IsolationLevel: isolationLevelToPB(worker.req.IsolationLevel), + Priority: priorityToPB(worker.req.Priority), + NotFillCache: worker.req.NotFillCache, + RecordTimeStat: true, + RecordScanStat: true, + TaskId: worker.req.TaskID, + ResourceGroupTag: worker.req.ResourceGroupTag, }) req.StoreTp = getEndPointType(task.storeType) startTime := time.Now() if worker.kvclient.Stats == nil { worker.kvclient.Stats = make(map[tikvrpc.CmdType]*tikv.RPCRuntimeStats) } + req.TxnScope = worker.req.TxnScope if worker.req.IsStaleness { req.EnableStaleRead() } - var ops []tikv.StoreSelectorOption + ops := make([]tikv.StoreSelectorOption, 0, 2) if len(worker.req.MatchStoreLabels) > 0 { ops = append(ops, tikv.WithMatchLabels(worker.req.MatchStoreLabels)) } resp, rpcCtx, storeAddr, err := worker.kvclient.SendReqCtx(bo.TiKVBackoffer(), req, task.region, tikv.ReadTimeoutMedium, getEndPointType(task.storeType), task.storeAddr, ops...) - err = txndriver.ToTiDBErr(err) + err = derr.ToTiDBErr(err) if err != nil { if task.storeType == kv.TiDB { err = worker.handleTiDBSendReqErr(err, task, ch) @@ -748,7 +752,7 @@ const ( minLogKVProcessTime = 100 ) -func (worker *copIteratorWorker) logTimeCopTask(costTime time.Duration, task *copTask, bo *backoffer, resp *tikvrpc.Response) { +func (worker *copIteratorWorker) logTimeCopTask(costTime time.Duration, task *copTask, bo *Backoffer, resp *tikvrpc.Response) { logStr := fmt.Sprintf("[TIME_COP_PROCESS] resp_time:%s txnStartTS:%d region_id:%d store_addr:%s", costTime, worker.req.StartTs, task.region.GetID(), task.storeAddr) if bo.GetTotalSleep() > minLogBackoffTime { backoffTypes := strings.Replace(fmt.Sprintf("%v", bo.TiKVBackoffer().GetTypes()), " ", ",", -1) @@ -781,6 +785,7 @@ func (worker *copIteratorWorker) logTimeCopTask(costTime time.Duration, task *co if timeDetail != nil { logStr += fmt.Sprintf(" kv_process_ms:%d", timeDetail.ProcessWallTimeMs) logStr += fmt.Sprintf(" kv_wait_ms:%d", timeDetail.WaitWallTimeMs) + logStr += fmt.Sprintf(" kv_read_ms:%d", timeDetail.KvReadWallTimeMs) if timeDetail.ProcessWallTimeMs <= minLogKVProcessTime { logStr = strings.Replace(logStr, "TIME_COP_PROCESS", "TIME_COP_WAIT", 1) } @@ -810,7 +815,7 @@ func appendScanDetail(logStr string, columnFamily string, scanInfo *kvrpcpb.Scan return logStr } -func (worker *copIteratorWorker) handleCopStreamResult(bo *backoffer, rpcCtx *tikv.RPCContext, stream *tikvrpc.CopStreamResponse, task *copTask, ch chan<- *copResponse, costTime time.Duration) ([]*copTask, error) { +func (worker *copIteratorWorker) handleCopStreamResult(bo *Backoffer, rpcCtx *tikv.RPCContext, stream *tikvrpc.CopStreamResponse, task *copTask, ch chan<- *copResponse, costTime time.Duration) ([]*copTask, error) { defer stream.Close() var resp *coprocessor.Response var lastRange *coprocessor.KeyRange @@ -830,11 +835,14 @@ func (worker *copIteratorWorker) handleCopStreamResult(bo *backoffer, rpcCtx *ti return nil, nil } - boRPCType := tikv.BoTiKVRPC + err1 := errors.Errorf("recv stream response error: %v, task: %s", err, task) if task.storeType == kv.TiFlash { - boRPCType = tikv.BoTiFlashRPC + err1 = bo.Backoff(tikv.BoTiFlashRPC(), err1) + } else { + err1 = bo.Backoff(tikv.BoTiKVRPC(), err1) } - if err1 := bo.Backoff(boRPCType, errors.Errorf("recv stream response error: %v, task: %s", err, task)); err1 != nil { + + if err1 != nil { return nil, errors.Trace(err) } @@ -856,7 +864,7 @@ func (worker *copIteratorWorker) handleCopStreamResult(bo *backoffer, rpcCtx *ti // returns more tasks when that happens, or handles the response if no error. // if we're handling streaming coprocessor response, lastRange is the range of last // successful response, otherwise it's nil. -func (worker *copIteratorWorker) handleCopResponse(bo *backoffer, rpcCtx *tikv.RPCContext, resp *copResponse, cacheKey []byte, cacheValue *coprCacheValue, task *copTask, ch chan<- *copResponse, lastRange *coprocessor.KeyRange, costTime time.Duration) ([]*copTask, error) { +func (worker *copIteratorWorker) handleCopResponse(bo *Backoffer, rpcCtx *tikv.RPCContext, resp *copResponse, cacheKey []byte, cacheValue *coprCacheValue, task *copTask, ch chan<- *copResponse, lastRange *coprocessor.KeyRange, costTime time.Duration) ([]*copTask, error) { if regionErr := resp.pbResp.GetRegionError(); regionErr != nil { if rpcCtx != nil && task.storeType == kv.TiDB { resp.err = errors.Errorf("error: %v", regionErr) @@ -865,7 +873,7 @@ func (worker *copIteratorWorker) handleCopResponse(bo *backoffer, rpcCtx *tikv.R } errStr := fmt.Sprintf("region_id:%v, region_ver:%v, store_type:%s, peer_addr:%s, error:%s", task.region.GetID(), task.region.GetVer(), task.storeType.Name(), task.storeAddr, regionErr.String()) - if err := bo.Backoff(tikv.BoRegionMiss, errors.New(errStr)); err != nil { + if err := bo.Backoff(tikv.BoRegionMiss(), errors.New(errStr)); err != nil { return nil, errors.Trace(err) } // We may meet RegionError at the first packet, but not during visiting the stream. @@ -875,12 +883,12 @@ func (worker *copIteratorWorker) handleCopResponse(bo *backoffer, rpcCtx *tikv.R logutil.BgLogger().Debug("coprocessor encounters", zap.Stringer("lock", lockErr)) msBeforeExpired, err1 := worker.kvclient.ResolveLocks(bo.TiKVBackoffer(), worker.req.StartTs, []*tikv.Lock{tikv.NewLock(lockErr)}) - err1 = txndriver.ToTiDBErr(err1) + err1 = derr.ToTiDBErr(err1) if err1 != nil { return nil, errors.Trace(err1) } if msBeforeExpired > 0 { - if err := bo.BackoffWithMaxSleep(tikv.BoTxnLockFast, int(msBeforeExpired), errors.New(lockErr.String())); err != nil { + if err := bo.BackoffWithMaxSleepTxnLockFast(int(msBeforeExpired), errors.New(lockErr.String())); err != nil { return nil, errors.Trace(err) } } @@ -899,7 +907,7 @@ func (worker *copIteratorWorker) handleCopResponse(bo *backoffer, rpcCtx *tikv.R if resp.pbResp.Range != nil { resp.startKey = resp.pbResp.Range.Start } else if task.ranges != nil && task.ranges.Len() > 0 { - resp.startKey = kv.Key(task.ranges.At(0).StartKey) + resp.startKey = task.ranges.At(0).StartKey } if resp.detail == nil { resp.detail = new(CopRuntimeStats) @@ -911,9 +919,8 @@ func (worker *copIteratorWorker) handleCopResponse(bo *backoffer, rpcCtx *tikv.R resp.detail.BackoffSleep = make(map[string]time.Duration, len(backoffTimes)) resp.detail.BackoffTimes = make(map[string]int, len(backoffTimes)) for backoff := range backoffTimes { - backoffName := backoff.String() - resp.detail.BackoffTimes[backoffName] = backoffTimes[backoff] - resp.detail.BackoffSleep[backoffName] = time.Duration(bo.GetBackoffSleepMS()[backoff]) * time.Millisecond + resp.detail.BackoffTimes[backoff] = backoffTimes[backoff] + resp.detail.BackoffSleep[backoff] = time.Duration(bo.GetBackoffSleepMS()[backoff]) * time.Millisecond } if rpcCtx != nil { resp.detail.CalleeAddress = rpcCtx.Addr @@ -983,11 +990,11 @@ type CopRuntimeStats struct { func (worker *copIteratorWorker) handleTiDBSendReqErr(err error, task *copTask, ch chan<- *copResponse) error { errCode := errno.ErrUnknown errMsg := err.Error() - if terror.ErrorEqual(err, txndriver.ErrTiKVServerTimeout) { + if terror.ErrorEqual(err, derr.ErrTiKVServerTimeout) { errCode = errno.ErrTiKVServerTimeout errMsg = "TiDB server timeout, address is " + task.storeAddr } - if terror.ErrorEqual(err, txndriver.ErrTiFlashServerTimeout) { + if terror.ErrorEqual(err, derr.ErrTiFlashServerTimeout) { errCode = errno.ErrTiFlashServerTimeout errMsg = "TiDB server timeout, address is " + task.storeAddr } @@ -1013,7 +1020,7 @@ func (worker *copIteratorWorker) handleTiDBSendReqErr(err error, task *copTask, return nil } -func (worker *copIteratorWorker) buildCopTasksFromRemain(bo *backoffer, lastRange *coprocessor.KeyRange, task *copTask) ([]*copTask, error) { +func (worker *copIteratorWorker) buildCopTasksFromRemain(bo *Backoffer, lastRange *coprocessor.KeyRange, task *copTask) ([]*copTask, error) { remainedRanges := task.ranges if worker.req.Streaming && lastRange != nil { remainedRanges = worker.calculateRemain(task.ranges, lastRange, worker.req.Desc) @@ -1028,7 +1035,7 @@ func (worker *copIteratorWorker) buildCopTasksFromRemain(bo *backoffer, lastRang // split: [s1 --> s2) // In normal scan order, all data before s1 is consumed, so the remain ranges should be [s1 --> r2) [r3 --> r4) // In reverse scan order, all data after s2 is consumed, so the remain ranges should be [r1 --> r2) [r3 --> s2) -func (worker *copIteratorWorker) calculateRemain(ranges *tikv.KeyRanges, split *coprocessor.KeyRange, desc bool) *tikv.KeyRanges { +func (worker *copIteratorWorker) calculateRemain(ranges *KeyRanges, split *coprocessor.KeyRange, desc bool) *KeyRanges { if desc { left, _ := ranges.Split(split.End) return left diff --git a/store/copr/coprocessor_test.go b/store/copr/coprocessor_test.go index d7a6d52c5b4bb..592eb33b0e5f3 100644 --- a/store/copr/coprocessor_test.go +++ b/store/copr/coprocessor_test.go @@ -19,8 +19,8 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/store/driver/backoff" "github.com/pingcap/tidb/store/tikv" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/mockstore/mocktikv" ) @@ -40,10 +40,10 @@ func (s *testCoprocessorSuite) TestBuildTasks(c *C) { cluster := mocktikv.NewCluster(mocktikv.MustNewMVCCStore()) _, regionIDs, _ := mocktikv.BootstrapWithMultiRegions(cluster, []byte("g"), []byte("n"), []byte("t")) pdCli := &tikv.CodecPDClient{Client: mocktikv.NewPDClient(cluster)} - cache := tikv.NewRegionCache(pdCli) + cache := NewRegionCache(tikv.NewRegionCache(pdCli)) defer cache.Close() - bo := newBackofferWithVars(context.Background(), 3000, nil) + bo := backoff.NewBackofferWithVars(context.Background(), 3000, nil) req := &kv.Request{} flashReq := &kv.Request{} @@ -157,48 +157,48 @@ func (s *testCoprocessorSuite) TestSplitRegionRanges(c *C) { cluster := mocktikv.NewCluster(mocktikv.MustNewMVCCStore()) mocktikv.BootstrapWithMultiRegions(cluster, []byte("g"), []byte("n"), []byte("t")) pdCli := &tikv.CodecPDClient{Client: mocktikv.NewPDClient(cluster)} - cache := tikv.NewRegionCache(pdCli) + cache := NewRegionCache(tikv.NewRegionCache(pdCli)) defer cache.Close() - bo := tikv.NewBackofferWithVars(context.Background(), 3000, nil) + bo := backoff.NewBackofferWithVars(context.Background(), 3000, nil) - ranges, err := tikv.SplitRegionRanges(bo, cache, buildKeyRanges("a", "c")) + ranges, err := cache.SplitRegionRanges(bo, buildKeyRanges("a", "c")) c.Assert(err, IsNil) c.Assert(ranges, HasLen, 1) s.rangeEqual(c, ranges, "a", "c") - ranges, err = tikv.SplitRegionRanges(bo, cache, buildKeyRanges("h", "y")) + ranges, err = cache.SplitRegionRanges(bo, buildKeyRanges("h", "y")) c.Assert(err, IsNil) c.Assert(len(ranges), Equals, 3) s.rangeEqual(c, ranges, "h", "n", "n", "t", "t", "y") - ranges, err = tikv.SplitRegionRanges(bo, cache, buildKeyRanges("s", "z")) + ranges, err = cache.SplitRegionRanges(bo, buildKeyRanges("s", "z")) c.Assert(err, IsNil) c.Assert(len(ranges), Equals, 2) s.rangeEqual(c, ranges, "s", "t", "t", "z") - ranges, err = tikv.SplitRegionRanges(bo, cache, buildKeyRanges("s", "s")) + ranges, err = cache.SplitRegionRanges(bo, buildKeyRanges("s", "s")) c.Assert(err, IsNil) c.Assert(len(ranges), Equals, 1) s.rangeEqual(c, ranges, "s", "s") - ranges, err = tikv.SplitRegionRanges(bo, cache, buildKeyRanges("t", "t")) + ranges, err = cache.SplitRegionRanges(bo, buildKeyRanges("t", "t")) c.Assert(err, IsNil) c.Assert(len(ranges), Equals, 1) s.rangeEqual(c, ranges, "t", "t") - ranges, err = tikv.SplitRegionRanges(bo, cache, buildKeyRanges("t", "u")) + ranges, err = cache.SplitRegionRanges(bo, buildKeyRanges("t", "u")) c.Assert(err, IsNil) c.Assert(len(ranges), Equals, 1) s.rangeEqual(c, ranges, "t", "u") - ranges, err = tikv.SplitRegionRanges(bo, cache, buildKeyRanges("u", "z")) + ranges, err = cache.SplitRegionRanges(bo, buildKeyRanges("u", "z")) c.Assert(err, IsNil) c.Assert(len(ranges), Equals, 1) s.rangeEqual(c, ranges, "u", "z") // min --> max - ranges, err = tikv.SplitRegionRanges(bo, cache, buildKeyRanges("a", "z")) + ranges, err = cache.SplitRegionRanges(bo, buildKeyRanges("a", "z")) c.Assert(err, IsNil) c.Assert(ranges, HasLen, 4) s.rangeEqual(c, ranges, "a", "g", "g", "n", "n", "t", "t", "z") @@ -210,9 +210,9 @@ func (s *testCoprocessorSuite) TestRebuild(c *C) { cluster := mocktikv.NewCluster(mocktikv.MustNewMVCCStore()) storeID, regionIDs, peerIDs := mocktikv.BootstrapWithMultiRegions(cluster, []byte("m")) pdCli := &tikv.CodecPDClient{Client: mocktikv.NewPDClient(cluster)} - cache := tikv.NewRegionCache(pdCli) + cache := NewRegionCache(tikv.NewRegionCache(pdCli)) defer cache.Close() - bo := newBackofferWithVars(context.Background(), 3000, nil) + bo := backoff.NewBackofferWithVars(context.Background(), 3000, nil) req := &kv.Request{} tasks, err := buildCopTasks(bo, cache, buildCopRanges("a", "z"), req) @@ -237,10 +237,10 @@ func (s *testCoprocessorSuite) TestRebuild(c *C) { s.taskEqual(c, tasks[0], regionIDs[2], "q", "z") } -func buildKeyRanges(keys ...string) []tikvstore.KeyRange { - var ranges []tikvstore.KeyRange +func buildKeyRanges(keys ...string) []kv.KeyRange { + var ranges []kv.KeyRange for i := 0; i < len(keys); i += 2 { - ranges = append(ranges, tikvstore.KeyRange{ + ranges = append(ranges, kv.KeyRange{ StartKey: []byte(keys[i]), EndKey: []byte(keys[i+1]), }) @@ -248,8 +248,8 @@ func buildKeyRanges(keys ...string) []tikvstore.KeyRange { return ranges } -func buildCopRanges(keys ...string) *tikv.KeyRanges { - return tikv.NewKeyRanges(buildKeyRanges(keys...)) +func buildCopRanges(keys ...string) *KeyRanges { + return NewKeyRanges(buildKeyRanges(keys...)) } func (s *testCoprocessorSuite) taskEqual(c *C, task *copTask, regionID uint64, keys ...string) { @@ -261,7 +261,7 @@ func (s *testCoprocessorSuite) taskEqual(c *C, task *copTask, regionID uint64, k } } -func (s *testCoprocessorSuite) rangeEqual(c *C, ranges []tikvstore.KeyRange, keys ...string) { +func (s *testCoprocessorSuite) rangeEqual(c *C, ranges []kv.KeyRange, keys ...string) { for i := 0; i < len(ranges); i++ { r := ranges[i] c.Assert(string(r.StartKey), Equals, keys[2*i]) diff --git a/store/tikv/key_ranges.go b/store/copr/key_ranges.go similarity index 62% rename from store/tikv/key_ranges.go rename to store/copr/key_ranges.go index e45364796fdf3..6b26d1026c785 100644 --- a/store/tikv/key_ranges.go +++ b/store/copr/key_ranges.go @@ -11,16 +11,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package tikv +package copr import ( "bytes" "fmt" "sort" - "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/coprocessor" - "github.com/pingcap/tidb/store/tikv/kv" + "github.com/pingcap/tidb/kv" ) // KeyRanges is like []kv.KeyRange, but may has extra elements at head/tail. @@ -142,69 +141,3 @@ func (r *KeyRanges) ToPBRanges() []*coprocessor.KeyRange { }) return ranges } - -// SplitRegionRanges get the split ranges from pd region. -func SplitRegionRanges(bo *Backoffer, cache *RegionCache, keyRanges []kv.KeyRange) ([]kv.KeyRange, error) { - ranges := NewKeyRanges(keyRanges) - - var ret []kv.KeyRange - appendRange := func(regionWithRangeInfo *KeyLocation, ranges *KeyRanges) { - for i := 0; i < ranges.Len(); i++ { - ret = append(ret, ranges.At(i)) - } - } - - err := SplitKeyRanges(bo, cache, ranges, appendRange) - if err != nil { - return nil, errors.Trace(err) - } - return ret, nil -} - -// SplitKeyRanges splits KeyRanges by the regions info from cache. -func SplitKeyRanges(bo *Backoffer, cache *RegionCache, ranges *KeyRanges, fn func(regionWithRangeInfo *KeyLocation, ranges *KeyRanges)) error { - for ranges.Len() > 0 { - loc, err := cache.LocateKey(bo, ranges.At(0).StartKey) - if err != nil { - return errors.Trace(err) - } - - // Iterate to the first range that is not complete in the region. - var i int - for ; i < ranges.Len(); i++ { - r := ranges.At(i) - if !(loc.Contains(r.EndKey) || bytes.Equal(loc.EndKey, r.EndKey)) { - break - } - } - // All rest ranges belong to the same region. - if i == ranges.Len() { - fn(loc, ranges) - break - } - - r := ranges.At(i) - if loc.Contains(r.StartKey) { - // Part of r is not in the region. We need to split it. - taskRanges := ranges.Slice(0, i) - taskRanges.last = &kv.KeyRange{ - StartKey: r.StartKey, - EndKey: loc.EndKey, - } - fn(loc, taskRanges) - - ranges = ranges.Slice(i+1, ranges.Len()) - ranges.first = &kv.KeyRange{ - StartKey: loc.EndKey, - EndKey: r.EndKey, - } - } else { - // rs[i] is not in the region. - taskRanges := ranges.Slice(0, i) - fn(loc, taskRanges) - ranges = ranges.Slice(i, ranges.Len()) - } - } - - return nil -} diff --git a/store/tikv/key_ranges_test.go b/store/copr/key_ranges_test.go similarity index 91% rename from store/tikv/key_ranges_test.go rename to store/copr/key_ranges_test.go index 40f924392acd6..eb1755c53a32f 100644 --- a/store/tikv/key_ranges_test.go +++ b/store/copr/key_ranges_test.go @@ -11,11 +11,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package tikv +package copr import ( . "github.com/pingcap/check" - "github.com/pingcap/tidb/store/tikv/kv" + "github.com/pingcap/tidb/kv" ) type testKeyRangesSuite struct { @@ -126,18 +126,3 @@ func (s *testKeyRangesSuite) testSplit(c *C, ranges *KeyRanges, checkLeft bool, } } } - -func buildKeyRanges(keys ...string) []kv.KeyRange { - var ranges []kv.KeyRange - for i := 0; i < len(keys); i += 2 { - ranges = append(ranges, kv.KeyRange{ - StartKey: []byte(keys[i]), - EndKey: []byte(keys[i+1]), - }) - } - return ranges -} - -func buildCopRanges(keys ...string) *KeyRanges { - return NewKeyRanges(buildKeyRanges(keys...)) -} diff --git a/store/copr/mpp.go b/store/copr/mpp.go index 3ea07e744f9b9..4be45c3288e23 100644 --- a/store/copr/mpp.go +++ b/store/copr/mpp.go @@ -27,9 +27,9 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/mpp" "github.com/pingcap/tidb/kv" - txndriver "github.com/pingcap/tidb/store/driver/txn" + "github.com/pingcap/tidb/store/driver/backoff" + derr "github.com/pingcap/tidb/store/driver/error" "github.com/pingcap/tidb/store/tikv" - tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/tikvrpc" "go.uber.org/zap" @@ -56,12 +56,12 @@ func (c *MPPClient) selectAllTiFlashStore() []kv.MPPTaskMeta { // ConstructMPPTasks receives ScheduleRequest, which are actually collects of kv ranges. We allocates MPPTaskMeta for them and returns. func (c *MPPClient) ConstructMPPTasks(ctx context.Context, req *kv.MPPBuildTasksRequest) ([]kv.MPPTaskMeta, error) { - ctx = context.WithValue(ctx, tikv.TxnStartKey, req.StartTS) - bo := newBackofferWithVars(ctx, copBuildTaskMaxBackoff, nil) + ctx = context.WithValue(ctx, tikv.TxnStartKey(), req.StartTS) + bo := backoff.NewBackofferWithVars(ctx, copBuildTaskMaxBackoff, nil) if req.KeyRanges == nil { return c.selectAllTiFlashStore(), nil } - ranges := toTiKVKeyRanges(req.KeyRanges) + ranges := NewKeyRanges(req.KeyRanges) tasks, err := buildBatchCopTasks(bo, c.store.GetRegionCache(), ranges, kv.TiFlash) if err != nil { return nil, errors.Trace(err) @@ -144,16 +144,12 @@ func (m *mppIterator) run(ctx context.Context) { break } m.mu.Lock() - switch task.State { - case kv.MppTaskReady: + if task.State == kv.MppTaskReady { task.State = kv.MppTaskRunning - m.mu.Unlock() - default: - m.mu.Unlock() - break } + m.mu.Unlock() m.wg.Add(1) - bo := newBackoffer(ctx, copNextMaxBackoff) + bo := backoff.NewBackoffer(ctx, copNextMaxBackoff) go m.handleDispatchReq(ctx, bo, task) } m.wg.Wait() @@ -177,21 +173,21 @@ func (m *mppIterator) sendToRespCh(resp *mppResponse) (exit bool) { // TODO:: Consider that which way is better: // - dispatch all tasks at once, and connect tasks at second. // - dispatch tasks and establish connection at the same time. -func (m *mppIterator) handleDispatchReq(ctx context.Context, bo *backoffer, req *kv.MPPDispatchRequest) { +func (m *mppIterator) handleDispatchReq(ctx context.Context, bo *Backoffer, req *kv.MPPDispatchRequest) { defer func() { m.wg.Done() }() var regionInfos []*coprocessor.RegionInfo originalTask, ok := req.Meta.(*batchCopTask) if ok { - for _, task := range originalTask.copTasks { + for _, ri := range originalTask.regionInfos { regionInfos = append(regionInfos, &coprocessor.RegionInfo{ - RegionId: task.task.region.GetID(), + RegionId: ri.Region.GetID(), RegionEpoch: &metapb.RegionEpoch{ - ConfVer: task.task.region.GetConfVer(), - Version: task.task.region.GetVer(), + ConfVer: ri.Region.GetConfVer(), + Version: ri.Region.GetVer(), }, - Ranges: task.task.ranges.ToPBRanges(), + Ranges: ri.Ranges.ToPBRanges(), }) } } @@ -219,14 +215,14 @@ func (m *mppIterator) handleDispatchReq(ctx context.Context, bo *backoffer, req // In that case if originalTask != nil { sender := NewRegionBatchRequestSender(m.store.GetRegionCache(), m.store.GetTiKVClient()) - rpcResp, _, _, err = sender.sendStreamReqToAddr(bo, originalTask.copTasks, wrappedReq, tikv.ReadTimeoutMedium) + rpcResp, _, _, err = sender.SendReqToAddr(bo, originalTask.ctx, originalTask.regionInfos, wrappedReq, tikv.ReadTimeoutMedium) // No matter what the rpc error is, we won't retry the mpp dispatch tasks. // TODO: If we want to retry, we must redo the plan fragment cutting and task scheduling. // That's a hard job but we can try it in the future. if sender.GetRPCError() != nil { logutil.BgLogger().Error("mpp dispatch meet io error", zap.String("error", sender.GetRPCError().Error())) // we return timeout to trigger tikv's fallback - m.sendError(txndriver.ErrTiFlashServerTimeout) + m.sendError(derr.ErrTiFlashServerTimeout) return } } else { @@ -236,13 +232,14 @@ func (m *mppIterator) handleDispatchReq(ctx context.Context, bo *backoffer, req if err != nil { logutil.BgLogger().Error("mpp dispatch meet error", zap.String("error", err.Error())) // we return timeout to trigger tikv's fallback - m.sendError(txndriver.ErrTiFlashServerTimeout) + m.sendError(derr.ErrTiFlashServerTimeout) return } realResp := rpcResp.Resp.(*mpp.DispatchTaskResponse) if realResp.Error != nil { + logutil.BgLogger().Error("mpp dispatch response meet error", zap.String("error", realResp.Error.Msg)) m.sendError(errors.New(realResp.Error.Msg)) return } @@ -256,7 +253,7 @@ func (m *mppIterator) handleDispatchReq(ctx context.Context, bo *backoffer, req failpoint.Inject("mppNonRootTaskError", func(val failpoint.Value) { if val.(bool) && !req.IsRoot { time.Sleep(1 * time.Second) - m.sendError(txndriver.ErrTiFlashServerTimeout) + m.sendError(derr.ErrTiFlashServerTimeout) return } }) @@ -300,7 +297,7 @@ func (m *mppIterator) cancelMppTasks() { } } -func (m *mppIterator) establishMPPConns(bo *backoffer, req *kv.MPPDispatchRequest, taskMeta *mpp.TaskMeta) { +func (m *mppIterator) establishMPPConns(bo *Backoffer, req *kv.MPPDispatchRequest, taskMeta *mpp.TaskMeta) { connReq := &mpp.EstablishMPPConnectionRequest{ SenderMeta: taskMeta, ReceiverMeta: &mpp.TaskMeta{ @@ -314,12 +311,12 @@ func (m *mppIterator) establishMPPConns(bo *backoffer, req *kv.MPPDispatchReques // Drain result from root task. // We don't need to process any special error. When we meet errors, just let it fail. - rpcResp, err := m.store.GetTiKVClient().SendRequest(bo.GetCtx(), req.Meta.GetAddress(), wrappedReq, tikv.ReadTimeoutUltraLong) + rpcResp, err := m.store.GetTiKVClient().SendRequest(bo.GetCtx(), req.Meta.GetAddress(), wrappedReq, readTimeoutUltraLong) if err != nil { logutil.BgLogger().Error("establish mpp connection meet error", zap.String("error", err.Error())) // we return timeout to trigger tikv's fallback - m.sendError(txndriver.ErrTiFlashServerTimeout) + m.sendError(derr.ErrTiFlashServerTimeout) return } @@ -344,14 +341,14 @@ func (m *mppIterator) establishMPPConns(bo *backoffer, req *kv.MPPDispatchReques return } - if err1 := bo.Backoff(tikv.BoTiKVRPC, errors.Errorf("recv stream response error: %v", err)); err1 != nil { + if err1 := bo.Backoff(tikv.BoTiKVRPC(), errors.Errorf("recv stream response error: %v", err)); err1 != nil { if errors.Cause(err) == context.Canceled { logutil.BgLogger().Info("stream recv timeout", zap.Error(err)) } else { logutil.BgLogger().Info("stream unknown error", zap.Error(err)) } } - m.sendError(txndriver.ErrTiFlashServerTimeout) + m.sendError(derr.ErrTiFlashServerTimeout) return } } @@ -367,7 +364,7 @@ func (m *mppIterator) Close() error { return nil } -func (m *mppIterator) handleMPPStreamResponse(bo *backoffer, response *mpp.MPPDataPacket, req *kv.MPPDispatchRequest) (err error) { +func (m *mppIterator) handleMPPStreamResponse(bo *Backoffer, response *mpp.MPPDataPacket, req *kv.MPPDispatchRequest) (err error) { if response.Error != nil { err = errors.Errorf("other error for mpp stream: %s", response.Error.Msg) logutil.BgLogger().Warn("other error", @@ -387,9 +384,8 @@ func (m *mppIterator) handleMPPStreamResponse(bo *backoffer, response *mpp.MPPDa resp.detail.BackoffSleep = make(map[string]time.Duration, len(backoffTimes)) resp.detail.BackoffTimes = make(map[string]int, len(backoffTimes)) for backoff := range backoffTimes { - backoffName := backoff.String() - resp.detail.BackoffTimes[backoffName] = backoffTimes[backoff] - resp.detail.BackoffSleep[backoffName] = time.Duration(bo.GetBackoffSleepMS()[backoff]) * time.Millisecond + resp.detail.BackoffTimes[backoff] = backoffTimes[backoff] + resp.detail.BackoffSleep[backoff] = time.Duration(bo.GetBackoffSleepMS()[backoff]) * time.Millisecond } resp.detail.CalleeAddress = req.Meta.GetAddress() @@ -406,7 +402,7 @@ func (m *mppIterator) nextImpl(ctx context.Context) (resp *mppResponse, ok bool, return case <-ticker.C: if m.vars != nil && m.vars.Killed != nil && atomic.LoadUint32(m.vars.Killed) == 1 { - err = tikverr.ErrQueryInterrupted + err = derr.ErrQueryInterrupted exit = true return } diff --git a/store/copr/region_cache.go b/store/copr/region_cache.go new file mode 100644 index 0000000000000..23c49725b40e6 --- /dev/null +++ b/store/copr/region_cache.go @@ -0,0 +1,123 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package copr + +import ( + "bytes" + + "github.com/pingcap/errors" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/store/tikv" + "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" +) + +// RegionCache wraps tikv.RegionCache. +type RegionCache struct { + *tikv.RegionCache +} + +// NewRegionCache returns a new RegionCache. +func NewRegionCache(rc *tikv.RegionCache) *RegionCache { + return &RegionCache{rc} +} + +// SplitRegionRanges gets the split ranges from pd region. +func (c *RegionCache) SplitRegionRanges(bo *Backoffer, keyRanges []kv.KeyRange) ([]kv.KeyRange, error) { + ranges := NewKeyRanges(keyRanges) + + locations, err := c.SplitKeyRangesByLocations(bo, ranges) + if err != nil { + return nil, errors.Trace(err) + } + var ret []kv.KeyRange + for _, loc := range locations { + for i := 0; i < loc.Ranges.Len(); i++ { + ret = append(ret, loc.Ranges.At(i)) + } + } + return ret, nil +} + +// LocationKeyRanges wrapps a real Location in PD and its logical ranges info. +type LocationKeyRanges struct { + // Location is the real location in PD. + Location *tikv.KeyLocation + // Ranges is the logic ranges the current Location contains. + Ranges *KeyRanges +} + +// SplitKeyRangesByLocations splits the KeyRanges by logical info in the cache. +func (c *RegionCache) SplitKeyRangesByLocations(bo *Backoffer, ranges *KeyRanges) ([]*LocationKeyRanges, error) { + res := make([]*LocationKeyRanges, 0) + for ranges.Len() > 0 { + loc, err := c.LocateKey(bo.TiKVBackoffer(), ranges.At(0).StartKey) + if err != nil { + return res, errors.Trace(err) + } + + // Iterate to the first range that is not complete in the region. + var i int + for ; i < ranges.Len(); i++ { + r := ranges.At(i) + if !(loc.Contains(r.EndKey) || bytes.Equal(loc.EndKey, r.EndKey)) { + break + } + } + // All rest ranges belong to the same region. + if i == ranges.Len() { + res = append(res, &LocationKeyRanges{Location: loc, Ranges: ranges}) + break + } + + r := ranges.At(i) + if loc.Contains(r.StartKey) { + // Part of r is not in the region. We need to split it. + taskRanges := ranges.Slice(0, i) + taskRanges.last = &kv.KeyRange{ + StartKey: r.StartKey, + EndKey: loc.EndKey, + } + res = append(res, &LocationKeyRanges{Location: loc, Ranges: taskRanges}) + + ranges = ranges.Slice(i+1, ranges.Len()) + ranges.first = &kv.KeyRange{ + StartKey: loc.EndKey, + EndKey: r.EndKey, + } + } else { + // rs[i] is not in the region. + taskRanges := ranges.Slice(0, i) + res = append(res, &LocationKeyRanges{Location: loc, Ranges: taskRanges}) + ranges = ranges.Slice(i, ranges.Len()) + } + } + + return res, nil +} + +// OnSendFailForBatchRegions handles send request fail logic. +func (c *RegionCache) OnSendFailForBatchRegions(bo *Backoffer, store *tikv.Store, regionInfos []RegionInfo, scheduleReload bool, err error) { + metrics.RegionCacheCounterWithSendFail.Add(float64(len(regionInfos))) + if !store.IsTiFlash() { + logutil.Logger(bo.GetCtx()).Info("Should not reach here, OnSendFailForBatchRegions only support TiFlash") + return + } + for _, ri := range regionInfos { + if ri.Meta == nil { + continue + } + c.OnSendFailForTiFlash(bo.TiKVBackoffer(), store, ri.Region, ri.Meta, scheduleReload, err) + } +} diff --git a/store/copr/store.go b/store/copr/store.go index 2cc10ee7bad38..5ef623adac0e3 100644 --- a/store/copr/store.go +++ b/store/copr/store.go @@ -21,7 +21,8 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tidb/kv" - txndriver "github.com/pingcap/tidb/store/driver/txn" + "github.com/pingcap/tidb/store/driver/backoff" + derr "github.com/pingcap/tidb/store/driver/error" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/config" "github.com/pingcap/tidb/store/tikv/tikvrpc" @@ -32,14 +33,14 @@ type kvStore struct { } // GetRegionCache returns the region cache instance. -func (s *kvStore) GetRegionCache() *tikv.RegionCache { - return s.store.GetRegionCache() +func (s *kvStore) GetRegionCache() *RegionCache { + return &RegionCache{s.store.GetRegionCache()} } // CheckVisibility checks if it is safe to read using given ts. func (s *kvStore) CheckVisibility(startTime uint64) error { err := s.store.CheckVisibility(startTime) - return txndriver.ToTiDBErr(err) + return derr.ToTiDBErr(err) } // GetTiKVClient gets the client instance. @@ -54,13 +55,13 @@ type tikvClient struct { func (c *tikvClient) Close() error { err := c.c.Close() - return txndriver.ToTiDBErr(err) + return derr.ToTiDBErr(err) } // SendRequest sends Request. func (c *tikvClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) { res, err := c.c.SendRequest(ctx, addr, req, timeout) - return res, txndriver.ToTiDBErr(err) + return res, derr.ToTiDBErr(err) } // Store wraps tikv.KVStore and provides coprocessor utilities. @@ -122,62 +123,5 @@ func getEndPointType(t kv.StoreType) tikvrpc.EndpointType { } } -// backoffer wraps tikv.Backoffer and converts the error which returns by the functions of tikv.Backoffer to tidb error. -type backoffer struct { - b *tikv.Backoffer -} - -// newBackofferWithVars creates a Backoffer with maximum sleep time(in ms) and kv.Variables. -func newBackofferWithVars(ctx context.Context, maxSleep int, vars *kv.Variables) *backoffer { - b := tikv.NewBackofferWithVars(ctx, maxSleep, vars) - return &backoffer{b: b} -} - -func newBackoffer(ctx context.Context, maxSleep int) *backoffer { - b := tikv.NewBackoffer(ctx, maxSleep) - return &backoffer{b: b} -} - -// TiKVBackoffer returns tikv.Backoffer. -func (b *backoffer) TiKVBackoffer() *tikv.Backoffer { - return b.b -} - -// Backoff sleeps a while base on the backoffType and records the error message. -// It returns a retryable error if total sleep time exceeds maxSleep. -func (b *backoffer) Backoff(typ tikv.BackoffType, err error) error { - e := b.b.Backoff(typ, err) - return txndriver.ToTiDBErr(e) -} - -// BackoffWithMaxSleep sleeps a while base on the backoffType and records the error message -// and never sleep more than maxSleepMs for each sleep. -func (b *backoffer) BackoffWithMaxSleep(typ tikv.BackoffType, maxSleepMs int, err error) error { - e := b.b.BackoffWithMaxSleep(typ, maxSleepMs, err) - return txndriver.ToTiDBErr(e) -} - -// GetBackoffTimes returns a map contains backoff time count by type. -func (b *backoffer) GetBackoffTimes() map[tikv.BackoffType]int { - return b.b.GetBackoffTimes() -} - -// GetCtx returns the binded context. -func (b *backoffer) GetCtx() context.Context { - return b.b.GetCtx() -} - -// GetVars returns the binded vars. -func (b *backoffer) GetVars() *tikv.Variables { - return b.b.GetVars() -} - -// GetBackoffSleepMS returns a map contains backoff sleep time by type. -func (b *backoffer) GetBackoffSleepMS() map[tikv.BackoffType]int { - return b.b.GetBackoffSleepMS() -} - -// GetTotalSleep returns total sleep time. -func (b *backoffer) GetTotalSleep() int { - return b.b.GetTotalSleep() -} +// Backoffer wraps tikv.Backoffer and converts the error which returns by the functions of tikv.Backoffer to tidb error. +type Backoffer = backoff.Backoffer diff --git a/store/driver/backoff/backoff.go b/store/driver/backoff/backoff.go new file mode 100644 index 0000000000000..35979edc638b4 --- /dev/null +++ b/store/driver/backoff/backoff.go @@ -0,0 +1,83 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package backoff + +import ( + "context" + + "github.com/pingcap/tidb/kv" + derr "github.com/pingcap/tidb/store/driver/error" + "github.com/pingcap/tidb/store/tikv" +) + +// Backoffer wraps tikv.Backoffer and converts the error which returns by the functions of tikv.Backoffer to tidb error. +type Backoffer struct { + b *tikv.Backoffer +} + +// NewBackofferWithVars creates a Backoffer with maximum sleep time(in ms) and kv.Variables. +func NewBackofferWithVars(ctx context.Context, maxSleep int, vars *kv.Variables) *Backoffer { + b := tikv.NewBackofferWithVars(ctx, maxSleep, vars) + return &Backoffer{b: b} +} + +// NewBackoffer creates a Backoffer with maximum sleep time(in ms). +func NewBackoffer(ctx context.Context, maxSleep int) *Backoffer { + b := tikv.NewBackoffer(ctx, maxSleep) + return &Backoffer{b: b} +} + +// TiKVBackoffer returns tikv.Backoffer. +func (b *Backoffer) TiKVBackoffer() *tikv.Backoffer { + return b.b +} + +// Backoff sleeps a while base on the BackoffConfig and records the error message. +// It returns a retryable error if total sleep time exceeds maxSleep. +func (b *Backoffer) Backoff(cfg *tikv.BackoffConfig, err error) error { + e := b.b.Backoff(cfg, err) + return derr.ToTiDBErr(e) +} + +// BackoffWithMaxSleepTxnLockFast sleeps a while for the operation TxnLockFast and records the error message +// and never sleep more than maxSleepMs for each sleep. +func (b *Backoffer) BackoffWithMaxSleepTxnLockFast(maxSleepMs int, err error) error { + e := b.b.BackoffWithMaxSleepTxnLockFast(maxSleepMs, err) + return derr.ToTiDBErr(e) +} + +// GetBackoffTimes returns a map contains backoff time count by type. +func (b *Backoffer) GetBackoffTimes() map[string]int { + return b.b.GetBackoffTimes() +} + +// GetCtx returns the binded context. +func (b *Backoffer) GetCtx() context.Context { + return b.b.GetCtx() +} + +// GetVars returns the binded vars. +func (b *Backoffer) GetVars() *tikv.Variables { + return b.b.GetVars() +} + +// GetBackoffSleepMS returns a map contains backoff sleep time by type. +func (b *Backoffer) GetBackoffSleepMS() map[string]int { + return b.b.GetBackoffSleepMS() +} + +// GetTotalSleep returns total sleep time. +func (b *Backoffer) GetTotalSleep() int { + return b.b.GetTotalSleep() +} diff --git a/store/driver/error/error.go b/store/driver/error/error.go new file mode 100644 index 0000000000000..17da8f7ef2fa3 --- /dev/null +++ b/store/driver/error/error.go @@ -0,0 +1,158 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package error + +import ( + "github.com/pingcap/errors" + "github.com/pingcap/tidb/errno" + "github.com/pingcap/tidb/kv" + tikverr "github.com/pingcap/tidb/store/tikv/error" + "github.com/pingcap/tidb/util/dbterror" +) + +// tikv error instance +var ( + // ErrTokenLimit is the error that token is up to the limit. + ErrTokenLimit = dbterror.ClassTiKV.NewStd(errno.ErrTiKVStoreLimit) + // ErrTiKVServerTimeout is the error when tikv server is timeout. + ErrTiKVServerTimeout = dbterror.ClassTiKV.NewStd(errno.ErrTiKVServerTimeout) + ErrTiFlashServerTimeout = dbterror.ClassTiKV.NewStd(errno.ErrTiFlashServerTimeout) + // ErrGCTooEarly is the error that GC life time is shorter than transaction duration + ErrGCTooEarly = dbterror.ClassTiKV.NewStd(errno.ErrGCTooEarly) + // ErrTiKVStaleCommand is the error that the command is stale in tikv. + ErrTiKVStaleCommand = dbterror.ClassTiKV.NewStd(errno.ErrTiKVStaleCommand) + // ErrQueryInterrupted is the error when the query is interrupted. + ErrQueryInterrupted = dbterror.ClassTiKV.NewStd(errno.ErrQueryInterrupted) + // ErrTiKVMaxTimestampNotSynced is the error that tikv's max timestamp is not synced. + ErrTiKVMaxTimestampNotSynced = dbterror.ClassTiKV.NewStd(errno.ErrTiKVMaxTimestampNotSynced) + // ErrLockAcquireFailAndNoWaitSet is the error that acquire the lock failed while no wait is setted. + ErrLockAcquireFailAndNoWaitSet = dbterror.ClassTiKV.NewStd(errno.ErrLockAcquireFailAndNoWaitSet) + ErrResolveLockTimeout = dbterror.ClassTiKV.NewStd(errno.ErrResolveLockTimeout) + // ErrLockWaitTimeout is the error that wait for the lock is timeout. + ErrLockWaitTimeout = dbterror.ClassTiKV.NewStd(errno.ErrLockWaitTimeout) + // ErrTiKVServerBusy is the error when tikv server is busy. + ErrTiKVServerBusy = dbterror.ClassTiKV.NewStd(errno.ErrTiKVServerBusy) + // ErrTiFlashServerBusy is the error that tiflash server is busy. + ErrTiFlashServerBusy = dbterror.ClassTiKV.NewStd(errno.ErrTiFlashServerBusy) + // ErrPDServerTimeout is the error when pd server is timeout. + ErrPDServerTimeout = dbterror.ClassTiKV.NewStd(errno.ErrPDServerTimeout) + // ErrRegionUnavailable is the error when region is not available. + ErrRegionUnavailable = dbterror.ClassTiKV.NewStd(errno.ErrRegionUnavailable) + // ErrUnknown is the unknow error. + ErrUnknown = dbterror.ClassTiKV.NewStd(errno.ErrUnknown) +) + +// Registers error returned from TiKV. +var ( + _ = dbterror.ClassTiKV.NewStd(errno.ErrDataOutOfRange) + _ = dbterror.ClassTiKV.NewStd(errno.ErrTruncatedWrongValue) + _ = dbterror.ClassTiKV.NewStd(errno.ErrDivisionByZero) +) + +// ToTiDBErr checks and converts a tikv error to a tidb error. +func ToTiDBErr(err error) error { + originErr := err + if err == nil { + return nil + } + err = errors.Cause(err) + if tikverr.IsErrNotFound(err) { + return kv.ErrNotExist + } + + if e, ok := err.(*tikverr.ErrWriteConflictInLatch); ok { + return kv.ErrWriteConflictInTiDB.FastGenByArgs(e.StartTS) + } + + if e, ok := err.(*tikverr.ErrTxnTooLarge); ok { + return kv.ErrTxnTooLarge.GenWithStackByArgs(e.Size) + } + + if errors.ErrorEqual(err, tikverr.ErrCannotSetNilValue) { + return kv.ErrCannotSetNilValue + } + + if e, ok := err.(*tikverr.ErrEntryTooLarge); ok { + return kv.ErrEntryTooLarge.GenWithStackByArgs(e.Limit, e.Size) + } + + if errors.ErrorEqual(err, tikverr.ErrInvalidTxn) { + return kv.ErrInvalidTxn + } + + if errors.ErrorEqual(err, tikverr.ErrTiKVServerTimeout) { + return ErrTiKVServerTimeout + } + + if e, ok := err.(*tikverr.ErrPDServerTimeout); ok { + if len(e.Error()) == 0 { + return ErrPDServerTimeout + } + return ErrPDServerTimeout.GenWithStackByArgs(e.Error()) + } + + if errors.ErrorEqual(err, tikverr.ErrTiFlashServerTimeout) { + return ErrTiFlashServerTimeout + } + + if errors.ErrorEqual(err, tikverr.ErrQueryInterrupted) { + return ErrQueryInterrupted + } + + if errors.ErrorEqual(err, tikverr.ErrTiKVServerBusy) { + return ErrTiKVServerBusy + } + + if errors.ErrorEqual(err, tikverr.ErrTiFlashServerBusy) { + return ErrTiFlashServerBusy + } + + if e, ok := err.(*tikverr.ErrGCTooEarly); ok { + return ErrGCTooEarly.GenWithStackByArgs(e.TxnStartTS, e.GCSafePoint) + } + + if errors.ErrorEqual(err, tikverr.ErrTiKVStaleCommand) { + return ErrTiKVStaleCommand + } + + if errors.ErrorEqual(err, tikverr.ErrTiKVMaxTimestampNotSynced) { + return ErrTiKVMaxTimestampNotSynced + } + + if errors.ErrorEqual(err, tikverr.ErrLockAcquireFailAndNoWaitSet) { + return ErrLockAcquireFailAndNoWaitSet + } + + if errors.ErrorEqual(err, tikverr.ErrResolveLockTimeout) { + return ErrResolveLockTimeout + } + + if errors.ErrorEqual(err, tikverr.ErrLockWaitTimeout) { + return ErrLockWaitTimeout + } + + if errors.ErrorEqual(err, tikverr.ErrRegionUnavailable) { + return ErrRegionUnavailable + } + + if e, ok := err.(*tikverr.ErrTokenLimit); ok { + return ErrTokenLimit.GenWithStackByArgs(e.StoreID) + } + + if errors.ErrorEqual(err, tikverr.ErrUnknown) { + return ErrUnknown + } + + return errors.Trace(originErr) +} diff --git a/store/tikv/error/errcode.go b/store/driver/options/options.go similarity index 52% rename from store/tikv/error/errcode.go rename to store/driver/options/options.go index bf0e3c7c4b91d..dc16f7793ed91 100644 --- a/store/tikv/error/errcode.go +++ b/store/driver/options/options.go @@ -11,16 +11,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -package error +package options -// MySQL error code. -// This value is numeric. It is not portable to other database systems. -const ( - CodeUnknown = 1105 - CodeLockWaitTimeout = 1205 - CodeQueryInterrupted = 1317 - CodeLockAcquireFailAndNoWaitSet = 3572 - - // TiKV/PD/TiFlash errors. - CodeTiKVStoreLimit = 9008 +import ( + "github.com/pingcap/tidb/kv" + storekv "github.com/pingcap/tidb/store/tikv/kv" ) + +// GetTiKVReplicaReadType maps kv.ReplicaReadType to tikv/kv.ReplicaReadType. +func GetTiKVReplicaReadType(t kv.ReplicaReadType) storekv.ReplicaReadType { + switch t { + case kv.ReplicaReadLeader: + return storekv.ReplicaReadLeader + case kv.ReplicaReadFollower: + return storekv.ReplicaReadFollower + case kv.ReplicaReadMixed: + return storekv.ReplicaReadMixed + } + return 0 +} diff --git a/store/driver/sql_fail_test.go b/store/driver/sql_fail_test.go index 903dcedcb1878..3618c5ec59b15 100644 --- a/store/driver/sql_fail_test.go +++ b/store/driver/sql_fail_test.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/store/tikv" + "github.com/pingcap/tidb/store/tikv/mockstore" "github.com/pingcap/tidb/util/mock" "github.com/pingcap/tidb/util/testkit" ) @@ -54,7 +55,7 @@ func (s *testSQLSuiteBase) SetUpSuite(c *C) { var err error s.store = NewTestStore(c) // actual this is better done in `OneByOneSuite.SetUpSuite`, but this would cause circle dependency - if *WithTiKV { + if *mockstore.WithTiKV { session.ResetStoreForWithTiKVTest(s.store) } @@ -100,7 +101,6 @@ func (s *testSQLSerialSuite) TestFailBusyServerCop(c *C) { } func TestMain(m *testing.M) { - tikv.ReadTimeoutMedium = 2 * time.Second os.Exit(m.Run()) } diff --git a/store/driver/tikv_driver.go b/store/driver/tikv_driver.go index 17308d33c6be3..689c6ea170678 100644 --- a/store/driver/tikv_driver.go +++ b/store/driver/tikv_driver.go @@ -24,13 +24,16 @@ import ( "time" "github.com/pingcap/errors" + deadlockpb "github.com/pingcap/kvproto/pkg/deadlock" + "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/store/copr" + derr "github.com/pingcap/tidb/store/driver/error" txn_driver "github.com/pingcap/tidb/store/driver/txn" "github.com/pingcap/tidb/store/gcworker" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/config" - "github.com/pingcap/tidb/store/tikv/oracle" + "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/store/tikv/util" "github.com/pingcap/tidb/util/logutil" pd "github.com/tikv/pd/client" @@ -206,6 +209,8 @@ var ( ldflagGetEtcdAddrsFromConfig = "0" // 1:Yes, otherwise:No ) +const getAllMembersBackoff = 5000 + // EtcdAddrs returns etcd server addresses. func (s *tikvStore) EtcdAddrs() ([]string, error) { if s.etcdAddrs == nil { @@ -220,7 +225,7 @@ func (s *tikvStore) EtcdAddrs() ([]string, error) { } ctx := context.Background() - bo := tikv.NewBackoffer(ctx, tikv.GetAllMembersBackoff) + bo := tikv.NewBackoffer(ctx, getAllMembersBackoff) etcdAddrs := make([]string, 0) pdClient := s.GetPDClient() if pdClient == nil { @@ -229,7 +234,7 @@ func (s *tikvStore) EtcdAddrs() ([]string, error) { for { members, err := pdClient.GetAllMembers(ctx) if err != nil { - err := bo.Backoff(tikv.BoRegionMiss, err) + err := bo.Backoff(tikv.BoRegionMiss(), err) if err != nil { return nil, err } @@ -262,7 +267,7 @@ func (s *tikvStore) StartGCWorker() error { gcWorker, err := gcworker.NewGCWorker(s, s.pdClient) if err != nil { - return txn_driver.ToTiDBErr(err) + return derr.ToTiDBErr(err) } gcWorker.Start() s.gcWorker = gcWorker @@ -287,7 +292,7 @@ func (s *tikvStore) Close() error { } s.coprStore.Close() err := s.KVStore.Close() - return txn_driver.ToTiDBErr(err) + return derr.ToTiDBErr(err) } // GetMemCache return memory manager of the storage @@ -299,34 +304,17 @@ func (s *tikvStore) GetMemCache() kv.MemManager { func (s *tikvStore) Begin() (kv.Transaction, error) { txn, err := s.KVStore.Begin() if err != nil { - return nil, txn_driver.ToTiDBErr(err) + return nil, derr.ToTiDBErr(err) } return txn_driver.NewTiKVTxn(txn), err } // BeginWithOption begins a transaction with given option -func (s *tikvStore) BeginWithOption(option kv.TransactionOption) (kv.Transaction, error) { - txnScope := option.TxnScope - if txnScope == "" { - txnScope = oracle.GlobalTxnScope - } - var txn *tikv.KVTxn - var err error - if option.StartTS != nil { - txn, err = s.BeginWithStartTS(txnScope, *option.StartTS) - } else if option.PrevSec != nil { - txn, err = s.BeginWithExactStaleness(txnScope, *option.PrevSec) - } else if option.MaxPrevSec != nil { - txn, err = s.BeginWithMaxPrevSec(txnScope, *option.MaxPrevSec) - } else if option.MinStartTS != nil { - txn, err = s.BeginWithMinStartTS(txnScope, *option.MinStartTS) - } else { - txn, err = s.BeginWithTxnScope(txnScope) - } +func (s *tikvStore) BeginWithOption(option tikv.StartTSOption) (kv.Transaction, error) { + txn, err := s.KVStore.BeginWithOption(option) if err != nil { - return nil, txn_driver.ToTiDBErr(err) + return nil, derr.ToTiDBErr(err) } - return txn_driver.NewTiKVTxn(txn), err } @@ -339,10 +327,30 @@ func (s *tikvStore) GetSnapshot(ver kv.Version) kv.Snapshot { // CurrentVersion returns current max committed version with the given txnScope (local or global). func (s *tikvStore) CurrentVersion(txnScope string) (kv.Version, error) { ver, err := s.KVStore.CurrentTimestamp(txnScope) - return kv.NewVersion(ver), txn_driver.ToTiDBErr(err) + return kv.NewVersion(ver), derr.ToTiDBErr(err) } // ShowStatus returns the specified status of the storage func (s *tikvStore) ShowStatus(ctx context.Context, key string) (interface{}, error) { return nil, kv.ErrNotImplemented } + +// GetLockWaits get return lock waits info +func (s *tikvStore) GetLockWaits() ([]*deadlockpb.WaitForEntry, error) { + stores := s.GetRegionCache().GetStoresByType(tikvrpc.TiKV) + var result []*deadlockpb.WaitForEntry + for _, store := range stores { + resp, err := s.GetTiKVClient().SendRequest(context.TODO(), store.GetAddr(), tikvrpc.NewRequest(tikvrpc.CmdLockWaitInfo, &kvrpcpb.GetLockWaitInfoRequest{}), time.Second*30) + if err != nil { + logutil.BgLogger().Warn("query lock wait info failed", zap.Error(err)) + continue + } + if resp.Resp == nil { + logutil.BgLogger().Warn("lock wait info from store is nil") + continue + } + entries := resp.Resp.(*kvrpcpb.GetLockWaitInfoResponse).Entries + result = append(result, entries...) + } + return result, nil +} diff --git a/store/driver/txn/error.go b/store/driver/txn/error.go index b5e69b4e522a9..cde513e9f1e7d 100644 --- a/store/driver/txn/error.go +++ b/store/driver/txn/error.go @@ -25,46 +25,16 @@ import ( "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" - "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/kv" + derr "github.com/pingcap/tidb/store/driver/error" tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" - "github.com/pingcap/tidb/util/dbterror" "go.uber.org/zap" ) -// tikv error instance -var ( - // ErrTiKVServerTimeout is the error when tikv server is timeout. - ErrTiKVServerTimeout = dbterror.ClassTiKV.NewStd(errno.ErrTiKVServerTimeout) - ErrTiFlashServerTimeout = dbterror.ClassTiKV.NewStd(errno.ErrTiFlashServerTimeout) - // ErrGCTooEarly is the error that GC life time is shorter than transaction duration - ErrGCTooEarly = dbterror.ClassTiKV.NewStd(errno.ErrGCTooEarly) - // ErrTiKVStaleCommand is the error that the command is stale in tikv. - ErrTiKVStaleCommand = dbterror.ClassTiKV.NewStd(errno.ErrTiKVStaleCommand) - // ErrTiKVMaxTimestampNotSynced is the error that tikv's max timestamp is not synced. - ErrTiKVMaxTimestampNotSynced = dbterror.ClassTiKV.NewStd(errno.ErrTiKVMaxTimestampNotSynced) - ErrResolveLockTimeout = dbterror.ClassTiKV.NewStd(errno.ErrResolveLockTimeout) - // ErrTiKVServerBusy is the error when tikv server is busy. - ErrTiKVServerBusy = dbterror.ClassTiKV.NewStd(errno.ErrTiKVServerBusy) - // ErrTiFlashServerBusy is the error that tiflash server is busy. - ErrTiFlashServerBusy = dbterror.ClassTiKV.NewStd(errno.ErrTiFlashServerBusy) - // ErrPDServerTimeout is the error when pd server is timeout. - ErrPDServerTimeout = dbterror.ClassTiKV.NewStd(errno.ErrPDServerTimeout) - // ErrRegionUnavailable is the error when region is not available. - ErrRegionUnavailable = dbterror.ClassTiKV.NewStd(errno.ErrRegionUnavailable) -) - -// Registers error returned from TiKV. -var ( - _ = dbterror.ClassTiKV.NewStd(errno.ErrDataOutOfRange) - _ = dbterror.ClassTiKV.NewStd(errno.ErrTruncatedWrongValue) - _ = dbterror.ClassTiKV.NewStd(errno.ErrDivisionByZero) -) - func genKeyExistsError(name string, value string, err error) error { if err != nil { logutil.BgLogger().Info("extractKeyExistsErr meets error", zap.Error(err)) @@ -124,6 +94,9 @@ func extractKeyExistsErrFromHandle(key kv.Key, value []byte, tblInfo *model.Tabl if err != nil { return genKeyExistsError(name, key.String(), err) } + if col.Length > 0 { + str = str[:col.Length] + } valueStr = append(valueStr, str) } return genKeyExistsError(name, strings.Join(valueStr, "-"), nil) @@ -176,84 +149,7 @@ func extractKeyErr(err error) error { notFoundDetail := prettyLockNotFoundKey(e.Retryable) return kv.ErrTxnRetryable.GenWithStackByArgs(e.Retryable + " " + notFoundDetail) } - return ToTiDBErr(err) -} - -// ToTiDBErr checks and converts a tikv error to a tidb error. -func ToTiDBErr(err error) error { - originErr := err - if err == nil { - return nil - } - err = errors.Cause(err) - if tikverr.IsErrNotFound(err) { - return kv.ErrNotExist - } - - if e, ok := err.(*tikverr.ErrWriteConflictInLatch); ok { - return kv.ErrWriteConflictInTiDB.FastGenByArgs(e.StartTS) - } - - if e, ok := err.(*tikverr.ErrTxnTooLarge); ok { - return kv.ErrTxnTooLarge.GenWithStackByArgs(e.Size) - } - - if errors.ErrorEqual(err, tikverr.ErrCannotSetNilValue) { - return kv.ErrCannotSetNilValue - } - - if e, ok := err.(*tikverr.ErrEntryTooLarge); ok { - return kv.ErrEntryTooLarge.GenWithStackByArgs(e.Limit, e.Size) - } - - if errors.ErrorEqual(err, tikverr.ErrInvalidTxn) { - return kv.ErrInvalidTxn - } - - if errors.ErrorEqual(err, tikverr.ErrTiKVServerTimeout) { - return ErrTiKVServerTimeout - } - - if e, ok := err.(*tikverr.ErrPDServerTimeout); ok { - if len(e.Error()) == 0 { - return ErrPDServerTimeout - } - return ErrPDServerTimeout.GenWithStackByArgs(e.Error()) - } - - if errors.ErrorEqual(err, tikverr.ErrTiFlashServerTimeout) { - return ErrTiFlashServerTimeout - } - - if errors.ErrorEqual(err, tikverr.ErrTiKVServerBusy) { - return ErrTiKVServerBusy - } - - if errors.ErrorEqual(err, tikverr.ErrTiFlashServerBusy) { - return ErrTiFlashServerBusy - } - - if e, ok := err.(*tikverr.ErrGCTooEarly); ok { - return ErrGCTooEarly.GenWithStackByArgs(e.TxnStartTS, e.GCSafePoint) - } - - if errors.ErrorEqual(err, tikverr.ErrTiKVStaleCommand) { - return ErrTiKVStaleCommand - } - - if errors.ErrorEqual(err, tikverr.ErrTiKVMaxTimestampNotSynced) { - return ErrTiKVMaxTimestampNotSynced - } - - if errors.ErrorEqual(err, tikverr.ErrResolveLockTimeout) { - return ErrResolveLockTimeout - } - - if errors.ErrorEqual(err, tikverr.ErrRegionUnavailable) { - return ErrRegionUnavailable - } - - return errors.Trace(originErr) + return derr.ToTiDBErr(err) } func newWriteConflictError(conflict *kvrpcpb.WriteConflict) error { diff --git a/store/driver/txn/snapshot.go b/store/driver/txn/snapshot.go index 1a3f7bf9c3bbc..257feb37938af 100644 --- a/store/driver/txn/snapshot.go +++ b/store/driver/txn/snapshot.go @@ -17,9 +17,11 @@ import ( "context" "unsafe" + "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/tidb/kv" + derr "github.com/pingcap/tidb/store/driver/error" + "github.com/pingcap/tidb/store/driver/options" "github.com/pingcap/tidb/store/tikv" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" ) type tikvSnapshot struct { @@ -48,7 +50,7 @@ func (s *tikvSnapshot) Get(ctx context.Context, k kv.Key) ([]byte, error) { func (s *tikvSnapshot) Iter(k kv.Key, upperBound kv.Key) (kv.Iterator, error) { scanner, err := s.KVSnapshot.Iter(k, upperBound) if err != nil { - return nil, ToTiDBErr(err) + return nil, derr.ToTiDBErr(err) } return &tikvScanner{scanner.(*tikv.Scanner)}, err } @@ -57,24 +59,41 @@ func (s *tikvSnapshot) Iter(k kv.Key, upperBound kv.Key) (kv.Iterator, error) { func (s *tikvSnapshot) IterReverse(k kv.Key) (kv.Iterator, error) { scanner, err := s.KVSnapshot.IterReverse(k) if err != nil { - return nil, ToTiDBErr(err) + return nil, derr.ToTiDBErr(err) } return &tikvScanner{scanner.(*tikv.Scanner)}, err } func (s *tikvSnapshot) SetOption(opt int, val interface{}) { switch opt { - case tikvstore.IsolationLevel: + case kv.IsolationLevel: level := getTiKVIsolationLevel(val.(kv.IsoLevel)) s.KVSnapshot.SetIsolationLevel(level) - case tikvstore.Priority: + case kv.Priority: s.KVSnapshot.SetPriority(getTiKVPriority(val.(int))) - case tikvstore.NotFillCache: + case kv.NotFillCache: s.KVSnapshot.SetNotFillCache(val.(bool)) - case tikvstore.SnapshotTS: + case kv.SnapshotTS: s.KVSnapshot.SetSnapshotTS(val.(uint64)) - default: - s.KVSnapshot.SetOption(opt, val) + case kv.ReplicaRead: + t := options.GetTiKVReplicaReadType(val.(kv.ReplicaReadType)) + s.KVSnapshot.SetReplicaRead(t) + case kv.SampleStep: + s.KVSnapshot.SetSampleStep(val.(uint32)) + case kv.TaskID: + s.KVSnapshot.SetTaskID(val.(uint64)) + case kv.CollectRuntimeStats: + if val == nil { + s.KVSnapshot.SetRuntimeStats(nil) + } else { + s.KVSnapshot.SetRuntimeStats(val.(*tikv.SnapshotRuntimeStats)) + } + case kv.IsStalenessReadOnly: + s.KVSnapshot.SetIsStatenessReadOnly(val.(bool)) + case kv.MatchStoreLabels: + s.KVSnapshot.SetMatchStoreLabels(val.([]*metapb.StoreLabel)) + case kv.ResourceGroupTag: + s.KVSnapshot.SetResourceGroupTag(val.([]byte)) } } diff --git a/store/driver/txn/txn_driver.go b/store/driver/txn/txn_driver.go index 8595b2fe874ae..886b0df39900a 100644 --- a/store/driver/txn/txn_driver.go +++ b/store/driver/txn/txn_driver.go @@ -19,9 +19,12 @@ import ( "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx/binloginfo" + derr "github.com/pingcap/tidb/store/driver/error" + "github.com/pingcap/tidb/store/driver/options" "github.com/pingcap/tidb/store/tikv" tikverr "github.com/pingcap/tidb/store/tikv/error" tikvstore "github.com/pingcap/tidb/store/tikv/kv" @@ -75,7 +78,7 @@ func (txn *tikvTxn) GetSnapshot() kv.Snapshot { // The Iterator must be Closed after use. func (txn *tikvTxn) Iter(k kv.Key, upperBound kv.Key) (kv.Iterator, error) { it, err := txn.KVTxn.Iter(k, upperBound) - return newKVIterator(it), ToTiDBErr(err) + return newKVIterator(it), derr.ToTiDBErr(err) } // IterReverse creates a reversed Iterator positioned on the first entry which key is less than k. @@ -84,7 +87,7 @@ func (txn *tikvTxn) Iter(k kv.Key, upperBound kv.Key) (kv.Iterator, error) { // TODO: Add lower bound limit func (txn *tikvTxn) IterReverse(k kv.Key) (kv.Iterator, error) { it, err := txn.KVTxn.IterReverse(k) - return newKVIterator(it), ToTiDBErr(err) + return newKVIterator(it), derr.ToTiDBErr(err) } // BatchGet gets kv from the memory buffer of statement and transaction, and the kv storage. @@ -101,68 +104,89 @@ func (txn *tikvTxn) BatchGet(ctx context.Context, keys []kv.Key) (map[string][]b func (txn *tikvTxn) Delete(k kv.Key) error { err := txn.KVTxn.Delete(k) - return ToTiDBErr(err) + return derr.ToTiDBErr(err) } func (txn *tikvTxn) Get(ctx context.Context, k kv.Key) ([]byte, error) { data, err := txn.KVTxn.Get(ctx, k) - return data, ToTiDBErr(err) + return data, derr.ToTiDBErr(err) } func (txn *tikvTxn) Set(k kv.Key, v []byte) error { err := txn.KVTxn.Set(k, v) - return ToTiDBErr(err) + return derr.ToTiDBErr(err) } func (txn *tikvTxn) GetMemBuffer() kv.MemBuffer { return newMemBuffer(txn.KVTxn.GetMemBuffer()) } -func (txn *tikvTxn) GetUnionStore() kv.UnionStore { - return &tikvUnionStore{txn.KVTxn.GetUnionStore()} -} - func (txn *tikvTxn) SetOption(opt int, val interface{}) { switch opt { - case tikvstore.BinlogInfo: + case kv.BinlogInfo: txn.SetBinlogExecutor(&binlogExecutor{ txn: txn.KVTxn, binInfo: val.(*binloginfo.BinlogInfo), // val cannot be other type. }) - case tikvstore.SchemaChecker: + case kv.SchemaChecker: txn.SetSchemaLeaseChecker(val.(tikv.SchemaLeaseChecker)) - case tikvstore.IsolationLevel: + case kv.IsolationLevel: level := getTiKVIsolationLevel(val.(kv.IsoLevel)) txn.KVTxn.GetSnapshot().SetIsolationLevel(level) - case tikvstore.Priority: + case kv.Priority: txn.KVTxn.SetPriority(getTiKVPriority(val.(int))) - case tikvstore.NotFillCache: + case kv.NotFillCache: txn.KVTxn.GetSnapshot().SetNotFillCache(val.(bool)) - case tikvstore.SyncLog: + case kv.SyncLog: txn.EnableForceSyncLog() - case tikvstore.Pessimistic: + case kv.Pessimistic: txn.SetPessimistic(val.(bool)) - case tikvstore.SnapshotTS: + case kv.SnapshotTS: txn.KVTxn.GetSnapshot().SetSnapshotTS(val.(uint64)) - case tikvstore.InfoSchema: + case kv.ReplicaRead: + t := options.GetTiKVReplicaReadType(val.(kv.ReplicaReadType)) + txn.KVTxn.GetSnapshot().SetReplicaRead(t) + case kv.TaskID: + txn.KVTxn.GetSnapshot().SetTaskID(val.(uint64)) + case kv.InfoSchema: txn.SetSchemaVer(val.(tikv.SchemaVer)) - case tikvstore.CommitHook: + case kv.CollectRuntimeStats: + if val == nil { + txn.KVTxn.GetSnapshot().SetRuntimeStats(nil) + } else { + txn.KVTxn.GetSnapshot().SetRuntimeStats(val.(*tikv.SnapshotRuntimeStats)) + } + case kv.SchemaAmender: + txn.SetSchemaAmender(val.(tikv.SchemaAmender)) + case kv.SampleStep: + txn.KVTxn.GetSnapshot().SetSampleStep(val.(uint32)) + case kv.CommitHook: txn.SetCommitCallback(val.(func(string, error))) - case tikvstore.Enable1PC: + case kv.EnableAsyncCommit: + txn.SetEnableAsyncCommit(val.(bool)) + case kv.Enable1PC: txn.SetEnable1PC(val.(bool)) - case tikvstore.TxnScope: + case kv.GuaranteeLinearizability: + txn.SetCausalConsistency(!val.(bool)) + case kv.TxnScope: txn.SetScope(val.(string)) - default: - txn.KVTxn.SetOption(opt, val) + case kv.IsStalenessReadOnly: + txn.KVTxn.GetSnapshot().SetIsStatenessReadOnly(val.(bool)) + case kv.MatchStoreLabels: + txn.KVTxn.GetSnapshot().SetMatchStoreLabels(val.([]*metapb.StoreLabel)) + case kv.ResourceGroupTag: + txn.KVTxn.SetResourceGroupTag(val.([]byte)) } } func (txn *tikvTxn) GetOption(opt int) interface{} { switch opt { - case tikvstore.TxnScope: + case kv.GuaranteeLinearizability: + return !txn.KVTxn.IsCasualConsistency() + case kv.TxnScope: return txn.KVTxn.GetScope() default: - return txn.KVTxn.GetOption(opt) + return nil } } diff --git a/store/driver/txn/unionstore_driver.go b/store/driver/txn/unionstore_driver.go index 9db2325a0148f..3023140cd2c92 100644 --- a/store/driver/txn/unionstore_driver.go +++ b/store/driver/txn/unionstore_driver.go @@ -17,6 +17,7 @@ import ( "context" "github.com/pingcap/tidb/kv" + derr "github.com/pingcap/tidb/store/driver/error" tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/unionstore" ) @@ -37,19 +38,19 @@ func (m *memBuffer) Delete(k kv.Key) error { return m.MemDB.Delete(k) } -func (m *memBuffer) DeleteWithFlags(k kv.Key, ops ...tikvstore.FlagsOp) error { - err := m.MemDB.DeleteWithFlags(k, ops...) - return ToTiDBErr(err) +func (m *memBuffer) DeleteWithFlags(k kv.Key, ops ...kv.FlagsOp) error { + err := m.MemDB.DeleteWithFlags(k, getTiKVFlagsOps(ops)...) + return derr.ToTiDBErr(err) } func (m *memBuffer) Get(_ context.Context, key kv.Key) ([]byte, error) { data, err := m.MemDB.Get(key) - return data, ToTiDBErr(err) + return data, derr.ToTiDBErr(err) } -func (m *memBuffer) GetFlags(key kv.Key) (tikvstore.KeyFlags, error) { +func (m *memBuffer) GetFlags(key kv.Key) (kv.KeyFlags, error) { data, err := m.MemDB.GetFlags(key) - return data, ToTiDBErr(err) + return getTiDBKeyFlags(data), derr.ToTiDBErr(err) } func (m *memBuffer) Staging() kv.StagingHandle { @@ -64,21 +65,21 @@ func (m *memBuffer) Release(h kv.StagingHandle) { m.MemDB.Release(int(h)) } -func (m *memBuffer) InspectStage(handle kv.StagingHandle, f func(kv.Key, tikvstore.KeyFlags, []byte)) { +func (m *memBuffer) InspectStage(handle kv.StagingHandle, f func(kv.Key, kv.KeyFlags, []byte)) { tf := func(key []byte, flag tikvstore.KeyFlags, value []byte) { - f(kv.Key(key), flag, value) + f(kv.Key(key), getTiDBKeyFlags(flag), value) } m.MemDB.InspectStage(int(handle), tf) } func (m *memBuffer) Set(key kv.Key, value []byte) error { err := m.MemDB.Set(key, value) - return ToTiDBErr(err) + return derr.ToTiDBErr(err) } func (m *memBuffer) SetWithFlags(key kv.Key, value []byte, ops ...kv.FlagsOp) error { - err := m.MemDB.SetWithFlags(key, value, ops...) - return ToTiDBErr(err) + err := m.MemDB.SetWithFlags(key, value, getTiKVFlagsOps(ops)...) + return derr.ToTiDBErr(err) } // Iter creates an Iterator positioned on the first entry that k <= entry's key. @@ -87,7 +88,7 @@ func (m *memBuffer) SetWithFlags(key kv.Key, value []byte, ops ...kv.FlagsOp) er // The Iterator must be Closed after use. func (m *memBuffer) Iter(k kv.Key, upperBound kv.Key) (kv.Iterator, error) { it, err := m.MemDB.Iter(k, upperBound) - return &tikvIterator{Iterator: it}, ToTiDBErr(err) + return &tikvIterator{Iterator: it}, derr.ToTiDBErr(err) } // IterReverse creates a reversed Iterator positioned on the first entry which key is less than k. @@ -96,7 +97,7 @@ func (m *memBuffer) Iter(k kv.Key, upperBound kv.Key) (kv.Iterator, error) { // TODO: Add lower bound limit func (m *memBuffer) IterReverse(k kv.Key) (kv.Iterator, error) { it, err := m.MemDB.IterReverse(k) - return &tikvIterator{Iterator: it}, ToTiDBErr(err) + return &tikvIterator{Iterator: it}, derr.ToTiDBErr(err) } // SnapshotIter returns a Iterator for a snapshot of MemBuffer. @@ -110,42 +111,6 @@ func (m *memBuffer) SnapshotGetter() kv.Getter { return newKVGetter(m.MemDB.SnapshotGetter()) } -//tikvUnionStore implements kv.UnionStore -type tikvUnionStore struct { - *unionstore.KVUnionStore -} - -func (u *tikvUnionStore) GetMemBuffer() kv.MemBuffer { - return newMemBuffer(u.KVUnionStore.GetMemBuffer()) -} - -func (u *tikvUnionStore) Get(ctx context.Context, k kv.Key) ([]byte, error) { - data, err := u.KVUnionStore.Get(ctx, k) - return data, ToTiDBErr(err) -} - -func (u *tikvUnionStore) HasPresumeKeyNotExists(k kv.Key) bool { - return u.KVUnionStore.HasPresumeKeyNotExists(k) -} - -func (u *tikvUnionStore) UnmarkPresumeKeyNotExists(k kv.Key) { - u.KVUnionStore.UnmarkPresumeKeyNotExists(k) -} - -func (u *tikvUnionStore) Iter(k kv.Key, upperBound kv.Key) (kv.Iterator, error) { - it, err := u.KVUnionStore.Iter(k, upperBound) - return newKVIterator(it), ToTiDBErr(err) -} - -// IterReverse creates a reversed Iterator positioned on the first entry which key is less than k. -// The returned iterator will iterate from greater key to smaller key. -// If k is nil, the returned iterator will be positioned at the last key. -// TODO: Add lower bound limit -func (u *tikvUnionStore) IterReverse(k kv.Key) (kv.Iterator, error) { - it, err := u.KVUnionStore.IterReverse(k) - return newKVIterator(it), ToTiDBErr(err) -} - type tikvGetter struct { unionstore.Getter } @@ -156,7 +121,7 @@ func newKVGetter(getter unionstore.Getter) kv.Getter { func (g *tikvGetter) Get(_ context.Context, k kv.Key) ([]byte, error) { data, err := g.Getter.Get(k) - return data, ToTiDBErr(err) + return data, derr.ToTiDBErr(err) } // tikvIterator wraps unionstore.Iterator as kv.Iterator @@ -174,3 +139,32 @@ func newKVIterator(it unionstore.Iterator) kv.Iterator { func (it *tikvIterator) Key() kv.Key { return kv.Key(it.Iterator.Key()) } + +func getTiDBKeyFlags(flag tikvstore.KeyFlags) kv.KeyFlags { + var v kv.KeyFlags + if flag.HasPresumeKeyNotExists() { + v = kv.ApplyFlagsOps(v, kv.SetPresumeKeyNotExists) + } + if flag.HasNeedLocked() { + v = kv.ApplyFlagsOps(v, kv.SetNeedLocked) + } + return v +} + +func getTiKVFlagsOp(op kv.FlagsOp) tikvstore.FlagsOp { + switch op { + case kv.SetPresumeKeyNotExists: + return tikvstore.SetPresumeKeyNotExists + case kv.SetNeedLocked: + return tikvstore.SetNeedLocked + } + return 0 +} + +func getTiKVFlagsOps(ops []kv.FlagsOp) []tikvstore.FlagsOp { + v := make([]tikvstore.FlagsOp, len(ops)) + for i := range ops { + v[i] = getTiKVFlagsOp(ops[i]) + } + return v +} diff --git a/store/driver/util_test.go b/store/driver/util_test.go index f46a482bd823e..a740b055b052b 100644 --- a/store/driver/util_test.go +++ b/store/driver/util_test.go @@ -17,7 +17,6 @@ import ( "context" "flag" "fmt" - "sync" "testing" . "github.com/pingcap/check" @@ -26,6 +25,7 @@ import ( "github.com/pingcap/tidb/store/copr" "github.com/pingcap/tidb/store/mockstore/unistore" "github.com/pingcap/tidb/store/tikv" + "github.com/pingcap/tidb/store/tikv/mockstore" ) func TestT(t *testing.T) { @@ -34,9 +34,7 @@ func TestT(t *testing.T) { } var ( - withTiKVGlobalLock sync.RWMutex - WithTiKV = flag.Bool("with-tikv", false, "run tests with TiKV cluster started. (not use the mock server)") - pdAddrs = flag.String("pd-addrs", "127.0.0.1:2379", "pd addrs") + pdAddrs = flag.String("pd-addrs", "127.0.0.1:2379", "pd addrs") ) // NewTestStore creates a kv.Storage for testing purpose. @@ -45,7 +43,7 @@ func NewTestStore(c *C) kv.Storage { flag.Parse() } - if *WithTiKV { + if *mockstore.WithTiKV { var d TiKVDriver store, err := d.Open(fmt.Sprintf("tikv://%s", *pdAddrs)) c.Assert(err, IsNil) @@ -82,20 +80,4 @@ func clearStorage(store kv.Storage) error { } // OneByOneSuite is a suite, When with-tikv flag is true, there is only one storage, so the test suite have to run one by one. -type OneByOneSuite struct{} - -func (s *OneByOneSuite) SetUpSuite(c *C) { - if *WithTiKV { - withTiKVGlobalLock.Lock() - } else { - withTiKVGlobalLock.RLock() - } -} - -func (s *OneByOneSuite) TearDownSuite(c *C) { - if *WithTiKV { - withTiKVGlobalLock.Unlock() - } else { - withTiKVGlobalLock.RUnlock() - } -} +type OneByOneSuite = mockstore.OneByOneSuite diff --git a/store/gcworker/gc_worker.go b/store/gcworker/gc_worker.go index 038efa30b92f3..5982f7e2dd0ca 100644 --- a/store/gcworker/gc_worker.go +++ b/store/gcworker/gc_worker.go @@ -74,7 +74,7 @@ type GCWorker struct { // NewGCWorker creates a GCWorker instance. func NewGCWorker(store kv.Storage, pdClient pd.Client) (*GCWorker, error) { - ver, err := store.CurrentVersion(oracle.GlobalTxnScope) + ver, err := store.CurrentVersion(kv.GlobalTxnScope) if err != nil { return nil, errors.Trace(err) } @@ -185,6 +185,12 @@ var gcVariableComments = map[string]string{ gcScanLockModeKey: "Mode of scanning locks, \"physical\" or \"legacy\"", } +const ( + unsafeDestroyRangeTimeout = 5 * time.Minute + accessLockObserverTimeout = 10 * time.Second + gcTimeout = 5 * time.Minute +) + func (w *GCWorker) start(ctx context.Context, wg *sync.WaitGroup) { logutil.Logger(ctx).Info("[gc worker] start", zap.String("uuid", w.uuid)) @@ -409,18 +415,27 @@ func (w *GCWorker) calcSafePointByMinStartTS(ctx context.Context, safePoint uint return safePoint } - if globalMinStartTS < safePoint { + // If the lock.ts <= max_ts(safePoint), it will be collected and resolved by the gc worker, + // the locks of ongoing pessimistic transactions could be resolved by the gc worker and then + // the transaction is aborted, decrement the value by 1 to avoid this. + globalMinStartAllowedTS := globalMinStartTS + if globalMinStartTS > 0 { + globalMinStartAllowedTS = globalMinStartTS - 1 + } + + if globalMinStartAllowedTS < safePoint { logutil.Logger(ctx).Info("[gc worker] gc safepoint blocked by a running session", zap.String("uuid", w.uuid), zap.Uint64("globalMinStartTS", globalMinStartTS), + zap.Uint64("globalMinStartAllowedTS", globalMinStartAllowedTS), zap.Uint64("safePoint", safePoint)) - safePoint = globalMinStartTS + safePoint = globalMinStartAllowedTS } return safePoint } func (w *GCWorker) getOracleTime() (time.Time, error) { - currentVer, err := w.store.CurrentVersion(oracle.GlobalTxnScope) + currentVer, err := w.store.CurrentVersion(kv.GlobalTxnScope) if err != nil { return time.Time{}, errors.Trace(err) } @@ -799,7 +814,7 @@ func (w *GCWorker) doUnsafeDestroyRangeRequest(ctx context.Context, startKey []b go func() { defer wg.Done() - resp, err1 := w.tikvStore.GetTiKVClient().SendRequest(ctx, address, req, tikv.UnsafeDestroyRangeTimeout) + resp, err1 := w.tikvStore.GetTiKVClient().SendRequest(ctx, address, req, unsafeDestroyRangeTimeout) if err1 == nil { if resp == nil || resp.Resp == nil { err1 = errors.Errorf("unsafe destroy range returns nil response from store %v", storeID) @@ -1050,7 +1065,7 @@ func (w *GCWorker) resolveLocksForRange(ctx context.Context, safePoint uint64, s var stat tikv.RangeTaskStat key := startKey - bo := tikv.NewBackofferWithVars(ctx, tikv.GcResolveLockMaxBackoff, nil) + bo := tikv.NewGcResolveLockMaxBackoffer(ctx) failpoint.Inject("setGcResolveMaxBackoff", func(v failpoint.Value) { sleep := v.(int) // cooperate with github.com/pingcap/tidb/store/tikv/invalidCacheAndRetry @@ -1080,7 +1095,7 @@ retryScanAndResolve: return stat, errors.Trace(err) } if regionErr != nil { - err = bo.Backoff(tikv.BoRegionMiss, errors.New(regionErr.String())) + err = bo.Backoff(tikv.BoRegionMiss(), errors.New(regionErr.String())) if err != nil { return stat, errors.Trace(err) } @@ -1116,7 +1131,7 @@ retryScanAndResolve: return stat, errors.Trace(err1) } if !ok { - err = bo.Backoff(tikv.BoTxnLock, errors.Errorf("remain locks: %d", len(locks))) + err = bo.Backoff(tikv.BoTxnLock(), errors.Errorf("remain locks: %d", len(locks))) if err != nil { return stat, errors.Trace(err) } @@ -1147,7 +1162,7 @@ retryScanAndResolve: if len(key) == 0 || (len(endKey) != 0 && bytes.Compare(key, endKey) >= 0) { break } - bo = tikv.NewBackofferWithVars(ctx, tikv.GcResolveLockMaxBackoff, nil) + bo = tikv.NewGcResolveLockMaxBackoffer(ctx) failpoint.Inject("setGcResolveMaxBackoff", func(v failpoint.Value) { sleep := v.(int) bo = tikv.NewBackofferWithVars(ctx, sleep, nil) @@ -1263,7 +1278,7 @@ func (w *GCWorker) registerLockObservers(ctx context.Context, safePoint uint64, for _, store := range stores { address := store.Address - resp, err := w.tikvStore.GetTiKVClient().SendRequest(ctx, address, req, tikv.AccessLockObserverTimeout) + resp, err := w.tikvStore.GetTiKVClient().SendRequest(ctx, address, req, accessLockObserverTimeout) if err != nil { return errors.Trace(err) } @@ -1303,7 +1318,7 @@ func (w *GCWorker) checkLockObservers(ctx context.Context, safePoint uint64, sto for _, store := range stores { address := store.Address - resp, err := w.tikvStore.GetTiKVClient().SendRequest(ctx, address, req, tikv.AccessLockObserverTimeout) + resp, err := w.tikvStore.GetTiKVClient().SendRequest(ctx, address, req, accessLockObserverTimeout) if err != nil { logError(store, err) continue @@ -1369,7 +1384,7 @@ func (w *GCWorker) removeLockObservers(ctx context.Context, safePoint uint64, st for _, store := range stores { address := store.Address - resp, err := w.tikvStore.GetTiKVClient().SendRequest(ctx, address, req, tikv.AccessLockObserverTimeout) + resp, err := w.tikvStore.GetTiKVClient().SendRequest(ctx, address, req, accessLockObserverTimeout) if err != nil { logError(store, err) continue @@ -1460,7 +1475,7 @@ func (w *GCWorker) resolveLocksAcrossRegions(ctx context.Context, locks []*tikv. failpoint.Return(errors.New("injectedError")) }) - bo := tikv.NewBackofferWithVars(ctx, tikv.GcResolveLockMaxBackoff, nil) + bo := tikv.NewGcResolveLockMaxBackoffer(ctx) for { if len(locks) == 0 { @@ -1488,7 +1503,7 @@ func (w *GCWorker) resolveLocksAcrossRegions(ctx context.Context, locks []*tikv. return errors.Trace(err) } if !ok { - err = bo.Backoff(tikv.BoTxnLock, errors.Errorf("remain locks: %d", len(locks))) + err = bo.Backoff(tikv.BoTxnLock(), errors.Errorf("remain locks: %d", len(locks))) if err != nil { return errors.Trace(err) } @@ -1496,25 +1511,27 @@ func (w *GCWorker) resolveLocksAcrossRegions(ctx context.Context, locks []*tikv. } // Recreate backoffer for next region - bo = tikv.NewBackofferWithVars(ctx, tikv.GcResolveLockMaxBackoff, nil) + bo = tikv.NewGcResolveLockMaxBackoffer(ctx) locks = locks[len(locksInRegion):] } return nil } +const gcOneRegionMaxBackoff = 20000 + func (w *GCWorker) uploadSafePointToPD(ctx context.Context, safePoint uint64) error { var newSafePoint uint64 var err error - bo := tikv.NewBackofferWithVars(ctx, tikv.GcOneRegionMaxBackoff, nil) + bo := tikv.NewBackofferWithVars(ctx, gcOneRegionMaxBackoff, nil) for { newSafePoint, err = w.pdClient.UpdateGCSafePoint(ctx, safePoint) if err != nil { if errors.Cause(err) == context.Canceled { return errors.Trace(err) } - err = bo.Backoff(tikv.BoPDRPC, errors.Errorf("failed to upload safe point to PD, err: %v", err)) + err = bo.Backoff(tikv.BoPDRPC(), errors.Errorf("failed to upload safe point to PD, err: %v", err)) if err != nil { return errors.Trace(err) } @@ -1544,7 +1561,7 @@ func (w *GCWorker) doGCForRange(ctx context.Context, startKey []byte, endKey []b }() key := startKey for { - bo := tikv.NewBackofferWithVars(ctx, tikv.GcOneRegionMaxBackoff, nil) + bo := tikv.NewBackofferWithVars(ctx, gcOneRegionMaxBackoff, nil) loc, err := w.tikvStore.GetRegionCache().LocateKey(bo, key) if err != nil { return stat, errors.Trace(err) @@ -1556,7 +1573,7 @@ func (w *GCWorker) doGCForRange(ctx context.Context, startKey []byte, endKey []b // we check regionErr here first, because we know 'regionErr' and 'err' should not return together, to keep it to // make the process correct. if regionErr != nil { - err = bo.Backoff(tikv.BoRegionMiss, errors.New(regionErr.String())) + err = bo.Backoff(tikv.BoRegionMiss(), errors.New(regionErr.String())) if err == nil { continue } @@ -1589,7 +1606,7 @@ func (w *GCWorker) doGCForRegion(bo *tikv.Backoffer, safePoint uint64, region ti SafePoint: safePoint, }) - resp, err := w.tikvStore.SendReq(bo, req, region, tikv.GCTimeout) + resp, err := w.tikvStore.SendReq(bo, req, region, gcTimeout) if err != nil { return nil, errors.Trace(err) } @@ -1973,7 +1990,7 @@ type MockGCWorker struct { // NewMockGCWorker creates a MockGCWorker instance ONLY for test. func NewMockGCWorker(store kv.Storage) (*MockGCWorker, error) { - ver, err := store.CurrentVersion(oracle.GlobalTxnScope) + ver, err := store.CurrentVersion(kv.GlobalTxnScope) if err != nil { return nil, errors.Trace(err) } diff --git a/store/gcworker/gc_worker_test.go b/store/gcworker/gc_worker_test.go index 3bfd616929aec..c3ec1c9cbb601 100644 --- a/store/gcworker/gc_worker_test.go +++ b/store/gcworker/gc_worker_test.go @@ -43,6 +43,7 @@ import ( "github.com/pingcap/tidb/store/tikv/mockstore/mocktikv" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/store/tikv/oracle/oracles" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" pd "github.com/tikv/pd/client" ) @@ -258,10 +259,10 @@ func (s *testGCWorkerSuite) TestMinStartTS(c *C) { strconv.FormatUint(now, 10)) c.Assert(err, IsNil) err = spkv.Put(fmt.Sprintf("%s/%s", infosync.ServerMinStartTSPath, "b"), - strconv.FormatUint(now-oracle.EncodeTSO(20000), 10)) + strconv.FormatUint(now-oracle.ComposeTS(20000, 0), 10)) c.Assert(err, IsNil) - sp = s.gcWorker.calcSafePointByMinStartTS(ctx, now-oracle.EncodeTSO(10000)) - c.Assert(sp, Equals, now-oracle.EncodeTSO(20000)) + sp = s.gcWorker.calcSafePointByMinStartTS(ctx, now-oracle.ComposeTS(10000, 0)) + c.Assert(sp, Equals, now-oracle.ComposeTS(20000, 0)-1) } func (s *testGCWorkerSuite) TestPrepareGC(c *C) { @@ -412,7 +413,7 @@ func (s *testGCWorkerSuite) TestStatusVars(c *C) { func (s *testGCWorkerSuite) TestDoGCForOneRegion(c *C) { ctx := context.Background() - bo := tikv.NewBackofferWithVars(ctx, tikv.GcOneRegionMaxBackoff, nil) + bo := tikv.NewBackofferWithVars(ctx, gcOneRegionMaxBackoff, nil) loc, err := s.tikvStore.GetRegionCache().LocateKey(bo, []byte("")) c.Assert(err, IsNil) var regionErr *errorpb.Error @@ -942,8 +943,8 @@ func (s *testGCWorkerSuite) TestResolveLockRangeMeetRegionEnlargeCausedByRegionM mCluster := s.cluster.(*mocktikv.Cluster) mCluster.Merge(s.initRegion.regionID, region2) regionMeta, _ := mCluster.GetRegion(s.initRegion.regionID) - err := s.tikvStore.GetRegionCache().OnRegionEpochNotMatch( - tikv.NewNoopBackoff(context.Background()), + _, err := s.tikvStore.GetRegionCache().OnRegionEpochNotMatch( + retry.NewNoopBackoff(context.Background()), &tikv.RPCContext{Region: regionID, Store: &tikv.Store{}}, []*metapb.Region{regionMeta}) c.Assert(err, IsNil) @@ -1588,3 +1589,52 @@ func (s *testGCWorkerSuite) TestGCPlacementRules(c *C) { c.Assert(pid, Equals, int64(1)) c.Assert(err, IsNil) } + +func (s *testGCWorkerSuite) TestGCWithPendingTxn(c *C) { + ctx := context.Background() + gcSafePointCacheInterval = 0 + err := s.gcWorker.saveValueToSysTable(gcEnableKey, booleanFalse) + c.Assert(err, IsNil) + + k1 := []byte("tk1") + v1 := []byte("v1") + txn, err := s.store.Begin() + c.Assert(err, IsNil) + txn.SetOption(kv.Pessimistic, true) + lockCtx := &kv.LockCtx{ForUpdateTS: txn.StartTS(), WaitStartTime: time.Now()} + + // Lock the key. + err = txn.Set(k1, v1) + c.Assert(err, IsNil) + err = txn.LockKeys(ctx, lockCtx, k1) + c.Assert(err, IsNil) + + // Prepare to run gc with txn's startTS as the safepoint ts. + spkv := s.tikvStore.GetSafePointKV() + err = spkv.Put(fmt.Sprintf("%s/%s", infosync.ServerMinStartTSPath, "a"), strconv.FormatUint(txn.StartTS(), 10)) + c.Assert(err, IsNil) + s.mustSetTiDBServiceSafePoint(c, txn.StartTS(), txn.StartTS()) + veryLong := gcDefaultLifeTime * 100 + err = s.gcWorker.saveTime(gcLastRunTimeKey, oracle.GetTimeFromTS(s.mustAllocTs(c)).Add(-veryLong)) + c.Assert(err, IsNil) + s.gcWorker.lastFinish = time.Now().Add(-veryLong) + s.oracle.AddOffset(time.Minute * 10) + err = s.gcWorker.saveValueToSysTable(gcEnableKey, booleanTrue) + c.Assert(err, IsNil) + + // Trigger the tick let the gc job start. + err = s.gcWorker.leaderTick(ctx) + c.Assert(err, IsNil) + // Wait for GC finish + select { + case err = <-s.gcWorker.done: + s.gcWorker.gcIsRunning = false + break + case <-time.After(time.Second * 10): + err = errors.New("receive from s.gcWorker.done timeout") + } + c.Assert(err, IsNil) + + err = txn.Commit(ctx) + c.Assert(err, IsNil) +} diff --git a/store/helper/helper.go b/store/helper/helper.go index e96ad4ae21851..533b1d66d576e 100644 --- a/store/helper/helper.go +++ b/store/helper/helper.go @@ -29,6 +29,7 @@ import ( "time" "github.com/pingcap/errors" + deadlockpb "github.com/pingcap/kvproto/pkg/deadlock" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/log" "github.com/pingcap/parser/model" @@ -48,7 +49,7 @@ import ( // Methods copied from kv.Storage and tikv.Storage due to limitation of go1.13. type Storage interface { Begin() (kv.Transaction, error) - BeginWithOption(option kv.TransactionOption) (kv.Transaction, error) + BeginWithOption(option tikv.StartTSOption) (kv.Transaction, error) GetSnapshot(ver kv.Version) kv.Snapshot GetClient() kv.Client GetMPPClient() kv.MPPClient @@ -71,6 +72,8 @@ type Storage interface { SetTiKVClient(client tikv.Client) GetTiKVClient() tikv.Client Closed() <-chan struct{} + GetMinSafeTS(txnScope string) uint64 + GetLockWaits() ([]*deadlockpb.WaitForEntry, error) } // Helper is a middleware to get some information from tikv/pd. It can be used for TiDB's http api or mem table. @@ -108,6 +111,78 @@ func (h *Helper) GetMvccByEncodedKey(encodedKey kv.Key) (*kvrpcpb.MvccGetByKeyRe return kvResp.Resp.(*kvrpcpb.MvccGetByKeyResponse), nil } +// MvccKV wraps the key's mvcc info in tikv. +type MvccKV struct { + Key string `json:"key"` + RegionID uint64 `json:"region_id"` + Value *kvrpcpb.MvccGetByKeyResponse `json:"value"` +} + +// GetMvccByStartTs gets Mvcc info by startTS from tikv. +func (h *Helper) GetMvccByStartTs(startTS uint64, startKey, endKey kv.Key) (*MvccKV, error) { + bo := tikv.NewBackofferWithVars(context.Background(), 5000, nil) + for { + curRegion, err := h.RegionCache.LocateKey(bo, startKey) + if err != nil { + logutil.BgLogger().Error("get MVCC by startTS failed", zap.Uint64("txnStartTS", startTS), + zap.Stringer("startKey", startKey), zap.Error(err)) + return nil, errors.Trace(err) + } + + tikvReq := tikvrpc.NewRequest(tikvrpc.CmdMvccGetByStartTs, &kvrpcpb.MvccGetByStartTsRequest{ + StartTs: startTS, + }) + tikvReq.Context.Priority = kvrpcpb.CommandPri_Low + kvResp, err := h.Store.SendReq(bo, tikvReq, curRegion.Region, time.Hour) + if err != nil { + logutil.BgLogger().Error("get MVCC by startTS failed", + zap.Uint64("txnStartTS", startTS), + zap.Stringer("startKey", startKey), + zap.Reflect("region", curRegion.Region), + zap.Stringer("curRegion", curRegion), + zap.Reflect("kvResp", kvResp), + zap.Error(err)) + return nil, errors.Trace(err) + } + data := kvResp.Resp.(*kvrpcpb.MvccGetByStartTsResponse) + if err := data.GetRegionError(); err != nil { + logutil.BgLogger().Warn("get MVCC by startTS failed", + zap.Uint64("txnStartTS", startTS), + zap.Stringer("startKey", startKey), + zap.Reflect("region", curRegion.Region), + zap.Stringer("curRegion", curRegion), + zap.Reflect("kvResp", kvResp), + zap.Stringer("error", err)) + continue + } + + if len(data.GetError()) > 0 { + logutil.BgLogger().Error("get MVCC by startTS failed", + zap.Uint64("txnStartTS", startTS), + zap.Stringer("startKey", startKey), + zap.Reflect("region", curRegion.Region), + zap.Stringer("curRegion", curRegion), + zap.Reflect("kvResp", kvResp), + zap.String("error", data.GetError())) + return nil, errors.New(data.GetError()) + } + + key := data.GetKey() + if len(key) > 0 { + resp := &kvrpcpb.MvccGetByKeyResponse{Info: data.Info, RegionError: data.RegionError, Error: data.Error} + return &MvccKV{Key: strings.ToUpper(hex.EncodeToString(key)), Value: resp, RegionID: curRegion.Region.GetID()}, nil + } + + if len(endKey) > 0 && curRegion.Contains(endKey) { + return nil, nil + } + if len(curRegion.EndKey) == 0 { + return nil, nil + } + startKey = kv.Key(curRegion.EndKey) + } +} + // StoreHotRegionInfos records all hog region stores. // it's the response of PD. type StoreHotRegionInfos struct { @@ -466,8 +541,8 @@ type RegionEpoch struct { // RegionPeerStat stores one field `DownSec` which indicates how long it's down than `RegionPeer`. type RegionPeerStat struct { - RegionPeer - DownSec int64 `json:"down_seconds"` + Peer RegionPeer `json:"peer"` + DownSec int64 `json:"down_seconds"` } // RegionInfo stores the information of one region. diff --git a/store/mockstore/mockcopr/executor_test.go b/store/mockstore/mockcopr/executor_test.go index af9ac45beae96..7437b8d995997 100644 --- a/store/mockstore/mockcopr/executor_test.go +++ b/store/mockstore/mockcopr/executor_test.go @@ -83,7 +83,7 @@ func (s *testExecutorSuite) TestResolvedLargeTxnLocks(c *C) { tk.MustExec("insert into t values (1, 1)") o := s.store.GetOracle() - tso, err := o.GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) + tso, err := o.GetTimestamp(context.Background(), &oracle.Option{TxnScope: kv.GlobalTxnScope}) c.Assert(err, IsNil) key := tablecodec.EncodeRowKeyWithHandle(tbl.Meta().ID, kv.IntHandle(1)) diff --git a/store/mockstore/mockstorage/storage.go b/store/mockstore/mockstorage/storage.go index 05ece29c57a6a..4fa049fc69f42 100644 --- a/store/mockstore/mockstorage/storage.go +++ b/store/mockstore/mockstorage/storage.go @@ -17,34 +17,40 @@ import ( "context" "crypto/tls" + deadlockpb "github.com/pingcap/kvproto/pkg/deadlock" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/store/copr" driver "github.com/pingcap/tidb/store/driver/txn" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/config" - "github.com/pingcap/tidb/store/tikv/oracle" ) // Wraps tikv.KVStore and make it compatible with kv.Storage. type mockStorage struct { *tikv.KVStore *copr.Store - memCache kv.MemManager + memCache kv.MemManager + LockWaits []*deadlockpb.WaitForEntry } // NewMockStorage wraps tikv.KVStore as kv.Storage. func NewMockStorage(tikvStore *tikv.KVStore) (kv.Storage, error) { + return NewMockStorageWithLockWaits(tikvStore, nil) +} + +// NewMockStorageWithLockWaits wraps tikv.KVStore as kv.Storage, with mock LockWaits. +func NewMockStorageWithLockWaits(tikvStore *tikv.KVStore, lockWaits []*deadlockpb.WaitForEntry) (kv.Storage, error) { coprConfig := config.DefaultConfig().TiKVClient.CoprCache coprStore, err := copr.NewStore(tikvStore, &coprConfig) if err != nil { return nil, err } return &mockStorage{ - KVStore: tikvStore, - Store: coprStore, - memCache: kv.NewCacheDB(), + KVStore: tikvStore, + Store: coprStore, + memCache: kv.NewCacheDB(), + LockWaits: lockWaits, }, nil - } func (s *mockStorage) EtcdAddrs() ([]string, error) { @@ -84,21 +90,8 @@ func (s *mockStorage) ShowStatus(ctx context.Context, key string) (interface{}, } // BeginWithOption begins a transaction with given option -func (s *mockStorage) BeginWithOption(option kv.TransactionOption) (kv.Transaction, error) { - txnScope := option.TxnScope - if txnScope == "" { - txnScope = oracle.GlobalTxnScope - } - if option.StartTS != nil { - return newTiKVTxn(s.BeginWithStartTS(txnScope, *option.StartTS)) - } else if option.PrevSec != nil { - return newTiKVTxn(s.BeginWithExactStaleness(txnScope, *option.PrevSec)) - } else if option.MaxPrevSec != nil { - return newTiKVTxn(s.BeginWithMaxPrevSec(txnScope, *option.MaxPrevSec)) - } else if option.MinStartTS != nil { - return newTiKVTxn(s.BeginWithMinStartTS(txnScope, *option.MinStartTS)) - } - return newTiKVTxn(s.BeginWithTxnScope(txnScope)) +func (s *mockStorage) BeginWithOption(option tikv.StartTSOption) (kv.Transaction, error) { + return newTiKVTxn(s.KVStore.BeginWithOption(option)) } // GetSnapshot gets a snapshot that is able to read any data which data is <= ver. @@ -113,6 +106,11 @@ func (s *mockStorage) CurrentVersion(txnScope string) (kv.Version, error) { return kv.NewVersion(ver), err } +// GetMinSafeTS return the minimal SafeTS of the storage with given txnScope. +func (s *mockStorage) GetMinSafeTS(txnScope string) uint64 { + return 0 +} + func newTiKVTxn(txn *tikv.KVTxn, err error) (kv.Transaction, error) { if err != nil { return nil, err @@ -120,6 +118,10 @@ func newTiKVTxn(txn *tikv.KVTxn, err error) (kv.Transaction, error) { return driver.NewTiKVTxn(txn), nil } +func (s *mockStorage) GetLockWaits() ([]*deadlockpb.WaitForEntry, error) { + return s.LockWaits, nil +} + func (s *mockStorage) Close() error { s.Store.Close() return s.KVStore.Close() diff --git a/store/mockstore/unistore/cophandler/analyze.go b/store/mockstore/unistore/cophandler/analyze.go index 4f309027edc4a..c7ce4630fee85 100644 --- a/store/mockstore/unistore/cophandler/analyze.go +++ b/store/mockstore/unistore/cophandler/analyze.go @@ -107,7 +107,7 @@ func handleAnalyzeIndexReq(dbReader *dbreader.DBReader, rans []kv.KeyRange, anal return nil, err } } - if statsVer == statistics.Version2 { + if statsVer >= statistics.Version2 { if processor.topNCurValuePair.Count != 0 { processor.topNValuePairs = append(processor.topNValuePairs, processor.topNCurValuePair) } @@ -126,7 +126,7 @@ func handleAnalyzeIndexReq(dbReader *dbreader.DBReader, rans []kv.KeyRange, anal hg := statistics.HistogramToProto(processor.statsBuilder.Hist()) var cm *tipb.CMSketch if processor.cms != nil { - if statsVer == statistics.Version2 { + if statsVer >= statistics.Version2 { for _, valueCnt := range processor.topNValuePairs { h1, h2 := murmur3.Sum128(valueCnt.Encoded) processor.cms.SubValue(h1, h2, valueCnt.Count) @@ -207,7 +207,7 @@ func (p *analyzeIndexProcessor) Process(key, _ []byte) error { } } - if p.statsVer == statistics.Version2 { + if p.statsVer >= statistics.Version2 { if bytes.Equal(p.topNCurValuePair.Encoded, p.rowBuf) { p.topNCurValuePair.Count++ } else { @@ -547,7 +547,7 @@ func handleAnalyzeMixedReq(dbReader *dbreader.DBReader, rans []kv.KeyRange, anal colResp.Collectors = append(colResp.Collectors, statistics.SampleCollectorToProto(c)) } // common handle - if statsVer == statistics.Version2 { + if statsVer >= statistics.Version2 { if e.topNCurValuePair.Count != 0 { e.topNValuePairs = append(e.topNValuePairs, e.topNCurValuePair) } @@ -566,7 +566,7 @@ func handleAnalyzeMixedReq(dbReader *dbreader.DBReader, rans []kv.KeyRange, anal hg := statistics.HistogramToProto(e.statsBuilder.Hist()) var cm *tipb.CMSketch if e.cms != nil { - if statsVer == statistics.Version2 { + if statsVer >= statistics.Version2 { for _, valueCnt := range e.topNValuePairs { h1, h2 := murmur3.Sum128(valueCnt.Encoded) e.cms.SubValue(h1, h2, valueCnt.Count) @@ -623,7 +623,7 @@ func (e *analyzeMixedExec) Process(key, value []byte) error { } } - if e.statsVer == statistics.Version2 { + if e.statsVer >= statistics.Version2 { if bytes.Equal(e.topNCurValuePair.Encoded, e.rowBuf) { e.topNCurValuePair.Count++ } else { diff --git a/store/mockstore/unistore/cophandler/closure_exec.go b/store/mockstore/unistore/cophandler/closure_exec.go index 152362c13d5e3..696a1497b068e 100644 --- a/store/mockstore/unistore/cophandler/closure_exec.go +++ b/store/mockstore/unistore/cophandler/closure_exec.go @@ -124,9 +124,7 @@ func buildClosureExecutorFromExecutorList(dagCtx *dagContext, executors []*tipb. outputFieldTypes = append(outputFieldTypes, originalOutputFieldTypes[idx]) } } else { - for _, tp := range originalOutputFieldTypes { - outputFieldTypes = append(outputFieldTypes, tp) - } + outputFieldTypes = append(outputFieldTypes, originalOutputFieldTypes...) } if len(executors) == 1 { ce.resultFieldType = outputFieldTypes diff --git a/store/mockstore/unistore/cophandler/cop_handler_test.go b/store/mockstore/unistore/cophandler/cop_handler_test.go index 5cbd24f08a581..c013d04c404a3 100644 --- a/store/mockstore/unistore/cophandler/cop_handler_test.go +++ b/store/mockstore/unistore/cophandler/cop_handler_test.go @@ -16,7 +16,6 @@ package cophandler import ( "errors" "fmt" - "io/ioutil" "math" "os" "path/filepath" @@ -443,11 +442,11 @@ func (ts *testStore) commit(keys [][]byte, startTS, commitTS uint64) error { } func newTestStore(dbPrefix string, logPrefix string) (*testStore, error) { - dbPath, err := ioutil.TempDir("", dbPrefix) + dbPath, err := os.MkdirTemp("", dbPrefix) if err != nil { return nil, err } - LogPath, err := ioutil.TempDir("", logPrefix) + LogPath, err := os.MkdirTemp("", logPrefix) if err != nil { return nil, err } diff --git a/store/mockstore/unistore/mock.go b/store/mockstore/unistore/mock.go index c37235488477f..818016436f922 100644 --- a/store/mockstore/unistore/mock.go +++ b/store/mockstore/unistore/mock.go @@ -14,7 +14,6 @@ package unistore import ( - "io/ioutil" "os" "github.com/pingcap/errors" @@ -28,7 +27,7 @@ func New(path string) (*RPCClient, pd.Client, *Cluster, error) { persistent := true if path == "" { var err error - if path, err = ioutil.TempDir("", "tidb-unistore-temp"); err != nil { + if path, err = os.MkdirTemp("", "tidb-unistore-temp"); err != nil { return nil, nil, nil, err } persistent = false diff --git a/store/mockstore/unistore/rpc.go b/store/mockstore/unistore/rpc.go index 378b6b23b56a4..51c26529d4593 100644 --- a/store/mockstore/unistore/rpc.go +++ b/store/mockstore/unistore/rpc.go @@ -59,6 +59,9 @@ type RPCClient struct { rpcCli Client } +// UnistoreRPCClientSendHook exports for test. +var UnistoreRPCClientSendHook func(*tikvrpc.Request) + // SendRequest sends a request to mock cluster. func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) { failpoint.Inject("rpcServerBusy", func(val failpoint.Value) { @@ -67,6 +70,12 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R } }) + failpoint.Inject("unistoreRPCClientSendHook", func(val failpoint.Value) { + if val.(bool) && UnistoreRPCClientSendHook != nil { + UnistoreRPCClientSendHook(req) + } + }) + if req.StoreTp == tikvrpc.TiDB { return c.redirectRequestToRPCServer(ctx, addr, req, timeout) } @@ -116,6 +125,8 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R failpoint.Inject("rpcPrewriteResult", func(val failpoint.Value) { if val != nil { switch val.(string) { + case "timeout": + failpoint.Return(nil, errors.New("timeout")) case "notLeader": failpoint.Return(&tikvrpc.Response{ Resp: &kvrpcpb.PrewriteResponse{RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}}, diff --git a/store/mockstore/unistore/tikv/dbreader/db_reader.go b/store/mockstore/unistore/tikv/dbreader/db_reader.go index 158f295cc1bdf..6e3909ada740f 100644 --- a/store/mockstore/unistore/tikv/dbreader/db_reader.go +++ b/store/mockstore/unistore/tikv/dbreader/db_reader.go @@ -165,7 +165,6 @@ func (r *DBReader) BatchGet(keys [][]byte, startTS uint64, f BatchGetFunc) { } f(key, val, err) } - return } // ErrScanBreak is returned by ScanFunc to break the scan loop. diff --git a/store/mockstore/unistore/tikv/deadlock.go b/store/mockstore/unistore/tikv/deadlock.go index 6641a500e2cc1..de2eaf8fa61d9 100644 --- a/store/mockstore/unistore/tikv/deadlock.go +++ b/store/mockstore/unistore/tikv/deadlock.go @@ -44,7 +44,10 @@ type DetectorServer struct { func (ds *DetectorServer) Detect(req *deadlockPb.DeadlockRequest) *deadlockPb.DeadlockResponse { switch req.Tp { case deadlockPb.DeadlockRequestType_Detect: - err := ds.Detector.Detect(req.Entry.Txn, req.Entry.WaitForTxn, req.Entry.KeyHash) + err := ds.Detector.Detect(req.Entry.Txn, req.Entry.WaitForTxn, req.Entry.KeyHash, diagnosticContext{ + key: req.Entry.Key, + resourceGroupTag: req.Entry.ResourceGroupTag, + }) if err != nil { resp := convertErrToResp(err, req.Entry.Txn, req.Entry.WaitForTxn, req.Entry.KeyHash) return resp @@ -178,30 +181,35 @@ func (dt *DetectorClient) recvLoop(streamCli deadlockPb.Deadlock_DetectClient) { } func (dt *DetectorClient) handleRemoteTask(requestType deadlockPb.DeadlockRequestType, - txnTs uint64, waitForTxnTs uint64, keyHash uint64) { + txnTs uint64, waitForTxnTs uint64, keyHash uint64, diagCtx diagnosticContext) { detectReq := &deadlockPb.DeadlockRequest{} detectReq.Tp = requestType detectReq.Entry.Txn = txnTs detectReq.Entry.WaitForTxn = waitForTxnTs detectReq.Entry.KeyHash = keyHash + detectReq.Entry.Key = diagCtx.key + detectReq.Entry.ResourceGroupTag = diagCtx.resourceGroupTag dt.sendCh <- detectReq } // CleanUp processes cleaup task on local detector // user interfaces func (dt *DetectorClient) CleanUp(startTs uint64) { - dt.handleRemoteTask(deadlockPb.DeadlockRequestType_CleanUp, startTs, 0, 0) + dt.handleRemoteTask(deadlockPb.DeadlockRequestType_CleanUp, startTs, 0, 0, diagnosticContext{}) } // CleanUpWaitFor cleans up the specific wait edge in detector's wait map func (dt *DetectorClient) CleanUpWaitFor(txnTs, waitForTxn, keyHash uint64) { - dt.handleRemoteTask(deadlockPb.DeadlockRequestType_CleanUpWaitFor, txnTs, waitForTxn, keyHash) + dt.handleRemoteTask(deadlockPb.DeadlockRequestType_CleanUpWaitFor, txnTs, waitForTxn, keyHash, diagnosticContext{}) } // Detect post the detection request to local deadlock detector or remote first region leader, // the caller should use `waiter.ch` to receive possible deadlock response -func (dt *DetectorClient) Detect(txnTs uint64, waitForTxnTs uint64, keyHash uint64) { - dt.handleRemoteTask(deadlockPb.DeadlockRequestType_Detect, txnTs, waitForTxnTs, keyHash) +func (dt *DetectorClient) Detect(txnTs uint64, waitForTxnTs uint64, keyHash uint64, key []byte, resourceGroupTag []byte) { + dt.handleRemoteTask(deadlockPb.DeadlockRequestType_Detect, txnTs, waitForTxnTs, keyHash, diagnosticContext{ + key: key, + resourceGroupTag: resourceGroupTag, + }) } // convertErrToResp converts `ErrDeadlock` to `DeadlockResponse` proto type @@ -213,6 +221,18 @@ func convertErrToResp(errDeadlock *ErrDeadlock, txnTs, waitForTxnTs, keyHash uin resp := &deadlockPb.DeadlockResponse{} resp.Entry = entry resp.DeadlockKeyHash = errDeadlock.DeadlockKeyHash + + resp.WaitChain = make([]*deadlockPb.WaitForEntry, 0, len(errDeadlock.WaitChain)) + for _, item := range errDeadlock.WaitChain { + resp.WaitChain = append(resp.WaitChain, &deadlockPb.WaitForEntry{ + Txn: item.Txn, + WaitForTxn: item.WaitForTxn, + KeyHash: item.KeyHash, + Key: item.Key, + ResourceGroupTag: item.ResourceGroupTag, + }) + } + return resp } diff --git a/store/mockstore/unistore/tikv/detector.go b/store/mockstore/unistore/tikv/detector.go index 0273bed5fe6a8..b86804696a670 100644 --- a/store/mockstore/unistore/tikv/detector.go +++ b/store/mockstore/unistore/tikv/detector.go @@ -30,6 +30,7 @@ import ( "sync" "time" + deadlockpb "github.com/pingcap/kvproto/pkg/deadlock" "github.com/pingcap/log" "go.uber.org/zap" ) @@ -54,13 +55,16 @@ type txnKeyHashPair struct { txn uint64 keyHash uint64 registerTime time.Time + diagCtx diagnosticContext +} + +type diagnosticContext struct { + key []byte + resourceGroupTag []byte } func (p *txnKeyHashPair) isExpired(ttl time.Duration, nowTime time.Time) bool { - if p.registerTime.Add(ttl).Before(nowTime) { - return true - } - return false + return p.registerTime.Add(ttl).Before(nowTime) } // NewDetector creates a new Detector. @@ -75,13 +79,27 @@ func NewDetector(ttl time.Duration, urgentSize uint64, expireInterval time.Durat } // Detect detects deadlock for the sourceTxn on a locked key. -func (d *Detector) Detect(sourceTxn, waitForTxn, keyHash uint64) *ErrDeadlock { +func (d *Detector) Detect(sourceTxn, waitForTxn, keyHash uint64, diagCtx diagnosticContext) *ErrDeadlock { d.lock.Lock() nowTime := time.Now() d.activeExpire(nowTime) err := d.doDetect(nowTime, sourceTxn, waitForTxn) if err == nil { - d.register(sourceTxn, waitForTxn, keyHash) + d.register(sourceTxn, waitForTxn, keyHash, diagCtx) + } else { + // Reverse the wait chain so that the order will be each one waiting for the next one, and append the current + // entry that finally caused the deadlock. + for i := 0; i < len(err.WaitChain)/2; i++ { + j := len(err.WaitChain) - i - 1 + err.WaitChain[i], err.WaitChain[j] = err.WaitChain[j], err.WaitChain[i] + } + err.WaitChain = append(err.WaitChain, &deadlockpb.WaitForEntry{ + Txn: sourceTxn, + Key: diagCtx.key, + KeyHash: keyHash, + ResourceGroupTag: diagCtx.resourceGroupTag, + WaitForTxn: waitForTxn, + }) } d.lock.Unlock() return err @@ -103,9 +121,26 @@ func (d *Detector) doDetect(nowTime time.Time, sourceTxn, waitForTxn uint64) *Er continue } if keyHashPair.txn == sourceTxn { - return &ErrDeadlock{DeadlockKeyHash: keyHashPair.keyHash} + return &ErrDeadlock{DeadlockKeyHash: keyHashPair.keyHash, + WaitChain: []*deadlockpb.WaitForEntry{ + { + Txn: waitForTxn, + Key: keyHashPair.diagCtx.key, + KeyHash: keyHashPair.keyHash, + ResourceGroupTag: keyHashPair.diagCtx.resourceGroupTag, + WaitForTxn: keyHashPair.txn, + }, + }, + } } if err := d.doDetect(nowTime, sourceTxn, keyHashPair.txn); err != nil { + err.WaitChain = append(err.WaitChain, &deadlockpb.WaitForEntry{ + Txn: waitForTxn, + Key: keyHashPair.diagCtx.key, + KeyHash: keyHashPair.keyHash, + ResourceGroupTag: keyHashPair.diagCtx.resourceGroupTag, + WaitForTxn: keyHashPair.txn, + }) return err } } @@ -115,9 +150,9 @@ func (d *Detector) doDetect(nowTime time.Time, sourceTxn, waitForTxn uint64) *Er return nil } -func (d *Detector) register(sourceTxn, waitForTxn, keyHash uint64) { +func (d *Detector) register(sourceTxn, waitForTxn, keyHash uint64, diagCtx diagnosticContext) { val := d.waitForMap[sourceTxn] - pair := txnKeyHashPair{txn: waitForTxn, keyHash: keyHash, registerTime: time.Now()} + pair := txnKeyHashPair{txn: waitForTxn, keyHash: keyHash, registerTime: time.Now(), diagCtx: diagCtx} if val == nil { newList := &txnList{txns: list.New()} newList.txns.PushBack(&pair) diff --git a/store/mockstore/unistore/tikv/detector_test.go b/store/mockstore/unistore/tikv/detector_test.go index 1768cc377ec7c..c47260f886275 100644 --- a/store/mockstore/unistore/tikv/detector_test.go +++ b/store/mockstore/unistore/tikv/detector_test.go @@ -26,11 +26,11 @@ package tikv import ( - "fmt" "testing" "time" . "github.com/pingcap/check" + deadlockpb "github.com/pingcap/kvproto/pkg/deadlock" ) func TestT(t *testing.T) { @@ -42,19 +42,38 @@ var _ = Suite(&testDeadlockSuite{}) type testDeadlockSuite struct{} func (s *testDeadlockSuite) TestDeadlock(c *C) { + makeDiagCtx := func(key string, resourceGroupTag string) diagnosticContext { + return diagnosticContext{ + key: []byte(key), + resourceGroupTag: []byte(resourceGroupTag), + } + } + checkWaitChainEntry := func(entry *deadlockpb.WaitForEntry, txn, waitForTxn uint64, key, resourceGroupTag string) { + c.Assert(entry.Txn, Equals, txn) + c.Assert(entry.WaitForTxn, Equals, waitForTxn) + c.Assert(string(entry.Key), Equals, key) + c.Assert(string(entry.ResourceGroupTag), Equals, resourceGroupTag) + } + ttl := 50 * time.Millisecond expireInterval := 100 * time.Millisecond urgentSize := uint64(1) detector := NewDetector(ttl, urgentSize, expireInterval) - err := detector.Detect(1, 2, 100) + err := detector.Detect(1, 2, 100, makeDiagCtx("k1", "tag1")) c.Assert(err, IsNil) c.Assert(detector.totalSize, Equals, uint64(1)) - err = detector.Detect(2, 3, 200) + err = detector.Detect(2, 3, 200, makeDiagCtx("k2", "tag2")) c.Assert(err, IsNil) c.Assert(detector.totalSize, Equals, uint64(2)) - err = detector.Detect(3, 1, 300) + err = detector.Detect(3, 1, 300, makeDiagCtx("k3", "tag3")) c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, fmt.Sprintf("deadlock")) + c.Assert(err.Error(), Equals, "deadlock") + c.Assert(len(err.WaitChain), Equals, 3) + // The order of entries in the wait chain is specific: each item is waiting for the next one. + checkWaitChainEntry(err.WaitChain[0], 1, 2, "k1", "tag1") + checkWaitChainEntry(err.WaitChain[1], 2, 3, "k2", "tag2") + checkWaitChainEntry(err.WaitChain[2], 3, 1, "k3", "tag3") + c.Assert(detector.totalSize, Equals, uint64(2)) detector.CleanUp(2) list2 := detector.waitForMap[2] @@ -62,20 +81,21 @@ func (s *testDeadlockSuite) TestDeadlock(c *C) { c.Assert(detector.totalSize, Equals, uint64(1)) // After cycle is broken, no deadlock now. - err = detector.Detect(3, 1, 300) + diagCtx := diagnosticContext{} + err = detector.Detect(3, 1, 300, diagCtx) c.Assert(err, IsNil) list3 := detector.waitForMap[3] c.Assert(list3.txns.Len(), Equals, 1) c.Assert(detector.totalSize, Equals, uint64(2)) // Different keyHash grows the list. - err = detector.Detect(3, 1, 400) + err = detector.Detect(3, 1, 400, diagCtx) c.Assert(err, IsNil) c.Assert(list3.txns.Len(), Equals, 2) c.Assert(detector.totalSize, Equals, uint64(3)) // Same waitFor and key hash doesn't grow the list. - err = detector.Detect(3, 1, 400) + err = detector.Detect(3, 1, 400, diagCtx) c.Assert(err, IsNil) c.Assert(list3.txns.Len(), Equals, 2) c.Assert(detector.totalSize, Equals, uint64(3)) @@ -90,7 +110,7 @@ func (s *testDeadlockSuite) TestDeadlock(c *C) { // after 100ms, all entries expired, detect non exist edges time.Sleep(100 * time.Millisecond) - err = detector.Detect(100, 200, 100) + err = detector.Detect(100, 200, 100, diagCtx) c.Assert(err, IsNil) c.Assert(detector.totalSize, Equals, uint64(1)) c.Assert(len(detector.waitForMap), Equals, 1) @@ -98,7 +118,7 @@ func (s *testDeadlockSuite) TestDeadlock(c *C) { // expired entry should not report deadlock, detect will remove this entry // not dependent on expire check interval time.Sleep(60 * time.Millisecond) - err = detector.Detect(200, 100, 200) + err = detector.Detect(200, 100, 200, diagCtx) c.Assert(err, IsNil) c.Assert(detector.totalSize, Equals, uint64(1)) c.Assert(len(detector.waitForMap), Equals, 1) diff --git a/store/mockstore/unistore/tikv/errors.go b/store/mockstore/unistore/tikv/errors.go index 01d28fb73c896..bce76319320c9 100644 --- a/store/mockstore/unistore/tikv/errors.go +++ b/store/mockstore/unistore/tikv/errors.go @@ -16,6 +16,7 @@ package tikv import ( "fmt" + deadlockpb "github.com/pingcap/kvproto/pkg/deadlock" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/store/mockstore/unistore/tikv/mvcc" ) @@ -90,6 +91,7 @@ type ErrDeadlock struct { LockKey []byte LockTS uint64 DeadlockKeyHash uint64 + WaitChain []*deadlockpb.WaitForEntry } func (e ErrDeadlock) Error() string { diff --git a/store/mockstore/unistore/tikv/mvcc.go b/store/mockstore/unistore/tikv/mvcc.go index 4e3eb4f7d7df8..fe5a75b549945 100644 --- a/store/mockstore/unistore/tikv/mvcc.go +++ b/store/mockstore/unistore/tikv/mvcc.go @@ -239,7 +239,11 @@ func (store *MVCCStore) PessimisticLock(reqCtx *requestCtx, req *kvrpcpb.Pessimi for _, m := range mutations { lock, err := store.checkConflictInLockStore(reqCtx, m, startTS) if err != nil { - return store.handleCheckPessimisticErr(startTS, err, req.IsFirstLock, req.WaitTimeout) + var resourceGroupTag []byte = nil + if req.Context != nil { + resourceGroupTag = req.Context.ResourceGroupTag + } + return store.handleCheckPessimisticErr(startTS, err, req.IsFirstLock, req.WaitTimeout, m.Key, resourceGroupTag) } if lock != nil { if lock.Op != uint8(kvrpcpb.Op_PessimisticLock) { @@ -533,11 +537,13 @@ func (store *MVCCStore) CheckSecondaryLocks(reqCtx *requestCtx, keys [][]byte, s func (store *MVCCStore) normalizeWaitTime(lockWaitTime int64) time.Duration { if lockWaitTime > store.conf.PessimisticTxn.WaitForLockTimeout { lockWaitTime = store.conf.PessimisticTxn.WaitForLockTimeout + } else if lockWaitTime == 0 { + lockWaitTime = store.conf.PessimisticTxn.WaitForLockTimeout } return time.Duration(lockWaitTime) * time.Millisecond } -func (store *MVCCStore) handleCheckPessimisticErr(startTS uint64, err error, isFirstLock bool, lockWaitTime int64) (*lockwaiter.Waiter, error) { +func (store *MVCCStore) handleCheckPessimisticErr(startTS uint64, err error, isFirstLock bool, lockWaitTime int64, key []byte, resourceGroupTag []byte) (*lockwaiter.Waiter, error) { if locked, ok := err.(*ErrLocked); ok { if lockWaitTime != lockwaiter.LockNoWait { keyHash := farm.Fingerprint64(locked.Key) @@ -546,7 +552,7 @@ func (store *MVCCStore) handleCheckPessimisticErr(startTS uint64, err error, isF log.S().Debugf("%d blocked by %d on key %d", startTS, lock.StartTS, keyHash) waiter := store.lockWaiterManager.NewWaiter(startTS, lock.StartTS, keyHash, waitTimeDuration) if !isFirstLock { - store.DeadlockDetectCli.Detect(startTS, lock.StartTS, keyHash) + store.DeadlockDetectCli.Detect(startTS, lock.StartTS, keyHash, key, resourceGroupTag) } return waiter, err } diff --git a/store/mockstore/unistore/tikv/mvcc_test.go b/store/mockstore/unistore/tikv/mvcc_test.go index 8819fe21ab398..c2564a5b4ebdf 100644 --- a/store/mockstore/unistore/tikv/mvcc_test.go +++ b/store/mockstore/unistore/tikv/mvcc_test.go @@ -16,7 +16,6 @@ package tikv import ( "bytes" "fmt" - "io/ioutil" "math" "os" "path/filepath" @@ -89,11 +88,11 @@ func CreateTestDB(dbPath, LogPath string) (*badger.DB, error) { } func NewTestStore(dbPrefix string, logPrefix string, c *C) (*TestStore, error) { - dbPath, err := ioutil.TempDir("", dbPrefix) + dbPath, err := os.MkdirTemp("", dbPrefix) if err != nil { return nil, err } - LogPath, err := ioutil.TempDir("", logPrefix) + LogPath, err := os.MkdirTemp("", logPrefix) if err != nil { return nil, err } diff --git a/store/mockstore/unistore/tikv/server.go b/store/mockstore/unistore/tikv/server.go index f571ff4fe963f..8d8d2553fc823 100644 --- a/store/mockstore/unistore/tikv/server.go +++ b/store/mockstore/unistore/tikv/server.go @@ -21,7 +21,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/coprocessor" - "github.com/pingcap/kvproto/pkg/coprocessor_v2" deadlockPb "github.com/pingcap/kvproto/pkg/deadlock" "github.com/pingcap/kvproto/pkg/errorpb" "github.com/pingcap/kvproto/pkg/kvrpcpb" @@ -217,6 +216,7 @@ func (svr *Server) KvPessimisticLock(ctx context.Context, req *kvrpcpb.Pessimist LockKey: errLocked.Key, LockTS: errLocked.Lock.StartTS, DeadlockKeyHash: result.DeadlockResp.DeadlockKeyHash, + WaitChain: result.DeadlockResp.WaitChain, } resp.Errors, resp.RegionError = convertToPBErrors(deadlockErr) return resp, nil @@ -617,6 +617,11 @@ func (svr *Server) BatchCoprocessor(req *coprocessor.BatchRequest, batchCopServe return nil } +// RawCoprocessor implements implements the tikvpb.TikvServer interface. +func (svr *Server) RawCoprocessor(context.Context, *kvrpcpb.RawCoprocessorRequest) (*kvrpcpb.RawCoprocessorResponse, error) { + panic("unimplemented") +} + func (mrm *MockRegionManager) removeMPPTaskHandler(taskID int64, storeID uint64) error { set := mrm.getMPPTaskSet(storeID) if set == nil { @@ -961,16 +966,16 @@ func (svr *Server) RawCompareAndSwap(context.Context, *kvrpcpb.RawCASRequest) (* panic("implement me") } -// CoprocessorV2 implements the tikvpb.TikvServer interface. -func (svr *Server) CoprocessorV2(context.Context, *coprocessor_v2.RawCoprocessorRequest) (*coprocessor_v2.RawCoprocessorResponse, error) { - panic("implement me") -} - // GetStoreSafeTS implements the tikvpb.TikvServer interface. func (svr *Server) GetStoreSafeTS(context.Context, *kvrpcpb.StoreSafeTSRequest) (*kvrpcpb.StoreSafeTSResponse, error) { return &kvrpcpb.StoreSafeTSResponse{}, nil } +// GetLockWaitInfo implements the tikvpb.TikvServer interface. +func (svr *Server) GetLockWaitInfo(context.Context, *kvrpcpb.GetLockWaitInfoRequest) (*kvrpcpb.GetLockWaitInfoResponse, error) { + panic("unimplemented") +} + func convertToKeyError(err error) *kvrpcpb.KeyError { if err == nil { return nil @@ -1006,6 +1011,7 @@ func convertToKeyError(err error) *kvrpcpb.KeyError { LockKey: x.LockKey, LockTs: x.LockTS, DeadlockKeyHash: x.DeadlockKeyHash, + WaitChain: x.WaitChain, }, } case *ErrCommitExpire: diff --git a/store/store_test.go b/store/store_test.go index 627a214badee7..3f4a44cecc189 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -26,7 +26,6 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/store/mockstore" - "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/testleak" ) @@ -543,7 +542,7 @@ func (s *testKVSuite) TestDBClose(c *C) { err = txn.Commit(context.Background()) c.Assert(err, IsNil) - ver, err := store.CurrentVersion(oracle.GlobalTxnScope) + ver, err := store.CurrentVersion(kv.GlobalTxnScope) c.Assert(err, IsNil) c.Assert(kv.MaxVersion.Cmp(ver), Equals, 1) diff --git a/store/tikv/2pc.go b/store/tikv/2pc.go index 8703b1861c65d..235d60203d940 100644 --- a/store/tikv/2pc.go +++ b/store/tikv/2pc.go @@ -27,14 +27,16 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" - pb "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/parser/terror" + "github.com/pingcap/tidb/store/tikv/client" "github.com/pingcap/tidb/store/tikv/config" tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/oracle" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/store/tikv/unionstore" "github.com/pingcap/tidb/store/tikv/util" @@ -42,6 +44,9 @@ import ( zap "go.uber.org/zap" ) +// If the duration of a single request exceeds the slowRequestThreshold, a warning log will be logged. +const slowRequestThreshold = time.Minute + type twoPhaseCommitAction interface { handleSingleBatch(*twoPhaseCommitter, *Backoffer, batchMutations) error tiKVTxnRegionsNumHistogram() prometheus.Observer @@ -61,7 +66,7 @@ type twoPhaseCommitter struct { mutations *memBufferMutations lockTTL uint64 commitTS uint64 - priority pb.CommandPri + priority kvrpcpb.CommandPri sessionID uint64 // sessionID is used for log. cleanWg sync.WaitGroup detail unsafe.Pointer @@ -96,6 +101,7 @@ type twoPhaseCommitter struct { maxCommitTS uint64 prewriteStarted bool prewriteCancelled uint32 + prewriteFailed uint32 useOnePC uint32 onePCCommitTS uint64 @@ -106,6 +112,8 @@ type twoPhaseCommitter struct { doingAmend bool binlog BinlogExecutor + + resourceGroupTag []byte } type memBufferMutations struct { @@ -141,8 +149,8 @@ func (m *memBufferMutations) GetValue(i int) []byte { return v } -func (m *memBufferMutations) GetOp(i int) pb.Op { - return pb.Op(m.handles[i].UserData >> 1) +func (m *memBufferMutations) GetOp(i int) kvrpcpb.Op { + return kvrpcpb.Op(m.handles[i].UserData >> 1) } func (m *memBufferMutations) IsPessimisticLock(i int) bool { @@ -156,7 +164,7 @@ func (m *memBufferMutations) Slice(from, to int) CommitterMutations { } } -func (m *memBufferMutations) Push(op pb.Op, isPessimisticLock bool, handle unionstore.MemKeyHandle) { +func (m *memBufferMutations) Push(op kvrpcpb.Op, isPessimisticLock bool, handle unionstore.MemKeyHandle) { aux := uint16(op) << 1 if isPessimisticLock { aux |= 1 @@ -170,7 +178,7 @@ type CommitterMutations interface { Len() int GetKey(i int) []byte GetKeys() [][]byte - GetOp(i int) pb.Op + GetOp(i int) kvrpcpb.Op GetValue(i int) []byte IsPessimisticLock(i int) bool Slice(from, to int) CommitterMutations @@ -178,7 +186,7 @@ type CommitterMutations interface { // PlainMutations contains transaction operations. type PlainMutations struct { - ops []pb.Op + ops []kvrpcpb.Op keys [][]byte values [][]byte isPessimisticLock []bool @@ -187,7 +195,7 @@ type PlainMutations struct { // NewPlainMutations creates a PlainMutations object with sizeHint reserved. func NewPlainMutations(sizeHint int) PlainMutations { return PlainMutations{ - ops: make([]pb.Op, 0, sizeHint), + ops: make([]kvrpcpb.Op, 0, sizeHint), keys: make([][]byte, 0, sizeHint), values: make([][]byte, 0, sizeHint), isPessimisticLock: make([]bool, 0, sizeHint), @@ -211,7 +219,7 @@ func (c *PlainMutations) Slice(from, to int) CommitterMutations { } // Push another mutation into mutations. -func (c *PlainMutations) Push(op pb.Op, key []byte, value []byte, isPessimisticLock bool) { +func (c *PlainMutations) Push(op kvrpcpb.Op, key []byte, value []byte, isPessimisticLock bool) { c.ops = append(c.ops, op) c.keys = append(c.keys, key) c.values = append(c.values, value) @@ -234,7 +242,7 @@ func (c *PlainMutations) GetKeys() [][]byte { } // GetOps returns the key ops. -func (c *PlainMutations) GetOps() []pb.Op { +func (c *PlainMutations) GetOps() []kvrpcpb.Op { return c.ops } @@ -249,7 +257,7 @@ func (c *PlainMutations) GetPessimisticFlags() []bool { } // GetOp returns the key op at index. -func (c *PlainMutations) GetOp(i int) pb.Op { +func (c *PlainMutations) GetOp(i int) kvrpcpb.Op { return c.ops[i] } @@ -268,7 +276,7 @@ func (c *PlainMutations) IsPessimisticLock(i int) bool { // PlainMutation represents a single transaction operation. type PlainMutation struct { - KeyOp pb.Op + KeyOp kvrpcpb.Op Key []byte Value []byte IsPessimisticLock bool @@ -335,13 +343,13 @@ func (c *twoPhaseCommitter) initKeysAndMutations() error { key := it.Key() flags := it.Flags() var value []byte - var op pb.Op + var op kvrpcpb.Op if !it.HasValue() { if !flags.HasLocked() { continue } - op = pb.Op_Lock + op = kvrpcpb.Op_Lock lockCnt++ } else { value = it.Value() @@ -354,12 +362,12 @@ func (c *twoPhaseCommitter) initKeysAndMutations() error { // If the key was locked before, we should prewrite the lock even if // the KV needn't be committed according to the filter. Otherwise, we // were forgetting removing pessimistic locks added before. - op = pb.Op_Lock + op = kvrpcpb.Op_Lock lockCnt++ } else { - op = pb.Op_Put + op = kvrpcpb.Op_Put if flags.HasPresumeKeyNotExists() { - op = pb.Op_Insert + op = kvrpcpb.Op_Insert } putCnt++ } @@ -367,13 +375,13 @@ func (c *twoPhaseCommitter) initKeysAndMutations() error { if !txn.IsPessimistic() && flags.HasPresumeKeyNotExists() { // delete-your-writes keys in optimistic txn need check not exists in prewrite-phase // due to `Op_CheckNotExists` doesn't prewrite lock, so mark those keys should not be used in commit-phase. - op = pb.Op_CheckNotExists + op = kvrpcpb.Op_CheckNotExists checkCnt++ memBuf.UpdateFlags(key, kv.SetPrewriteOnly) } else { // normal delete keys in optimistic txn can be delete without not exists checking // delete-your-writes keys in pessimistic txn can ensure must be no exists so can directly delete them - op = pb.Op_Del + op = kvrpcpb.Op_Del delCnt++ } } @@ -386,7 +394,7 @@ func (c *twoPhaseCommitter) initKeysAndMutations() error { c.mutations.Push(op, isPessimistic, it.Handle()) size += len(key) + len(value) - if len(c.primaryKey) == 0 && op != pb.Op_CheckNotExists { + if len(c.primaryKey) == 0 && op != kvrpcpb.Op_CheckNotExists { c.primaryKey = key } } @@ -427,6 +435,7 @@ func (c *twoPhaseCommitter) initKeysAndMutations() error { c.lockTTL = txnLockTTL(txn.startTime, size) c.priority = txn.priority.ToPB() c.syncLog = txn.syncLog + c.resourceGroupTag = txn.resourceGroupTag c.setDetail(commitDetail) return nil } @@ -443,7 +452,7 @@ func (c *twoPhaseCommitter) asyncSecondaries() [][]byte { secondaries := make([][]byte, 0, c.mutations.Len()) for i := 0; i < c.mutations.Len(); i++ { k := c.mutations.GetKey(i) - if bytes.Equal(k, c.primary()) || c.mutations.GetOp(i) == pb.Op_CheckNotExists { + if bytes.Equal(k, c.primary()) || c.mutations.GetOp(i) == kvrpcpb.Op_CheckNotExists { continue } secondaries = append(secondaries, k) @@ -499,9 +508,46 @@ func (c *twoPhaseCommitter) doActionOnMutations(bo *Backoffer, action twoPhaseCo return c.doActionOnGroupMutations(bo, action, groups) } +type groupedMutations struct { + region RegionVerID + mutations CommitterMutations +} + +// groupSortedMutationsByRegion separates keys into groups by their belonging Regions. +func groupSortedMutationsByRegion(c *RegionCache, bo *retry.Backoffer, m CommitterMutations) ([]groupedMutations, error) { + var ( + groups []groupedMutations + lastLoc *KeyLocation + ) + lastUpperBound := 0 + for i := 0; i < m.Len(); i++ { + if lastLoc == nil || !lastLoc.Contains(m.GetKey(i)) { + if lastLoc != nil { + groups = append(groups, groupedMutations{ + region: lastLoc.Region, + mutations: m.Slice(lastUpperBound, i), + }) + lastUpperBound = i + } + var err error + lastLoc, err = c.LocateKey(bo, m.GetKey(i)) + if err != nil { + return nil, errors.Trace(err) + } + } + } + if lastLoc != nil { + groups = append(groups, groupedMutations{ + region: lastLoc.Region, + mutations: m.Slice(lastUpperBound, m.Len()), + }) + } + return groups, nil +} + // groupMutations groups mutations by region, then checks for any large groups and in that case pre-splits the region. func (c *twoPhaseCommitter) groupMutations(bo *Backoffer, mutations CommitterMutations) ([]groupedMutations, error) { - groups, err := c.store.regionCache.groupSortedMutationsByRegion(bo, mutations) + groups, err := groupSortedMutationsByRegion(c.store.regionCache, bo, mutations) if err != nil { return nil, errors.Trace(err) } @@ -515,15 +561,14 @@ func (c *twoPhaseCommitter) groupMutations(bo *Backoffer, mutations CommitterMut logutil.BgLogger().Info("2PC detect large amount of mutations on a single region", zap.Uint64("region", group.region.GetID()), zap.Int("mutations count", group.mutations.Len())) - // Use context.Background, this time should not add up to Backoffer. - if c.store.preSplitRegion(context.Background(), group) { + if c.store.preSplitRegion(bo.GetCtx(), group) { didPreSplit = true } } } // Reload region cache again. if didPreSplit { - groups, err = c.store.regionCache.groupSortedMutationsByRegion(bo, mutations) + groups, err = groupSortedMutationsByRegion(c.store.regionCache, bo, mutations) if err != nil { return nil, errors.Trace(err) } @@ -542,7 +587,7 @@ func (c *twoPhaseCommitter) doActionOnGroupMutations(bo *Backoffer, action twoPh switch act := action.(type) { case actionPrewrite: // Do not update regionTxnSize on retries. They are not used when building a PrewriteRequest. - if len(bo.errors) == 0 { + if !act.retry { for _, group := range groups { c.regionTxnSize[group.region.id] = group.mutations.Len() } @@ -572,7 +617,7 @@ func (c *twoPhaseCommitter) doActionOnGroupMutations(bo *Backoffer, action twoPh valStr, ok := val.(string) if ok && c.sessionID > 0 { if firstIsPrimary && actionIsPessimiticLock { - logutil.Logger(bo.ctx).Warn("pessimisticLock failpoint", zap.String("valStr", valStr)) + logutil.Logger(bo.GetCtx()).Warn("pessimisticLock failpoint", zap.String("valStr", valStr)) switch valStr { case "pessimisticLockSkipPrimary": err = c.doActionOnBatches(bo, action, batchBuilder.allBatches()) @@ -587,7 +632,7 @@ func (c *twoPhaseCommitter) doActionOnGroupMutations(bo *Backoffer, action twoPh failpoint.Inject("pessimisticRollbackDoNth", func() { _, actionIsPessimisticRollback := action.(actionPessimisticRollback) if actionIsPessimisticRollback && c.sessionID > 0 { - logutil.Logger(bo.ctx).Warn("pessimisticRollbackDoNth failpoint") + logutil.Logger(bo.GetCtx()).Warn("pessimisticRollbackDoNth failpoint") failpoint.Return(nil) } }) @@ -607,16 +652,16 @@ func (c *twoPhaseCommitter) doActionOnGroupMutations(bo *Backoffer, action twoPh } // Already spawned a goroutine for async commit transaction. if actionIsCommit && !actionCommit.retry && !c.isAsyncCommit() { - secondaryBo := NewBackofferWithVars(context.Background(), int(atomic.LoadUint64(&CommitMaxBackoff)), c.txn.vars) + secondaryBo := retry.NewBackofferWithVars(context.Background(), CommitSecondaryMaxBackoff, c.txn.vars) go func() { if c.sessionID > 0 { failpoint.Inject("beforeCommitSecondaries", func(v failpoint.Value) { if s, ok := v.(string); !ok { - logutil.Logger(bo.ctx).Info("[failpoint] sleep 2s before commit secondary keys", + logutil.Logger(bo.GetCtx()).Info("[failpoint] sleep 2s before commit secondary keys", zap.Uint64("sessionID", c.sessionID), zap.Uint64("txnStartTS", c.startTS), zap.Uint64("txnCommitTS", c.commitTS)) time.Sleep(2 * time.Second) } else if s == "skip" { - logutil.Logger(bo.ctx).Info("[failpoint] injected skip committing secondaries", + logutil.Logger(bo.GetCtx()).Info("[failpoint] injected skip committing secondaries", zap.Uint64("sessionID", c.sessionID), zap.Uint64("txnStartTS", c.startTS), zap.Uint64("txnCommitTS", c.commitTS)) failpoint.Return() } @@ -722,10 +767,15 @@ func (tm *ttlManager) close() { close(tm.ch) } +const keepAliveMaxBackoff = 20000 // 20 seconds +const pessimisticLockMaxBackoff = 600000 // 10 minutes +const maxConsecutiveFailure = 10 + func (tm *ttlManager) keepAlive(c *twoPhaseCommitter) { // Ticker is set to 1/2 of the ManagedLockTTL. ticker := time.NewTicker(time.Duration(atomic.LoadUint64(&ManagedLockTTL)) * time.Millisecond / 2) defer ticker.Stop() + keepFail := 0 for { select { case <-tm.ch: @@ -735,23 +785,19 @@ func (tm *ttlManager) keepAlive(c *twoPhaseCommitter) { if tm.lockCtx != nil && tm.lockCtx.Killed != nil && atomic.LoadUint32(tm.lockCtx.Killed) != 0 { return } - bo := NewBackofferWithVars(context.Background(), pessimisticLockMaxBackoff, c.txn.vars) - now, err := c.store.GetOracle().GetTimestamp(bo.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) + bo := retry.NewBackofferWithVars(context.Background(), keepAliveMaxBackoff, c.txn.vars) + now, err := c.store.getTimestampWithRetry(bo, c.txn.GetScope()) if err != nil { - err1 := bo.Backoff(BoPDRPC, err) - if err1 != nil { - logutil.Logger(bo.ctx).Warn("keepAlive get tso fail", - zap.Error(err)) - return - } - continue + logutil.Logger(bo.GetCtx()).Warn("keepAlive get tso fail", + zap.Error(err)) + return } uptime := uint64(oracle.ExtractPhysical(now) - oracle.ExtractPhysical(c.startTS)) if uptime > config.GetGlobalConfig().MaxTxnTTL { // Checks maximum lifetime for the ttlManager, so when something goes wrong // the key will not be locked forever. - logutil.Logger(bo.ctx).Info("ttlManager live up to its lifetime", + logutil.Logger(bo.GetCtx()).Info("ttlManager live up to its lifetime", zap.Uint64("txnStartTS", c.startTS), zap.Uint64("uptime", uptime), zap.Uint64("maxTxnTTL", config.GetGlobalConfig().MaxTxnTTL)) @@ -765,24 +811,33 @@ func (tm *ttlManager) keepAlive(c *twoPhaseCommitter) { } newTTL := uptime + atomic.LoadUint64(&ManagedLockTTL) - logutil.Logger(bo.ctx).Info("send TxnHeartBeat", + logutil.Logger(bo.GetCtx()).Info("send TxnHeartBeat", zap.Uint64("startTS", c.startTS), zap.Uint64("newTTL", newTTL)) startTime := time.Now() - _, err = sendTxnHeartBeat(bo, c.store, c.primary(), c.startTS, newTTL) + _, stopHeartBeat, err := sendTxnHeartBeat(bo, c.store, c.primary(), c.startTS, newTTL) if err != nil { + keepFail++ metrics.TxnHeartBeatHistogramError.Observe(time.Since(startTime).Seconds()) - logutil.Logger(bo.ctx).Warn("send TxnHeartBeat failed", + logutil.Logger(bo.GetCtx()).Debug("send TxnHeartBeat failed", zap.Error(err), zap.Uint64("txnStartTS", c.startTS)) - return + if stopHeartBeat || keepFail > maxConsecutiveFailure { + logutil.Logger(bo.GetCtx()).Warn("stop TxnHeartBeat", + zap.Error(err), + zap.Int("consecutiveFailure", keepFail), + zap.Uint64("txnStartTS", c.startTS)) + return + } + continue } + keepFail = 0 metrics.TxnHeartBeatHistogramOK.Observe(time.Since(startTime).Seconds()) } } } -func sendTxnHeartBeat(bo *Backoffer, store *KVStore, primary []byte, startTS, ttl uint64) (uint64, error) { - req := tikvrpc.NewRequest(tikvrpc.CmdTxnHeartBeat, &pb.TxnHeartBeatRequest{ +func sendTxnHeartBeat(bo *Backoffer, store *KVStore, primary []byte, startTS, ttl uint64) (newTTL uint64, stopHeartBeat bool, err error) { + req := tikvrpc.NewRequest(tikvrpc.CmdTxnHeartBeat, &kvrpcpb.TxnHeartBeatRequest{ PrimaryLock: primary, StartVersion: startTS, AdviseLockTtl: ttl, @@ -790,31 +845,36 @@ func sendTxnHeartBeat(bo *Backoffer, store *KVStore, primary []byte, startTS, tt for { loc, err := store.GetRegionCache().LocateKey(bo, primary) if err != nil { - return 0, errors.Trace(err) + return 0, false, errors.Trace(err) } - resp, err := store.SendReq(bo, req, loc.Region, ReadTimeoutShort) + resp, err := store.SendReq(bo, req, loc.Region, client.ReadTimeoutShort) if err != nil { - return 0, errors.Trace(err) + return 0, false, errors.Trace(err) } regionErr, err := resp.GetRegionError() if err != nil { - return 0, errors.Trace(err) + return 0, false, errors.Trace(err) } if regionErr != nil { - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) - if err != nil { - return 0, errors.Trace(err) + // For other region error and the fake region error, backoff because + // there's something wrong. + // For the real EpochNotMatch error, don't backoff. + if regionErr.GetEpochNotMatch() == nil || isFakeRegionError(regionErr) { + err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) + if err != nil { + return 0, false, errors.Trace(err) + } } continue } if resp.Resp == nil { - return 0, errors.Trace(tikverr.ErrBodyMissing) + return 0, false, errors.Trace(tikverr.ErrBodyMissing) } - cmdResp := resp.Resp.(*pb.TxnHeartBeatResponse) + cmdResp := resp.Resp.(*kvrpcpb.TxnHeartBeatResponse) if keyErr := cmdResp.GetError(); keyErr != nil { - return 0, errors.Errorf("txn %d heartbeat fail, primary key = %v, err = %s", startTS, hex.EncodeToString(primary), extractKeyErr(keyErr)) + return 0, true, errors.Errorf("txn %d heartbeat fail, primary key = %v, err = %s", startTS, hex.EncodeToString(primary), extractKeyErr(keyErr)) } - return cmdResp.GetLockTtl(), nil + return cmdResp.GetLockTtl(), false, nil } } @@ -825,12 +885,10 @@ func (c *twoPhaseCommitter) checkAsyncCommit() bool { return false } - enableAsyncCommitOption := c.txn.us.GetOption(kv.EnableAsyncCommit) - enableAsyncCommit := enableAsyncCommitOption != nil && enableAsyncCommitOption.(bool) asyncCommitCfg := config.GetGlobalConfig().TiKVClient.AsyncCommit // TODO the keys limit need more tests, this value makes the unit test pass by now. // Async commit is not compatible with Binlog because of the non unique timestamp issue. - if c.sessionID > 0 && enableAsyncCommit && + if c.sessionID > 0 && c.txn.enableAsyncCommit && uint(c.mutations.Len()) <= asyncCommitCfg.KeysLimit && !c.shouldWriteBinlog() { totalKeySize := uint64(0) @@ -856,9 +914,7 @@ func (c *twoPhaseCommitter) checkOnePC() bool { } func (c *twoPhaseCommitter) needLinearizability() bool { - GuaranteeLinearizabilityOption := c.txn.us.GetOption(kv.GuaranteeLinearizability) - // by default, guarantee - return GuaranteeLinearizabilityOption == nil || GuaranteeLinearizabilityOption.(bool) + return !c.txn.causalConsistency } func (c *twoPhaseCommitter) isAsyncCommit() bool { @@ -893,6 +949,14 @@ func (c *twoPhaseCommitter) checkOnePCFallBack(action twoPhaseCommitAction, batc } } +const ( + cleanupMaxBackoff = 20000 + tsoMaxBackoff = 15000 +) + +// VeryLongMaxBackoff is the max sleep time of transaction commit. +var VeryLongMaxBackoff = uint64(600000) // 10mins + func (c *twoPhaseCommitter) cleanup(ctx context.Context) { c.cleanWg.Add(1) go func() { @@ -903,12 +967,12 @@ func (c *twoPhaseCommitter) cleanup(ctx context.Context) { failpoint.Return() }) - cleanupKeysCtx := context.WithValue(context.Background(), TxnStartKey, ctx.Value(TxnStartKey)) + cleanupKeysCtx := context.WithValue(context.Background(), retry.TxnStartKey, ctx.Value(retry.TxnStartKey)) var err error if !c.isOnePC() { - err = c.cleanupMutations(NewBackofferWithVars(cleanupKeysCtx, cleanupMaxBackoff, c.txn.vars), c.mutations) + err = c.cleanupMutations(retry.NewBackofferWithVars(cleanupKeysCtx, cleanupMaxBackoff, c.txn.vars), c.mutations) } else if c.isPessimistic { - err = c.pessimisticRollbackMutations(NewBackofferWithVars(cleanupKeysCtx, cleanupMaxBackoff, c.txn.vars), c.mutations) + err = c.pessimisticRollbackMutations(retry.NewBackofferWithVars(cleanupKeysCtx, cleanupMaxBackoff, c.txn.vars), c.mutations) } if err != nil { @@ -989,20 +1053,30 @@ func (c *twoPhaseCommitter) execute(ctx context.Context) (err error) { c.setOnePC(true) c.hasTriedOnePC = true } + + // TODO(youjiali1995): It's better to use different maxSleep for different operations + // and distinguish permanent errors from temporary errors, for example: + // - If all PDs are down, all requests to PD will fail due to network error. + // The maxSleep should't be very long in this case. + // - If the region isn't found in PD, it's possible the reason is write-stall. + // The maxSleep can be long in this case. + bo := retry.NewBackofferWithVars(ctx, int(atomic.LoadUint64(&VeryLongMaxBackoff)), c.txn.vars) + // If we want to use async commit or 1PC and also want linearizability across // all nodes, we have to make sure the commit TS of this transaction is greater // than the snapshot TS of all existent readers. So we get a new timestamp - // from PD as our MinCommitTS. + // from PD and plus one as our MinCommitTS. if commitTSMayBeCalculated && c.needLinearizability() { failpoint.Inject("getMinCommitTSFromTSO", nil) - minCommitTS, err := c.store.oracle.GetTimestamp(ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) + latestTS, err := c.store.getTimestampWithRetry(bo, c.txn.GetScope()) // If we fail to get a timestamp from PD, we just propagate the failure // instead of falling back to the normal 2PC because a normal 2PC will // also be likely to fail due to the same timestamp issue. if err != nil { return errors.Trace(err) } - c.minCommitTS = minCommitTS + // Plus 1 to avoid producing the same commit TS with previously committed transactions + c.minCommitTS = latestTS + 1 } // Calculate maxCommitTS if necessary if commitTSMayBeCalculated { @@ -1020,14 +1094,17 @@ func (c *twoPhaseCommitter) execute(ctx context.Context) (err error) { if c.shouldWriteBinlog() { binlogChan = c.binlog.Prewrite(ctx, c.primary()) } - prewriteBo := NewBackofferWithVars(ctx, PrewriteMaxBackoff, c.txn.vars) + start := time.Now() - err = c.prewriteMutations(prewriteBo, c.mutations) + err = c.prewriteMutations(bo, c.mutations) + // Return an undetermined error only if we don't know the transaction fails. + // If it fails due to a write conflict or a already existed unique key, we + // needn't return an undetermined error even if such an error is set. + if atomic.LoadUint32(&c.prewriteFailed) == 1 { + c.setUndeterminedErr(nil) + } if err != nil { - // TODO: Now we return an undetermined error as long as one of the prewrite - // RPCs fails. However, if there are multiple errors and some of the errors - // are not RPC failures, we can return the actual error instead of undetermined. if undeterminedErr := c.getUndeterminedErr(); undeterminedErr != nil { logutil.Logger(ctx).Error("2PC commit result undetermined", zap.Error(err), @@ -1039,12 +1116,16 @@ func (c *twoPhaseCommitter) execute(ctx context.Context) (err error) { commitDetail := c.getDetail() commitDetail.PrewriteTime = time.Since(start) - if prewriteBo.totalSleep > 0 { - atomic.AddInt64(&commitDetail.CommitBackoffTime, int64(prewriteBo.totalSleep)*int64(time.Millisecond)) + if bo.GetTotalSleep() > 0 { + boSleep := int64(bo.GetTotalSleep()) * int64(time.Millisecond) commitDetail.Mu.Lock() - commitDetail.Mu.BackoffTypes = append(commitDetail.Mu.BackoffTypes, prewriteBo.types...) + if boSleep > commitDetail.Mu.CommitBackoffTime { + commitDetail.Mu.CommitBackoffTime = boSleep + commitDetail.Mu.BackoffTypes = bo.GetTypes() + } commitDetail.Mu.Unlock() } + if binlogChan != nil { startWaitBinlog := time.Now() binlogWriteResult := <-binlogChan @@ -1096,7 +1177,7 @@ func (c *twoPhaseCommitter) execute(ctx context.Context) (err error) { } else { start = time.Now() logutil.Event(ctx, "start get commit ts") - commitTS, err = c.store.getTimestampWithRetry(NewBackofferWithVars(ctx, tsoMaxBackoff, c.txn.vars), c.txn.GetScope()) + commitTS, err = c.store.getTimestampWithRetry(retry.NewBackofferWithVars(ctx, tsoMaxBackoff, c.txn.vars), c.txn.GetScope()) if err != nil { logutil.Logger(ctx).Warn("2PC get commitTS failed", zap.Error(err), @@ -1179,7 +1260,7 @@ func (c *twoPhaseCommitter) execute(ctx context.Context) (err error) { failpoint.Inject("asyncCommitDoNothing", func() { failpoint.Return() }) - commitBo := NewBackofferWithVars(ctx, int(atomic.LoadUint64(&CommitMaxBackoff)), c.txn.vars) + commitBo := retry.NewBackofferWithVars(ctx, CommitSecondaryMaxBackoff, c.txn.vars) err := c.commitMutations(commitBo, c.mutations) if err != nil { logutil.Logger(ctx).Warn("2PC async commit failed", zap.Uint64("sessionID", c.sessionID), @@ -1195,13 +1276,14 @@ func (c *twoPhaseCommitter) commitTxn(ctx context.Context, commitDetail *util.Co c.txn.GetMemBuffer().DiscardValues() start := time.Now() - commitBo := NewBackofferWithVars(ctx, int(atomic.LoadUint64(&CommitMaxBackoff)), c.txn.vars) + // Use the VeryLongMaxBackoff to commit the primary key. + commitBo := retry.NewBackofferWithVars(ctx, int(atomic.LoadUint64(&VeryLongMaxBackoff)), c.txn.vars) err := c.commitMutations(commitBo, c.mutations) commitDetail.CommitTime = time.Since(start) - if commitBo.totalSleep > 0 { - atomic.AddInt64(&commitDetail.CommitBackoffTime, int64(commitBo.totalSleep)*int64(time.Millisecond)) + if commitBo.GetTotalSleep() > 0 { commitDetail.Mu.Lock() - commitDetail.Mu.BackoffTypes = append(commitDetail.Mu.BackoffTypes, commitBo.types...) + commitDetail.Mu.CommitBackoffTime += int64(commitBo.GetTotalSleep()) * int64(time.Millisecond) + commitDetail.Mu.BackoffTypes = append(commitDetail.Mu.BackoffTypes, commitBo.GetTypes()...) commitDetail.Mu.Unlock() } if err != nil { @@ -1286,7 +1368,7 @@ func (c *twoPhaseCommitter) amendPessimisticLock(ctx context.Context, addMutatio retryLimit := config.GetGlobalConfig().PessimisticTxn.MaxRetryCount var err error for tryTimes < retryLimit { - pessimisticLockBo := NewBackofferWithVars(ctx, pessimisticLockMaxBackoff, c.txn.vars) + pessimisticLockBo := retry.NewBackofferWithVars(ctx, pessimisticLockMaxBackoff, c.txn.vars) err = c.pessimisticLockMutations(pessimisticLockBo, lCtx, &keysNeedToLock) if err != nil { // KeysNeedToLock won't change, so don't async rollback pessimistic locks here for write conflict. @@ -1332,7 +1414,7 @@ func (c *twoPhaseCommitter) tryAmendTxn(ctx context.Context, startInfoSchema Sch return false, err } if c.prewriteStarted { - prewriteBo := NewBackofferWithVars(ctx, PrewriteMaxBackoff, c.txn.vars) + prewriteBo := retry.NewBackofferWithVars(ctx, PrewriteMaxBackoff, c.txn.vars) err = c.prewriteMutations(prewriteBo, addMutations) if err != nil { logutil.Logger(ctx).Warn("amend prewrite has failed", zap.Error(err), zap.Uint64("txnStartTS", c.startTS)) @@ -1346,7 +1428,7 @@ func (c *twoPhaseCommitter) tryAmendTxn(ctx context.Context, startInfoSchema Sch key := addMutations.GetKey(i) op := addMutations.GetOp(i) var err error - if op == pb.Op_Del { + if op == kvrpcpb.Op_Del { err = memBuf.Delete(key) } else { err = memBuf.Set(key, addMutations.GetValue(i)) @@ -1365,7 +1447,7 @@ func (c *twoPhaseCommitter) tryAmendTxn(ctx context.Context, startInfoSchema Sch func (c *twoPhaseCommitter) getCommitTS(ctx context.Context, commitDetail *util.CommitDetails) (uint64, error) { start := time.Now() logutil.Event(ctx, "start get commit ts") - commitTS, err := c.store.getTimestampWithRetry(NewBackofferWithVars(ctx, tsoMaxBackoff, c.txn.vars), c.txn.GetScope()) + commitTS, err := c.store.getTimestampWithRetry(retry.NewBackofferWithVars(ctx, tsoMaxBackoff, c.txn.vars), c.txn.GetScope()) if err != nil { logutil.Logger(ctx).Warn("2PC get commitTS failed", zap.Error(err), @@ -1427,7 +1509,7 @@ func (c *twoPhaseCommitter) checkSchemaValid(ctx context.Context, checkTS uint64 func (c *twoPhaseCommitter) calculateMaxCommitTS(ctx context.Context) error { // Amend txn with current time first, then we can make sure we have another SafeWindow time to commit - currentTS := oracle.EncodeTSO(int64(time.Since(c.txn.startTime)/time.Millisecond)) + c.startTS + currentTS := oracle.ComposeTS(int64(time.Since(c.txn.startTime)/time.Millisecond), 0) + c.startTS _, _, err := c.checkSchemaValid(ctx, currentTS, c.txn.schemaVer, true) if err != nil { logutil.Logger(ctx).Info("Schema changed for async commit txn", @@ -1437,7 +1519,7 @@ func (c *twoPhaseCommitter) calculateMaxCommitTS(ctx context.Context) error { } safeWindow := config.GetGlobalConfig().TiKVClient.AsyncCommit.SafeWindow - maxCommitTS := oracle.EncodeTSO(int64(safeWindow/time.Millisecond)) + currentTS + maxCommitTS := oracle.ComposeTS(int64(safeWindow/time.Millisecond), 0) + currentTS logutil.BgLogger().Debug("calculate MaxCommitTS", zap.Time("startTime", c.txn.startTime), zap.Duration("safeWindow", safeWindow), @@ -1461,6 +1543,20 @@ type batchMutations struct { mutations CommitterMutations isPrimary bool } + +func (b *batchMutations) relocate(bo *Backoffer, c *RegionCache) (bool, error) { + begin, end := b.mutations.GetKey(0), b.mutations.GetKey(b.mutations.Len()-1) + loc, err := c.LocateKey(bo, begin) + if err != nil { + return false, errors.Trace(err) + } + if !loc.Contains(end) { + return false, nil + } + b.region = loc.Region + return true, nil +} + type batched struct { batches []batchMutations primaryIdx int @@ -1545,7 +1641,7 @@ type batchExecutor struct { func newBatchExecutor(rateLimit int, committer *twoPhaseCommitter, action twoPhaseCommitAction, backoffer *Backoffer) *batchExecutor { return &batchExecutor{rateLimit, nil, committer, - action, backoffer, 1 * time.Millisecond} + action, backoffer, 0} } // initUtils do initialize batchExecutor related policies like rateLimit util @@ -1579,20 +1675,23 @@ func (batchExe *batchExecutor) startWorker(exitCh chan struct{}, ch chan error, singleBatchBackoffer, singleBatchCancel = batchExe.backoffer.Fork() defer singleBatchCancel() } - beforeSleep := singleBatchBackoffer.totalSleep ch <- batchExe.action.handleSingleBatch(batchExe.committer, singleBatchBackoffer, batch) commitDetail := batchExe.committer.getDetail() - if commitDetail != nil { // lock operations of pessimistic-txn will let commitDetail be nil - if delta := singleBatchBackoffer.totalSleep - beforeSleep; delta > 0 { - atomic.AddInt64(&commitDetail.CommitBackoffTime, int64(singleBatchBackoffer.totalSleep-beforeSleep)*int64(time.Millisecond)) - commitDetail.Mu.Lock() - commitDetail.Mu.BackoffTypes = append(commitDetail.Mu.BackoffTypes, singleBatchBackoffer.types...) - commitDetail.Mu.Unlock() + // For prewrite, we record the max backoff time + if _, ok := batchExe.action.(actionPrewrite); ok { + commitDetail.Mu.Lock() + boSleep := int64(singleBatchBackoffer.GetTotalSleep()) * int64(time.Millisecond) + if boSleep > commitDetail.Mu.CommitBackoffTime { + commitDetail.Mu.CommitBackoffTime = boSleep + commitDetail.Mu.BackoffTypes = singleBatchBackoffer.GetTypes() } + commitDetail.Mu.Unlock() } + // Backoff time in the 2nd phase of a non-async-commit txn is added + // in the commitTxn method, so we don't add it here. }() } else { - logutil.Logger(batchExe.backoffer.ctx).Info("break startWorker", + logutil.Logger(batchExe.backoffer.GetCtx()).Info("break startWorker", zap.Stringer("action", batchExe.action), zap.Int("batch size", len(batches)), zap.Int("index", idx)) break @@ -1605,7 +1704,7 @@ func (batchExe *batchExecutor) process(batches []batchMutations) error { var err error err = batchExe.initUtils() if err != nil { - logutil.Logger(batchExe.backoffer.ctx).Error("batchExecutor initUtils failed", zap.Error(err)) + logutil.Logger(batchExe.backoffer.GetCtx()).Error("batchExecutor initUtils failed", zap.Error(err)) return err } @@ -1622,14 +1721,14 @@ func (batchExe *batchExecutor) process(batches []batchMutations) error { // check results for i := 0; i < len(batches); i++ { if e := <-ch; e != nil { - logutil.Logger(batchExe.backoffer.ctx).Debug("2PC doActionOnBatch failed", + logutil.Logger(batchExe.backoffer.GetCtx()).Debug("2PC doActionOnBatch failed", zap.Uint64("session", batchExe.committer.sessionID), zap.Stringer("action type", batchExe.action), zap.Error(e), zap.Uint64("txnStartTS", batchExe.committer.startTS)) // Cancel other requests and return the first error. if cancel != nil { - logutil.Logger(batchExe.backoffer.ctx).Debug("2PC doActionOnBatch to cancel other actions", + logutil.Logger(batchExe.backoffer.GetCtx()).Debug("2PC doActionOnBatch to cancel other actions", zap.Uint64("session", batchExe.committer.sessionID), zap.Stringer("action type", batchExe.action), zap.Uint64("txnStartTS", batchExe.committer.startTS)) @@ -1642,7 +1741,9 @@ func (batchExe *batchExecutor) process(batches []batchMutations) error { } } close(exitCh) - metrics.TiKVTokenWaitDuration.Observe(float64(batchExe.tokenWaitDuration.Nanoseconds())) + if batchExe.tokenWaitDuration > 0 { + metrics.TiKVTokenWaitDuration.Observe(float64(batchExe.tokenWaitDuration.Nanoseconds())) + } return err } diff --git a/store/tikv/backoff.go b/store/tikv/backoff.go index e0115c9e3904a..34285a8a8d204 100644 --- a/store/tikv/backoff.go +++ b/store/tikv/backoff.go @@ -1,4 +1,4 @@ -// Copyright 2016 PingCAP, Inc. +// Copyright 2021 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,431 +15,72 @@ package tikv import ( "context" - "fmt" - "math" - "math/rand" - "strings" - "sync/atomic" - "time" - "github.com/opentracing/opentracing-go" - "github.com/pingcap/errors" - "github.com/pingcap/log" - tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/kv" - "github.com/pingcap/tidb/store/tikv/logutil" - "github.com/pingcap/tidb/store/tikv/metrics" - "github.com/pingcap/tidb/store/tikv/util" - "github.com/prometheus/client_golang/prometheus" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" + "github.com/pingcap/tidb/store/tikv/retry" ) -const ( - // NoJitter makes the backoff sequence strict exponential. - NoJitter = 1 + iota - // FullJitter applies random factors to strict exponential. - FullJitter - // EqualJitter is also randomized, but prevents very short sleeps. - EqualJitter - // DecorrJitter increases the maximum jitter based on the last random value. - DecorrJitter -) - -func (t BackoffType) metric() prometheus.Observer { - switch t { - // TODO: distinguish tikv and tiflash in metrics - case BoTiKVRPC, BoTiFlashRPC: - return metrics.BackoffHistogramRPC - case BoTxnLock: - return metrics.BackoffHistogramLock - case BoTxnLockFast: - return metrics.BackoffHistogramLockFast - case BoPDRPC: - return metrics.BackoffHistogramPD - case BoRegionMiss: - return metrics.BackoffHistogramRegionMiss - case boTiKVServerBusy, boTiFlashServerBusy: - return metrics.BackoffHistogramServerBusy - case boStaleCmd: - return metrics.BackoffHistogramStaleCmd - } - return metrics.BackoffHistogramEmpty -} - -// NewBackoffFn creates a backoff func which implements exponential backoff with -// optional jitters. -// See http://www.awsarchitectureblog.com/2015/03/backoff.html -func NewBackoffFn(base, cap, jitter int) func(ctx context.Context, maxSleepMs int) int { - if base < 2 { - // Top prevent panic in 'rand.Intn'. - base = 2 - } - attempts := 0 - lastSleep := base - return func(ctx context.Context, maxSleepMs int) int { - var sleep int - switch jitter { - case NoJitter: - sleep = expo(base, cap, attempts) - case FullJitter: - v := expo(base, cap, attempts) - sleep = rand.Intn(v) - case EqualJitter: - v := expo(base, cap, attempts) - sleep = v/2 + rand.Intn(v/2) - case DecorrJitter: - sleep = int(math.Min(float64(cap), float64(base+rand.Intn(lastSleep*3-base)))) - } - logutil.BgLogger().Debug("backoff", - zap.Int("base", base), - zap.Int("sleep", sleep), - zap.Int("attempts", attempts)) - - realSleep := sleep - // when set maxSleepMs >= 0 in `tikv.BackoffWithMaxSleep` will force sleep maxSleepMs milliseconds. - if maxSleepMs >= 0 && realSleep > maxSleepMs { - realSleep = maxSleepMs - } - select { - case <-time.After(time.Duration(realSleep) * time.Millisecond): - attempts++ - lastSleep = sleep - return realSleep - case <-ctx.Done(): - return 0 - } - } -} - -func expo(base, cap, n int) int { - return int(math.Min(float64(cap), float64(base)*math.Pow(2.0, float64(n)))) -} - -// BackoffType defines the backoff type. -type BackoffType int - -// Back off types. -const ( - BoTiKVRPC BackoffType = iota - BoTiFlashRPC - BoTxnLock - BoTxnLockFast - BoPDRPC - BoRegionMiss - boTiKVServerBusy - boTiFlashServerBusy - boTxnNotFound - boStaleCmd - boMaxTsNotSynced -) - -func (t BackoffType) createFn(vars *kv.Variables) func(context.Context, int) int { - if vars.Hook != nil { - vars.Hook(t.String(), vars) - } - switch t { - case BoTiKVRPC, BoTiFlashRPC: - return NewBackoffFn(100, 2000, EqualJitter) - case BoTxnLock: - return NewBackoffFn(200, 3000, EqualJitter) - case BoTxnLockFast: - return NewBackoffFn(vars.BackoffLockFast, 3000, EqualJitter) - case BoPDRPC: - return NewBackoffFn(500, 3000, EqualJitter) - case BoRegionMiss: - // change base time to 2ms, because it may recover soon. - return NewBackoffFn(2, 500, NoJitter) - case boTxnNotFound: - return NewBackoffFn(2, 500, NoJitter) - case boTiKVServerBusy, boTiFlashServerBusy: - return NewBackoffFn(2000, 10000, EqualJitter) - case boStaleCmd: - return NewBackoffFn(2, 1000, NoJitter) - case boMaxTsNotSynced: - return NewBackoffFn(2, 500, NoJitter) - } - return nil -} - -func (t BackoffType) String() string { - switch t { - case BoTiKVRPC: - return "tikvRPC" - case BoTiFlashRPC: - return "tiflashRPC" - case BoTxnLock: - return "txnLock" - case BoTxnLockFast: - return "txnLockFast" - case BoPDRPC: - return "pdRPC" - case BoRegionMiss: - return "regionMiss" - case boTiKVServerBusy: - return "tikvServerBusy" - case boTiFlashServerBusy: - return "tiflashServerBusy" - case boStaleCmd: - return "staleCommand" - case boTxnNotFound: - return "txnNotFound" - case boMaxTsNotSynced: - return "maxTsNotSynced" - } - return "" -} +// Backoffer is a utility for retrying queries. +type Backoffer = retry.Backoffer -// TError returns pingcap/error of the backoff type. -func (t BackoffType) TError() error { - switch t { - case BoTiKVRPC: - return tikverr.ErrTiKVServerTimeout - case BoTiFlashRPC: - return tikverr.ErrTiFlashServerTimeout - case BoTxnLock, BoTxnLockFast, boTxnNotFound: - return tikverr.ErrResolveLockTimeout - case BoPDRPC: - return tikverr.NewErrPDServerTimeout("") - case BoRegionMiss: - return tikverr.ErrRegionUnavailable - case boTiKVServerBusy: - return tikverr.ErrTiKVServerBusy - case boTiFlashServerBusy: - return tikverr.ErrTiFlashServerBusy - case boStaleCmd: - return tikverr.ErrTiKVStaleCommand - case boMaxTsNotSynced: - return tikverr.ErrTiKVMaxTimestampNotSynced - } - return tikverr.ErrUnknown -} +// BackoffConfig defines the backoff configuration. +type BackoffConfig = retry.Config // Maximum total sleep time(in ms) for kv/cop commands. const ( - GetAllMembersBackoff = 5000 - tsoMaxBackoff = 15000 - scannerNextMaxBackoff = 20000 - batchGetMaxBackoff = 20000 - getMaxBackoff = 20000 - cleanupMaxBackoff = 20000 - GcOneRegionMaxBackoff = 20000 - GcResolveLockMaxBackoff = 100000 - deleteRangeOneRegionMaxBackoff = 100000 - rawkvMaxBackoff = 20000 - splitRegionBackoff = 20000 - maxSplitRegionsBackoff = 120000 - waitScatterRegionFinishBackoff = 120000 - locateRegionMaxBackoff = 20000 - pessimisticLockMaxBackoff = 20000 - pessimisticRollbackMaxBackoff = 20000 + gcResolveLockMaxBackoff = 100000 + // CommitSecondaryMaxBackoff is max sleep time of the 'commit' command + CommitSecondaryMaxBackoff = 41000 ) var ( // CommitMaxBackoff is max sleep time of the 'commit' command CommitMaxBackoff = uint64(41000) - // PrewriteMaxBackoff is max sleep time of the `pre-write` command. PrewriteMaxBackoff = 20000 ) -// Backoffer is a utility for retrying queries. -type Backoffer struct { - ctx context.Context - - fn map[BackoffType]func(context.Context, int) int - maxSleep int - totalSleep int - errors []error - types []fmt.Stringer - vars *kv.Variables - noop bool - - backoffSleepMS map[BackoffType]int - backoffTimes map[BackoffType]int -} - -type txnStartCtxKeyType struct{} - -// TxnStartKey is a key for transaction start_ts info in context.Context. -var TxnStartKey interface{} = txnStartCtxKeyType{} - -// NewBackoffer (Deprecated) creates a Backoffer with maximum sleep time(in ms). -func NewBackoffer(ctx context.Context, maxSleep int) *Backoffer { - return &Backoffer{ - ctx: ctx, - maxSleep: maxSleep, - vars: kv.DefaultVars, - } -} - // NewBackofferWithVars creates a Backoffer with maximum sleep time(in ms) and kv.Variables. func NewBackofferWithVars(ctx context.Context, maxSleep int, vars *kv.Variables) *Backoffer { - return NewBackoffer(ctx, maxSleep).withVars(vars) -} - -// NewNoopBackoff create a Backoffer do nothing just return error directly -func NewNoopBackoff(ctx context.Context) *Backoffer { - return &Backoffer{ctx: ctx, noop: true} -} - -// withVars sets the kv.Variables to the Backoffer and return it. -func (b *Backoffer) withVars(vars *kv.Variables) *Backoffer { - if vars != nil { - b.vars = vars - } - // maxSleep is the max sleep time in millisecond. - // When it is multiplied by BackOffWeight, it should not be greater than MaxInt32. - if math.MaxInt32/b.vars.BackOffWeight >= b.maxSleep { - b.maxSleep *= b.vars.BackOffWeight - } - return b -} - -// Backoff sleeps a while base on the backoffType and records the error message. -// It returns a retryable error if total sleep time exceeds maxSleep. -func (b *Backoffer) Backoff(typ BackoffType, err error) error { - if span := opentracing.SpanFromContext(b.ctx); span != nil && span.Tracer() != nil { - span1 := span.Tracer().StartSpan(fmt.Sprintf("tikv.backoff.%s", typ), opentracing.ChildOf(span.Context())) - defer span1.Finish() - opentracing.ContextWithSpan(b.ctx, span1) - } - return b.BackoffWithMaxSleep(typ, -1, err) + return retry.NewBackofferWithVars(ctx, maxSleep, vars) } -// BackoffWithMaxSleep sleeps a while base on the backoffType and records the error message -// and never sleep more than maxSleepMs for each sleep. -func (b *Backoffer) BackoffWithMaxSleep(typ BackoffType, maxSleepMs int, err error) error { - if strings.Contains(err.Error(), tikverr.MismatchClusterID) { - logutil.BgLogger().Fatal("critical error", zap.Error(err)) - } - select { - case <-b.ctx.Done(): - return errors.Trace(err) - default: - } - - b.errors = append(b.errors, errors.Errorf("%s at %s", err.Error(), time.Now().Format(time.RFC3339Nano))) - b.types = append(b.types, typ) - if b.noop || (b.maxSleep > 0 && b.totalSleep >= b.maxSleep) { - errMsg := fmt.Sprintf("%s backoffer.maxSleep %dms is exceeded, errors:", typ.String(), b.maxSleep) - for i, err := range b.errors { - // Print only last 3 errors for non-DEBUG log levels. - if log.GetLevel() == zapcore.DebugLevel || i >= len(b.errors)-3 { - errMsg += "\n" + err.Error() - } - } - logutil.BgLogger().Warn(errMsg) - // Use the first backoff type to generate a MySQL error. - return b.types[0].(BackoffType).TError() - } - - // Lazy initialize. - if b.fn == nil { - b.fn = make(map[BackoffType]func(context.Context, int) int) - } - f, ok := b.fn[typ] - if !ok { - f = typ.createFn(b.vars) - b.fn[typ] = f - } - - realSleep := f(b.ctx, maxSleepMs) - typ.metric().Observe(float64(realSleep) / 1000) - b.totalSleep += realSleep - if b.backoffSleepMS == nil { - b.backoffSleepMS = make(map[BackoffType]int) - } - b.backoffSleepMS[typ] += realSleep - if b.backoffTimes == nil { - b.backoffTimes = make(map[BackoffType]int) - } - b.backoffTimes[typ]++ - - stmtExec := b.ctx.Value(util.ExecDetailsKey) - if stmtExec != nil { - detail := stmtExec.(*util.ExecDetails) - atomic.AddInt64(&detail.BackoffDuration, int64(realSleep)*int64(time.Millisecond)) - atomic.AddInt64(&detail.BackoffCount, 1) - } - - if b.vars != nil && b.vars.Killed != nil { - if atomic.LoadUint32(b.vars.Killed) == 1 { - return tikverr.ErrQueryInterrupted - } - } - - var startTs interface{} - if ts := b.ctx.Value(TxnStartKey); ts != nil { - startTs = ts - } - logutil.Logger(b.ctx).Debug("retry later", - zap.Error(err), - zap.Int("totalSleep", b.totalSleep), - zap.Int("maxSleep", b.maxSleep), - zap.Stringer("type", typ), - zap.Reflect("txnStartTS", startTs)) - return nil -} - -func (b *Backoffer) String() string { - if b.totalSleep == 0 { - return "" - } - return fmt.Sprintf(" backoff(%dms %v)", b.totalSleep, b.types) -} - -// Clone creates a new Backoffer which keeps current Backoffer's sleep time and errors, and shares -// current Backoffer's context. -func (b *Backoffer) Clone() *Backoffer { - return &Backoffer{ - ctx: b.ctx, - maxSleep: b.maxSleep, - totalSleep: b.totalSleep, - errors: b.errors, - vars: b.vars, - } +// NewBackoffer creates a Backoffer with maximum sleep time(in ms). +func NewBackoffer(ctx context.Context, maxSleep int) *Backoffer { + return retry.NewBackoffer(ctx, maxSleep) } -// Fork creates a new Backoffer which keeps current Backoffer's sleep time and errors, and holds -// a child context of current Backoffer's context. -func (b *Backoffer) Fork() (*Backoffer, context.CancelFunc) { - ctx, cancel := context.WithCancel(b.ctx) - return &Backoffer{ - ctx: ctx, - maxSleep: b.maxSleep, - totalSleep: b.totalSleep, - errors: b.errors, - vars: b.vars, - }, cancel +// TxnStartKey is a key for transaction start_ts info in context.Context. +func TxnStartKey() interface{} { + return retry.TxnStartKey } -// GetVars returns the binded vars. -func (b *Backoffer) GetVars() *kv.Variables { - return b.vars +// BoRegionMiss returns the default backoff config for RegionMiss. +func BoRegionMiss() *BackoffConfig { + return retry.BoRegionMiss } -// GetTotalSleep returns total sleep time. -func (b *Backoffer) GetTotalSleep() int { - return b.totalSleep +// BoTiFlashRPC returns the default backoff config for TiFlashRPC. +func BoTiFlashRPC() *BackoffConfig { + return retry.BoTiFlashRPC } -// GetTypes returns type list. -func (b *Backoffer) GetTypes() []fmt.Stringer { - return b.types +// BoTxnLock returns the default backoff config for TxnLock. +func BoTxnLock() *BackoffConfig { + return retry.BoTxnLock } -// GetCtx returns the binded context. -func (b *Backoffer) GetCtx() context.Context { - return b.ctx +// BoPDRPC returns the default backoff config for PDRPC. +func BoPDRPC() *BackoffConfig { + return retry.BoPDRPC } -// GetBackoffTimes returns a map contains backoff time count by type. -func (b *Backoffer) GetBackoffTimes() map[BackoffType]int { - return b.backoffTimes +// BoTiKVRPC returns the default backoff config for TiKVRPC. +func BoTiKVRPC() *BackoffConfig { + return retry.BoTiKVRPC } -// GetBackoffSleepMS returns a map contains backoff sleep time by type. -func (b *Backoffer) GetBackoffSleepMS() map[BackoffType]int { - return b.backoffSleepMS +// NewGcResolveLockMaxBackoffer creates a Backoffer for Gc to resolve lock. +func NewGcResolveLockMaxBackoffer(ctx context.Context) *Backoffer { + return retry.NewBackofferWithVars(ctx, gcResolveLockMaxBackoff, nil) } diff --git a/store/tikv/cleanup.go b/store/tikv/cleanup.go index dc96ed32ab54c..08f65e4434236 100644 --- a/store/tikv/cleanup.go +++ b/store/tikv/cleanup.go @@ -15,9 +15,11 @@ package tikv import ( "github.com/pingcap/errors" - pb "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/tidb/store/tikv/client" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -36,11 +38,11 @@ func (actionCleanup) tiKVTxnRegionsNumHistogram() prometheus.Observer { } func (actionCleanup) handleSingleBatch(c *twoPhaseCommitter, bo *Backoffer, batch batchMutations) error { - req := tikvrpc.NewRequest(tikvrpc.CmdBatchRollback, &pb.BatchRollbackRequest{ + req := tikvrpc.NewRequest(tikvrpc.CmdBatchRollback, &kvrpcpb.BatchRollbackRequest{ Keys: batch.mutations.GetKeys(), StartVersion: c.startTS, - }, pb.Context{Priority: c.priority, SyncLog: c.syncLog}) - resp, err := c.store.SendReq(bo, req, batch.region, ReadTimeoutShort) + }, kvrpcpb.Context{Priority: c.priority, SyncLog: c.syncLog, ResourceGroupTag: c.resourceGroupTag}) + resp, err := c.store.SendReq(bo, req, batch.region, client.ReadTimeoutShort) if err != nil { return errors.Trace(err) } @@ -49,14 +51,14 @@ func (actionCleanup) handleSingleBatch(c *twoPhaseCommitter, bo *Backoffer, batc return errors.Trace(err) } if regionErr != nil { - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) if err != nil { return errors.Trace(err) } err = c.cleanupMutations(bo, batch.mutations) return errors.Trace(err) } - if keyErr := resp.Resp.(*pb.BatchRollbackResponse).GetError(); keyErr != nil { + if keyErr := resp.Resp.(*kvrpcpb.BatchRollbackResponse).GetError(); keyErr != nil { err = errors.Errorf("session %d 2PC cleanup failed: %s", c.sessionID, keyErr) logutil.BgLogger().Debug("2PC failed cleanup key", zap.Error(err), diff --git a/store/tikv/client.go b/store/tikv/client.go index 63b18496d04c6..cb446d4a300e0 100644 --- a/store/tikv/client.go +++ b/store/tikv/client.go @@ -1,4 +1,4 @@ -// Copyright 2016 PingCAP, Inc. +// Copyright 2021 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,509 +11,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package tikv provides tcp connection to kvserver. package tikv import ( - "context" - "fmt" - "io" - "math" - "runtime/trace" - "strconv" - "sync" - "sync/atomic" - "time" - - grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing" - "github.com/opentracing/opentracing-go" - "github.com/pingcap/errors" - "github.com/pingcap/kvproto/pkg/coprocessor" - "github.com/pingcap/kvproto/pkg/debugpb" - "github.com/pingcap/kvproto/pkg/mpp" - "github.com/pingcap/kvproto/pkg/tikvpb" - "github.com/pingcap/parser/terror" + "github.com/pingcap/tidb/store/tikv/client" "github.com/pingcap/tidb/store/tikv/config" - "github.com/pingcap/tidb/store/tikv/logutil" - "github.com/pingcap/tidb/store/tikv/metrics" - "github.com/pingcap/tidb/store/tikv/tikvrpc" - "github.com/pingcap/tidb/store/tikv/util" - "github.com/prometheus/client_golang/prometheus" - "google.golang.org/grpc" - "google.golang.org/grpc/backoff" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/encoding/gzip" - "google.golang.org/grpc/keepalive" - "google.golang.org/grpc/metadata" ) -// MaxRecvMsgSize set max gRPC receive message size received from server. If any message size is larger than -// current value, an error will be reported from gRPC. -var MaxRecvMsgSize = math.MaxInt64 +// Client is a client that sends RPC. +// It should not be used after calling Close(). +type Client = client.Client // Timeout durations. -var ( - dialTimeout = 5 * time.Second - ReadTimeoutShort = 20 * time.Second // For requests that read/write several key-values. - ReadTimeoutMedium = 60 * time.Second // For requests that may need scan region. - ReadTimeoutLong = 150 * time.Second // For requests that may need scan region multiple times. - ReadTimeoutUltraLong = 3600 * time.Second // For requests that may scan many regions for tiflash. - GCTimeout = 5 * time.Minute - UnsafeDestroyRangeTimeout = 5 * time.Minute - AccessLockObserverTimeout = 10 * time.Second -) - const ( - grpcInitialWindowSize = 1 << 30 - grpcInitialConnWindowSize = 1 << 30 + ReadTimeoutMedium = client.ReadTimeoutMedium + ReadTimeoutShort = client.ReadTimeoutShort ) -// forwardMetadataKey is the key of gRPC metadata which represents a forwarded request. -const forwardMetadataKey = "tikv-forwarded-host" - -// Client is a client that sends RPC. -// It should not be used after calling Close(). -type Client interface { - // Close should release all data. - Close() error - // SendRequest sends Request. - SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) -} - -type connArray struct { - // The target host. - target string - - index uint32 - v []*grpc.ClientConn - // streamTimeout binds with a background goroutine to process coprocessor streaming timeout. - streamTimeout chan *tikvrpc.Lease - dialTimeout time.Duration - // batchConn is not null when batch is enabled. - *batchConn - done chan struct{} -} - -func newConnArray(maxSize uint, addr string, security config.Security, idleNotify *uint32, enableBatch bool, dialTimeout time.Duration) (*connArray, error) { - a := &connArray{ - index: 0, - v: make([]*grpc.ClientConn, maxSize), - streamTimeout: make(chan *tikvrpc.Lease, 1024), - done: make(chan struct{}), - dialTimeout: dialTimeout, - } - if err := a.Init(addr, security, idleNotify, enableBatch); err != nil { - return nil, err - } - return a, nil -} - -func (a *connArray) Init(addr string, security config.Security, idleNotify *uint32, enableBatch bool) error { - a.target = addr - - opt := grpc.WithInsecure() - if len(security.ClusterSSLCA) != 0 { - tlsConfig, err := security.ToTLSConfig() - if err != nil { - return errors.Trace(err) - } - opt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)) - } - - cfg := config.GetGlobalConfig() - var ( - unaryInterceptor grpc.UnaryClientInterceptor - streamInterceptor grpc.StreamClientInterceptor - ) - if cfg.OpenTracingEnable { - unaryInterceptor = grpc_opentracing.UnaryClientInterceptor() - streamInterceptor = grpc_opentracing.StreamClientInterceptor() - } - - allowBatch := (cfg.TiKVClient.MaxBatchSize > 0) && enableBatch - if allowBatch { - a.batchConn = newBatchConn(uint(len(a.v)), cfg.TiKVClient.MaxBatchSize, idleNotify) - a.pendingRequests = metrics.TiKVBatchPendingRequests.WithLabelValues(a.target) - a.batchSize = metrics.TiKVBatchRequests.WithLabelValues(a.target) - } - keepAlive := cfg.TiKVClient.GrpcKeepAliveTime - keepAliveTimeout := cfg.TiKVClient.GrpcKeepAliveTimeout - for i := range a.v { - ctx, cancel := context.WithTimeout(context.Background(), a.dialTimeout) - var callOptions []grpc.CallOption - callOptions = append(callOptions, grpc.MaxCallRecvMsgSize(MaxRecvMsgSize)) - if cfg.TiKVClient.GrpcCompressionType == gzip.Name { - callOptions = append(callOptions, grpc.UseCompressor(gzip.Name)) - } - conn, err := grpc.DialContext( - ctx, - addr, - opt, - grpc.WithInitialWindowSize(grpcInitialWindowSize), - grpc.WithInitialConnWindowSize(grpcInitialConnWindowSize), - grpc.WithUnaryInterceptor(unaryInterceptor), - grpc.WithStreamInterceptor(streamInterceptor), - grpc.WithDefaultCallOptions(callOptions...), - grpc.WithConnectParams(grpc.ConnectParams{ - Backoff: backoff.Config{ - BaseDelay: 100 * time.Millisecond, // Default was 1s. - Multiplier: 1.6, // Default - Jitter: 0.2, // Default - MaxDelay: 3 * time.Second, // Default was 120s. - }, - MinConnectTimeout: a.dialTimeout, - }), - grpc.WithKeepaliveParams(keepalive.ClientParameters{ - Time: time.Duration(keepAlive) * time.Second, - Timeout: time.Duration(keepAliveTimeout) * time.Second, - PermitWithoutStream: true, - }), - ) - cancel() - if err != nil { - // Cleanup if the initialization fails. - a.Close() - return errors.Trace(err) - } - a.v[i] = conn - - if allowBatch { - batchClient := &batchCommandsClient{ - target: a.target, - conn: conn, - forwardedClients: make(map[string]*batchCommandsStream), - batched: sync.Map{}, - epoch: 0, - closed: 0, - tikvClientCfg: cfg.TiKVClient, - tikvLoad: &a.tikvTransportLayerLoad, - dialTimeout: a.dialTimeout, - tryLock: tryLock{sync.NewCond(new(sync.Mutex)), false}, - } - a.batchCommandsClients = append(a.batchCommandsClients, batchClient) - } - } - go tikvrpc.CheckStreamTimeoutLoop(a.streamTimeout, a.done) - if allowBatch { - go a.batchSendLoop(cfg.TiKVClient) - } - - return nil -} - -func (a *connArray) Get() *grpc.ClientConn { - next := atomic.AddUint32(&a.index, 1) % uint32(len(a.v)) - return a.v[next] -} - -func (a *connArray) Close() { - if a.batchConn != nil { - a.batchConn.Close() - } - - for i, c := range a.v { - if c != nil { - err := c.Close() - terror.Log(errors.Trace(err)) - a.v[i] = nil - } - } - - close(a.done) -} - -// RPCClient is RPC client struct. -// TODO: Add flow control between RPC clients in TiDB ond RPC servers in TiKV. -// Since we use shared client connection to communicate to the same TiKV, it's possible -// that there are too many concurrent requests which overload the service of TiKV. -type RPCClient struct { - sync.RWMutex - - conns map[string]*connArray - security config.Security - - idleNotify uint32 - // recycleMu protect the conns from being modified during a connArray is taken out and used. - // That means recycleIdleConnArray() will wait until nobody doing sendBatchRequest() - recycleMu sync.RWMutex - // Periodically check whether there is any connection that is idle and then close and remove these connections. - // Implement background cleanup. - isClosed bool - dialTimeout time.Duration -} - -// NewRPCClient creates a client that manages connections and rpc calls with tikv-servers. -func NewRPCClient(security config.Security, opts ...func(c *RPCClient)) *RPCClient { - cli := &RPCClient{ - conns: make(map[string]*connArray), - security: security, - dialTimeout: dialTimeout, - } - for _, opt := range opts { - opt(cli) - } - return cli -} - // NewTestRPCClient is for some external tests. func NewTestRPCClient(security config.Security) Client { - return NewRPCClient(security) -} - -func (c *RPCClient) getConnArray(addr string, enableBatch bool, opt ...func(cfg *config.TiKVClient)) (*connArray, error) { - c.RLock() - if c.isClosed { - c.RUnlock() - return nil, errors.Errorf("rpcClient is closed") - } - array, ok := c.conns[addr] - c.RUnlock() - if !ok { - var err error - array, err = c.createConnArray(addr, enableBatch, opt...) - if err != nil { - return nil, err - } - } - return array, nil -} - -func (c *RPCClient) createConnArray(addr string, enableBatch bool, opts ...func(cfg *config.TiKVClient)) (*connArray, error) { - c.Lock() - defer c.Unlock() - array, ok := c.conns[addr] - if !ok { - var err error - client := config.GetGlobalConfig().TiKVClient - for _, opt := range opts { - opt(&client) - } - array, err = newConnArray(client.GrpcConnectionCount, addr, c.security, &c.idleNotify, enableBatch, c.dialTimeout) - if err != nil { - return nil, err - } - c.conns[addr] = array - } - return array, nil -} - -func (c *RPCClient) closeConns() { - c.Lock() - if !c.isClosed { - c.isClosed = true - // close all connections - for _, array := range c.conns { - array.Close() - } - } - c.Unlock() -} - -var sendReqHistCache sync.Map - -type sendReqHistCacheKey struct { - tp tikvrpc.CmdType - id uint64 -} - -func (c *RPCClient) updateTiKVSendReqHistogram(req *tikvrpc.Request, start time.Time) { - key := sendReqHistCacheKey{ - req.Type, - req.Context.GetPeer().GetStoreId(), - } - - v, ok := sendReqHistCache.Load(key) - if !ok { - reqType := req.Type.String() - storeID := strconv.FormatUint(req.Context.GetPeer().GetStoreId(), 10) - v = metrics.TiKVSendReqHistogram.WithLabelValues(reqType, storeID) - sendReqHistCache.Store(key, v) - } - - v.(prometheus.Observer).Observe(time.Since(start).Seconds()) -} - -// SendRequest sends a Request to server and receives Response. -func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) { - if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { - span1 := span.Tracer().StartSpan(fmt.Sprintf("rpcClient.SendRequest, region ID: %d, type: %s", req.RegionId, req.Type), opentracing.ChildOf(span.Context())) - defer span1.Finish() - ctx = opentracing.ContextWithSpan(ctx, span1) - } - - start := time.Now() - defer func() { - stmtExec := ctx.Value(util.ExecDetailsKey) - if stmtExec != nil { - detail := stmtExec.(*util.ExecDetails) - atomic.AddInt64(&detail.WaitKVRespDuration, int64(time.Since(start))) - } - c.updateTiKVSendReqHistogram(req, start) - }() - - if atomic.CompareAndSwapUint32(&c.idleNotify, 1, 0) { - c.recycleMu.Lock() - c.recycleIdleConnArray() - c.recycleMu.Unlock() - } - - // TiDB will not send batch commands to TiFlash, to resolve the conflict with Batch Cop Request. - enableBatch := req.StoreTp != tikvrpc.TiDB && req.StoreTp != tikvrpc.TiFlash - c.recycleMu.RLock() - defer c.recycleMu.RUnlock() - connArray, err := c.getConnArray(addr, enableBatch) - if err != nil { - return nil, errors.Trace(err) - } - - // TiDB RPC server supports batch RPC, but batch connection will send heart beat, It's not necessary since - // request to TiDB is not high frequency. - if config.GetGlobalConfig().TiKVClient.MaxBatchSize > 0 && enableBatch { - if batchReq := req.ToBatchCommandsRequest(); batchReq != nil { - defer trace.StartRegion(ctx, req.Type.String()).End() - return sendBatchRequest(ctx, addr, req.ForwardedHost, connArray.batchConn, batchReq, timeout) - } - } - - clientConn := connArray.Get() - if state := clientConn.GetState(); state == connectivity.TransientFailure { - storeID := strconv.FormatUint(req.Context.GetPeer().GetStoreId(), 10) - metrics.TiKVGRPCConnTransientFailureCounter.WithLabelValues(addr, storeID).Inc() - } - - if req.IsDebugReq() { - client := debugpb.NewDebugClient(clientConn) - ctx1, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - return tikvrpc.CallDebugRPC(ctx1, client, req) - } - - client := tikvpb.NewTikvClient(clientConn) - - // Set metadata for request forwarding. Needn't forward DebugReq. - if req.ForwardedHost != "" { - ctx = metadata.AppendToOutgoingContext(ctx, forwardMetadataKey, req.ForwardedHost) - } - switch req.Type { - case tikvrpc.CmdBatchCop: - return c.getBatchCopStreamResponse(ctx, client, req, timeout, connArray) - case tikvrpc.CmdCopStream: - return c.getCopStreamResponse(ctx, client, req, timeout, connArray) - case tikvrpc.CmdMPPConn: - return c.getMPPStreamResponse(ctx, client, req, timeout, connArray) - } - // Or else it's a unary call. - ctx1, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - return tikvrpc.CallRPC(ctx1, client, req) -} - -func (c *RPCClient) getCopStreamResponse(ctx context.Context, client tikvpb.TikvClient, req *tikvrpc.Request, timeout time.Duration, connArray *connArray) (*tikvrpc.Response, error) { - // Coprocessor streaming request. - // Use context to support timeout for grpc streaming client. - ctx1, cancel := context.WithCancel(ctx) - // Should NOT call defer cancel() here because it will cancel further stream.Recv() - // We put it in copStream.Lease.Cancel call this cancel at copStream.Close - // TODO: add unit test for SendRequest. - resp, err := tikvrpc.CallRPC(ctx1, client, req) - if err != nil { - cancel() - return nil, errors.Trace(err) - } - - // Put the lease object to the timeout channel, so it would be checked periodically. - copStream := resp.Resp.(*tikvrpc.CopStreamResponse) - copStream.Timeout = timeout - copStream.Lease.Cancel = cancel - connArray.streamTimeout <- &copStream.Lease - - // Read the first streaming response to get CopStreamResponse. - // This can make error handling much easier, because SendReq() retry on - // region error automatically. - var first *coprocessor.Response - first, err = copStream.Recv() - if err != nil { - if errors.Cause(err) != io.EOF { - return nil, errors.Trace(err) - } - logutil.BgLogger().Debug("copstream returns nothing for the request.") - } - copStream.Response = first - return resp, nil - + return client.NewTestRPCClient(security) } -func (c *RPCClient) getBatchCopStreamResponse(ctx context.Context, client tikvpb.TikvClient, req *tikvrpc.Request, timeout time.Duration, connArray *connArray) (*tikvrpc.Response, error) { - // Coprocessor streaming request. - // Use context to support timeout for grpc streaming client. - ctx1, cancel := context.WithCancel(ctx) - // Should NOT call defer cancel() here because it will cancel further stream.Recv() - // We put it in copStream.Lease.Cancel call this cancel at copStream.Close - // TODO: add unit test for SendRequest. - resp, err := tikvrpc.CallRPC(ctx1, client, req) - if err != nil { - cancel() - return nil, errors.Trace(err) - } - - // Put the lease object to the timeout channel, so it would be checked periodically. - copStream := resp.Resp.(*tikvrpc.BatchCopStreamResponse) - copStream.Timeout = timeout - copStream.Lease.Cancel = cancel - connArray.streamTimeout <- &copStream.Lease - - // Read the first streaming response to get CopStreamResponse. - // This can make error handling much easier, because SendReq() retry on - // region error automatically. - var first *coprocessor.BatchResponse - first, err = copStream.Recv() - if err != nil { - if errors.Cause(err) != io.EOF { - return nil, errors.Trace(err) - } - logutil.BgLogger().Debug("batch copstream returns nothing for the request.") - } - copStream.BatchResponse = first - return resp, nil -} - -func (c *RPCClient) getMPPStreamResponse(ctx context.Context, client tikvpb.TikvClient, req *tikvrpc.Request, timeout time.Duration, connArray *connArray) (*tikvrpc.Response, error) { - // MPP streaming request. - // Use context to support timeout for grpc streaming client. - ctx1, cancel := context.WithCancel(ctx) - // Should NOT call defer cancel() here because it will cancel further stream.Recv() - // We put it in copStream.Lease.Cancel call this cancel at copStream.Close - // TODO: add unit test for SendRequest. - resp, err := tikvrpc.CallRPC(ctx1, client, req) - if err != nil { - cancel() - return nil, errors.Trace(err) - } - - // Put the lease object to the timeout channel, so it would be checked periodically. - copStream := resp.Resp.(*tikvrpc.MPPStreamResponse) - copStream.Timeout = timeout - copStream.Lease.Cancel = cancel - connArray.streamTimeout <- &copStream.Lease - - // Read the first streaming response to get CopStreamResponse. - // This can make error handling much easier, because SendReq() retry on - // region error automatically. - var first *mpp.MPPDataPacket - first, err = copStream.Recv() - if err != nil { - if errors.Cause(err) != io.EOF { - return nil, errors.Trace(err) - } - } - copStream.MPPDataPacket = first - return resp, nil -} - -// Close closes all connections. -func (c *RPCClient) Close() error { - // TODO: add a unit test for SendRequest After Closed - c.closeConns() - return nil +// NewRPCClient creates a client that manages connections and rpc calls with tikv-servers. +func NewRPCClient(security config.Security, opts ...func(c *client.RPCClient)) *client.RPCClient { + return client.NewRPCClient(security, opts...) } diff --git a/store/tikv/client/client.go b/store/tikv/client/client.go new file mode 100644 index 0000000000000..b2652aeabb977 --- /dev/null +++ b/store/tikv/client/client.go @@ -0,0 +1,515 @@ +// Copyright 2016 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package client provides tcp connection to kvserver. +package client + +import ( + "context" + "fmt" + "io" + "math" + "runtime/trace" + "strconv" + "sync" + "sync/atomic" + "time" + + grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing" + "github.com/opentracing/opentracing-go" + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/coprocessor" + "github.com/pingcap/kvproto/pkg/debugpb" + "github.com/pingcap/kvproto/pkg/mpp" + "github.com/pingcap/kvproto/pkg/tikvpb" + "github.com/pingcap/parser/terror" + "github.com/pingcap/tidb/store/tikv/config" + "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/pingcap/tidb/store/tikv/tikvrpc" + "github.com/pingcap/tidb/store/tikv/util" + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/encoding/gzip" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/metadata" +) + +// MaxRecvMsgSize set max gRPC receive message size received from server. If any message size is larger than +// current value, an error will be reported from gRPC. +var MaxRecvMsgSize = math.MaxInt64 + +// Timeout durations. +const ( + dialTimeout = 5 * time.Second + ReadTimeoutShort = 20 * time.Second // For requests that read/write several key-values. + ReadTimeoutMedium = 60 * time.Second // For requests that may need scan region. +) + +// Grpc window size +const ( + GrpcInitialWindowSize = 1 << 30 + GrpcInitialConnWindowSize = 1 << 30 +) + +// forwardMetadataKey is the key of gRPC metadata which represents a forwarded request. +const forwardMetadataKey = "tikv-forwarded-host" + +// Client is a client that sends RPC. +// It should not be used after calling Close(). +type Client interface { + // Close should release all data. + Close() error + // SendRequest sends Request. + SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) +} + +type connArray struct { + // The target host. + target string + + index uint32 + v []*grpc.ClientConn + // streamTimeout binds with a background goroutine to process coprocessor streaming timeout. + streamTimeout chan *tikvrpc.Lease + dialTimeout time.Duration + // batchConn is not null when batch is enabled. + *batchConn + done chan struct{} +} + +func newConnArray(maxSize uint, addr string, security config.Security, idleNotify *uint32, enableBatch bool, dialTimeout time.Duration) (*connArray, error) { + a := &connArray{ + index: 0, + v: make([]*grpc.ClientConn, maxSize), + streamTimeout: make(chan *tikvrpc.Lease, 1024), + done: make(chan struct{}), + dialTimeout: dialTimeout, + } + if err := a.Init(addr, security, idleNotify, enableBatch); err != nil { + return nil, err + } + return a, nil +} + +func (a *connArray) Init(addr string, security config.Security, idleNotify *uint32, enableBatch bool) error { + a.target = addr + + opt := grpc.WithInsecure() + if len(security.ClusterSSLCA) != 0 { + tlsConfig, err := security.ToTLSConfig() + if err != nil { + return errors.Trace(err) + } + opt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)) + } + + cfg := config.GetGlobalConfig() + var ( + unaryInterceptor grpc.UnaryClientInterceptor + streamInterceptor grpc.StreamClientInterceptor + ) + if cfg.OpenTracingEnable { + unaryInterceptor = grpc_opentracing.UnaryClientInterceptor() + streamInterceptor = grpc_opentracing.StreamClientInterceptor() + } + + allowBatch := (cfg.TiKVClient.MaxBatchSize > 0) && enableBatch + if allowBatch { + a.batchConn = newBatchConn(uint(len(a.v)), cfg.TiKVClient.MaxBatchSize, idleNotify) + a.pendingRequests = metrics.TiKVBatchPendingRequests.WithLabelValues(a.target) + a.batchSize = metrics.TiKVBatchRequests.WithLabelValues(a.target) + } + keepAlive := cfg.TiKVClient.GrpcKeepAliveTime + keepAliveTimeout := cfg.TiKVClient.GrpcKeepAliveTimeout + for i := range a.v { + ctx, cancel := context.WithTimeout(context.Background(), a.dialTimeout) + var callOptions []grpc.CallOption + callOptions = append(callOptions, grpc.MaxCallRecvMsgSize(MaxRecvMsgSize)) + if cfg.TiKVClient.GrpcCompressionType == gzip.Name { + callOptions = append(callOptions, grpc.UseCompressor(gzip.Name)) + } + conn, err := grpc.DialContext( + ctx, + addr, + opt, + grpc.WithInitialWindowSize(GrpcInitialWindowSize), + grpc.WithInitialConnWindowSize(GrpcInitialConnWindowSize), + grpc.WithUnaryInterceptor(unaryInterceptor), + grpc.WithStreamInterceptor(streamInterceptor), + grpc.WithDefaultCallOptions(callOptions...), + grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: backoff.Config{ + BaseDelay: 100 * time.Millisecond, // Default was 1s. + Multiplier: 1.6, // Default + Jitter: 0.2, // Default + MaxDelay: 3 * time.Second, // Default was 120s. + }, + MinConnectTimeout: a.dialTimeout, + }), + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: time.Duration(keepAlive) * time.Second, + Timeout: time.Duration(keepAliveTimeout) * time.Second, + PermitWithoutStream: true, + }), + ) + cancel() + if err != nil { + // Cleanup if the initialization fails. + a.Close() + return errors.Trace(err) + } + a.v[i] = conn + + if allowBatch { + batchClient := &batchCommandsClient{ + target: a.target, + conn: conn, + forwardedClients: make(map[string]*batchCommandsStream), + batched: sync.Map{}, + epoch: 0, + closed: 0, + tikvClientCfg: cfg.TiKVClient, + tikvLoad: &a.tikvTransportLayerLoad, + dialTimeout: a.dialTimeout, + tryLock: tryLock{sync.NewCond(new(sync.Mutex)), false}, + } + a.batchCommandsClients = append(a.batchCommandsClients, batchClient) + } + } + go tikvrpc.CheckStreamTimeoutLoop(a.streamTimeout, a.done) + if allowBatch { + go a.batchSendLoop(cfg.TiKVClient) + } + + return nil +} + +func (a *connArray) Get() *grpc.ClientConn { + next := atomic.AddUint32(&a.index, 1) % uint32(len(a.v)) + return a.v[next] +} + +func (a *connArray) Close() { + if a.batchConn != nil { + a.batchConn.Close() + } + + for i, c := range a.v { + if c != nil { + err := c.Close() + terror.Log(errors.Trace(err)) + a.v[i] = nil + } + } + + close(a.done) +} + +// RPCClient is RPC client struct. +// TODO: Add flow control between RPC clients in TiDB ond RPC servers in TiKV. +// Since we use shared client connection to communicate to the same TiKV, it's possible +// that there are too many concurrent requests which overload the service of TiKV. +type RPCClient struct { + sync.RWMutex + + conns map[string]*connArray + security config.Security + + idleNotify uint32 + // recycleMu protect the conns from being modified during a connArray is taken out and used. + // That means recycleIdleConnArray() will wait until nobody doing sendBatchRequest() + recycleMu sync.RWMutex + // Periodically check whether there is any connection that is idle and then close and remove these connections. + // Implement background cleanup. + isClosed bool + dialTimeout time.Duration +} + +// NewRPCClient creates a client that manages connections and rpc calls with tikv-servers. +func NewRPCClient(security config.Security, opts ...func(c *RPCClient)) *RPCClient { + cli := &RPCClient{ + conns: make(map[string]*connArray), + security: security, + dialTimeout: dialTimeout, + } + for _, opt := range opts { + opt(cli) + } + return cli +} + +// NewTestRPCClient is for some external tests. +func NewTestRPCClient(security config.Security) Client { + return NewRPCClient(security) +} + +func (c *RPCClient) getConnArray(addr string, enableBatch bool, opt ...func(cfg *config.TiKVClient)) (*connArray, error) { + c.RLock() + if c.isClosed { + c.RUnlock() + return nil, errors.Errorf("rpcClient is closed") + } + array, ok := c.conns[addr] + c.RUnlock() + if !ok { + var err error + array, err = c.createConnArray(addr, enableBatch, opt...) + if err != nil { + return nil, err + } + } + return array, nil +} + +func (c *RPCClient) createConnArray(addr string, enableBatch bool, opts ...func(cfg *config.TiKVClient)) (*connArray, error) { + c.Lock() + defer c.Unlock() + array, ok := c.conns[addr] + if !ok { + var err error + client := config.GetGlobalConfig().TiKVClient + for _, opt := range opts { + opt(&client) + } + array, err = newConnArray(client.GrpcConnectionCount, addr, c.security, &c.idleNotify, enableBatch, c.dialTimeout) + if err != nil { + return nil, err + } + c.conns[addr] = array + } + return array, nil +} + +func (c *RPCClient) closeConns() { + c.Lock() + if !c.isClosed { + c.isClosed = true + // close all connections + for _, array := range c.conns { + array.Close() + } + } + c.Unlock() +} + +var sendReqHistCache sync.Map + +type sendReqHistCacheKey struct { + tp tikvrpc.CmdType + id uint64 +} + +func (c *RPCClient) updateTiKVSendReqHistogram(req *tikvrpc.Request, start time.Time) { + key := sendReqHistCacheKey{ + req.Type, + req.Context.GetPeer().GetStoreId(), + } + + v, ok := sendReqHistCache.Load(key) + if !ok { + reqType := req.Type.String() + storeID := strconv.FormatUint(req.Context.GetPeer().GetStoreId(), 10) + v = metrics.TiKVSendReqHistogram.WithLabelValues(reqType, storeID) + sendReqHistCache.Store(key, v) + } + + v.(prometheus.Observer).Observe(time.Since(start).Seconds()) +} + +// SendRequest sends a Request to server and receives Response. +func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) { + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan(fmt.Sprintf("rpcClient.SendRequest, region ID: %d, type: %s", req.RegionId, req.Type), opentracing.ChildOf(span.Context())) + defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) + } + + start := time.Now() + defer func() { + stmtExec := ctx.Value(util.ExecDetailsKey) + if stmtExec != nil { + detail := stmtExec.(*util.ExecDetails) + atomic.AddInt64(&detail.WaitKVRespDuration, int64(time.Since(start))) + } + c.updateTiKVSendReqHistogram(req, start) + }() + + if atomic.CompareAndSwapUint32(&c.idleNotify, 1, 0) { + c.recycleMu.Lock() + c.recycleIdleConnArray() + c.recycleMu.Unlock() + } + + // TiDB will not send batch commands to TiFlash, to resolve the conflict with Batch Cop Request. + enableBatch := req.StoreTp != tikvrpc.TiDB && req.StoreTp != tikvrpc.TiFlash + c.recycleMu.RLock() + defer c.recycleMu.RUnlock() + connArray, err := c.getConnArray(addr, enableBatch) + if err != nil { + return nil, errors.Trace(err) + } + + // TiDB RPC server supports batch RPC, but batch connection will send heart beat, It's not necessary since + // request to TiDB is not high frequency. + if config.GetGlobalConfig().TiKVClient.MaxBatchSize > 0 && enableBatch { + if batchReq := req.ToBatchCommandsRequest(); batchReq != nil { + defer trace.StartRegion(ctx, req.Type.String()).End() + return sendBatchRequest(ctx, addr, req.ForwardedHost, connArray.batchConn, batchReq, timeout) + } + } + + clientConn := connArray.Get() + if state := clientConn.GetState(); state == connectivity.TransientFailure { + storeID := strconv.FormatUint(req.Context.GetPeer().GetStoreId(), 10) + metrics.TiKVGRPCConnTransientFailureCounter.WithLabelValues(addr, storeID).Inc() + } + + if req.IsDebugReq() { + client := debugpb.NewDebugClient(clientConn) + ctx1, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + return tikvrpc.CallDebugRPC(ctx1, client, req) + } + + client := tikvpb.NewTikvClient(clientConn) + + // Set metadata for request forwarding. Needn't forward DebugReq. + if req.ForwardedHost != "" { + ctx = metadata.AppendToOutgoingContext(ctx, forwardMetadataKey, req.ForwardedHost) + } + switch req.Type { + case tikvrpc.CmdBatchCop: + return c.getBatchCopStreamResponse(ctx, client, req, timeout, connArray) + case tikvrpc.CmdCopStream: + return c.getCopStreamResponse(ctx, client, req, timeout, connArray) + case tikvrpc.CmdMPPConn: + return c.getMPPStreamResponse(ctx, client, req, timeout, connArray) + } + // Or else it's a unary call. + ctx1, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + return tikvrpc.CallRPC(ctx1, client, req) +} + +func (c *RPCClient) getCopStreamResponse(ctx context.Context, client tikvpb.TikvClient, req *tikvrpc.Request, timeout time.Duration, connArray *connArray) (*tikvrpc.Response, error) { + // Coprocessor streaming request. + // Use context to support timeout for grpc streaming client. + ctx1, cancel := context.WithCancel(ctx) + // Should NOT call defer cancel() here because it will cancel further stream.Recv() + // We put it in copStream.Lease.Cancel call this cancel at copStream.Close + // TODO: add unit test for SendRequest. + resp, err := tikvrpc.CallRPC(ctx1, client, req) + if err != nil { + cancel() + return nil, errors.Trace(err) + } + + // Put the lease object to the timeout channel, so it would be checked periodically. + copStream := resp.Resp.(*tikvrpc.CopStreamResponse) + copStream.Timeout = timeout + copStream.Lease.Cancel = cancel + connArray.streamTimeout <- &copStream.Lease + + // Read the first streaming response to get CopStreamResponse. + // This can make error handling much easier, because SendReq() retry on + // region error automatically. + var first *coprocessor.Response + first, err = copStream.Recv() + if err != nil { + if errors.Cause(err) != io.EOF { + return nil, errors.Trace(err) + } + logutil.BgLogger().Debug("copstream returns nothing for the request.") + } + copStream.Response = first + return resp, nil + +} + +func (c *RPCClient) getBatchCopStreamResponse(ctx context.Context, client tikvpb.TikvClient, req *tikvrpc.Request, timeout time.Duration, connArray *connArray) (*tikvrpc.Response, error) { + // Coprocessor streaming request. + // Use context to support timeout for grpc streaming client. + ctx1, cancel := context.WithCancel(ctx) + // Should NOT call defer cancel() here because it will cancel further stream.Recv() + // We put it in copStream.Lease.Cancel call this cancel at copStream.Close + // TODO: add unit test for SendRequest. + resp, err := tikvrpc.CallRPC(ctx1, client, req) + if err != nil { + cancel() + return nil, errors.Trace(err) + } + + // Put the lease object to the timeout channel, so it would be checked periodically. + copStream := resp.Resp.(*tikvrpc.BatchCopStreamResponse) + copStream.Timeout = timeout + copStream.Lease.Cancel = cancel + connArray.streamTimeout <- &copStream.Lease + + // Read the first streaming response to get CopStreamResponse. + // This can make error handling much easier, because SendReq() retry on + // region error automatically. + var first *coprocessor.BatchResponse + first, err = copStream.Recv() + if err != nil { + if errors.Cause(err) != io.EOF { + return nil, errors.Trace(err) + } + logutil.BgLogger().Debug("batch copstream returns nothing for the request.") + } + copStream.BatchResponse = first + return resp, nil +} + +func (c *RPCClient) getMPPStreamResponse(ctx context.Context, client tikvpb.TikvClient, req *tikvrpc.Request, timeout time.Duration, connArray *connArray) (*tikvrpc.Response, error) { + // MPP streaming request. + // Use context to support timeout for grpc streaming client. + ctx1, cancel := context.WithCancel(ctx) + // Should NOT call defer cancel() here because it will cancel further stream.Recv() + // We put it in copStream.Lease.Cancel call this cancel at copStream.Close + // TODO: add unit test for SendRequest. + resp, err := tikvrpc.CallRPC(ctx1, client, req) + if err != nil { + cancel() + return nil, errors.Trace(err) + } + + // Put the lease object to the timeout channel, so it would be checked periodically. + copStream := resp.Resp.(*tikvrpc.MPPStreamResponse) + copStream.Timeout = timeout + copStream.Lease.Cancel = cancel + connArray.streamTimeout <- &copStream.Lease + + // Read the first streaming response to get CopStreamResponse. + // This can make error handling much easier, because SendReq() retry on + // region error automatically. + var first *mpp.MPPDataPacket + first, err = copStream.Recv() + if err != nil { + if errors.Cause(err) != io.EOF { + return nil, errors.Trace(err) + } + } + copStream.MPPDataPacket = first + return resp, nil +} + +// Close closes all connections. +func (c *RPCClient) Close() error { + // TODO: add a unit test for SendRequest After Closed + c.closeConns() + return nil +} diff --git a/store/tikv/client_batch.go b/store/tikv/client/client_batch.go similarity index 98% rename from store/tikv/client_batch.go rename to store/tikv/client/client_batch.go index 886d20abf0e46..5cac70f38b260 100644 --- a/store/tikv/client_batch.go +++ b/store/tikv/client/client_batch.go @@ -11,8 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package tikv provides tcp connection to kvserver. -package tikv +// Package client provides tcp connection to kvserver. +package client import ( "context" @@ -29,6 +29,7 @@ import ( "github.com/pingcap/tidb/store/tikv/config" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -662,7 +663,7 @@ func (c *batchCommandsClient) recreateStreamingClient(err error, streamClient *b *epoch++ c.failPendingRequests(err) // fail all pending requests. - b := NewBackofferWithVars(context.Background(), math.MaxInt32, nil) + b := retry.NewBackofferWithVars(context.Background(), math.MaxInt32, nil) for { // try to re-create the streaming in the loop. if c.isStopped() { return true @@ -672,7 +673,7 @@ func (c *batchCommandsClient) recreateStreamingClient(err error, streamClient *b break } - err2 := b.Backoff(BoTiKVRPC, err1) + err2 := b.Backoff(retry.BoTiKVRPC, err1) // As timeout is set to math.MaxUint32, err2 should always be nil. // This line is added to make the 'make errcheck' pass. terror.Log(err2) @@ -768,6 +769,7 @@ func sendBatchRequest( zap.String("to", addr), zap.String("cause", ctx.Err().Error())) return nil, errors.Trace(ctx.Err()) case <-timer.C: + atomic.StoreInt32(&entry.canceled, 1) return nil, errors.SuspendStack(errors.Annotate(context.DeadlineExceeded, "wait recvLoop")) } } diff --git a/store/tikv/client_collapse.go b/store/tikv/client/client_collapse.go similarity index 93% rename from store/tikv/client_collapse.go rename to store/tikv/client/client_collapse.go index e7f9cfadcf08b..159b3dd1ef50b 100644 --- a/store/tikv/client_collapse.go +++ b/store/tikv/client/client_collapse.go @@ -11,8 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package tikv provides tcp connection to kvserver. -package tikv +// Package client provides tcp connection to kvserver. +package client import ( "context" @@ -32,6 +32,10 @@ type reqCollapse struct { Client } +// NewReqCollapse creates a reqCollapse. +func NewReqCollapse(client Client) Client { + return &reqCollapse{client} +} func (r reqCollapse) Close() error { if r.Client == nil { panic("client should not be nil") diff --git a/store/tikv/client_fail_test.go b/store/tikv/client/client_fail_test.go similarity index 92% rename from store/tikv/client_fail_test.go rename to store/tikv/client/client_fail_test.go index 25b7e36d2c95b..e443d46d9184e 100644 --- a/store/tikv/client_fail_test.go +++ b/store/tikv/client/client_fail_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package tikv +package client import ( "context" @@ -33,16 +33,16 @@ type testClientFailSuite struct { func (s *testClientFailSuite) SetUpSuite(_ *C) { // This lock make testClientFailSuite runs exclusively. - withTiKVGlobalLock.Lock() + s.LockGlobalTiKV() } func (s testClientFailSuite) TearDownSuite(_ *C) { - withTiKVGlobalLock.Unlock() + s.UnLockGlobalTiKV() } func (s *testClientFailSuite) TestPanicInRecvLoop(c *C) { - c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/panicInFailPendingRequests", `panic`), IsNil) - c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/gotErrorInRecvLoop", `return("0")`), IsNil) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/client/panicInFailPendingRequests", `panic`), IsNil) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/client/gotErrorInRecvLoop", `return("0")`), IsNil) server, port := startMockTikvService() c.Assert(port > 0, IsTrue) @@ -61,8 +61,8 @@ func (s *testClientFailSuite) TestPanicInRecvLoop(c *C) { _, err = rpcClient.SendRequest(context.Background(), addr, req, time.Second/2) c.Assert(err, NotNil) - c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/gotErrorInRecvLoop"), IsNil) - c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/panicInFailPendingRequests"), IsNil) + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/client/gotErrorInRecvLoop"), IsNil) + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/client/panicInFailPendingRequests"), IsNil) time.Sleep(time.Second * 2) req = tikvrpc.NewRequest(tikvrpc.CmdEmpty, &tikvpb.BatchCommandsEmptyRequest{}) @@ -115,7 +115,7 @@ func (s *testClientFailSuite) TestRecvErrorInMultipleRecvLoops(c *C) { } epoch := atomic.LoadUint64(&batchClient.epoch) - fp := "github.com/pingcap/tidb/store/tikv/gotErrorInRecvLoop" + fp := "github.com/pingcap/tidb/store/tikv/client/gotErrorInRecvLoop" // Send a request to each stream to trigger reconnection. for _, forwardedHost := range forwardedHosts { c.Assert(failpoint.Enable(fp, `1*return("0")`), IsNil) diff --git a/store/tikv/client_test.go b/store/tikv/client/client_test.go similarity index 98% rename from store/tikv/client_test.go rename to store/tikv/client/client_test.go index 3828422b02328..b3645d990723a 100644 --- a/store/tikv/client_test.go +++ b/store/tikv/client/client_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package tikv +package client import ( "context" @@ -28,10 +28,14 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/tikvpb" "github.com/pingcap/tidb/store/tikv/config" + "github.com/pingcap/tidb/store/tikv/mockstore" "github.com/pingcap/tidb/store/tikv/tikvrpc" "google.golang.org/grpc/metadata" ) +// OneByOneSuite is a suite, When with-tikv flag is true, there is only one storage, so the test suite have to run one by one. +type OneByOneSuite = mockstore.OneByOneSuite + func TestT(t *testing.T) { CustomVerboseFlag = true TestingT(t) diff --git a/store/tikv/mock_tikv_service_test.go b/store/tikv/client/mock_tikv_service_test.go similarity index 99% rename from store/tikv/mock_tikv_service_test.go rename to store/tikv/client/mock_tikv_service_test.go index 5ad7b023b1e9d..b2b5682457818 100644 --- a/store/tikv/mock_tikv_service_test.go +++ b/store/tikv/client/mock_tikv_service_test.go @@ -1,4 +1,4 @@ -package tikv +package client import ( "context" diff --git a/store/tikv/commit.go b/store/tikv/commit.go index ce9df6a927355..9eb60d41d474f 100644 --- a/store/tikv/commit.go +++ b/store/tikv/commit.go @@ -15,13 +15,16 @@ package tikv import ( "encoding/hex" + "time" "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" - pb "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/tidb/store/tikv/client" tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -41,103 +44,130 @@ func (actionCommit) tiKVTxnRegionsNumHistogram() prometheus.Observer { func (actionCommit) handleSingleBatch(c *twoPhaseCommitter, bo *Backoffer, batch batchMutations) error { keys := batch.mutations.GetKeys() - req := tikvrpc.NewRequest(tikvrpc.CmdCommit, &pb.CommitRequest{ + req := tikvrpc.NewRequest(tikvrpc.CmdCommit, &kvrpcpb.CommitRequest{ StartVersion: c.startTS, Keys: keys, CommitVersion: c.commitTS, - }, pb.Context{Priority: c.priority, SyncLog: c.syncLog}) - - sender := NewRegionRequestSender(c.store.regionCache, c.store.client) - resp, err := sender.SendReq(bo, req, batch.region, ReadTimeoutShort) - - // If we fail to receive response for the request that commits primary key, it will be undetermined whether this - // transaction has been successfully committed. - // Under this circumstance, we can not declare the commit is complete (may lead to data lost), nor can we throw - // an error (may lead to the duplicated key error when upper level restarts the transaction). Currently the best - // solution is to populate this error and let upper layer drop the connection to the corresponding mysql client. - if batch.isPrimary && sender.rpcError != nil { - c.setUndeterminedErr(errors.Trace(sender.rpcError)) - } + }, kvrpcpb.Context{Priority: c.priority, SyncLog: c.syncLog, ResourceGroupTag: c.resourceGroupTag}) - if err != nil { - return errors.Trace(err) - } - regionErr, err := resp.GetRegionError() - if err != nil { - return errors.Trace(err) - } - if regionErr != nil { - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + tBegin := time.Now() + attempts := 0 + + sender := NewRegionRequestSender(c.store.regionCache, c.store.GetTiKVClient()) + for { + attempts++ + if time.Since(tBegin) > slowRequestThreshold { + logutil.BgLogger().Warn("slow commit request", zap.Uint64("startTS", c.startTS), zap.Stringer("region", &batch.region), zap.Int("attempts", attempts)) + tBegin = time.Now() + } + + resp, err := sender.SendReq(bo, req, batch.region, client.ReadTimeoutShort) + // If we fail to receive response for the request that commits primary key, it will be undetermined whether this + // transaction has been successfully committed. + // Under this circumstance, we can not declare the commit is complete (may lead to data lost), nor can we throw + // an error (may lead to the duplicated key error when upper level restarts the transaction). Currently the best + // solution is to populate this error and let upper layer drop the connection to the corresponding mysql client. + if batch.isPrimary && sender.rpcError != nil && !c.isAsyncCommit() { + c.setUndeterminedErr(errors.Trace(sender.rpcError)) + } + + // Unexpected error occurs, return it. if err != nil { return errors.Trace(err) } - // re-split keys and commit again. - err = c.doActionOnMutations(bo, actionCommit{retry: true}, batch.mutations) - return errors.Trace(err) - } - if resp.Resp == nil { - return errors.Trace(tikverr.ErrBodyMissing) - } - commitResp := resp.Resp.(*pb.CommitResponse) - // Here we can make sure tikv has processed the commit primary key request. So - // we can clean undetermined error. - if batch.isPrimary { - c.setUndeterminedErr(nil) - } - if keyErr := commitResp.GetError(); keyErr != nil { - if rejected := keyErr.GetCommitTsExpired(); rejected != nil { - logutil.Logger(bo.ctx).Info("2PC commitTS rejected by TiKV, retry with a newer commitTS", - zap.Uint64("txnStartTS", c.startTS), - zap.Stringer("info", logutil.Hex(rejected))) - - // Do not retry for a txn which has a too large MinCommitTs - // 3600000 << 18 = 943718400000 - if rejected.MinCommitTs-rejected.AttemptedCommitTs > 943718400000 { - err := errors.Errorf("2PC MinCommitTS is too large, we got MinCommitTS: %d, and AttemptedCommitTS: %d", - rejected.MinCommitTs, rejected.AttemptedCommitTs) - return errors.Trace(err) - } - // Update commit ts and retry. - commitTS, err := c.store.getTimestampWithRetry(bo, c.txn.GetScope()) + regionErr, err := resp.GetRegionError() + if err != nil { + return errors.Trace(err) + } + if regionErr != nil { + // For other region error and the fake region error, backoff because + // there's something wrong. + // For the real EpochNotMatch error, don't backoff. + if regionErr.GetEpochNotMatch() == nil || isFakeRegionError(regionErr) { + err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) + if err != nil { + return errors.Trace(err) + } + } + same, err := batch.relocate(bo, c.store.regionCache) if err != nil { - logutil.Logger(bo.ctx).Warn("2PC get commitTS failed", - zap.Error(err), - zap.Uint64("txnStartTS", c.startTS)) return errors.Trace(err) } + if same { + continue + } + err = c.doActionOnMutations(bo, actionCommit{true}, batch.mutations) + return errors.Trace(err) + } - c.mu.Lock() - c.commitTS = commitTS - c.mu.Unlock() - return c.commitMutations(bo, batch.mutations) + if resp.Resp == nil { + return errors.Trace(tikverr.ErrBodyMissing) } + commitResp := resp.Resp.(*kvrpcpb.CommitResponse) + // Here we can make sure tikv has processed the commit primary key request. So + // we can clean undetermined error. + if batch.isPrimary && !c.isAsyncCommit() { + c.setUndeterminedErr(nil) + } + if keyErr := commitResp.GetError(); keyErr != nil { + if rejected := keyErr.GetCommitTsExpired(); rejected != nil { + logutil.Logger(bo.GetCtx()).Info("2PC commitTS rejected by TiKV, retry with a newer commitTS", + zap.Uint64("txnStartTS", c.startTS), + zap.Stringer("info", logutil.Hex(rejected))) + + // Do not retry for a txn which has a too large MinCommitTs + // 3600000 << 18 = 943718400000 + if rejected.MinCommitTs-rejected.AttemptedCommitTs > 943718400000 { + err := errors.Errorf("2PC MinCommitTS is too large, we got MinCommitTS: %d, and AttemptedCommitTS: %d", + rejected.MinCommitTs, rejected.AttemptedCommitTs) + return errors.Trace(err) + } - c.mu.RLock() - defer c.mu.RUnlock() - err = extractKeyErr(keyErr) - if c.mu.committed { - // No secondary key could be rolled back after it's primary key is committed. - // There must be a serious bug somewhere. - hexBatchKeys := func(keys [][]byte) []string { - var res []string - for _, k := range keys { - res = append(res, hex.EncodeToString(k)) + // Update commit ts and retry. + commitTS, err := c.store.getTimestampWithRetry(bo, c.txn.GetScope()) + if err != nil { + logutil.Logger(bo.GetCtx()).Warn("2PC get commitTS failed", + zap.Error(err), + zap.Uint64("txnStartTS", c.startTS)) + return errors.Trace(err) } - return res + + c.mu.Lock() + c.commitTS = commitTS + c.mu.Unlock() + // Update the commitTS of the request and retry. + req.Commit().CommitVersion = commitTS + continue } - logutil.Logger(bo.ctx).Error("2PC failed commit key after primary key committed", + + c.mu.RLock() + defer c.mu.RUnlock() + err = extractKeyErr(keyErr) + if c.mu.committed { + // No secondary key could be rolled back after it's primary key is committed. + // There must be a serious bug somewhere. + hexBatchKeys := func(keys [][]byte) []string { + var res []string + for _, k := range keys { + res = append(res, hex.EncodeToString(k)) + } + return res + } + logutil.Logger(bo.GetCtx()).Error("2PC failed commit key after primary key committed", + zap.Error(err), + zap.Uint64("txnStartTS", c.startTS), + zap.Uint64("commitTS", c.commitTS), + zap.Strings("keys", hexBatchKeys(keys))) + return errors.Trace(err) + } + // The transaction maybe rolled back by concurrent transactions. + logutil.Logger(bo.GetCtx()).Debug("2PC failed commit primary key", zap.Error(err), - zap.Uint64("txnStartTS", c.startTS), - zap.Uint64("commitTS", c.commitTS), - zap.Strings("keys", hexBatchKeys(keys))) - return errors.Trace(err) + zap.Uint64("txnStartTS", c.startTS)) + return err } - // The transaction maybe rolled back by concurrent transactions. - logutil.Logger(bo.ctx).Debug("2PC failed commit primary key", - zap.Error(err), - zap.Uint64("txnStartTS", c.startTS)) - return err + break } c.mu.Lock() @@ -149,10 +179,10 @@ func (actionCommit) handleSingleBatch(c *twoPhaseCommitter, bo *Backoffer, batch } func (c *twoPhaseCommitter) commitMutations(bo *Backoffer, mutations CommitterMutations) error { - if span := opentracing.SpanFromContext(bo.ctx); span != nil && span.Tracer() != nil { + if span := opentracing.SpanFromContext(bo.GetCtx()); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("twoPhaseCommitter.commitMutations", opentracing.ChildOf(span.Context())) defer span1.Finish() - bo.ctx = opentracing.ContextWithSpan(bo.ctx, span1) + bo.SetCtx(opentracing.ContextWithSpan(bo.GetCtx(), span1)) } return c.doActionOnMutations(bo, actionCommit{}, mutations) diff --git a/store/tikv/config/config.go b/store/tikv/config/config.go index 6fe0c018498ea..09c559581076a 100644 --- a/store/tikv/config/config.go +++ b/store/tikv/config/config.go @@ -45,7 +45,6 @@ func init() { type Config struct { CommitterConcurrency int MaxTxnTTL uint64 - ServerMemoryQuota uint64 TiKVClient TiKVClient Security Security PDClient PDClient @@ -64,7 +63,6 @@ func DefaultConfig() Config { return Config{ CommitterConcurrency: 128, MaxTxnTTL: 60 * 60 * 1000, // 1hour - ServerMemoryQuota: 0, TiKVClient: DefaultTiKVClient(), PDClient: DefaultPDClient(), TxnLocalLatches: DefaultTxnLocalLatches(), diff --git a/store/tikv/config/config_test.go b/store/tikv/config/config_test.go index f79f2d09c22c8..a47e1e7e5030b 100644 --- a/store/tikv/config/config_test.go +++ b/store/tikv/config/config_test.go @@ -34,20 +34,19 @@ func (s *testConfigSuite) TestParsePath(c *C) { } func (s *testConfigSuite) TestTxnScopeValue(c *C) { - - failpoint.Enable("github.com/pingcap/tidb/store/tikv/config/injectTxnScope", `return("bj")`) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/config/injectTxnScope", `return("bj")`), IsNil) isGlobal, v := GetTxnScopeFromConfig() c.Assert(isGlobal, IsFalse) c.Assert(v, Equals, "bj") - failpoint.Disable("github.com/pingcap/tidb/store/tikv/config/injectTxnScope") - failpoint.Enable("github.com/pingcap/tidb/store/tikv/config/injectTxnScope", `return("")`) + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/config/injectTxnScope"), IsNil) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/config/injectTxnScope", `return("")`), IsNil) isGlobal, v = GetTxnScopeFromConfig() c.Assert(isGlobal, IsTrue) c.Assert(v, Equals, "global") - failpoint.Disable("github.com/pingcap/tidb/store/tikv/config/injectTxnScope") - failpoint.Enable("github.com/pingcap/tidb/store/tikv/config/injectTxnScope", `return("global")`) + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/config/injectTxnScope"), IsNil) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/config/injectTxnScope", `return("global")`), IsNil) isGlobal, v = GetTxnScopeFromConfig() c.Assert(isGlobal, IsFalse) c.Assert(v, Equals, "global") - failpoint.Disable("github.com/pingcap/tidb/store/tikv/config/injectTxnScope") + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/config/injectTxnScope"), IsNil) } diff --git a/store/tikv/config/security.go b/store/tikv/config/security.go index 4cb414c050675..513dc40791509 100644 --- a/store/tikv/config/security.go +++ b/store/tikv/config/security.go @@ -16,7 +16,7 @@ package config import ( "crypto/tls" "crypto/x509" - "io/ioutil" + "os" "github.com/pingcap/errors" ) @@ -45,7 +45,7 @@ func (s *Security) ToTLSConfig() (tlsConfig *tls.Config, err error) { certPool := x509.NewCertPool() // Create a certificate pool from the certificate authority var ca []byte - ca, err = ioutil.ReadFile(s.ClusterSSLCA) + ca, err = os.ReadFile(s.ClusterSSLCA) if err != nil { err = errors.Errorf("could not read ca certificate: %s", err) return diff --git a/store/tikv/delete_range.go b/store/tikv/delete_range.go index 4cbe9fc039749..f42d5a94ccdae 100644 --- a/store/tikv/delete_range.go +++ b/store/tikv/delete_range.go @@ -19,8 +19,10 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/tidb/store/tikv/client" tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/kv" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" ) @@ -78,6 +80,8 @@ func (t *DeleteRangeTask) Execute(ctx context.Context) error { return err } +const deleteRangeOneRegionMaxBackoff = 100000 + // Execute performs the delete range operation. func (t *DeleteRangeTask) sendReqOnRange(ctx context.Context, r kv.KeyRange) (RangeTaskStat, error) { startKey, rangeEndKey := r.StartKey, r.EndKey @@ -93,7 +97,7 @@ func (t *DeleteRangeTask) sendReqOnRange(ctx context.Context, r kv.KeyRange) (Ra break } - bo := NewBackofferWithVars(ctx, deleteRangeOneRegionMaxBackoff, nil) + bo := retry.NewBackofferWithVars(ctx, deleteRangeOneRegionMaxBackoff, nil) loc, err := t.store.GetRegionCache().LocateKey(bo, startKey) if err != nil { return stat, errors.Trace(err) @@ -112,7 +116,7 @@ func (t *DeleteRangeTask) sendReqOnRange(ctx context.Context, r kv.KeyRange) (Ra NotifyOnly: t.notifyOnly, }) - resp, err := t.store.SendReq(bo, req, loc.Region, ReadTimeoutMedium) + resp, err := t.store.SendReq(bo, req, loc.Region, client.ReadTimeoutMedium) if err != nil { return stat, errors.Trace(err) } @@ -121,7 +125,7 @@ func (t *DeleteRangeTask) sendReqOnRange(ctx context.Context, r kv.KeyRange) (Ra return stat, errors.Trace(err) } if regionErr != nil { - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) if err != nil { return stat, errors.Trace(err) } diff --git a/store/tikv/error/error.go b/store/tikv/error/error.go index 8ed21d8db192f..898354cc11a2d 100644 --- a/store/tikv/error/error.go +++ b/store/tikv/error/error.go @@ -20,7 +20,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/pdpb" - "github.com/pingcap/tidb/util/dbterror" ) var ( @@ -38,32 +37,31 @@ var ( ErrTiKVServerTimeout = errors.New("tikv server timeout") // ErrTiFlashServerTimeout is the error when tiflash server is timeout. ErrTiFlashServerTimeout = errors.New("tiflash server timeout") + // ErrQueryInterrupted is the error when the query is interrupted. + ErrQueryInterrupted = errors.New("query interruppted") // ErrTiKVStaleCommand is the error that the command is stale in tikv. ErrTiKVStaleCommand = errors.New("tikv stale command") // ErrTiKVMaxTimestampNotSynced is the error that tikv's max timestamp is not synced. ErrTiKVMaxTimestampNotSynced = errors.New("tikv max timestamp not synced") + // ErrLockAcquireFailAndNoWaitSet is the error that acquire the lock failed while no wait is setted. + ErrLockAcquireFailAndNoWaitSet = errors.New("lock acquired failed and no wait is setted") // ErrResolveLockTimeout is the error that resolve lock timeout. ErrResolveLockTimeout = errors.New("resolve lock timeout") + // ErrLockWaitTimeout is the error that wait for the lock is timeout. + ErrLockWaitTimeout = errors.New("lock wait timeout") // ErrTiKVServerBusy is the error when tikv server is busy. ErrTiKVServerBusy = errors.New("tikv server busy") // ErrTiFlashServerBusy is the error that tiflash server is busy. ErrTiFlashServerBusy = errors.New("tiflash server busy") // ErrRegionUnavailable is the error when region is not available. ErrRegionUnavailable = errors.New("region unavailable") + // ErrUnknown is the unknow error. + ErrUnknown = errors.New("unknow") ) // MismatchClusterID represents the message that the cluster ID of the PD client does not match the PD. const MismatchClusterID = "mismatch cluster id" -// error instances. -var ( - ErrQueryInterrupted = dbterror.ClassTiKV.NewStd(CodeQueryInterrupted) - ErrLockAcquireFailAndNoWaitSet = dbterror.ClassTiKV.NewStd(CodeLockAcquireFailAndNoWaitSet) - ErrLockWaitTimeout = dbterror.ClassTiKV.NewStd(CodeLockWaitTimeout) - ErrTokenLimit = dbterror.ClassTiKV.NewStd(CodeTiKVStoreLimit) - ErrUnknown = dbterror.ClassTiKV.NewStd(CodeUnknown) -) - // IsErrNotFound checks if err is a kind of NotFound error. func IsErrNotFound(err error) bool { return errors.ErrorEqual(err, ErrNotExist) @@ -190,3 +188,12 @@ type ErrGCTooEarly struct { func (e *ErrGCTooEarly) Error() string { return fmt.Sprintf("GC life time is shorter than transaction duration, transaction starts at %v, GC safe point is %v", e.TxnStartTS, e.GCSafePoint) } + +// ErrTokenLimit is the error that token is up to the limit. +type ErrTokenLimit struct { + StoreID uint64 +} + +func (e *ErrTokenLimit) Error() string { + return fmt.Sprintf("Store token is up to the limit, store id = %d.", e.StoreID) +} diff --git a/store/tikv/kv.go b/store/tikv/kv.go index 0f4824a785ecb..8adf5c1f58782 100644 --- a/store/tikv/kv.go +++ b/store/tikv/kv.go @@ -18,6 +18,7 @@ import ( "crypto/tls" "math" "math/rand" + "strconv" "sync" "sync/atomic" "time" @@ -27,6 +28,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/tidb/store/tikv/client" "github.com/pingcap/tidb/store/tikv/config" tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/kv" @@ -35,6 +37,7 @@ import ( "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/store/tikv/oracle/oracles" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" pd "github.com/tikv/pd/client" "go.etcd.io/etcd/clientv3" @@ -65,10 +68,13 @@ var oracleUpdateInterval = 2000 // KVStore contains methods to interact with a TiKV cluster. type KVStore struct { - clusterID uint64 - uuid string - oracle oracle.Oracle - client Client + clusterID uint64 + uuid string + oracle oracle.Oracle + clientMu struct { + sync.RWMutex + client Client + } pdClient pd.Client regionCache *RegionCache lockResolver *LockResolver @@ -80,12 +86,12 @@ type KVStore struct { safePoint uint64 spTime time.Time spMutex sync.RWMutex // this is used to update safePoint and spTime - closed chan struct{} // this is used to nofity when the store is closed + closed chan struct{} // this is used to notify when the store is closed - resolveTSMu struct { - sync.RWMutex - resolveTS map[uint64]uint64 // storeID -> resolveTS - } + // storeID -> safeTS, stored as map[uint64]uint64 + // safeTS here will be used during the Stale Read process, + // it indicates the safe timestamp point that can be used to read consistent but may not the latest data. + safeTSMap sync.Map replicaReadSeed uint32 // this is used to load balance followers / learners when replica read is enabled } @@ -123,7 +129,7 @@ func (s *KVStore) CheckVisibility(startTime uint64) error { } // NewKVStore creates a new TiKV store instance. -func NewKVStore(uuid string, pdClient pd.Client, spkv SafePointKV, client Client) (*KVStore, error) { +func NewKVStore(uuid string, pdClient pd.Client, spkv SafePointKV, tikvclient Client) (*KVStore, error) { o, err := oracles.NewPdOracle(pdClient, time.Duration(oracleUpdateInterval)*time.Millisecond) if err != nil { return nil, errors.Trace(err) @@ -132,7 +138,6 @@ func NewKVStore(uuid string, pdClient pd.Client, spkv SafePointKV, client Client clusterID: pdClient.GetClusterID(context.TODO()), uuid: uuid, oracle: o, - client: reqCollapse{client}, pdClient: pdClient, regionCache: NewRegionCache(pdClient), kv: spkv, @@ -141,8 +146,8 @@ func NewKVStore(uuid string, pdClient pd.Client, spkv SafePointKV, client Client closed: make(chan struct{}), replicaReadSeed: rand.Uint32(), } + store.clientMu.client = client.NewReqCollapse(tikvclient) store.lockResolver = newLockResolver(store) - store.resolveTSMu.resolveTS = make(map[uint64]uint64) go store.runSafePointChecker() go store.safeTSUpdater() @@ -184,72 +189,12 @@ func (s *KVStore) runSafePointChecker() { // Begin a global transaction. func (s *KVStore) Begin() (*KVTxn, error) { - return s.BeginWithTxnScope(oracle.GlobalTxnScope) + return s.BeginWithOption(DefaultStartTSOption()) } -// BeginWithTxnScope begins a transaction with the given txnScope (local or global) -func (s *KVStore) BeginWithTxnScope(txnScope string) (*KVTxn, error) { - txn, err := newTiKVTxn(s, txnScope) - if err != nil { - return nil, errors.Trace(err) - } - return txn, nil -} - -// BeginWithStartTS begins a transaction with startTS. -func (s *KVStore) BeginWithStartTS(txnScope string, startTS uint64) (*KVTxn, error) { - txn, err := newTiKVTxnWithStartTS(s, txnScope, startTS, s.nextReplicaReadSeed()) - if err != nil { - return nil, errors.Trace(err) - } - return txn, nil -} - -// BeginWithExactStaleness begins transaction with given staleness -func (s *KVStore) BeginWithExactStaleness(txnScope string, prevSec uint64) (*KVTxn, error) { - txn, err := newTiKVTxnWithExactStaleness(s, txnScope, prevSec) - if err != nil { - return nil, errors.Trace(err) - } - return txn, nil -} - -// BeginWithMinStartTS begins transaction with the least startTS -func (s *KVStore) BeginWithMinStartTS(txnScope string, minStartTS uint64) (*KVTxn, error) { - stores := make([]*Store, 0) - allStores := s.regionCache.getStoresByType(tikvrpc.TiKV) - if txnScope != oracle.GlobalTxnScope { - for _, store := range allStores { - if store.IsLabelsMatch([]*metapb.StoreLabel{ - { - Key: DCLabelKey, - Value: txnScope, - }, - }) { - stores = append(stores, store) - } - } - } else { - stores = allStores - } - resolveTS := s.getMinResolveTSByStores(stores) - startTS := minStartTS - // If the resolveTS is larger than the minStartTS, we will use resolveTS as StartTS, otherwise we will use - // minStartTS directly. - if oracle.CompareTS(startTS, resolveTS) < 0 { - startTS = resolveTS - } - return s.BeginWithStartTS(txnScope, startTS) -} - -// BeginWithMaxPrevSec begins transaction with given max previous seconds for startTS -func (s *KVStore) BeginWithMaxPrevSec(txnScope string, maxPrevSec uint64) (*KVTxn, error) { - bo := NewBackofferWithVars(context.Background(), tsoMaxBackoff, nil) - minStartTS, err := s.getStalenessTimestamp(bo, txnScope, maxPrevSec) - if err != nil { - return nil, errors.Trace(err) - } - return s.BeginWithMinStartTS(txnScope, minStartTS) +// BeginWithOption begins a transaction with the given StartTSOption +func (s *KVStore) BeginWithOption(options StartTSOption) (*KVTxn, error) { + return newTiKVTxnWithOptions(s, options) } // GetSnapshot gets a snapshot that is able to read any data which data is <= ver. @@ -265,7 +210,7 @@ func (s *KVStore) Close() error { s.pdClient.Close() close(s.closed) - if err := s.client.Close(); err != nil { + if err := s.GetTiKVClient().Close(); err != nil { return errors.Trace(err) } @@ -287,7 +232,7 @@ func (s *KVStore) UUID() string { // CurrentTimestamp returns current timestamp with the given txnScope (local or global). func (s *KVStore) CurrentTimestamp(txnScope string) (uint64, error) { - bo := NewBackofferWithVars(context.Background(), tsoMaxBackoff, nil) + bo := retry.NewBackofferWithVars(context.Background(), tsoMaxBackoff, nil) startTS, err := s.getTimestampWithRetry(bo, txnScope) if err != nil { return 0, errors.Trace(err) @@ -296,14 +241,14 @@ func (s *KVStore) CurrentTimestamp(txnScope string) (uint64, error) { } func (s *KVStore) getTimestampWithRetry(bo *Backoffer, txnScope string) (uint64, error) { - if span := opentracing.SpanFromContext(bo.ctx); span != nil && span.Tracer() != nil { + if span := opentracing.SpanFromContext(bo.GetCtx()); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("TiKVStore.getTimestampWithRetry", opentracing.ChildOf(span.Context())) defer span1.Finish() - bo.ctx = opentracing.ContextWithSpan(bo.ctx, span1) + bo.SetCtx(opentracing.ContextWithSpan(bo.GetCtx(), span1)) } for { - startTS, err := s.oracle.GetTimestamp(bo.ctx, &oracle.Option{TxnScope: txnScope}) + startTS, err := s.oracle.GetTimestamp(bo.GetCtx(), &oracle.Option{TxnScope: txnScope}) // mockGetTSErrorInRetry should wait MockCommitErrorOnce first, then will run into retry() logic. // Then mockGetTSErrorInRetry will return retryable error when first retry. // Before PR #8743, we don't cleanup txn after meet error such as error like: PD server timeout @@ -317,20 +262,7 @@ func (s *KVStore) getTimestampWithRetry(bo *Backoffer, txnScope string) (uint64, if err == nil { return startTS, nil } - err = bo.Backoff(BoPDRPC, errors.Errorf("get timestamp failed: %v", err)) - if err != nil { - return 0, errors.Trace(err) - } - } -} - -func (s *KVStore) getStalenessTimestamp(bo *Backoffer, txnScope string, prevSec uint64) (uint64, error) { - for { - startTS, err := s.oracle.GetStaleTimestamp(bo.ctx, txnScope, prevSec) - if err == nil { - return startTS, nil - } - err = bo.Backoff(BoPDRPC, errors.Errorf("get staleness timestamp failed: %v", err)) + err = bo.Backoff(retry.BoPDRPC, errors.Errorf("get timestamp failed: %v", err)) if err != nil { return 0, errors.Trace(err) } @@ -358,7 +290,7 @@ func (s *KVStore) SupportDeleteRange() (supported bool) { // SendReq sends a request to region. func (s *KVStore) SendReq(bo *Backoffer, req *tikvrpc.Request, regionID RegionVerID, timeout time.Duration) (*tikvrpc.Response, error) { - sender := NewRegionRequestSender(s.regionCache, s.client) + sender := NewRegionRequestSender(s.regionCache, s.GetTiKVClient()) return sender.SendReq(bo, req, regionID, timeout) } @@ -389,28 +321,64 @@ func (s *KVStore) SetOracle(oracle oracle.Oracle) { // SetTiKVClient resets the client instance. func (s *KVStore) SetTiKVClient(client Client) { - s.client = client + s.clientMu.Lock() + defer s.clientMu.Unlock() + s.clientMu.client = client } // GetTiKVClient gets the client instance. func (s *KVStore) GetTiKVClient() (client Client) { - return s.client + s.clientMu.RLock() + defer s.clientMu.RUnlock() + return s.clientMu.client +} + +// GetMinSafeTS return the minimal safeTS of the storage with given txnScope. +func (s *KVStore) GetMinSafeTS(txnScope string) uint64 { + stores := make([]*Store, 0) + allStores := s.regionCache.GetStoresByType(tikvrpc.TiKV) + if txnScope != oracle.GlobalTxnScope { + for _, store := range allStores { + if store.IsLabelsMatch([]*metapb.StoreLabel{ + { + Key: DCLabelKey, + Value: txnScope, + }, + }) { + stores = append(stores, store) + } + } + } else { + stores = allStores + } + return s.getMinSafeTSByStores(stores) +} + +func (s *KVStore) getSafeTS(storeID uint64) uint64 { + safeTS, ok := s.safeTSMap.Load(storeID) + if !ok { + return 0 + } + return safeTS.(uint64) +} + +// setSafeTS sets safeTs for store storeID, export for testing +func (s *KVStore) setSafeTS(storeID, safeTS uint64) { + s.safeTSMap.Store(storeID, safeTS) } -func (s *KVStore) getMinResolveTSByStores(stores []*Store) uint64 { - failpoint.Inject("injectResolveTS", func(val failpoint.Value) { +func (s *KVStore) getMinSafeTSByStores(stores []*Store) uint64 { + failpoint.Inject("injectSafeTS", func(val failpoint.Value) { injectTS := val.(int) failpoint.Return(uint64(injectTS)) }) minSafeTS := uint64(math.MaxUint64) - s.resolveTSMu.RLock() - defer s.resolveTSMu.RUnlock() // when there is no store, return 0 in order to let minStartTS become startTS directly if len(stores) < 1 { return 0 } for _, store := range stores { - safeTS := s.resolveTSMu.resolveTS[store.storeID] + safeTS := s.getSafeTS(store.storeID) if safeTS < minSafeTS { minSafeTS = safeTS } @@ -428,13 +396,13 @@ func (s *KVStore) safeTSUpdater() { case <-s.Closed(): return case <-t.C: - s.updateResolveTS(ctx) + s.updateSafeTS(ctx) } } } -func (s *KVStore) updateResolveTS(ctx context.Context) { - stores := s.regionCache.getStoresByType(tikvrpc.TiKV) +func (s *KVStore) updateSafeTS(ctx context.Context) { + stores := s.regionCache.GetStoresByType(tikvrpc.TiKV) tikvClient := s.GetTiKVClient() wg := &sync.WaitGroup{} wg.Add(len(stores)) @@ -443,19 +411,20 @@ func (s *KVStore) updateResolveTS(ctx context.Context) { storeAddr := store.addr go func(ctx context.Context, wg *sync.WaitGroup, storeID uint64, storeAddr string) { defer wg.Done() - // TODO: add metrics for updateSafeTS resp, err := tikvClient.SendRequest(ctx, storeAddr, tikvrpc.NewRequest(tikvrpc.CmdStoreSafeTS, &kvrpcpb.StoreSafeTSRequest{KeyRange: &kvrpcpb.KeyRange{ StartKey: []byte(""), EndKey: []byte(""), - }}), ReadTimeoutShort) + }}), client.ReadTimeoutShort) + storeIDStr := strconv.Itoa(int(storeID)) if err != nil { - logutil.BgLogger().Debug("update resolveTS failed", zap.Error(err), zap.Uint64("store-id", storeID)) + metrics.TiKVSafeTSUpdateCounter.WithLabelValues("fail", storeIDStr).Inc() + logutil.BgLogger().Debug("update safeTS failed", zap.Error(err), zap.Uint64("store-id", storeID)) return } safeTSResp := resp.Resp.(*kvrpcpb.StoreSafeTSResponse) - s.resolveTSMu.Lock() - s.resolveTSMu.resolveTS[storeID] = safeTSResp.GetSafeTs() - s.resolveTSMu.Unlock() + s.setSafeTS(storeID, safeTSResp.GetSafeTs()) + metrics.TiKVSafeTSUpdateCounter.WithLabelValues("success", storeIDStr).Inc() + metrics.TiKVSafeTSUpdateStats.WithLabelValues(storeIDStr).Set(float64(safeTSResp.GetSafeTs())) }(ctx, wg, storeID, storeAddr) } wg.Wait() diff --git a/store/tikv/kv/kv.go b/store/tikv/kv/kv.go index 2b7e87ecd2e47..0d900d6facddb 100644 --- a/store/tikv/kv/kv.go +++ b/store/tikv/kv/kv.go @@ -4,6 +4,7 @@ import ( "sync" "time" + tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/util" ) @@ -27,4 +28,34 @@ type LockCtx struct { ValuesLock sync.Mutex LockExpired *uint32 Stats *util.LockKeysDetails + ResourceGroupTag []byte + OnDeadlock func(*tikverr.ErrDeadlock) +} + +// InitReturnValues creates the map to store returned value. +func (ctx *LockCtx) InitReturnValues(valueLen int) { + ctx.ReturnValues = true + ctx.Values = make(map[string]ReturnedValue, valueLen) +} + +// GetValueNotLocked returns a value if the key is not already locked. +// (nil, false) means already locked. +func (ctx *LockCtx) GetValueNotLocked(key []byte) ([]byte, bool) { + rv := ctx.Values[string(key)] + if !rv.AlreadyLocked { + return rv.Value, true + } + return nil, false +} + +// IterateValuesNotLocked applies f to all key-values that are not already +// locked. +func (ctx *LockCtx) IterateValuesNotLocked(f func([]byte, []byte)) { + ctx.ValuesLock.Lock() + defer ctx.ValuesLock.Unlock() + for key, val := range ctx.Values { + if !val.AlreadyLocked { + f([]byte(key), val.Value) + } + } } diff --git a/store/tikv/kv/store_vars.go b/store/tikv/kv/store_vars.go index 5f65f927bffb9..02d87018213a9 100644 --- a/store/tikv/kv/store_vars.go +++ b/store/tikv/kv/store_vars.go @@ -25,15 +25,14 @@ type ReplicaReadType byte const ( // ReplicaReadLeader stands for 'read from leader'. - ReplicaReadLeader ReplicaReadType = 1 << iota + ReplicaReadLeader ReplicaReadType = iota // ReplicaReadFollower stands for 'read from follower'. ReplicaReadFollower // ReplicaReadMixed stands for 'read from leader and follower and learner'. ReplicaReadMixed ) -// IsFollowerRead checks if leader is going to be used to read data. +// IsFollowerRead checks if follower is going to be used to read data. func (r ReplicaReadType) IsFollowerRead() bool { - // In some cases the default value is 0, which should be treated as `ReplicaReadLeader`. - return r != ReplicaReadLeader && r != 0 + return r != ReplicaReadLeader } diff --git a/store/tikv/kv/variables.go b/store/tikv/kv/variables.go index b722023bcae08..5e7a4c83b669a 100644 --- a/store/tikv/kv/variables.go +++ b/store/tikv/kv/variables.go @@ -21,9 +21,6 @@ type Variables struct { // BackOffWeight specifies the weight of the max back off time duration. BackOffWeight int - // Hook is used for test to verify the variable take effect. - Hook func(name string, vars *Variables) - // Pointer to SessionVars.Killed // Killed is a flag to indicate that this query is killed. Killed *uint32 diff --git a/store/tikv/latch/latch_test.go b/store/tikv/latch/latch_test.go index 4b10c118883a6..ce53794d44f12 100644 --- a/store/tikv/latch/latch_test.go +++ b/store/tikv/latch/latch_test.go @@ -110,7 +110,7 @@ func (s *testLatchSuite) TestFirstAcquireFailedWithStale(c *C) { func (s *testLatchSuite) TestRecycle(c *C) { latches := NewLatches(8) now := time.Now() - startTS := oracle.ComposeTS(oracle.GetPhysical(now), 0) + startTS := oracle.GoTimeToTS(now) lock := latches.genLock(startTS, [][]byte{ []byte("a"), []byte("b"), }) @@ -142,7 +142,7 @@ func (s *testLatchSuite) TestRecycle(c *C) { } c.Assert(allEmpty, IsFalse) - currentTS := oracle.ComposeTS(oracle.GetPhysical(now.Add(expireDuration)), 3) + currentTS := oracle.GoTimeToTS(now.Add(expireDuration)) + 3 latches.recycle(currentTS) for i := 0; i < len(latches.slots); i++ { diff --git a/store/tikv/lock_resolver.go b/store/tikv/lock_resolver.go index 05feee6d31adb..a41cf522224ef 100644 --- a/store/tikv/lock_resolver.go +++ b/store/tikv/lock_resolver.go @@ -26,11 +26,13 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/tidb/store/tikv/client" "github.com/pingcap/tidb/store/tikv/config" tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/oracle" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/store/tikv/util" pd "github.com/tikv/pd/client" @@ -94,7 +96,7 @@ func NewLockResolver(etcdAddrs []string, security config.Security, opts ...pd.Cl return nil, errors.Trace(err) } - s, err := NewKVStore(uuid, &CodecPDClient{pdCli}, spkv, NewRPCClient(security)) + s, err := NewKVStore(uuid, &CodecPDClient{pdCli}, spkv, client.NewRPCClient(security)) if err != nil { return nil, errors.Trace(err) } @@ -228,11 +230,6 @@ func (lr *LockResolver) BatchResolveLocks(bo *Backoffer, locks []*Lock, loc Regi // locks have been cleaned before GC. expiredLocks := locks - callerStartTS, err := lr.store.GetOracle().GetTimestamp(bo.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) - if err != nil { - return false, errors.Trace(err) - } - txnInfos := make(map[uint64]uint64) startTime := time.Now() for _, l := range expiredLocks { @@ -242,7 +239,7 @@ func (lr *LockResolver) BatchResolveLocks(bo *Backoffer, locks []*Lock, loc Regi metrics.LockResolverCountWithExpired.Inc() // Use currentTS = math.MaxUint64 means rollback the txn, no matter the lock is expired or not! - status, err := lr.getTxnStatus(bo, l.TxnID, l.Primary, callerStartTS, math.MaxUint64, true, false, l) + status, err := lr.getTxnStatus(bo, l.TxnID, l.Primary, 0, math.MaxUint64, true, false, l) if err != nil { return false, err } @@ -256,7 +253,7 @@ func (lr *LockResolver) BatchResolveLocks(bo *Backoffer, locks []*Lock, loc Regi continue } if _, ok := errors.Cause(err).(*nonAsyncCommitLock); ok { - status, err = lr.getTxnStatus(bo, l.TxnID, l.Primary, callerStartTS, math.MaxUint64, true, true, l) + status, err = lr.getTxnStatus(bo, l.TxnID, l.Primary, 0, math.MaxUint64, true, true, l) if err != nil { return false, err } @@ -286,7 +283,7 @@ func (lr *LockResolver) BatchResolveLocks(bo *Backoffer, locks []*Lock, loc Regi req := tikvrpc.NewRequest(tikvrpc.CmdResolveLock, &kvrpcpb.ResolveLockRequest{TxnInfos: listTxnInfos}) startTime = time.Now() - resp, err := lr.store.SendReq(bo, req, loc, ReadTimeoutShort) + resp, err := lr.store.SendReq(bo, req, loc, client.ReadTimeoutShort) if err != nil { return false, errors.Trace(err) } @@ -297,7 +294,7 @@ func (lr *LockResolver) BatchResolveLocks(bo *Backoffer, locks []*Lock, loc Regi } if regionErr != nil { - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) if err != nil { return false, errors.Trace(err) } @@ -473,8 +470,8 @@ func (t *txnExpireTime) value() int64 { // seconds before calling it after Prewrite. func (lr *LockResolver) GetTxnStatus(txnID uint64, callerStartTS uint64, primary []byte) (TxnStatus, error) { var status TxnStatus - bo := NewBackoffer(context.Background(), cleanupMaxBackoff) - currentTS, err := lr.store.GetOracle().GetLowResolutionTimestamp(bo.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) + bo := retry.NewBackoffer(context.Background(), cleanupMaxBackoff) + currentTS, err := lr.store.GetOracle().GetLowResolutionTimestamp(bo.GetCtx(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) if err != nil { return status, err } @@ -493,7 +490,7 @@ func (lr *LockResolver) getTxnStatusFromLock(bo *Backoffer, l *Lock, callerStart // Set currentTS to max uint64 to make the lock expired. currentTS = math.MaxUint64 } else { - currentTS, err = lr.store.GetOracle().GetLowResolutionTimestamp(bo.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) + currentTS, err = lr.store.GetOracle().GetLowResolutionTimestamp(bo.GetCtx(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) if err != nil { return TxnStatus{}, err } @@ -522,12 +519,12 @@ func (lr *LockResolver) getTxnStatusFromLock(bo *Backoffer, l *Lock, callerStart // getTxnStatus() returns it when the secondary locks exist while the primary lock doesn't. // This is likely to happen in the concurrently prewrite when secondary regions // success before the primary region. - if err := bo.Backoff(boTxnNotFound, err); err != nil { - logutil.Logger(bo.ctx).Warn("getTxnStatusFromLock backoff fail", zap.Error(err)) + if err := bo.Backoff(retry.BoTxnNotFound, err); err != nil { + logutil.Logger(bo.GetCtx()).Warn("getTxnStatusFromLock backoff fail", zap.Error(err)) } if lr.store.GetOracle().UntilExpired(l.TxnID, l.TTL, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) <= 0 { - logutil.Logger(bo.ctx).Warn("lock txn not found, lock has expired", + logutil.Logger(bo.GetCtx()).Warn("lock txn not found, lock has expired", zap.Uint64("CallerStartTs", callerStartTS), zap.Stringer("lock str", l)) if l.LockType == kvrpcpb.Op_PessimisticLock { @@ -536,7 +533,7 @@ func (lr *LockResolver) getTxnStatusFromLock(bo *Backoffer, l *Lock, callerStart errors.New("error txn not found and lock expired")) }) } - // For pessimistic lock resolving, if the primary lock dose not exist and rollbackIfNotExist is true, + // For pessimistic lock resolving, if the primary lock does not exist and rollbackIfNotExist is true, // The Action_LockNotExistDoNothing will be returned as the status. rollbackIfNotExist = true } else { @@ -590,7 +587,7 @@ func (lr *LockResolver) getTxnStatus(bo *Backoffer, txnID uint64, primary []byte if err != nil { return status, errors.Trace(err) } - resp, err := lr.store.SendReq(bo, req, loc.Region, ReadTimeoutShort) + resp, err := lr.store.SendReq(bo, req, loc.Region, client.ReadTimeoutShort) if err != nil { return status, errors.Trace(err) } @@ -599,7 +596,7 @@ func (lr *LockResolver) getTxnStatus(bo *Backoffer, txnID uint64, primary []byte return status, errors.Trace(err) } if regionErr != nil { - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) if err != nil { return status, errors.Trace(err) } @@ -726,7 +723,7 @@ func (lr *LockResolver) checkSecondaries(bo *Backoffer, txnID uint64, curKeys [] } req := tikvrpc.NewRequest(tikvrpc.CmdCheckSecondaryLocks, checkReq) metrics.LockResolverCountWithQueryCheckSecondaryLocks.Inc() - resp, err := lr.store.SendReq(bo, req, curRegionID, ReadTimeoutShort) + resp, err := lr.store.SendReq(bo, req, curRegionID, client.ReadTimeoutShort) if err != nil { return errors.Trace(err) } @@ -735,7 +732,7 @@ func (lr *LockResolver) checkSecondaries(bo *Backoffer, txnID uint64, curKeys [] return errors.Trace(err) } if regionErr != nil { - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) if err != nil { return errors.Trace(err) } @@ -856,7 +853,7 @@ func (lr *LockResolver) resolveRegionLocks(bo *Backoffer, l *Lock, region Region lreq.Keys = keys req := tikvrpc.NewRequest(tikvrpc.CmdResolveLock, lreq) - resp, err := lr.store.SendReq(bo, req, region, ReadTimeoutShort) + resp, err := lr.store.SendReq(bo, req, region, client.ReadTimeoutShort) if err != nil { return errors.Trace(err) } @@ -866,7 +863,7 @@ func (lr *LockResolver) resolveRegionLocks(bo *Backoffer, l *Lock, region Region return errors.Trace(err) } if regionErr != nil { - err := bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + err := bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) if err != nil { return errors.Trace(err) } @@ -925,7 +922,7 @@ func (lr *LockResolver) resolveLock(bo *Backoffer, l *Lock, status TxnStatus, li lreq.Keys = [][]byte{l.Key} } req := tikvrpc.NewRequest(tikvrpc.CmdResolveLock, lreq) - resp, err := lr.store.SendReq(bo, req, loc.Region, ReadTimeoutShort) + resp, err := lr.store.SendReq(bo, req, loc.Region, client.ReadTimeoutShort) if err != nil { return errors.Trace(err) } @@ -934,7 +931,7 @@ func (lr *LockResolver) resolveLock(bo *Backoffer, l *Lock, status TxnStatus, li return errors.Trace(err) } if regionErr != nil { - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) if err != nil { return errors.Trace(err) } @@ -976,7 +973,7 @@ func (lr *LockResolver) resolvePessimisticLock(bo *Backoffer, l *Lock, cleanRegi Keys: [][]byte{l.Key}, } req := tikvrpc.NewRequest(tikvrpc.CmdPessimisticRollback, pessimisticRollbackReq) - resp, err := lr.store.SendReq(bo, req, loc.Region, ReadTimeoutShort) + resp, err := lr.store.SendReq(bo, req, loc.Region, client.ReadTimeoutShort) if err != nil { return errors.Trace(err) } @@ -985,7 +982,7 @@ func (lr *LockResolver) resolvePessimisticLock(bo *Backoffer, l *Lock, cleanRegi return errors.Trace(err) } if regionErr != nil { - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) if err != nil { return errors.Trace(err) } @@ -997,7 +994,7 @@ func (lr *LockResolver) resolvePessimisticLock(bo *Backoffer, l *Lock, cleanRegi cmdResp := resp.Resp.(*kvrpcpb.PessimisticRollbackResponse) if keyErr := cmdResp.GetErrors(); len(keyErr) > 0 { err = errors.Errorf("unexpected resolve pessimistic lock err: %s, lock: %v", keyErr[0], l) - logutil.Logger(bo.ctx).Error("resolveLock error", zap.Error(err)) + logutil.Logger(bo.GetCtx()).Error("resolveLock error", zap.Error(err)) return err } return nil diff --git a/store/tikv/metrics/metrics.go b/store/tikv/metrics/metrics.go index 8d71582fa2522..7f6cb97b9dde8 100644 --- a/store/tikv/metrics/metrics.go +++ b/store/tikv/metrics/metrics.go @@ -59,6 +59,12 @@ var ( TiKVPanicCounter *prometheus.CounterVec TiKVForwardRequestCounter *prometheus.CounterVec TiKVTSFutureWaitDuration prometheus.Histogram + TiKVSafeTSUpdateCounter *prometheus.CounterVec + TiKVSafeTSUpdateStats *prometheus.GaugeVec + TiKVReplicaSelectorFailureCounter *prometheus.CounterVec + TiKVRequestRetryTimesHistogram prometheus.Histogram + TiKVTxnCommitBackoffSeconds prometheus.Histogram + TiKVTxnCommitBackoffCount prometheus.Histogram ) // Label constants. @@ -414,6 +420,53 @@ func initMetrics(namespace, subsystem string) { Buckets: prometheus.ExponentialBuckets(0.000005, 2, 30), // 5us ~ 2560s }) + TiKVSafeTSUpdateCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "safets_update_counter", + Help: "Counter of tikv safe_ts being updated.", + }, []string{LblResult, LblStore}) + + TiKVSafeTSUpdateStats = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "safets_update_stats", + Help: "stat of tikv updating safe_ts stats", + }, []string{LblStore}) + TiKVReplicaSelectorFailureCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "replica_selector_failure_counter", + Help: "Counter of the reason why the replica selector cannot yield a potential leader.", + }, []string{LblType}) + TiKVRequestRetryTimesHistogram = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "request_retry_times", + Help: "Bucketed histogram of how many times a region request retries.", + Buckets: []float64{1, 2, 3, 4, 8, 16, 32, 64, 128, 256}, + }) + TiKVTxnCommitBackoffSeconds = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "txn_commit_backoff_seconds", + Help: "Bucketed histogram of the total backoff duration in committing a transaction.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 22), // 1ms ~ 2097s + }) + TiKVTxnCommitBackoffCount = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "txn_commit_backoff_count", + Help: "Bucketed histogram of the backoff count in committing a transaction.", + Buckets: prometheus.ExponentialBuckets(1, 2, 12), // 1 ~ 2048 + }) + initShortcuts() } @@ -468,6 +521,12 @@ func RegisterMetrics() { prometheus.MustRegister(TiKVPanicCounter) prometheus.MustRegister(TiKVForwardRequestCounter) prometheus.MustRegister(TiKVTSFutureWaitDuration) + prometheus.MustRegister(TiKVSafeTSUpdateCounter) + prometheus.MustRegister(TiKVSafeTSUpdateStats) + prometheus.MustRegister(TiKVReplicaSelectorFailureCounter) + prometheus.MustRegister(TiKVRequestRetryTimesHistogram) + prometheus.MustRegister(TiKVTxnCommitBackoffSeconds) + prometheus.MustRegister(TiKVTxnCommitBackoffCount) } // readCounter reads the value of a prometheus.Counter. diff --git a/store/tikv/metrics/shortcuts.go b/store/tikv/metrics/shortcuts.go index 01d4da8983cc7..d38c891b4c649 100644 --- a/store/tikv/metrics/shortcuts.go +++ b/store/tikv/metrics/shortcuts.go @@ -33,14 +33,15 @@ var ( RawkvSizeHistogramWithKey prometheus.Observer RawkvSizeHistogramWithValue prometheus.Observer - BackoffHistogramRPC prometheus.Observer - BackoffHistogramLock prometheus.Observer - BackoffHistogramLockFast prometheus.Observer - BackoffHistogramPD prometheus.Observer - BackoffHistogramRegionMiss prometheus.Observer - BackoffHistogramServerBusy prometheus.Observer - BackoffHistogramStaleCmd prometheus.Observer - BackoffHistogramEmpty prometheus.Observer + BackoffHistogramRPC prometheus.Observer + BackoffHistogramLock prometheus.Observer + BackoffHistogramLockFast prometheus.Observer + BackoffHistogramPD prometheus.Observer + BackoffHistogramRegionMiss prometheus.Observer + BackoffHistogramRegionScheduling prometheus.Observer + BackoffHistogramServerBusy prometheus.Observer + BackoffHistogramStaleCmd prometheus.Observer + BackoffHistogramEmpty prometheus.Observer TxnRegionsNumHistogramWithSnapshot prometheus.Observer TxnRegionsNumHistogramPrewrite prometheus.Observer @@ -120,6 +121,7 @@ func initShortcuts() { BackoffHistogramLockFast = TiKVBackoffHistogram.WithLabelValues("tikvLockFast") BackoffHistogramPD = TiKVBackoffHistogram.WithLabelValues("pdRPC") BackoffHistogramRegionMiss = TiKVBackoffHistogram.WithLabelValues("regionMiss") + BackoffHistogramRegionScheduling = TiKVBackoffHistogram.WithLabelValues("regionScheduling") BackoffHistogramServerBusy = TiKVBackoffHistogram.WithLabelValues("serverBusy") BackoffHistogramStaleCmd = TiKVBackoffHistogram.WithLabelValues("staleCommand") BackoffHistogramEmpty = TiKVBackoffHistogram.WithLabelValues("") diff --git a/store/tikv/mockstore/mocktikv/pd.go b/store/tikv/mockstore/mocktikv/pd.go index b2db0de8fb343..6531a4ca71cc3 100644 --- a/store/tikv/mockstore/mocktikv/pd.go +++ b/store/tikv/mockstore/mocktikv/pd.go @@ -15,6 +15,7 @@ package mocktikv import ( "context" + "fmt" "math" "sync" "time" @@ -126,6 +127,13 @@ func (c *pdClient) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, default: } store := c.cluster.GetStore(storeID) + // It's same as PD's implementation. + if store == nil { + return nil, fmt.Errorf("invalid store ID %d, not found", storeID) + } + if store.GetState() == metapb.StoreState_Tombstone { + return nil, nil + } return store, nil } diff --git a/store/tikv/mockstore/test_suite.go b/store/tikv/mockstore/test_suite.go new file mode 100644 index 0000000000000..9ec7a0fa8aeef --- /dev/null +++ b/store/tikv/mockstore/test_suite.go @@ -0,0 +1,58 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package mockstore + +import ( + "flag" + "sync" + + "github.com/pingcap/check" +) + +var ( + withTiKVGlobalLock sync.RWMutex + // WithTiKV is the flag which indicates whether it runs with tikv. + WithTiKV = flag.Bool("with-tikv", false, "run tests with TiKV cluster started. (not use the mock server)") +) + +// OneByOneSuite is a suite, When with-tikv flag is true, there is only one storage, so the test suite have to run one by one. +type OneByOneSuite struct{} + +// SetUpSuite implements the interface check.Suite. +func (s *OneByOneSuite) SetUpSuite(c *check.C) { + if *WithTiKV { + withTiKVGlobalLock.Lock() + } else { + withTiKVGlobalLock.RLock() + } +} + +// TearDownSuite implements the interface check.Suite. +func (s *OneByOneSuite) TearDownSuite(c *check.C) { + if *WithTiKV { + withTiKVGlobalLock.Unlock() + } else { + withTiKVGlobalLock.RUnlock() + } +} + +// LockGlobalTiKV locks withTiKVGlobalLock. +func (s *OneByOneSuite) LockGlobalTiKV() { + withTiKVGlobalLock.Lock() +} + +// UnLockGlobalTiKV unlocks withTiKVGlobalLock +func (s *OneByOneSuite) UnLockGlobalTiKV() { + withTiKVGlobalLock.Unlock() +} diff --git a/store/tikv/oracle/oracle.go b/store/tikv/oracle/oracle.go index 0a6865cf59039..fd95f1b357013 100644 --- a/store/tikv/oracle/oracle.go +++ b/store/tikv/oracle/oracle.go @@ -16,11 +16,6 @@ package oracle import ( "context" "time" - - "github.com/pingcap/failpoint" - "github.com/pingcap/tidb/store/tikv/config" - "github.com/pingcap/tidb/store/tikv/logutil" - "go.uber.org/zap" ) // Option represents available options for the oracle.Oracle. @@ -45,74 +40,15 @@ type Future interface { Wait() (uint64, error) } -// TxnScope indicates the used txnScope for oracle -type TxnScope struct { - // varValue indicates the value of @@txn_scope, which can only be `global` or `local` - varValue string - // txnScope indicates the value which the tidb-server holds to request tso to pd - txnScope string -} - -// GetTxnScope gets oracle.TxnScope from config -func GetTxnScope() TxnScope { - isGlobal, location := config.GetTxnScopeFromConfig() - if isGlobal { - return NewGlobalTxnScope() - } - return NewLocalTxnScope(location) -} - -// NewGlobalTxnScope creates a Global TxnScope -func NewGlobalTxnScope() TxnScope { - return newTxnScope(GlobalTxnScope, GlobalTxnScope) -} - -// NewLocalTxnScope creates a Local TxnScope with given real txnScope value. -func NewLocalTxnScope(txnScope string) TxnScope { - return newTxnScope(LocalTxnScope, txnScope) -} - -// GetVarValue returns the value of @@txn_scope which can only be `global` or `local` -func (t TxnScope) GetVarValue() string { - return t.varValue -} - -// GetTxnScope returns the value of the tidb-server holds to request tso to pd. -func (t TxnScope) GetTxnScope() string { - return t.txnScope -} - -func newTxnScope(varValue string, txnScope string) TxnScope { - return TxnScope{ - varValue: varValue, - txnScope: txnScope, - } -} - const ( physicalShiftBits = 18 logicalBits = (1 << physicalShiftBits) - 1 // GlobalTxnScope is the default transaction scope for a Oracle service. GlobalTxnScope = "global" - // LocalTxnScope indicates the local txn scope for a Oracle service. - LocalTxnScope = "local" ) // ComposeTS creates a ts from physical and logical parts. func ComposeTS(physical, logical int64) uint64 { - failpoint.Inject("changeTSFromPD", func(val failpoint.Value) { - valInt, ok := val.(int) - if ok { - origPhyTS := physical - logical := logical - newPhyTs := origPhyTS + int64(valInt) - origTS := uint64((physical << physicalShiftBits) + logical) - newTS := uint64((newPhyTs << physicalShiftBits) + logical) - logutil.BgLogger().Warn("ComposeTS failpoint", zap.Uint64("origTS", origTS), - zap.Int("valInt", valInt), zap.Uint64("ts", newTS)) - failpoint.Return(newTS) - } - }) return uint64((physical << physicalShiftBits) + logical) } @@ -131,11 +67,6 @@ func GetPhysical(t time.Time) int64 { return t.UnixNano() / int64(time.Millisecond) } -// EncodeTSO encodes a millisecond into tso. -func EncodeTSO(ts int64) uint64 { - return uint64(ts) << physicalShiftBits -} - // GetTimeFromTS extracts time.Time from a timestamp. func GetTimeFromTS(ts uint64) time.Time { ms := ExtractPhysical(ts) @@ -148,25 +79,6 @@ func GoTimeToTS(t time.Time) uint64 { return uint64(ts) } -// CompareTS is used to compare two timestamps. -// If tsoOne > tsoTwo, returns 1. -// If tsoOne = tsoTwo, returns 0. -// If tsoOne < tsoTwo, returns -1. -func CompareTS(tsoOne, tsoTwo uint64) int { - tsOnePhy := ExtractPhysical(tsoOne) - tsOneLog := ExtractLogical(tsoOne) - tsTwoPhy := ExtractPhysical(tsoTwo) - tsTwoLog := ExtractLogical(tsoTwo) - - if tsOnePhy > tsTwoPhy || (tsOnePhy == tsTwoPhy && tsOneLog > tsTwoLog) { - return 1 - } - if tsOnePhy == tsTwoPhy && tsOneLog == tsTwoLog { - return 0 - } - return -1 -} - // GoTimeToLowerLimitStartTS returns the min start_ts of the uncommitted transaction. // maxTxnTimeUse means the max time a Txn May use (in ms) from its begin to commit. func GoTimeToLowerLimitStartTS(now time.Time, maxTxnTimeUse int64) uint64 { diff --git a/store/tikv/oracle/oracles/local.go b/store/tikv/oracle/oracles/local.go index 4fcd7cbc51d78..8c7f8a30de645 100644 --- a/store/tikv/oracle/oracles/local.go +++ b/store/tikv/oracle/oracles/local.go @@ -42,7 +42,8 @@ func (l *localOracle) IsExpired(lockTS, TTL uint64, _ *oracle.Option) bool { if l.hook != nil { now = l.hook.currentTime } - return oracle.GetPhysical(now) >= oracle.ExtractPhysical(lockTS)+int64(TTL) + expire := oracle.GetTimeFromTS(lockTS).Add(time.Duration(TTL) * time.Millisecond) + return !now.Before(expire) } func (l *localOracle) GetTimestamp(ctx context.Context, _ *oracle.Option) (uint64, error) { @@ -52,8 +53,7 @@ func (l *localOracle) GetTimestamp(ctx context.Context, _ *oracle.Option) (uint6 if l.hook != nil { now = l.hook.currentTime } - physical := oracle.GetPhysical(now) - ts := oracle.ComposeTS(physical, 0) + ts := oracle.GoTimeToTS(now) if l.lastTimeStampTS == ts { l.n++ return ts + l.n, nil @@ -80,9 +80,7 @@ func (l *localOracle) GetLowResolutionTimestampAsync(ctx context.Context, opt *o // GetStaleTimestamp return physical func (l *localOracle) GetStaleTimestamp(ctx context.Context, txnScope string, prevSecond uint64) (ts uint64, err error) { - physical := oracle.GetPhysical(time.Now().Add(-time.Second * time.Duration(prevSecond))) - ts = oracle.ComposeTS(physical, 0) - return ts, nil + return oracle.GoTimeToTS(time.Now().Add(-time.Second * time.Duration(prevSecond))), nil } type future struct { diff --git a/store/tikv/oracle/oracles/mock.go b/store/tikv/oracle/oracles/mock.go index 2afd962fb5c42..b1eabe57feb37 100644 --- a/store/tikv/oracle/oracles/mock.go +++ b/store/tikv/oracle/oracles/mock.go @@ -62,9 +62,8 @@ func (o *MockOracle) GetTimestamp(ctx context.Context, _ *oracle.Option) (uint64 if o.stop { return 0, errors.Trace(errStopped) } - physical := oracle.GetPhysical(time.Now().Add(o.offset)) - ts := oracle.ComposeTS(physical, 0) - if oracle.ExtractPhysical(o.lastTS) == physical { + ts := oracle.GoTimeToTS(time.Now().Add(o.offset)) + if oracle.ExtractPhysical(o.lastTS) == oracle.ExtractPhysical(ts) { ts = o.lastTS + 1 } o.lastTS = ts @@ -73,9 +72,7 @@ func (o *MockOracle) GetTimestamp(ctx context.Context, _ *oracle.Option) (uint64 // GetStaleTimestamp implements oracle.Oracle interface. func (o *MockOracle) GetStaleTimestamp(ctx context.Context, txnScope string, prevSecond uint64) (ts uint64, err error) { - physical := oracle.GetPhysical(time.Now().Add(-time.Second * time.Duration(prevSecond))) - ts = oracle.ComposeTS(physical, 0) - return ts, nil + return oracle.GoTimeToTS(time.Now().Add(-time.Second * time.Duration(prevSecond))), nil } type mockOracleFuture struct { @@ -106,15 +103,16 @@ func (o *MockOracle) GetLowResolutionTimestampAsync(ctx context.Context, opt *or func (o *MockOracle) IsExpired(lockTimestamp, TTL uint64, _ *oracle.Option) bool { o.RLock() defer o.RUnlock() - - return oracle.GetPhysical(time.Now().Add(o.offset)) >= oracle.ExtractPhysical(lockTimestamp)+int64(TTL) + expire := oracle.GetTimeFromTS(lockTimestamp).Add(time.Duration(TTL) * time.Millisecond) + return !time.Now().Add(o.offset).Before(expire) } // UntilExpired implement oracle.Oracle interface. func (o *MockOracle) UntilExpired(lockTimeStamp, TTL uint64, _ *oracle.Option) int64 { o.RLock() defer o.RUnlock() - return oracle.ExtractPhysical(lockTimeStamp) + int64(TTL) - oracle.GetPhysical(time.Now().Add(o.offset)) + expire := oracle.GetTimeFromTS(lockTimeStamp).Add(time.Duration(TTL) * time.Millisecond) + return expire.Sub(time.Now().Add(o.offset)).Milliseconds() } // Close implements oracle.Oracle interface. diff --git a/store/tikv/oracle/oracles/pd.go b/store/tikv/oracle/oracles/pd.go index 063f73e343ce0..907dc278d71cb 100644 --- a/store/tikv/oracle/oracles/pd.go +++ b/store/tikv/oracle/oracles/pd.go @@ -135,7 +135,7 @@ func (o *pdOracle) getTimestamp(ctx context.Context, txnScope string) (uint64, e } func (o *pdOracle) getArrivalTimestamp() uint64 { - return oracle.ComposeTS(oracle.GetPhysical(time.Now()), 0) + return oracle.GoTimeToTS(time.Now()) } func (o *pdOracle) setLastTS(ts uint64, txnScope string) { @@ -288,7 +288,7 @@ func (o *pdOracle) getStaleTimestamp(txnScope string, prevSecond uint64) (uint64 staleTime := physicalTime.Add(-arrivalTime.Sub(time.Now().Add(-time.Duration(prevSecond) * time.Second))) - return oracle.ComposeTS(oracle.GetPhysical(staleTime), 0), nil + return oracle.GoTimeToTS(staleTime), nil } // GetStaleTimestamp generate a TSO which represents for the TSO prevSecond secs ago. diff --git a/store/tikv/oracle/oracles/pd_test.go b/store/tikv/oracle/oracles/pd_test.go index 4e881a82126b5..2894a782e505f 100644 --- a/store/tikv/oracle/oracles/pd_test.go +++ b/store/tikv/oracle/oracles/pd_test.go @@ -36,8 +36,8 @@ func (s *testPDSuite) TestPDOracle_UntilExpired(c *C) { lockAfter, lockExp := 10, 15 o := oracles.NewEmptyPDOracle() start := time.Now() - oracles.SetEmptyPDOracleLastTs(o, oracle.ComposeTS(oracle.GetPhysical(start), 0)) - lockTs := oracle.ComposeTS(oracle.GetPhysical(start.Add(time.Duration(lockAfter)*time.Millisecond)), 1) + oracles.SetEmptyPDOracleLastTs(o, oracle.GoTimeToTS(start)) + lockTs := oracle.GoTimeToTS((start.Add(time.Duration(lockAfter) * time.Millisecond))) + 1 waitTs := o.UntilExpired(lockTs, uint64(lockExp), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(waitTs, Equals, int64(lockAfter+lockExp), Commentf("waitTs shoulb be %d but got %d", int64(lockAfter+lockExp), waitTs)) } @@ -45,7 +45,7 @@ func (s *testPDSuite) TestPDOracle_UntilExpired(c *C) { func (s *testPDSuite) TestPdOracle_GetStaleTimestamp(c *C) { o := oracles.NewEmptyPDOracle() start := time.Now() - oracles.SetEmptyPDOracleLastTs(o, oracle.ComposeTS(oracle.GetPhysical(start), 0)) + oracles.SetEmptyPDOracleLastTs(o, oracle.GoTimeToTS(start)) ts, err := o.GetStaleTimestamp(context.Background(), oracle.GlobalTxnScope, 10) c.Assert(err, IsNil) @@ -75,7 +75,7 @@ func (s *testPDSuite) TestPdOracle_GetStaleTimestamp(c *C) { for _, testcase := range testcases { comment := Commentf("%s", testcase.name) start = time.Now() - oracles.SetEmptyPDOracleLastTs(o, oracle.ComposeTS(oracle.GetPhysical(start), 0)) + oracles.SetEmptyPDOracleLastTs(o, oracle.GoTimeToTS(start)) ts, err = o.GetStaleTimestamp(context.Background(), oracle.GlobalTxnScope, testcase.preSec) if testcase.expectErr == "" { c.Assert(err, IsNil, comment) diff --git a/store/tikv/pessimistic.go b/store/tikv/pessimistic.go index 475efc7ad2a8b..235535be8379b 100644 --- a/store/tikv/pessimistic.go +++ b/store/tikv/pessimistic.go @@ -22,11 +22,13 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" - pb "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/tidb/store/tikv/client" tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -68,14 +70,14 @@ func (actionPessimisticRollback) tiKVTxnRegionsNumHistogram() prometheus.Observe func (action actionPessimisticLock) handleSingleBatch(c *twoPhaseCommitter, bo *Backoffer, batch batchMutations) error { m := batch.mutations - mutations := make([]*pb.Mutation, m.Len()) + mutations := make([]*kvrpcpb.Mutation, m.Len()) for i := 0; i < m.Len(); i++ { - mut := &pb.Mutation{ - Op: pb.Op_PessimisticLock, + mut := &kvrpcpb.Mutation{ + Op: kvrpcpb.Op_PessimisticLock, Key: m.GetKey(i), } - if c.txn.us.HasPresumeKeyNotExists(m.GetKey(i)) || (c.doingAmend && m.GetOp(i) == pb.Op_Insert) { - mut.Assertion = pb.Assertion_NotExist + if c.txn.us.HasPresumeKeyNotExists(m.GetKey(i)) || (c.doingAmend && m.GetOp(i) == kvrpcpb.Op_Insert) { + mut.Assertion = kvrpcpb.Assertion_NotExist } mutations[i] = mut } @@ -90,7 +92,7 @@ func (action actionPessimisticLock) handleSingleBatch(c *twoPhaseCommitter, bo * logutil.BgLogger().Info("[failpoint] injected lock ttl = 1 on pessimistic lock", zap.Uint64("txnStartTS", c.startTS), zap.Strings("keys", keys)) }) - req := tikvrpc.NewRequest(tikvrpc.CmdPessimisticLock, &pb.PessimisticLockRequest{ + req := tikvrpc.NewRequest(tikvrpc.CmdPessimisticLock, &kvrpcpb.PessimisticLockRequest{ Mutations: mutations, PrimaryLock: c.primary(), StartVersion: c.startTS, @@ -100,7 +102,7 @@ func (action actionPessimisticLock) handleSingleBatch(c *twoPhaseCommitter, bo * WaitTimeout: action.LockWaitTime, ReturnValues: action.ReturnValues, MinCommitTs: c.forUpdateTS + 1, - }, pb.Context{Priority: c.priority, SyncLog: c.syncLog}) + }, kvrpcpb.Context{Priority: c.priority, SyncLog: c.syncLog, ResourceGroupTag: action.LockCtx.ResourceGroupTag}) lockWaitStartTime := action.WaitStartTime for { // if lockWaitTime set, refine the request `WaitTimeout` field based on timeout limit @@ -117,7 +119,7 @@ func (action actionPessimisticLock) handleSingleBatch(c *twoPhaseCommitter, bo * return &tikverr.ErrWriteConflict{WriteConflict: nil} }) startTime := time.Now() - resp, err := c.store.SendReq(bo, req, batch.region, ReadTimeoutShort) + resp, err := c.store.SendReq(bo, req, batch.region, client.ReadTimeoutShort) if action.LockCtx.Stats != nil { atomic.AddInt64(&action.LockCtx.Stats.LockRPCTime, int64(time.Since(startTime))) atomic.AddInt64(&action.LockCtx.Stats.LockRPCCount, 1) @@ -130,17 +132,29 @@ func (action actionPessimisticLock) handleSingleBatch(c *twoPhaseCommitter, bo * return errors.Trace(err) } if regionErr != nil { - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + // For other region error and the fake region error, backoff because + // there's something wrong. + // For the real EpochNotMatch error, don't backoff. + if regionErr.GetEpochNotMatch() == nil || isFakeRegionError(regionErr) { + err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) + if err != nil { + return errors.Trace(err) + } + } + same, err := batch.relocate(bo, c.store.regionCache) if err != nil { return errors.Trace(err) } + if same { + continue + } err = c.pessimisticLockMutations(bo, action.LockCtx, batch.mutations) return errors.Trace(err) } if resp.Resp == nil { return errors.Trace(tikverr.ErrBodyMissing) } - lockResp := resp.Resp.(*pb.PessimisticLockResponse) + lockResp := resp.Resp.(*kvrpcpb.PessimisticLockResponse) keyErrs := lockResp.GetErrors() if len(keyErrs) == 0 { if action.ReturnValues { @@ -214,12 +228,12 @@ func (action actionPessimisticLock) handleSingleBatch(c *twoPhaseCommitter, bo * } func (actionPessimisticRollback) handleSingleBatch(c *twoPhaseCommitter, bo *Backoffer, batch batchMutations) error { - req := tikvrpc.NewRequest(tikvrpc.CmdPessimisticRollback, &pb.PessimisticRollbackRequest{ + req := tikvrpc.NewRequest(tikvrpc.CmdPessimisticRollback, &kvrpcpb.PessimisticRollbackRequest{ StartVersion: c.startTS, ForUpdateTs: c.forUpdateTS, Keys: batch.mutations.GetKeys(), }) - resp, err := c.store.SendReq(bo, req, batch.region, ReadTimeoutShort) + resp, err := c.store.SendReq(bo, req, batch.region, client.ReadTimeoutShort) if err != nil { return errors.Trace(err) } @@ -228,7 +242,7 @@ func (actionPessimisticRollback) handleSingleBatch(c *twoPhaseCommitter, bo *Bac return errors.Trace(err) } if regionErr != nil { - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) if err != nil { return errors.Trace(err) } @@ -247,11 +261,11 @@ func (c *twoPhaseCommitter) pessimisticLockMutations(bo *Backoffer, lockCtx *kv. for _, action := range strings.Split(v, ",") { if action == "delay" { duration := time.Duration(rand.Int63n(int64(time.Second) * 5)) - logutil.Logger(bo.ctx).Info("[failpoint] injected delay at pessimistic lock", + logutil.Logger(bo.GetCtx()).Info("[failpoint] injected delay at pessimistic lock", zap.Uint64("txnStartTS", c.startTS), zap.Duration("duration", duration)) time.Sleep(duration) } else if action == "fail" { - logutil.Logger(bo.ctx).Info("[failpoint] injected failure at pessimistic lock", + logutil.Logger(bo.GetCtx()).Info("[failpoint] injected failure at pessimistic lock", zap.Uint64("txnStartTS", c.startTS)) failpoint.Return(errors.New("injected failure at pessimistic lock")) } diff --git a/store/tikv/prewrite.go b/store/tikv/prewrite.go index 1fea4ba467341..c221ad4b52c44 100644 --- a/store/tikv/prewrite.go +++ b/store/tikv/prewrite.go @@ -22,17 +22,19 @@ import ( "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/failpoint" - pb "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/tidb/store/tikv/client" "github.com/pingcap/tidb/store/tikv/config" tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" ) -type actionPrewrite struct{} +type actionPrewrite struct{ retry bool } var _ twoPhaseCommitAction = actionPrewrite{} @@ -46,10 +48,10 @@ func (actionPrewrite) tiKVTxnRegionsNumHistogram() prometheus.Observer { func (c *twoPhaseCommitter) buildPrewriteRequest(batch batchMutations, txnSize uint64) *tikvrpc.Request { m := batch.mutations - mutations := make([]*pb.Mutation, m.Len()) + mutations := make([]*kvrpcpb.Mutation, m.Len()) isPessimisticLock := make([]bool, m.Len()) for i := 0; i < m.Len(); i++ { - mutations[i] = &pb.Mutation{ + mutations[i] = &kvrpcpb.Mutation{ Op: m.GetOp(i), Key: m.GetKey(i), Value: m.GetValue(i), @@ -86,7 +88,7 @@ func (c *twoPhaseCommitter) buildPrewriteRequest(batch batchMutations, txnSize u }) } - req := &pb.PrewriteRequest{ + req := &kvrpcpb.PrewriteRequest{ Mutations: mutations, PrimaryLock: c.primary(), StartVersion: c.startTS, @@ -115,10 +117,10 @@ func (c *twoPhaseCommitter) buildPrewriteRequest(batch batchMutations, txnSize u req.TryOnePc = true } - return tikvrpc.NewRequest(tikvrpc.CmdPrewrite, req, pb.Context{Priority: c.priority, SyncLog: c.syncLog}) + return tikvrpc.NewRequest(tikvrpc.CmdPrewrite, req, kvrpcpb.Context{Priority: c.priority, SyncLog: c.syncLog, ResourceGroupTag: c.resourceGroupTag}) } -func (action actionPrewrite) handleSingleBatch(c *twoPhaseCommitter, bo *Backoffer, batch batchMutations) error { +func (action actionPrewrite) handleSingleBatch(c *twoPhaseCommitter, bo *Backoffer, batch batchMutations) (err error) { // WARNING: This function only tries to send a single request to a single region, so it don't // need to unset the `useOnePC` flag when it fails. A special case is that when TiKV returns // regionErr, it's uncertain if the request will be splitted into multiple and sent to multiple @@ -130,7 +132,7 @@ func (action actionPrewrite) handleSingleBatch(c *twoPhaseCommitter, bo *Backoff failpoint.Inject("prewritePrimaryFail", func() { // Delay to avoid cancelling other normally ongoing prewrite requests. time.Sleep(time.Millisecond * 50) - logutil.Logger(bo.ctx).Info("[failpoint] injected error on prewriting primary batch", + logutil.Logger(bo.GetCtx()).Info("[failpoint] injected error on prewriting primary batch", zap.Uint64("txnStartTS", c.startTS)) failpoint.Return(errors.New("injected error on prewriting primary batch")) }) @@ -139,7 +141,7 @@ func (action actionPrewrite) handleSingleBatch(c *twoPhaseCommitter, bo *Backoff failpoint.Inject("prewriteSecondaryFail", func() { // Delay to avoid cancelling other normally ongoing prewrite requests. time.Sleep(time.Millisecond * 50) - logutil.Logger(bo.ctx).Info("[failpoint] injected error on prewriting secondary batch", + logutil.Logger(bo.GetCtx()).Info("[failpoint] injected error on prewriting secondary batch", zap.Uint64("txnStartTS", c.startTS)) failpoint.Return(errors.New("injected error on prewriting secondary batch")) }) @@ -150,42 +152,69 @@ func (action actionPrewrite) handleSingleBatch(c *twoPhaseCommitter, bo *Backoff txnSize := uint64(c.regionTxnSize[batch.region.id]) // When we retry because of a region miss, we don't know the transaction size. We set the transaction size here // to MaxUint64 to avoid unexpected "resolve lock lite". - if len(bo.errors) > 0 { + if action.retry { txnSize = math.MaxUint64 } + tBegin := time.Now() + attempts := 0 + req := c.buildPrewriteRequest(batch, txnSize) + sender := NewRegionRequestSender(c.store.regionCache, c.store.GetTiKVClient()) + defer func() { + if err != nil { + // If we fail to receive response for async commit prewrite, it will be undetermined whether this + // transaction has been successfully committed. + // If prewrite has been cancelled, all ongoing prewrite RPCs will become errors, we needn't set undetermined + // errors. + if (c.isAsyncCommit() || c.isOnePC()) && sender.rpcError != nil && atomic.LoadUint32(&c.prewriteCancelled) == 0 { + c.setUndeterminedErr(errors.Trace(sender.rpcError)) + } + } + }() for { - sender := NewRegionRequestSender(c.store.regionCache, c.store.client) - resp, err := sender.SendReq(bo, req, batch.region, ReadTimeoutShort) - - // If we fail to receive response for async commit prewrite, it will be undetermined whether this - // transaction has been successfully committed. - // If prewrite has been cancelled, all ongoing prewrite RPCs will become errors, we needn't set undetermined - // errors. - if (c.isAsyncCommit() || c.isOnePC()) && sender.rpcError != nil && atomic.LoadUint32(&c.prewriteCancelled) == 0 { - c.setUndeterminedErr(errors.Trace(sender.rpcError)) + attempts++ + if time.Since(tBegin) > slowRequestThreshold { + logutil.BgLogger().Warn("slow prewrite request", zap.Uint64("startTS", c.startTS), zap.Stringer("region", &batch.region), zap.Int("attempts", attempts)) + tBegin = time.Now() } + resp, err := sender.SendReq(bo, req, batch.region, client.ReadTimeoutShort) + // Unexpected error occurs, return it if err != nil { return errors.Trace(err) } + regionErr, err := resp.GetRegionError() if err != nil { return errors.Trace(err) } if regionErr != nil { - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + // For other region error and the fake region error, backoff because + // there's something wrong. + // For the real EpochNotMatch error, don't backoff. + if regionErr.GetEpochNotMatch() == nil || isFakeRegionError(regionErr) { + err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) + if err != nil { + return errors.Trace(err) + } + } + same, err := batch.relocate(bo, c.store.regionCache) if err != nil { return errors.Trace(err) } - err = c.prewriteMutations(bo, batch.mutations) + failpoint.Inject("forceRecursion", func() { same = false }) + if same { + continue + } + err = c.doActionOnMutations(bo, actionPrewrite{true}, batch.mutations) return errors.Trace(err) } + if resp.Resp == nil { return errors.Trace(tikverr.ErrBodyMissing) } - prewriteResp := resp.Resp.(*pb.PrewriteResponse) + prewriteResp := resp.Resp.(*kvrpcpb.PrewriteResponse) keyErrs := prewriteResp.GetErrors() if len(keyErrs) == 0 { if batch.isPrimary { @@ -203,7 +232,7 @@ func (action actionPrewrite) handleSingleBatch(c *twoPhaseCommitter, bo *Backoff if prewriteResp.MinCommitTs != 0 { return errors.Trace(errors.New("MinCommitTs must be 0 when 1pc falls back to 2pc")) } - logutil.Logger(bo.ctx).Warn("1pc failed and fallbacks to normal commit procedure", + logutil.Logger(bo.GetCtx()).Warn("1pc failed and fallbacks to normal commit procedure", zap.Uint64("startTS", c.startTS)) metrics.OnePCTxnCounterFallback.Inc() c.setOnePC(false) @@ -212,14 +241,14 @@ func (action actionPrewrite) handleSingleBatch(c *twoPhaseCommitter, bo *Backoff // For 1PC, there's no racing to access to access `onePCCommmitTS` so it's safe // not to lock the mutex. if c.onePCCommitTS != 0 { - logutil.Logger(bo.ctx).Fatal("one pc happened multiple times", + logutil.Logger(bo.GetCtx()).Fatal("one pc happened multiple times", zap.Uint64("startTS", c.startTS)) } c.onePCCommitTS = prewriteResp.OnePcCommitTs } return nil } else if prewriteResp.OnePcCommitTs != 0 { - logutil.Logger(bo.ctx).Fatal("tikv committed a non-1pc transaction with 1pc protocol", + logutil.Logger(bo.GetCtx()).Fatal("tikv committed a non-1pc transaction with 1pc protocol", zap.Uint64("startTS", c.startTS)) } if c.isAsyncCommit() { @@ -230,7 +259,7 @@ func (action actionPrewrite) handleSingleBatch(c *twoPhaseCommitter, bo *Backoff if c.testingKnobs.noFallBack { return nil } - logutil.Logger(bo.ctx).Warn("async commit cannot proceed since the returned minCommitTS is zero, "+ + logutil.Logger(bo.GetCtx()).Warn("async commit cannot proceed since the returned minCommitTS is zero, "+ "fallback to normal path", zap.Uint64("startTS", c.startTS)) c.setAsyncCommit(false) } else { @@ -248,12 +277,17 @@ func (action actionPrewrite) handleSingleBatch(c *twoPhaseCommitter, bo *Backoff // Check already exists error if alreadyExist := keyErr.GetAlreadyExist(); alreadyExist != nil { e := &tikverr.ErrKeyExist{AlreadyExist: alreadyExist} - return c.extractKeyExistsErr(e) + err = c.extractKeyExistsErr(e) + if err != nil { + atomic.StoreUint32(&c.prewriteFailed, 1) + } + return err } // Extract lock from key error lock, err1 := extractLockFromKeyErr(keyErr) if err1 != nil { + atomic.StoreUint32(&c.prewriteFailed, 1) return errors.Trace(err1) } logutil.BgLogger().Info("prewrite encounters lock", @@ -268,7 +302,7 @@ func (action actionPrewrite) handleSingleBatch(c *twoPhaseCommitter, bo *Backoff } atomic.AddInt64(&c.getDetail().ResolveLockTime, int64(time.Since(start))) if msBeforeExpired > 0 { - err = bo.BackoffWithMaxSleep(BoTxnLock, int(msBeforeExpired), errors.Errorf("2PC prewrite lockedKeys: %d", len(locks))) + err = bo.BackoffWithCfgAndMaxSleep(retry.BoTxnLock, int(msBeforeExpired), errors.Errorf("2PC prewrite lockedKeys: %d", len(locks))) if err != nil { return errors.Trace(err) } @@ -277,10 +311,10 @@ func (action actionPrewrite) handleSingleBatch(c *twoPhaseCommitter, bo *Backoff } func (c *twoPhaseCommitter) prewriteMutations(bo *Backoffer, mutations CommitterMutations) error { - if span := opentracing.SpanFromContext(bo.ctx); span != nil && span.Tracer() != nil { + if span := opentracing.SpanFromContext(bo.GetCtx()); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("twoPhaseCommitter.prewriteMutations", opentracing.ChildOf(span.Context())) defer span1.Finish() - bo.ctx = opentracing.ContextWithSpan(bo.ctx, span1) + bo.SetCtx(opentracing.ContextWithSpan(bo.GetCtx(), span1)) } // `doActionOnMutations` will unset `useOnePC` if the mutations is splitted into multiple batches. diff --git a/store/tikv/range_task.go b/store/tikv/range_task.go index 1395fac0609a2..bc9b8fa9999c1 100644 --- a/store/tikv/range_task.go +++ b/store/tikv/range_task.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/pingcap/tidb/store/tikv/retry" "go.uber.org/zap" ) @@ -91,6 +92,8 @@ func (s *RangeTaskRunner) SetRegionsPerTask(regionsPerTask int) { s.regionsPerTask = regionsPerTask } +const locateRegionMaxBackoff = 20000 + // RunOnRange runs the task on the given range. // Empty startKey or endKey means unbounded. func (s *RangeTaskRunner) RunOnRange(ctx context.Context, startKey, endKey []byte) error { @@ -157,7 +160,7 @@ Loop: default: } - bo := NewBackofferWithVars(ctx, locateRegionMaxBackoff, nil) + bo := retry.NewBackofferWithVars(ctx, locateRegionMaxBackoff, nil) rangeEndKey, err := s.store.GetRegionCache().BatchLoadRegionsFromKey(bo, key, s.regionsPerTask) if err != nil { diff --git a/store/tikv/rawkv.go b/store/tikv/rawkv.go index 2a80d26917a20..22c47a854f6f9 100644 --- a/store/tikv/rawkv.go +++ b/store/tikv/rawkv.go @@ -20,9 +20,11 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/tidb/store/tikv/client" "github.com/pingcap/tidb/store/tikv/config" tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" pd "github.com/tikv/pd/client" ) @@ -64,7 +66,7 @@ func NewRawKVClient(pdAddrs []string, security config.Security, opts ...pd.Clien clusterID: pdCli.GetClusterID(context.TODO()), regionCache: NewRegionCache(pdCli), pdClient: pdCli, - rpcClient: NewRPCClient(security), + rpcClient: client.NewRPCClient(security), }, nil } @@ -110,6 +112,8 @@ func (c *RawKVClient) Get(key []byte) ([]byte, error) { return cmdResp.Value, nil } +const rawkvMaxBackoff = 20000 + // BatchGet queries values with the keys. func (c *RawKVClient) BatchGet(keys [][]byte) ([][]byte, error) { start := time.Now() @@ -117,7 +121,7 @@ func (c *RawKVClient) BatchGet(keys [][]byte) ([][]byte, error) { metrics.RawkvCmdHistogramWithBatchGet.Observe(time.Since(start).Seconds()) }() - bo := NewBackofferWithVars(context.Background(), rawkvMaxBackoff, nil) + bo := retry.NewBackofferWithVars(context.Background(), rawkvMaxBackoff, nil) resp, err := c.sendBatchReq(bo, keys, tikvrpc.CmdRawBatchGet) if err != nil { return nil, errors.Trace(err) @@ -184,7 +188,7 @@ func (c *RawKVClient) BatchPut(keys, values [][]byte) error { return errors.New("empty value is not supported") } } - bo := NewBackofferWithVars(context.Background(), rawkvMaxBackoff, nil) + bo := retry.NewBackofferWithVars(context.Background(), rawkvMaxBackoff, nil) err := c.sendBatchPut(bo, keys, values) return errors.Trace(err) } @@ -218,7 +222,7 @@ func (c *RawKVClient) BatchDelete(keys [][]byte) error { metrics.RawkvCmdHistogramWithBatchDelete.Observe(time.Since(start).Seconds()) }() - bo := NewBackofferWithVars(context.Background(), rawkvMaxBackoff, nil) + bo := retry.NewBackofferWithVars(context.Background(), rawkvMaxBackoff, nil) resp, err := c.sendBatchReq(bo, keys, tikvrpc.CmdRawBatchDelete) if err != nil { return errors.Trace(err) @@ -350,7 +354,7 @@ func (c *RawKVClient) ReverseScan(startKey, endKey []byte, limit int) (keys [][] } func (c *RawKVClient) sendReq(key []byte, req *tikvrpc.Request, reverse bool) (*tikvrpc.Response, *KeyLocation, error) { - bo := NewBackofferWithVars(context.Background(), rawkvMaxBackoff, nil) + bo := retry.NewBackofferWithVars(context.Background(), rawkvMaxBackoff, nil) sender := NewRegionRequestSender(c.regionCache, c.rpcClient) for { var loc *KeyLocation @@ -363,7 +367,7 @@ func (c *RawKVClient) sendReq(key []byte, req *tikvrpc.Request, reverse bool) (* if err != nil { return nil, nil, errors.Trace(err) } - resp, err := sender.SendReq(bo, req, loc.Region, ReadTimeoutShort) + resp, err := sender.SendReq(bo, req, loc.Region, client.ReadTimeoutShort) if err != nil { return nil, nil, errors.Trace(err) } @@ -372,7 +376,7 @@ func (c *RawKVClient) sendReq(key []byte, req *tikvrpc.Request, reverse bool) (* return nil, nil, errors.Trace(err) } if regionErr != nil { - err := bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + err := bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) if err != nil { return nil, nil, errors.Trace(err) } @@ -443,7 +447,7 @@ func (c *RawKVClient) doBatchReq(bo *Backoffer, batch batch, cmdType tikvrpc.Cmd } sender := NewRegionRequestSender(c.regionCache, c.rpcClient) - resp, err := sender.SendReq(bo, req, batch.regionID, ReadTimeoutShort) + resp, err := sender.SendReq(bo, req, batch.regionID, client.ReadTimeoutShort) batchResp := singleBatchResp{} if err != nil { @@ -456,7 +460,7 @@ func (c *RawKVClient) doBatchReq(bo *Backoffer, batch batch, cmdType tikvrpc.Cmd return batchResp } if regionErr != nil { - err := bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + err := bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) if err != nil { batchResp.err = errors.Trace(err) return batchResp @@ -490,7 +494,7 @@ func (c *RawKVClient) doBatchReq(bo *Backoffer, batch batch, cmdType tikvrpc.Cmd // We can't use sendReq directly, because we need to know the end of the region before we send the request // TODO: Is there any better way to avoid duplicating code with func `sendReq` ? func (c *RawKVClient) sendDeleteRangeReq(startKey []byte, endKey []byte) (*tikvrpc.Response, []byte, error) { - bo := NewBackofferWithVars(context.Background(), rawkvMaxBackoff, nil) + bo := retry.NewBackofferWithVars(context.Background(), rawkvMaxBackoff, nil) sender := NewRegionRequestSender(c.regionCache, c.rpcClient) for { loc, err := c.regionCache.LocateKey(bo, startKey) @@ -508,7 +512,7 @@ func (c *RawKVClient) sendDeleteRangeReq(startKey []byte, endKey []byte) (*tikvr EndKey: actualEndKey, }) - resp, err := sender.SendReq(bo, req, loc.Region, ReadTimeoutShort) + resp, err := sender.SendReq(bo, req, loc.Region, client.ReadTimeoutShort) if err != nil { return nil, nil, errors.Trace(err) } @@ -517,7 +521,7 @@ func (c *RawKVClient) sendDeleteRangeReq(startKey []byte, endKey []byte) (*tikvr return nil, nil, errors.Trace(err) } if regionErr != nil { - err := bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + err := bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) if err != nil { return nil, nil, errors.Trace(err) } @@ -613,7 +617,7 @@ func (c *RawKVClient) doBatchPut(bo *Backoffer, batch batch) error { req := tikvrpc.NewRequest(tikvrpc.CmdRawBatchPut, &kvrpcpb.RawBatchPutRequest{Pairs: kvPair}) sender := NewRegionRequestSender(c.regionCache, c.rpcClient) - resp, err := sender.SendReq(bo, req, batch.regionID, ReadTimeoutShort) + resp, err := sender.SendReq(bo, req, batch.regionID, client.ReadTimeoutShort) if err != nil { return errors.Trace(err) } @@ -622,7 +626,7 @@ func (c *RawKVClient) doBatchPut(bo *Backoffer, batch batch) error { return errors.Trace(err) } if regionErr != nil { - err := bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + err := bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) if err != nil { return errors.Trace(err) } diff --git a/store/tikv/rawkv_test.go b/store/tikv/rawkv_test.go new file mode 100644 index 0000000000000..2783db1e9945f --- /dev/null +++ b/store/tikv/rawkv_test.go @@ -0,0 +1,169 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package tikv + +import ( + "context" + "fmt" + + . "github.com/pingcap/check" + "github.com/pingcap/tidb/store/tikv/kv" + "github.com/pingcap/tidb/store/tikv/mockstore/mocktikv" + "github.com/pingcap/tidb/store/tikv/retry" +) + +type testRawkvSuite struct { + OneByOneSuite + cluster *mocktikv.Cluster + store1 uint64 // store1 is leader + store2 uint64 // store2 is follower + peer1 uint64 // peer1 is leader + peer2 uint64 // peer2 is follower + region1 uint64 + bo *retry.Backoffer +} + +var _ = Suite(&testRawkvSuite{}) + +func (s *testRawkvSuite) SetUpTest(c *C) { + s.cluster = mocktikv.NewCluster(mocktikv.MustNewMVCCStore()) + storeIDs, peerIDs, regionID, _ := mocktikv.BootstrapWithMultiStores(s.cluster, 2) + s.region1 = regionID + s.store1 = storeIDs[0] + s.store2 = storeIDs[1] + s.peer1 = peerIDs[0] + s.peer2 = peerIDs[1] + s.bo = retry.NewBackofferWithVars(context.Background(), 5000, nil) +} + +func (s *testRawkvSuite) storeAddr(id uint64) string { + return fmt.Sprintf("store%d", id) +} + +func (s *testRawkvSuite) TestReplaceAddrWithNewStore(c *C) { + mvccStore := mocktikv.MustNewMVCCStore() + defer mvccStore.Close() + + client := &RawKVClient{ + clusterID: 0, + regionCache: NewRegionCache(mocktikv.NewPDClient(s.cluster)), + rpcClient: mocktikv.NewRPCClient(s.cluster, mvccStore, nil), + } + defer client.Close() + testKey := []byte("test_key") + testValue := []byte("test_value") + err := client.Put(testKey, testValue) + c.Assert(err, IsNil) + + // make store2 using store1's addr and store1 offline + store1Addr := s.storeAddr(s.store1) + s.cluster.UpdateStoreAddr(s.store1, s.storeAddr(s.store2)) + s.cluster.UpdateStoreAddr(s.store2, store1Addr) + s.cluster.RemoveStore(s.store1) + s.cluster.ChangeLeader(s.region1, s.peer2) + s.cluster.RemovePeer(s.region1, s.peer1) + + getVal, err := client.Get(testKey) + + c.Assert(err, IsNil) + c.Assert(getVal, BytesEquals, testValue) +} + +func (s *testRawkvSuite) TestUpdateStoreAddr(c *C) { + mvccStore := mocktikv.MustNewMVCCStore() + defer mvccStore.Close() + + client := &RawKVClient{ + clusterID: 0, + regionCache: NewRegionCache(mocktikv.NewPDClient(s.cluster)), + rpcClient: mocktikv.NewRPCClient(s.cluster, mvccStore, nil), + } + defer client.Close() + testKey := []byte("test_key") + testValue := []byte("test_value") + err := client.Put(testKey, testValue) + c.Assert(err, IsNil) + // tikv-server reports `StoreNotMatch` And retry + store1Addr := s.storeAddr(s.store1) + s.cluster.UpdateStoreAddr(s.store1, s.storeAddr(s.store2)) + s.cluster.UpdateStoreAddr(s.store2, store1Addr) + + getVal, err := client.Get(testKey) + + c.Assert(err, IsNil) + c.Assert(getVal, BytesEquals, testValue) +} + +func (s *testRawkvSuite) TestReplaceNewAddrAndOldOfflineImmediately(c *C) { + mvccStore := mocktikv.MustNewMVCCStore() + defer mvccStore.Close() + + client := &RawKVClient{ + clusterID: 0, + regionCache: NewRegionCache(mocktikv.NewPDClient(s.cluster)), + rpcClient: mocktikv.NewRPCClient(s.cluster, mvccStore, nil), + } + defer client.Close() + testKey := []byte("test_key") + testValue := []byte("test_value") + err := client.Put(testKey, testValue) + c.Assert(err, IsNil) + + // pre-load store2's address into cache via follower-read. + loc, err := client.regionCache.LocateKey(s.bo, testKey) + c.Assert(err, IsNil) + fctx, err := client.regionCache.GetTiKVRPCContext(s.bo, loc.Region, kv.ReplicaReadFollower, 0) + c.Assert(err, IsNil) + c.Assert(fctx.Store.storeID, Equals, s.store2) + c.Assert(fctx.Addr, Equals, "store2") + + // make store2 using store1's addr and store1 offline + store1Addr := s.storeAddr(s.store1) + s.cluster.UpdateStoreAddr(s.store1, s.storeAddr(s.store2)) + s.cluster.UpdateStoreAddr(s.store2, store1Addr) + s.cluster.RemoveStore(s.store1) + s.cluster.ChangeLeader(s.region1, s.peer2) + s.cluster.RemovePeer(s.region1, s.peer1) + + getVal, err := client.Get(testKey) + c.Assert(err, IsNil) + c.Assert(getVal, BytesEquals, testValue) +} + +func (s *testRawkvSuite) TestReplaceStore(c *C) { + mvccStore := mocktikv.MustNewMVCCStore() + defer mvccStore.Close() + + client := &RawKVClient{ + clusterID: 0, + regionCache: NewRegionCache(mocktikv.NewPDClient(s.cluster)), + rpcClient: mocktikv.NewRPCClient(s.cluster, mvccStore, nil), + } + defer client.Close() + testKey := []byte("test_key") + testValue := []byte("test_value") + err := client.Put(testKey, testValue) + c.Assert(err, IsNil) + + s.cluster.MarkTombstone(s.store1) + store3 := s.cluster.AllocID() + peer3 := s.cluster.AllocID() + s.cluster.AddStore(store3, s.storeAddr(s.store1)) + s.cluster.AddPeer(s.region1, store3, peer3) + s.cluster.RemovePeer(s.region1, s.peer1) + s.cluster.ChangeLeader(s.region1, peer3) + + err = client.Put(testKey, testValue) + c.Assert(err, IsNil) +} diff --git a/store/tikv/region_cache.go b/store/tikv/region_cache.go index a73684fdf49c5..f2c8a1b16f419 100644 --- a/store/tikv/region_cache.go +++ b/store/tikv/region_cache.go @@ -31,10 +31,12 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/parser/terror" + "github.com/pingcap/tidb/store/tikv/client" "github.com/pingcap/tidb/store/tikv/config" "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" pd "github.com/tikv/pd/client" atomic2 "go.uber.org/atomic" @@ -53,8 +55,13 @@ const ( defaultRegionsPerBatch = 128 ) -// RegionCacheTTLSec is the max idle time for regions in the region cache. -var RegionCacheTTLSec int64 = 600 +// regionCacheTTLSec is the max idle time for regions in the region cache. +var regionCacheTTLSec int64 = 600 + +// SetRegionCacheTTLSec sets regionCacheTTLSec to t. +func SetRegionCacheTTLSec(t int64) { + regionCacheTTLSec = t +} const ( updated int32 = iota // region is updated and no need to reload. @@ -95,30 +102,39 @@ type Region struct { // AccessIndex represent the index for accessIndex array type AccessIndex int -// RegionStore represents region stores info +// regionStore represents region stores info // it will be store as unsafe.Pointer and be load at once -type RegionStore struct { +type regionStore struct { workTiKVIdx AccessIndex // point to current work peer in meta.Peers and work store in stores(same idx) for tikv peer proxyTiKVIdx AccessIndex // point to the tikv peer that can forward requests to the leader. -1 means not using proxy workTiFlashIdx int32 // point to current work peer in meta.Peers and work store in stores(same idx) for tiflash peer stores []*Store // stores in this region storeEpochs []uint32 // snapshots of store's epoch, need reload when `storeEpochs[curr] != stores[cur].fail` - accessIndex [NumAccessMode][]int // AccessMode => idx in stores + accessIndex [numAccessMode][]int // AccessMode => idx in stores } -func (r *RegionStore) accessStore(mode AccessMode, idx AccessIndex) (int, *Store) { +func (r *regionStore) accessStore(mode accessMode, idx AccessIndex) (int, *Store) { sidx := r.accessIndex[mode][idx] return sidx, r.stores[sidx] } -func (r *RegionStore) accessStoreNum(mode AccessMode) int { +func (r *regionStore) getAccessIndex(mode accessMode, store *Store) AccessIndex { + for index, sidx := range r.accessIndex[mode] { + if r.stores[sidx].storeID == store.storeID { + return AccessIndex(index) + } + } + return -1 +} + +func (r *regionStore) accessStoreNum(mode accessMode) int { return len(r.accessIndex[mode]) } // clone clones region store struct. -func (r *RegionStore) clone() *RegionStore { +func (r *regionStore) clone() *regionStore { storeEpochs := make([]uint32, len(r.stores)) - rs := &RegionStore{ + rs := ®ionStore{ workTiFlashIdx: r.workTiFlashIdx, proxyTiKVIdx: r.proxyTiKVIdx, workTiKVIdx: r.workTiKVIdx, @@ -126,7 +142,7 @@ func (r *RegionStore) clone() *RegionStore { storeEpochs: storeEpochs, } copy(storeEpochs, r.storeEpochs) - for i := 0; i < int(NumAccessMode); i++ { + for i := 0; i < int(numAccessMode); i++ { rs.accessIndex[i] = make([]int, len(r.accessIndex[i])) copy(rs.accessIndex[i], r.accessIndex[i]) } @@ -134,8 +150,8 @@ func (r *RegionStore) clone() *RegionStore { } // return next follower store's index -func (r *RegionStore) follower(seed uint32, op *storeSelectorOp) AccessIndex { - l := uint32(r.accessStoreNum(TiKVOnly)) +func (r *regionStore) follower(seed uint32, op *storeSelectorOp) AccessIndex { + l := uint32(r.accessStoreNum(tiKVOnly)) if l <= 1 { return r.workTiKVIdx } @@ -145,7 +161,7 @@ func (r *RegionStore) follower(seed uint32, op *storeSelectorOp) AccessIndex { if followerIdx >= r.workTiKVIdx { followerIdx++ } - storeIdx, s := r.accessStore(TiKVOnly, followerIdx) + storeIdx, s := r.accessStore(tiKVOnly, followerIdx) if r.storeEpochs[storeIdx] == atomic.LoadUint32(&s.epoch) && r.filterStoreCandidate(followerIdx, op) { return followerIdx } @@ -155,38 +171,44 @@ func (r *RegionStore) follower(seed uint32, op *storeSelectorOp) AccessIndex { } // return next leader or follower store's index -func (r *RegionStore) kvPeer(seed uint32, op *storeSelectorOp) AccessIndex { - candidates := make([]AccessIndex, 0, r.accessStoreNum(TiKVOnly)) - for i := 0; i < r.accessStoreNum(TiKVOnly); i++ { - storeIdx, s := r.accessStore(TiKVOnly, AccessIndex(i)) - if r.storeEpochs[storeIdx] != atomic.LoadUint32(&s.epoch) || !r.filterStoreCandidate(AccessIndex(i), op) { +func (r *regionStore) kvPeer(seed uint32, op *storeSelectorOp) AccessIndex { + if op.leaderOnly { + return r.workTiKVIdx + } + candidates := make([]AccessIndex, 0, r.accessStoreNum(tiKVOnly)) + for i := 0; i < r.accessStoreNum(tiKVOnly); i++ { + accessIdx := AccessIndex(i) + storeIdx, s := r.accessStore(tiKVOnly, accessIdx) + if r.storeEpochs[storeIdx] != atomic.LoadUint32(&s.epoch) || !r.filterStoreCandidate(accessIdx, op) { continue } - candidates = append(candidates, AccessIndex(i)) + candidates = append(candidates, accessIdx) } + // If there is no candidates, send to current workTiKVIdx which generally is the leader. if len(candidates) == 0 { return r.workTiKVIdx } return candidates[seed%uint32(len(candidates))] } -func (r *RegionStore) filterStoreCandidate(aidx AccessIndex, op *storeSelectorOp) bool { - _, s := r.accessStore(TiKVOnly, aidx) +func (r *regionStore) filterStoreCandidate(aidx AccessIndex, op *storeSelectorOp) bool { + _, s := r.accessStore(tiKVOnly, aidx) // filter label unmatched store return s.IsLabelsMatch(op.labels) } // init initializes region after constructed. -func (r *Region) init(c *RegionCache) error { +func (r *Region) init(bo *Backoffer, c *RegionCache) error { // region store pull used store from global store map // to avoid acquire storeMu in later access. - rs := &RegionStore{ + rs := ®ionStore{ workTiKVIdx: 0, proxyTiKVIdx: -1, workTiFlashIdx: 0, stores: make([]*Store, 0, len(r.meta.Peers)), storeEpochs: make([]uint32, 0, len(r.meta.Peers)), } + availablePeers := r.meta.GetPeers()[:0] for _, p := range r.meta.Peers { c.storeMu.RLock() store, exists := c.storeMu.stores[p.StoreId] @@ -194,19 +216,31 @@ func (r *Region) init(c *RegionCache) error { if !exists { store = c.getStoreByStoreID(p.StoreId) } - _, err := store.initResolve(NewNoopBackoff(context.Background()), c) + addr, err := store.initResolve(bo, c) if err != nil { return err } + // Filter the peer on a tombstone store. + if addr == "" { + continue + } + availablePeers = append(availablePeers, p) switch store.storeType { case tikvrpc.TiKV: - rs.accessIndex[TiKVOnly] = append(rs.accessIndex[TiKVOnly], len(rs.stores)) + rs.accessIndex[tiKVOnly] = append(rs.accessIndex[tiKVOnly], len(rs.stores)) case tikvrpc.TiFlash: - rs.accessIndex[TiFlashOnly] = append(rs.accessIndex[TiFlashOnly], len(rs.stores)) + rs.accessIndex[tiFlashOnly] = append(rs.accessIndex[tiFlashOnly], len(rs.stores)) } rs.stores = append(rs.stores, store) rs.storeEpochs = append(rs.storeEpochs, atomic.LoadUint32(&store.epoch)) } + // TODO(youjiali1995): It's possible the region info in PD is stale for now but it can recover. + // Maybe we need backoff here. + if len(availablePeers) == 0 { + return errors.Errorf("no available peers, region: {%v}", r.meta) + } + r.meta.Peers = availablePeers + atomic.StorePointer(&r.store, unsafe.Pointer(rs)) // mark region has been init accessed. @@ -214,12 +248,12 @@ func (r *Region) init(c *RegionCache) error { return nil } -func (r *Region) getStore() (store *RegionStore) { - store = (*RegionStore)(atomic.LoadPointer(&r.store)) +func (r *Region) getStore() (store *regionStore) { + store = (*regionStore)(atomic.LoadPointer(&r.store)) return } -func (r *Region) compareAndSwapStore(oldStore, newStore *RegionStore) bool { +func (r *Region) compareAndSwapStore(oldStore, newStore *regionStore) bool { return atomic.CompareAndSwapPointer(&r.store, unsafe.Pointer(oldStore), unsafe.Pointer(newStore)) } @@ -230,7 +264,7 @@ func (r *Region) checkRegionCacheTTL(ts int64) bool { }) for { lastAccess := atomic.LoadInt64(&r.lastAccess) - if ts-lastAccess > RegionCacheTTLSec { + if ts-lastAccess > regionCacheTTLSec { return false } if atomic.CompareAndSwapInt64(&r.lastAccess, lastAccess, ts) { @@ -269,6 +303,10 @@ func (r *Region) checkNeedReload() bool { return v != updated } +func (r *Region) isValid() bool { + return r != nil && !r.checkNeedReload() && r.checkRegionCacheTTL(time.Now().Unix()) +} + // RegionCache caches Regions loaded from PD. type RegionCache struct { pdClient pd.Client @@ -311,6 +349,18 @@ func NewRegionCache(pdClient pd.Client) *RegionCache { return c } +// clear clears all cached data in the RegionCache. It's only used in tests. +func (c *RegionCache) clear() { + c.mu.Lock() + c.mu.regions = make(map[RegionVerID]*Region) + c.mu.latestVersions = make(map[uint64]RegionVerID) + c.mu.sorted = btree.New(btreeDegree) + c.mu.Unlock() + c.storeMu.Lock() + c.storeMu.stores = make(map[uint64]*Store) + c.storeMu.Unlock() +} + // Close releases region cache's resource. func (c *RegionCache) Close() { close(c.closeCh) @@ -322,32 +372,29 @@ func (c *RegionCache) asyncCheckAndResolveLoop(interval time.Duration) { defer ticker.Stop() var needCheckStores []*Store for { + needCheckStores = needCheckStores[:0] select { case <-c.closeCh: return case <-c.notifyCheckCh: - needCheckStores = needCheckStores[:0] - c.checkAndResolve(needCheckStores) + c.checkAndResolve(needCheckStores, func(s *Store) bool { + return s.getResolveState() == needCheck + }) case <-ticker.C: - // refresh store once a minute to update labels - var stores []*Store - c.storeMu.RLock() - stores = make([]*Store, 0, len(c.storeMu.stores)) - for _, s := range c.storeMu.stores { - stores = append(stores, s) - } - c.storeMu.RUnlock() - for _, store := range stores { - _, err := store.reResolve(c) - terror.Log(err) - } + // refresh store to update labels. + c.checkAndResolve(needCheckStores, func(s *Store) bool { + state := s.getResolveState() + // Only valid stores should be reResolved. In fact, it's impossible + // there's a deleted store in the stores map which guaranteed by reReslve(). + return state != unresolved && state != tombstone && state != deleted + }) } } } // checkAndResolve checks and resolve addr of failed stores. // this method isn't thread-safe and only be used by one goroutine. -func (c *RegionCache) checkAndResolve(needCheckStores []*Store) { +func (c *RegionCache) checkAndResolve(needCheckStores []*Store, needCheck func(*Store) bool) { defer func() { r := recover() if r != nil { @@ -359,8 +406,7 @@ func (c *RegionCache) checkAndResolve(needCheckStores []*Store) { c.storeMu.RLock() for _, store := range c.storeMu.stores { - state := store.getResolveState() - if state == needCheck { + if needCheck(store) { needCheckStores = append(needCheckStores, store) } } @@ -380,11 +426,13 @@ type RPCContext struct { AccessIdx AccessIndex Store *Store Addr string - AccessMode AccessMode + AccessMode accessMode ProxyStore *Store // nil means proxy is not used ProxyAccessIdx AccessIndex // valid when ProxyStore is not nil ProxyAddr string // valid when ProxyStore is not nil TiKVNum int // Number of TiKV nodes among the region's peers. Assuming non-TiKV peers are all TiFlash peers. + + tryTimes int } func (c *RPCContext) String() string { @@ -401,16 +449,24 @@ func (c *RPCContext) String() string { } type storeSelectorOp struct { - labels []*metapb.StoreLabel + leaderOnly bool + labels []*metapb.StoreLabel } // StoreSelectorOption configures storeSelectorOp. type StoreSelectorOption func(*storeSelectorOp) -// WithMatchLabels indicates selecting stores with matched labels +// WithMatchLabels indicates selecting stores with matched labels. func WithMatchLabels(labels []*metapb.StoreLabel) StoreSelectorOption { return func(op *storeSelectorOp) { - op.labels = labels + op.labels = append(op.labels, labels...) + } +} + +// WithLeaderOnly indicates selecting stores with leader only. +func WithLeaderOnly() StoreSelectorOption { + return func(op *storeSelectorOp) { + op.leaderOnly = true } } @@ -419,14 +475,12 @@ func WithMatchLabels(labels []*metapb.StoreLabel) StoreSelectorOption { func (c *RegionCache) GetTiKVRPCContext(bo *Backoffer, id RegionVerID, replicaRead kv.ReplicaReadType, followerStoreSeed uint32, opts ...StoreSelectorOption) (*RPCContext, error) { ts := time.Now().Unix() - cachedRegion := c.getCachedRegionWithRLock(id) + cachedRegion := c.GetCachedRegionWithRLock(id) if cachedRegion == nil { return nil, nil } if cachedRegion.checkNeedReload() { - // TODO: This may cause a fake EpochNotMatch error, and reload the region after a backoff. It's better to reload - // the region directly here. return nil, nil } @@ -465,7 +519,7 @@ func (c *RegionCache) GetTiKVRPCContext(bo *Backoffer, id RegionVerID, replicaRe isLeaderReq = true store, peer, accessIdx, storeIdx = cachedRegion.WorkStorePeer(regionStore) } - addr, err := c.getStoreAddr(bo, cachedRegion, store, storeIdx) + addr, err := c.getStoreAddr(bo, cachedRegion, store) if err != nil { return nil, err } @@ -494,15 +548,14 @@ func (c *RegionCache) GetTiKVRPCContext(bo *Backoffer, id RegionVerID, replicaRe proxyStore *Store proxyAddr string proxyAccessIdx AccessIndex - proxyStoreIdx int ) if c.enableForwarding && isLeaderReq { if atomic.LoadInt32(&store.needForwarding) == 0 { regionStore.unsetProxyStoreIfNeeded(cachedRegion) } else { - proxyStore, proxyAccessIdx, proxyStoreIdx = c.getProxyStore(cachedRegion, store, regionStore, accessIdx) + proxyStore, proxyAccessIdx, _ = c.getProxyStore(cachedRegion, store, regionStore, accessIdx) if proxyStore != nil { - proxyAddr, err = c.getStoreAddr(bo, cachedRegion, proxyStore, proxyStoreIdx) + proxyAddr, err = c.getStoreAddr(bo, cachedRegion, proxyStore) if err != nil { return nil, err } @@ -517,21 +570,55 @@ func (c *RegionCache) GetTiKVRPCContext(bo *Backoffer, id RegionVerID, replicaRe AccessIdx: accessIdx, Store: store, Addr: addr, - AccessMode: TiKVOnly, + AccessMode: tiKVOnly, ProxyStore: proxyStore, ProxyAccessIdx: proxyAccessIdx, ProxyAddr: proxyAddr, - TiKVNum: regionStore.accessStoreNum(TiKVOnly), + TiKVNum: regionStore.accessStoreNum(tiKVOnly), }, nil } +// GetAllValidTiFlashStores returns the store ids of all valid TiFlash stores, the store id of currentStore is always the first one +func (c *RegionCache) GetAllValidTiFlashStores(id RegionVerID, currentStore *Store) []uint64 { + // set the cap to 2 because usually, TiFlash table will have 2 replicas + allStores := make([]uint64, 0, 2) + // make sure currentStore id is always the first in allStores + allStores = append(allStores, currentStore.storeID) + ts := time.Now().Unix() + cachedRegion := c.GetCachedRegionWithRLock(id) + if cachedRegion == nil { + return allStores + } + if !cachedRegion.checkRegionCacheTTL(ts) { + return allStores + } + regionStore := cachedRegion.getStore() + currentIndex := regionStore.getAccessIndex(tiFlashOnly, currentStore) + if currentIndex == -1 { + return allStores + } + for startOffset := 1; startOffset < regionStore.accessStoreNum(tiFlashOnly); startOffset++ { + accessIdx := AccessIndex((int(currentIndex) + startOffset) % regionStore.accessStoreNum(tiFlashOnly)) + storeIdx, store := regionStore.accessStore(tiFlashOnly, accessIdx) + if store.getResolveState() == needCheck { + continue + } + storeFailEpoch := atomic.LoadUint32(&store.epoch) + if storeFailEpoch != regionStore.storeEpochs[storeIdx] { + continue + } + allStores = append(allStores, store.storeID) + } + return allStores +} + // GetTiFlashRPCContext returns RPCContext for a region must access flash store. If it returns nil, the region // must be out of date and already dropped from cache or not flash store found. // `loadBalance` is an option. For MPP and batch cop, it is pointless and might cause try the failed store repeatly. func (c *RegionCache) GetTiFlashRPCContext(bo *Backoffer, id RegionVerID, loadBalance bool) (*RPCContext, error) { ts := time.Now().Unix() - cachedRegion := c.getCachedRegionWithRLock(id) + cachedRegion := c.GetCachedRegionWithRLock(id) if cachedRegion == nil { return nil, nil } @@ -548,10 +635,10 @@ func (c *RegionCache) GetTiFlashRPCContext(bo *Backoffer, id RegionVerID, loadBa } else { sIdx = int(atomic.LoadInt32(®ionStore.workTiFlashIdx)) } - for i := 0; i < regionStore.accessStoreNum(TiFlashOnly); i++ { - accessIdx := AccessIndex((sIdx + i) % regionStore.accessStoreNum(TiFlashOnly)) - storeIdx, store := regionStore.accessStore(TiFlashOnly, accessIdx) - addr, err := c.getStoreAddr(bo, cachedRegion, store, storeIdx) + for i := 0; i < regionStore.accessStoreNum(tiFlashOnly); i++ { + accessIdx := AccessIndex((sIdx + i) % regionStore.accessStoreNum(tiFlashOnly)) + storeIdx, store := regionStore.accessStore(tiFlashOnly, accessIdx) + addr, err := c.getStoreAddr(bo, cachedRegion, store) if err != nil { return nil, err } @@ -581,8 +668,8 @@ func (c *RegionCache) GetTiFlashRPCContext(bo *Backoffer, id RegionVerID, loadBa AccessIdx: accessIdx, Store: store, Addr: addr, - AccessMode: TiFlashOnly, - TiKVNum: regionStore.accessStoreNum(TiKVOnly), + AccessMode: tiFlashOnly, + TiKVNum: regionStore.accessStoreNum(tiKVOnly), }, nil } @@ -644,7 +731,7 @@ func (c *RegionCache) findRegionByKey(bo *Backoffer, key []byte, isEndKey bool) // no region data, return error if failure. return nil, err } - logutil.Eventf(bo.ctx, "load region %d from pd, due to cache-miss", lr.GetID()) + logutil.Eventf(bo.GetCtx(), "load region %d from pd, due to cache-miss", lr.GetID()) r = lr c.mu.Lock() c.insertRegionToCache(r) @@ -654,10 +741,10 @@ func (c *RegionCache) findRegionByKey(bo *Backoffer, key []byte, isEndKey bool) lr, err := c.loadRegion(bo, key, isEndKey) if err != nil { // ignore error and use old region info. - logutil.Logger(bo.ctx).Error("load region failure", + logutil.Logger(bo.GetCtx()).Error("load region failure", zap.ByteString("key", key), zap.Error(err)) } else { - logutil.Eventf(bo.ctx, "load region %d from pd, due to need-reload", lr.GetID()) + logutil.Eventf(bo.GetCtx(), "load region %d from pd, due to need-reload", lr.GetID()) r = lr c.mu.Lock() c.insertRegionToCache(r) @@ -667,90 +754,143 @@ func (c *RegionCache) findRegionByKey(bo *Backoffer, key []byte, isEndKey bool) return r, nil } +// OnSendFailForTiFlash handles send request fail logic for tiflash. +func (c *RegionCache) OnSendFailForTiFlash(bo *Backoffer, store *Store, region RegionVerID, prev *metapb.Region, scheduleReload bool, err error) { + + r := c.GetCachedRegionWithRLock(region) + if r == nil { + return + } + + rs := r.getStore() + peersNum := len(r.GetMeta().Peers) + if len(prev.Peers) != peersNum { + logutil.Logger(bo.GetCtx()).Info("retry and refresh current region after send request fail and up/down stores length changed", + zap.Stringer("region", ®ion), + zap.Bool("needReload", scheduleReload), + zap.Reflect("oldPeers", prev.Peers), + zap.Reflect("newPeers", r.GetMeta().Peers), + zap.Error(err)) + return + } + + accessMode := tiFlashOnly + accessIdx := rs.getAccessIndex(accessMode, store) + if accessIdx == -1 { + logutil.Logger(bo.GetCtx()).Warn("can not get access index for region " + region.String()) + return + } + if err != nil { + storeIdx, s := rs.accessStore(accessMode, accessIdx) + c.markRegionNeedBeRefill(s, storeIdx, rs) + } + + // try next peer + rs.switchNextFlashPeer(r, accessIdx) + logutil.Logger(bo.GetCtx()).Info("switch region tiflash peer to next due to send request fail", + zap.Stringer("region", ®ion), + zap.Bool("needReload", scheduleReload), + zap.Error(err)) + + // force reload region when retry all known peers in region. + if scheduleReload { + r.scheduleReload() + } +} + +func (c *RegionCache) markRegionNeedBeRefill(s *Store, storeIdx int, rs *regionStore) int { + incEpochStoreIdx := -1 + // invalidate regions in store. + epoch := rs.storeEpochs[storeIdx] + if atomic.CompareAndSwapUint32(&s.epoch, epoch, epoch+1) { + logutil.BgLogger().Info("mark store's regions need be refill", zap.String("store", s.addr)) + incEpochStoreIdx = storeIdx + metrics.RegionCacheCounterWithInvalidateStoreRegionsOK.Inc() + } + // schedule a store addr resolve. + s.markNeedCheck(c.notifyCheckCh) + return incEpochStoreIdx +} + // OnSendFail handles send request fail logic. func (c *RegionCache) OnSendFail(bo *Backoffer, ctx *RPCContext, scheduleReload bool, err error) { metrics.RegionCacheCounterWithSendFail.Inc() - r := c.getCachedRegionWithRLock(ctx.Region) - if r != nil { - peersNum := len(r.meta.Peers) - if len(ctx.Meta.Peers) != peersNum { - logutil.Logger(bo.ctx).Info("retry and refresh current ctx after send request fail and up/down stores length changed", - zap.Stringer("current", ctx), - zap.Bool("needReload", scheduleReload), - zap.Reflect("oldPeers", ctx.Meta.Peers), - zap.Reflect("newPeers", r.meta.Peers), - zap.Error(err)) - return - } - - rs := r.getStore() - startForwarding := false - incEpochStoreIdx := -1 + r := c.GetCachedRegionWithRLock(ctx.Region) + if r == nil { + return + } + peersNum := len(r.meta.Peers) + if len(ctx.Meta.Peers) != peersNum { + logutil.Logger(bo.GetCtx()).Info("retry and refresh current ctx after send request fail and up/down stores length changed", + zap.Stringer("current", ctx), + zap.Bool("needReload", scheduleReload), + zap.Reflect("oldPeers", ctx.Meta.Peers), + zap.Reflect("newPeers", r.meta.Peers), + zap.Error(err)) + return + } - if err != nil { - storeIdx, s := rs.accessStore(ctx.AccessMode, ctx.AccessIdx) - leaderReq := ctx.Store.storeType == tikvrpc.TiKV && rs.workTiKVIdx == ctx.AccessIdx - - // Mark the store as failure if it's not a redirection request because we - // can't know the status of the proxy store by it. - if ctx.ProxyStore == nil { - // send fail but store is reachable, keep retry current peer for replica leader request. - // but we still need switch peer for follower-read or learner-read(i.e. tiflash) - if leaderReq { - if s.requestLiveness(bo, c) == reachable { - return - } else if c.enableForwarding { - s.startHealthCheckLoopIfNeeded(c) - startForwarding = true - } - } + rs := r.getStore() + startForwarding := false + incEpochStoreIdx := -1 - // invalidate regions in store. - epoch := rs.storeEpochs[storeIdx] - if atomic.CompareAndSwapUint32(&s.epoch, epoch, epoch+1) { - logutil.BgLogger().Info("mark store's regions need be refill", zap.String("store", s.addr)) - incEpochStoreIdx = storeIdx - metrics.RegionCacheCounterWithInvalidateStoreRegionsOK.Inc() + if err != nil { + storeIdx, s := rs.accessStore(ctx.AccessMode, ctx.AccessIdx) + leaderReq := ctx.Store.storeType == tikvrpc.TiKV && rs.workTiKVIdx == ctx.AccessIdx + + // Mark the store as failure if it's not a redirection request because we + // can't know the status of the proxy store by it. + if ctx.ProxyStore == nil { + // send fail but store is reachable, keep retry current peer for replica leader request. + // but we still need switch peer for follower-read or learner-read(i.e. tiflash) + if leaderReq { + if s.requestLiveness(bo, c) == reachable { + return + } else if c.enableForwarding { + s.startHealthCheckLoopIfNeeded(c) + startForwarding = true } - // schedule a store addr resolve. - s.markNeedCheck(c.notifyCheckCh) } + + // invalidate regions in store. + incEpochStoreIdx = c.markRegionNeedBeRefill(s, storeIdx, rs) } + } - // try next peer to found new leader. - if ctx.AccessMode == TiKVOnly { - if startForwarding || ctx.ProxyStore != nil { - var currentProxyIdx AccessIndex = -1 - if ctx.ProxyStore != nil { - currentProxyIdx = ctx.ProxyAccessIdx - } - // In case the epoch of the store is increased, try to avoid reloading the current region by also - // increasing the epoch stored in `rs`. - rs.switchNextProxyStore(r, currentProxyIdx, incEpochStoreIdx) - logutil.Logger(bo.ctx).Info("switch region proxy peer to next due to send request fail", - zap.Stringer("current", ctx), - zap.Bool("needReload", scheduleReload), - zap.Error(err)) - } else { - rs.switchNextTiKVPeer(r, ctx.AccessIdx) - logutil.Logger(bo.ctx).Info("switch region peer to next due to send request fail", - zap.Stringer("current", ctx), - zap.Bool("needReload", scheduleReload), - zap.Error(err)) + // try next peer to found new leader. + if ctx.AccessMode == tiKVOnly { + if startForwarding || ctx.ProxyStore != nil { + var currentProxyIdx AccessIndex = -1 + if ctx.ProxyStore != nil { + currentProxyIdx = ctx.ProxyAccessIdx } + // In case the epoch of the store is increased, try to avoid reloading the current region by also + // increasing the epoch stored in `rs`. + rs.switchNextProxyStore(r, currentProxyIdx, incEpochStoreIdx) + logutil.Logger(bo.GetCtx()).Info("switch region proxy peer to next due to send request fail", + zap.Stringer("current", ctx), + zap.Bool("needReload", scheduleReload), + zap.Error(err)) } else { - rs.switchNextFlashPeer(r, ctx.AccessIdx) - logutil.Logger(bo.ctx).Info("switch region tiflash peer to next due to send request fail", + rs.switchNextTiKVPeer(r, ctx.AccessIdx) + logutil.Logger(bo.GetCtx()).Info("switch region peer to next due to send request fail", zap.Stringer("current", ctx), zap.Bool("needReload", scheduleReload), zap.Error(err)) } + } else { + rs.switchNextFlashPeer(r, ctx.AccessIdx) + logutil.Logger(bo.GetCtx()).Info("switch region tiflash peer to next due to send request fail", + zap.Stringer("current", ctx), + zap.Bool("needReload", scheduleReload), + zap.Error(err)) + } - // force reload region when retry all known peers in region. - if scheduleReload { - r.scheduleReload() - } + // force reload region when retry all known peers in region. + if scheduleReload { + r.scheduleReload() } + } // LocateRegionByID searches for the region with ID. @@ -763,7 +903,7 @@ func (c *RegionCache) LocateRegionByID(bo *Backoffer, regionID uint64) (*KeyLoca lr, err := c.loadRegionByID(bo, regionID) if err != nil { // ignore error and use old region info. - logutil.Logger(bo.ctx).Error("load region failure", + logutil.Logger(bo.GetCtx()).Error("load region failure", zap.Uint64("regionID", regionID), zap.Error(err)) } else { r = lr @@ -823,43 +963,6 @@ func (c *RegionCache) GroupKeysByRegion(bo *Backoffer, keys [][]byte, filter fun return groups, first, nil } -type groupedMutations struct { - region RegionVerID - mutations CommitterMutations -} - -// groupSortedMutationsByRegion separates keys into groups by their belonging Regions. -func (c *RegionCache) groupSortedMutationsByRegion(bo *Backoffer, m CommitterMutations) ([]groupedMutations, error) { - var ( - groups []groupedMutations - lastLoc *KeyLocation - ) - lastUpperBound := 0 - for i := 0; i < m.Len(); i++ { - if lastLoc == nil || !lastLoc.Contains(m.GetKey(i)) { - if lastLoc != nil { - groups = append(groups, groupedMutations{ - region: lastLoc.Region, - mutations: m.Slice(lastUpperBound, i), - }) - lastUpperBound = i - } - var err error - lastLoc, err = c.LocateKey(bo, m.GetKey(i)) - if err != nil { - return nil, errors.Trace(err) - } - } - } - if lastLoc != nil { - groups = append(groups, groupedMutations{ - region: lastLoc.Region, - mutations: m.Slice(lastUpperBound, m.Len()), - }) - } - return groups, nil -} - // ListRegionIDsInKeyRange lists ids of regions in [start_key,end_key]. func (c *RegionCache) ListRegionIDsInKeyRange(bo *Backoffer, startKey, endKey []byte) (regionIDs []uint64, err error) { for { @@ -938,7 +1041,7 @@ func (c *RegionCache) InvalidateCachedRegion(id RegionVerID) { // InvalidateCachedRegionWithReason removes a cached Region with the reason why it's invalidated. func (c *RegionCache) InvalidateCachedRegionWithReason(id RegionVerID, reason InvalidReason) { - cachedRegion := c.getCachedRegionWithRLock(id) + cachedRegion := c.GetCachedRegionWithRLock(id) if cachedRegion == nil { return } @@ -946,16 +1049,15 @@ func (c *RegionCache) InvalidateCachedRegionWithReason(id RegionVerID, reason In } // UpdateLeader update some region cache with newer leader info. -func (c *RegionCache) UpdateLeader(regionID RegionVerID, leaderStoreID uint64, currentPeerIdx AccessIndex) { - r := c.getCachedRegionWithRLock(regionID) +func (c *RegionCache) UpdateLeader(regionID RegionVerID, leader *metapb.Peer, currentPeerIdx AccessIndex) { + r := c.GetCachedRegionWithRLock(regionID) if r == nil { logutil.BgLogger().Debug("regionCache: cannot find region when updating leader", - zap.Uint64("regionID", regionID.GetID()), - zap.Uint64("leaderStoreID", leaderStoreID)) + zap.Uint64("regionID", regionID.GetID())) return } - if leaderStoreID == 0 { + if leader == nil { rs := r.getStore() rs.switchNextTiKVPeer(r, currentPeerIdx) logutil.BgLogger().Info("switch region peer to next due to NotLeader with NULL leader", @@ -964,17 +1066,17 @@ func (c *RegionCache) UpdateLeader(regionID RegionVerID, leaderStoreID uint64, c return } - if !c.switchWorkLeaderToPeer(r, leaderStoreID) { + if !c.switchWorkLeaderToPeer(r, leader) { logutil.BgLogger().Info("invalidate region cache due to cannot find peer when updating leader", zap.Uint64("regionID", regionID.GetID()), zap.Int("currIdx", int(currentPeerIdx)), - zap.Uint64("leaderStoreID", leaderStoreID)) + zap.Uint64("leaderStoreID", leader.GetStoreId())) r.invalidate(StoreNotFound) } else { logutil.BgLogger().Info("switch region leader to specific leader due to kv return NotLeader", zap.Uint64("regionID", regionID.GetID()), zap.Int("currIdx", int(currentPeerIdx)), - zap.Uint64("leaderStoreID", leaderStoreID)) + zap.Uint64("leaderStoreID", leader.GetStoreId())) } } @@ -995,14 +1097,18 @@ func (c *RegionCache) insertRegionToCache(cachedRegion *Region) { store := cachedRegion.getStore() oldRegion := old.(*btreeItem).cachedRegion oldRegionStore := oldRegion.getStore() + // TODO(youjiali1995): remove this because the new retry logic can handle this issue. + // // Joint consensus is enabled in v5.0, which is possible to make a leader step down as a learner during a conf change. // And if hibernate region is enabled, after the leader step down, there can be a long time that there is no leader // in the region and the leader info in PD is stale until requests are sent to followers or hibernate timeout. // To solve it, one solution is always to try a different peer if the invalid reason of the old cached region is no-leader. // There is a small probability that the current peer who reports no-leader becomes a leader and TiDB has to retry once in this case. if InvalidReason(atomic.LoadInt32((*int32)(&oldRegion.invalidReason))) == NoLeader { - store.workTiKVIdx = (oldRegionStore.workTiKVIdx + 1) % AccessIndex(store.accessStoreNum(TiKVOnly)) + store.workTiKVIdx = (oldRegionStore.workTiKVIdx + 1) % AccessIndex(store.accessStoreNum(tiKVOnly)) } + // Invalidate the old region in case it's not invalidated and some requests try with the stale region information. + oldRegion.invalidate(Other) // Don't refresh TiFlash work idx for region. Otherwise, it will always goto a invalid store which // is under transferring regions. store.workTiFlashIdx = atomic.LoadInt32(&oldRegionStore.workTiFlashIdx) @@ -1061,7 +1167,7 @@ func (c *RegionCache) getRegionByIDFromCache(regionID uint64) *Region { return nil } lastAccess := atomic.LoadInt64(&latestRegion.lastAccess) - if ts-lastAccess > RegionCacheTTLSec { + if ts-lastAccess > regionCacheTTLSec { return nil } if latestRegion != nil { @@ -1070,8 +1176,9 @@ func (c *RegionCache) getRegionByIDFromCache(regionID uint64) *Region { return latestRegion } +// GetStoresByType gets stores by type `typ` // TODO: revise it by get store by closure. -func (c *RegionCache) getStoresByType(typ tikvrpc.EndpointType) []*Store { +func (c *RegionCache) GetStoresByType(typ tikvrpc.EndpointType) []*Store { c.storeMu.Lock() defer c.storeMu.Unlock() stores := make([]*Store, 0) @@ -1115,9 +1222,6 @@ func filterUnavailablePeers(region *pd.Region) { new = append(new, p) } } - for i := len(new); i < len(region.Meta.Peers); i++ { - region.Meta.Peers[i] = nil - } region.Meta.Peers = new } @@ -1125,7 +1229,7 @@ func filterUnavailablePeers(region *pd.Region) { // If the given key is the end key of the region that you want, you may set the second argument to true. This is useful // when processing in reverse order. func (c *RegionCache) loadRegion(bo *Backoffer, key []byte, isEndKey bool) (*Region, error) { - ctx := bo.ctx + ctx := bo.GetCtx() if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("loadRegion", opentracing.ChildOf(span.Context())) defer span1.Finish() @@ -1136,7 +1240,7 @@ func (c *RegionCache) loadRegion(bo *Backoffer, key []byte, isEndKey bool) (*Reg searchPrev := false for { if backoffErr != nil { - err := bo.Backoff(BoPDRPC, backoffErr) + err := bo.Backoff(retry.BoPDRPC, backoffErr) if err != nil { return nil, errors.Trace(err) } @@ -1170,12 +1274,12 @@ func (c *RegionCache) loadRegion(bo *Backoffer, key []byte, isEndKey bool) (*Reg continue } region := &Region{meta: reg.Meta} - err = region.init(c) + err = region.init(bo, c) if err != nil { return nil, err } if reg.Leader != nil { - c.switchWorkLeaderToPeer(region, reg.Leader.StoreId) + c.switchWorkLeaderToPeer(region, reg.Leader) } return region, nil } @@ -1183,7 +1287,7 @@ func (c *RegionCache) loadRegion(bo *Backoffer, key []byte, isEndKey bool) (*Reg // loadRegionByID loads region from pd client, and picks the first peer as leader. func (c *RegionCache) loadRegionByID(bo *Backoffer, regionID uint64) (*Region, error) { - ctx := bo.ctx + ctx := bo.GetCtx() if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("loadRegionByID", opentracing.ChildOf(span.Context())) defer span1.Finish() @@ -1192,7 +1296,7 @@ func (c *RegionCache) loadRegionByID(bo *Backoffer, regionID uint64) (*Region, e var backoffErr error for { if backoffErr != nil { - err := bo.Backoff(BoPDRPC, backoffErr) + err := bo.Backoff(retry.BoPDRPC, backoffErr) if err != nil { return nil, errors.Trace(err) } @@ -1215,12 +1319,12 @@ func (c *RegionCache) loadRegionByID(bo *Backoffer, regionID uint64) (*Region, e return nil, errors.New("receive Region with no available peer") } region := &Region{meta: reg.Meta} - err = region.init(c) + err = region.init(bo, c) if err != nil { return nil, err } if reg.Leader != nil { - c.switchWorkLeaderToPeer(region, reg.Leader.GetStoreId()) + c.switchWorkLeaderToPeer(region, reg.Leader) } return region, nil } @@ -1232,7 +1336,7 @@ func (c *RegionCache) scanRegions(bo *Backoffer, startKey, endKey []byte, limit if limit == 0 { return nil, nil } - ctx := bo.ctx + ctx := bo.GetCtx() if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("scanRegions", opentracing.ChildOf(span.Context())) defer span1.Finish() @@ -1242,7 +1346,7 @@ func (c *RegionCache) scanRegions(bo *Backoffer, startKey, endKey []byte, limit var backoffErr error for { if backoffErr != nil { - err := bo.Backoff(BoPDRPC, backoffErr) + err := bo.Backoff(retry.BoPDRPC, backoffErr) if err != nil { return nil, errors.Trace(err) } @@ -1266,14 +1370,14 @@ func (c *RegionCache) scanRegions(bo *Backoffer, startKey, endKey []byte, limit regions := make([]*Region, 0, len(regionsInfo)) for _, r := range regionsInfo { region := &Region{meta: r.Meta} - err := region.init(c) + err := region.init(bo, c) if err != nil { return nil, err } leader := r.Leader // Leader id = 0 indicates no leader. - if leader.GetId() != 0 { - c.switchWorkLeaderToPeer(region, leader.GetStoreId()) + if leader != nil && leader.GetId() != 0 { + c.switchWorkLeaderToPeer(region, leader) regions = append(regions, region) } } @@ -1288,14 +1392,15 @@ func (c *RegionCache) scanRegions(bo *Backoffer, startKey, endKey []byte, limit } } -func (c *RegionCache) getCachedRegionWithRLock(regionID RegionVerID) (r *Region) { +// GetCachedRegionWithRLock returns region with lock. +func (c *RegionCache) GetCachedRegionWithRLock(regionID RegionVerID) (r *Region) { c.mu.RLock() r = c.mu.regions[regionID] c.mu.RUnlock() return } -func (c *RegionCache) getStoreAddr(bo *Backoffer, region *Region, store *Store, storeIdx int) (addr string, err error) { +func (c *RegionCache) getStoreAddr(bo *Backoffer, region *Region, store *Store) (addr string, err error) { state := store.getResolveState() switch state { case resolved, needCheck: @@ -1305,24 +1410,26 @@ func (c *RegionCache) getStoreAddr(bo *Backoffer, region *Region, store *Store, addr, err = store.initResolve(bo, c) return case deleted: - addr = c.changeToActiveStore(region, store, storeIdx) + addr = c.changeToActiveStore(region, store) return + case tombstone: + return "", nil default: panic("unsupported resolve state") } } -func (c *RegionCache) getProxyStore(region *Region, store *Store, rs *RegionStore, workStoreIdx AccessIndex) (proxyStore *Store, proxyAccessIdx AccessIndex, proxyStoreIdx int) { +func (c *RegionCache) getProxyStore(region *Region, store *Store, rs *regionStore, workStoreIdx AccessIndex) (proxyStore *Store, proxyAccessIdx AccessIndex, proxyStoreIdx int) { if !c.enableForwarding || store.storeType != tikvrpc.TiKV || atomic.LoadInt32(&store.needForwarding) == 0 { return } if rs.proxyTiKVIdx >= 0 { - storeIdx, proxyStore := rs.accessStore(TiKVOnly, rs.proxyTiKVIdx) + storeIdx, proxyStore := rs.accessStore(tiKVOnly, rs.proxyTiKVIdx) return proxyStore, rs.proxyTiKVIdx, storeIdx } - tikvNum := rs.accessStoreNum(TiKVOnly) + tikvNum := rs.accessStoreNum(tiKVOnly) if tikvNum <= 1 { return } @@ -1341,7 +1448,7 @@ func (c *RegionCache) getProxyStore(region *Region, store *Store, rs *RegionStor if index == int(workStoreIdx) { continue } - storeIdx, store := rs.accessStore(TiKVOnly, AccessIndex(index)) + storeIdx, store := rs.accessStore(tiKVOnly, AccessIndex(index)) // Skip unreachable stores. if atomic.LoadInt32(&store.needForwarding) != 0 { continue @@ -1354,7 +1461,9 @@ func (c *RegionCache) getProxyStore(region *Region, store *Store, rs *RegionStor return nil, 0, 0 } -func (c *RegionCache) changeToActiveStore(region *Region, store *Store, storeIdx int) (addr string) { +// changeToActiveStore replace the deleted store in the region by an up-to-date store in the stores map. +// The order is guaranteed by reResolve() which adds the new store before marking old store deleted. +func (c *RegionCache) changeToActiveStore(region *Region, store *Store) (addr string) { c.storeMu.RLock() store = c.storeMu.stores[store.storeID] c.storeMu.RUnlock() @@ -1362,8 +1471,8 @@ func (c *RegionCache) changeToActiveStore(region *Region, store *Store, storeIdx oldRegionStore := region.getStore() newRegionStore := oldRegionStore.clone() newRegionStore.stores = make([]*Store, 0, len(oldRegionStore.stores)) - for i, s := range oldRegionStore.stores { - if i == storeIdx { + for _, s := range oldRegionStore.stores { + if s.storeID == store.storeID { newRegionStore.stores = append(newRegionStore.stores, store) } else { newRegionStore.stores = append(newRegionStore.stores, s) @@ -1404,7 +1513,13 @@ func (c *RegionCache) getStoresByLabels(labels []*metapb.StoreLabel) []*Store { } // OnRegionEpochNotMatch removes the old region and inserts new regions into the cache. -func (c *RegionCache) OnRegionEpochNotMatch(bo *Backoffer, ctx *RPCContext, currentRegions []*metapb.Region) error { +// It returns whether retries the request because it's possible the region epoch is ahead of TiKV's due to slow appling. +func (c *RegionCache) OnRegionEpochNotMatch(bo *Backoffer, ctx *RPCContext, currentRegions []*metapb.Region) (bool, error) { + if len(currentRegions) == 0 { + c.InvalidateCachedRegionWithReason(ctx.Region, EpochNotMatch) + return false, nil + } + // Find whether the region epoch in `ctx` is ahead of TiKV's. If so, backoff. for _, meta := range currentRegions { if meta.GetId() == ctx.Region.id && @@ -1412,45 +1527,49 @@ func (c *RegionCache) OnRegionEpochNotMatch(bo *Backoffer, ctx *RPCContext, curr meta.GetRegionEpoch().GetVersion() < ctx.Region.ver) { err := errors.Errorf("region epoch is ahead of tikv. rpc ctx: %+v, currentRegions: %+v", ctx, currentRegions) logutil.BgLogger().Info("region epoch is ahead of tikv", zap.Error(err)) - return bo.Backoff(BoRegionMiss, err) + return true, bo.Backoff(retry.BoRegionMiss, err) } } - c.mu.Lock() - defer c.mu.Unlock() needInvalidateOld := true + newRegions := make([]*Region, 0, len(currentRegions)) // If the region epoch is not ahead of TiKV's, replace region meta in region cache. for _, meta := range currentRegions { if _, ok := c.pdClient.(*CodecPDClient); ok { var err error if meta, err = decodeRegionMetaKeyWithShallowCopy(meta); err != nil { - return errors.Errorf("newRegion's range key is not encoded: %v, %v", meta, err) + return false, errors.Errorf("newRegion's range key is not encoded: %v, %v", meta, err) } } region := &Region{meta: meta} - err := region.init(c) + err := region.init(bo, c) if err != nil { - return err + return false, err } - var initLeader uint64 + var initLeaderStoreID uint64 if ctx.Store.storeType == tikvrpc.TiFlash { - initLeader = region.findElectableStoreID() + initLeaderStoreID = region.findElectableStoreID() } else { - initLeader = ctx.Store.storeID + initLeaderStoreID = ctx.Store.storeID } - c.switchWorkLeaderToPeer(region, initLeader) - c.insertRegionToCache(region) + c.switchWorkLeaderToPeer(region, region.getPeerOnStore(initLeaderStoreID)) + newRegions = append(newRegions, region) if ctx.Region == region.VerID() { needInvalidateOld = false } } + c.mu.Lock() + for _, region := range newRegions { + c.insertRegionToCache(region) + } if needInvalidateOld { cachedRegion, ok := c.mu.regions[ctx.Region] if ok { cachedRegion.invalidate(EpochNotMatch) } } - return nil + c.mu.Unlock() + return false, nil } // PDClient returns the pd.Client in RegionCache. @@ -1507,42 +1626,42 @@ func (r *Region) GetMeta() *metapb.Region { // GetLeaderPeerID returns leader peer ID. func (r *Region) GetLeaderPeerID() uint64 { store := r.getStore() - if int(store.workTiKVIdx) >= store.accessStoreNum(TiKVOnly) { + if int(store.workTiKVIdx) >= store.accessStoreNum(tiKVOnly) { return 0 } - storeIdx, _ := store.accessStore(TiKVOnly, store.workTiKVIdx) + storeIdx, _ := store.accessStore(tiKVOnly, store.workTiKVIdx) return r.meta.Peers[storeIdx].Id } // GetLeaderStoreID returns the store ID of the leader region. func (r *Region) GetLeaderStoreID() uint64 { store := r.getStore() - if int(store.workTiKVIdx) >= store.accessStoreNum(TiKVOnly) { + if int(store.workTiKVIdx) >= store.accessStoreNum(tiKVOnly) { return 0 } - storeIdx, _ := store.accessStore(TiKVOnly, store.workTiKVIdx) + storeIdx, _ := store.accessStore(tiKVOnly, store.workTiKVIdx) return r.meta.Peers[storeIdx].StoreId } -func (r *Region) getKvStorePeer(rs *RegionStore, aidx AccessIndex) (store *Store, peer *metapb.Peer, accessIdx AccessIndex, storeIdx int) { - storeIdx, store = rs.accessStore(TiKVOnly, aidx) +func (r *Region) getKvStorePeer(rs *regionStore, aidx AccessIndex) (store *Store, peer *metapb.Peer, accessIdx AccessIndex, storeIdx int) { + storeIdx, store = rs.accessStore(tiKVOnly, aidx) peer = r.meta.Peers[storeIdx] accessIdx = aidx return } // WorkStorePeer returns current work store with work peer. -func (r *Region) WorkStorePeer(rs *RegionStore) (store *Store, peer *metapb.Peer, accessIdx AccessIndex, storeIdx int) { +func (r *Region) WorkStorePeer(rs *regionStore) (store *Store, peer *metapb.Peer, accessIdx AccessIndex, storeIdx int) { return r.getKvStorePeer(rs, rs.workTiKVIdx) } // FollowerStorePeer returns a follower store with follower peer. -func (r *Region) FollowerStorePeer(rs *RegionStore, followerStoreSeed uint32, op *storeSelectorOp) (store *Store, peer *metapb.Peer, accessIdx AccessIndex, storeIdx int) { +func (r *Region) FollowerStorePeer(rs *regionStore, followerStoreSeed uint32, op *storeSelectorOp) (store *Store, peer *metapb.Peer, accessIdx AccessIndex, storeIdx int) { return r.getKvStorePeer(rs, rs.follower(followerStoreSeed, op)) } // AnyStorePeer returns a leader or follower store with the associated peer. -func (r *Region) AnyStorePeer(rs *RegionStore, followerStoreSeed uint32, op *storeSelectorOp) (store *Store, peer *metapb.Peer, accessIdx AccessIndex, storeIdx int) { +func (r *Region) AnyStorePeer(rs *regionStore, followerStoreSeed uint32, op *storeSelectorOp) (store *Store, peer *metapb.Peer, accessIdx AccessIndex, storeIdx int) { return r.getKvStorePeer(rs, rs.kvPeer(followerStoreSeed, op)) } @@ -1603,14 +1722,17 @@ func (r *Region) EndKey() []byte { } // switchWorkLeaderToPeer switches current store to the one on specific store. It returns -// false if no peer matches the storeID. -func (c *RegionCache) switchWorkLeaderToPeer(r *Region, targetStoreID uint64) (found bool) { - globalStoreIdx, found := c.getPeerStoreIndex(r, targetStoreID) +// false if no peer matches the peer. +func (c *RegionCache) switchWorkLeaderToPeer(r *Region, peer *metapb.Peer) (found bool) { + globalStoreIdx, found := c.getPeerStoreIndex(r, peer) + if !found { + return + } retry: // switch to new leader. oldRegionStore := r.getStore() var leaderIdx AccessIndex - for i, gIdx := range oldRegionStore.accessIndex[TiKVOnly] { + for i, gIdx := range oldRegionStore.accessIndex[tiKVOnly] { if gIdx == globalStoreIdx { leaderIdx = AccessIndex(i) } @@ -1626,18 +1748,18 @@ retry: return } -func (r *RegionStore) switchNextFlashPeer(rr *Region, currentPeerIdx AccessIndex) { - nextIdx := (currentPeerIdx + 1) % AccessIndex(r.accessStoreNum(TiFlashOnly)) +func (r *regionStore) switchNextFlashPeer(rr *Region, currentPeerIdx AccessIndex) { + nextIdx := (currentPeerIdx + 1) % AccessIndex(r.accessStoreNum(tiFlashOnly)) newRegionStore := r.clone() newRegionStore.workTiFlashIdx = int32(nextIdx) rr.compareAndSwapStore(r, newRegionStore) } -func (r *RegionStore) switchNextTiKVPeer(rr *Region, currentPeerIdx AccessIndex) { +func (r *regionStore) switchNextTiKVPeer(rr *Region, currentPeerIdx AccessIndex) { if r.workTiKVIdx != currentPeerIdx { return } - nextIdx := (currentPeerIdx + 1) % AccessIndex(r.accessStoreNum(TiKVOnly)) + nextIdx := (currentPeerIdx + 1) % AccessIndex(r.accessStoreNum(tiKVOnly)) newRegionStore := r.clone() newRegionStore.workTiKVIdx = nextIdx rr.compareAndSwapStore(r, newRegionStore) @@ -1646,12 +1768,12 @@ func (r *RegionStore) switchNextTiKVPeer(rr *Region, currentPeerIdx AccessIndex) // switchNextProxyStore switches the index of the peer that will forward requests to the leader to the next peer. // If proxy is currently not used on this region, the value of `currentProxyIdx` should be -1, and a random peer will // be select in this case. -func (r *RegionStore) switchNextProxyStore(rr *Region, currentProxyIdx AccessIndex, incEpochStoreIdx int) { +func (r *regionStore) switchNextProxyStore(rr *Region, currentProxyIdx AccessIndex, incEpochStoreIdx int) { if r.proxyTiKVIdx != currentProxyIdx { return } - tikvNum := r.accessStoreNum(TiKVOnly) + tikvNum := r.accessStoreNum(tiKVOnly) var nextIdx AccessIndex // If the region is not using proxy before, randomly select a non-leader peer for the first try. @@ -1678,7 +1800,7 @@ func (r *RegionStore) switchNextProxyStore(rr *Region, currentProxyIdx AccessInd rr.compareAndSwapStore(r, newRegionStore) } -func (r *RegionStore) setProxyStoreIdx(rr *Region, idx AccessIndex) { +func (r *regionStore) setProxyStoreIdx(rr *Region, idx AccessIndex) { if r.proxyTiKVIdx == idx { return } @@ -1692,7 +1814,7 @@ func (r *RegionStore) setProxyStoreIdx(rr *Region, idx AccessIndex) { zap.Bool("success", success)) } -func (r *RegionStore) unsetProxyStoreIfNeeded(rr *Region) { +func (r *regionStore) unsetProxyStoreIfNeeded(rr *Region) { r.setProxyStoreIdx(rr, -1) } @@ -1708,12 +1830,21 @@ func (r *Region) findElectableStoreID() uint64 { return 0 } -func (c *RegionCache) getPeerStoreIndex(r *Region, id uint64) (idx int, found bool) { - if len(r.meta.Peers) == 0 { +func (r *Region) getPeerOnStore(storeID uint64) *metapb.Peer { + for _, p := range r.meta.Peers { + if p.StoreId == storeID { + return p + } + } + return nil +} + +func (c *RegionCache) getPeerStoreIndex(r *Region, peer *metapb.Peer) (idx int, found bool) { + if len(r.meta.Peers) == 0 || peer == nil { return } for i, p := range r.meta.Peers { - if p.GetStoreId() == id { + if isSamePeer(p, peer) { idx = i found = true return @@ -1758,58 +1889,73 @@ type Store struct { type resolveState uint64 const ( + // The store is just created and normally is being resolved. + // Store in this state will only be resolved by initResolve(). unresolved resolveState = iota + // The store is resolved and its address is valid. resolved + // Request failed on this store and it will be re-resolved by asyncCheckAndResolveLoop(). needCheck + // The store's address or label is changed and marked deleted. + // There is a new store struct replaced it in the RegionCache and should + // call changeToActiveStore() to get the new struct. deleted + // The store is a tombstone. Should invalidate the region if tries to access it. + tombstone ) -// initResolve resolves addr for store that never resolved. +// IsTiFlash returns true if the storeType is TiFlash +func (s *Store) IsTiFlash() bool { + return s.storeType == tikvrpc.TiFlash +} + +// initResolve resolves the address of the store that never resolved and returns an +// empty string if it's a tombstone. func (s *Store) initResolve(bo *Backoffer, c *RegionCache) (addr string, err error) { s.resolveMutex.Lock() state := s.getResolveState() defer s.resolveMutex.Unlock() if state != unresolved { - addr = s.addr + if state != tombstone { + addr = s.addr + } return } var store *metapb.Store for { - store, err = c.pdClient.GetStore(bo.ctx, s.storeID) + store, err = c.pdClient.GetStore(bo.GetCtx(), s.storeID) if err != nil { metrics.RegionCacheCounterWithGetStoreError.Inc() } else { metrics.RegionCacheCounterWithGetStoreOK.Inc() } - if err != nil { + if bo.GetCtx().Err() != nil && errors.Cause(bo.GetCtx().Err()) == context.Canceled { + return + } + if err != nil && !isStoreNotFoundError(err) { // TODO: more refine PD error status handle. - if errors.Cause(err) == context.Canceled { - return - } err = errors.Errorf("loadStore from PD failed, id: %d, err: %v", s.storeID, err) - if err = bo.Backoff(BoPDRPC, err); err != nil { + if err = bo.Backoff(retry.BoPDRPC, err); err != nil { return } continue } + // The store is a tombstone. if store == nil { - return + s.setResolveState(tombstone) + return "", nil } addr = store.GetAddress() + if addr == "" { + return "", errors.Errorf("empty store(%d) address", s.storeID) + } s.addr = addr s.saddr = store.GetStatusAddress() - s.storeType = GetStoreTypeByMeta(store) + s.storeType = tikvrpc.GetStoreTypeByMeta(store) s.labels = store.GetLabels() - retry: - state = s.getResolveState() - if state != unresolved { - addr = s.addr - return - } - if !s.compareAndSwapState(state, resolved) { - goto retry - } - return + // Shouldn't have other one changing its state concurrently, but we still use changeResolveStateTo for safety. + s.changeResolveStateTo(unresolved, resolved) + return s.addr, nil } } @@ -1842,41 +1988,22 @@ func (s *Store) reResolve(c *RegionCache) (bool, error) { logutil.BgLogger().Info("invalidate regions in removed store", zap.Uint64("store", s.storeID), zap.String("add", s.addr)) atomic.AddUint32(&s.epoch, 1) - atomic.StoreUint64(&s.state, uint64(deleted)) + s.setResolveState(tombstone) metrics.RegionCacheCounterWithInvalidateStoreRegionsOK.Inc() return false, nil } - storeType := GetStoreTypeByMeta(store) + storeType := tikvrpc.GetStoreTypeByMeta(store) addr = store.GetAddress() if s.addr != addr || !s.IsSameLabels(store.GetLabels()) { - state := resolved - newStore := &Store{storeID: s.storeID, addr: addr, saddr: store.GetStatusAddress(), storeType: storeType, labels: store.GetLabels()} - newStore.state = *(*uint64)(&state) + newStore := &Store{storeID: s.storeID, addr: addr, saddr: store.GetStatusAddress(), storeType: storeType, labels: store.GetLabels(), state: uint64(resolved)} c.storeMu.Lock() c.storeMu.stores[newStore.storeID] = newStore c.storeMu.Unlock() - retryMarkDel: - // all region used those - oldState := s.getResolveState() - if oldState == deleted { - return false, nil - } - newState := deleted - if !s.compareAndSwapState(oldState, newState) { - goto retryMarkDel - } + s.setResolveState(deleted) return false, nil } -retryMarkResolved: - oldState := s.getResolveState() - if oldState != needCheck { - return true, nil - } - newState := resolved - if !s.compareAndSwapState(oldState, newState) { - goto retryMarkResolved - } + s.changeResolveStateTo(needCheck, resolved) return true, nil } @@ -1888,23 +2015,35 @@ func (s *Store) getResolveState() resolveState { return resolveState(atomic.LoadUint64(&s.state)) } -func (s *Store) compareAndSwapState(oldState, newState resolveState) bool { - return atomic.CompareAndSwapUint64(&s.state, uint64(oldState), uint64(newState)) +func (s *Store) setResolveState(state resolveState) { + atomic.StoreUint64(&s.state, uint64(state)) +} + +// changeResolveStateTo changes the store resolveState from the old state to the new state. +// Returns true if it changes the state successfully, and false if the store's state +// is changed by another one. +func (s *Store) changeResolveStateTo(from, to resolveState) bool { + for { + state := s.getResolveState() + if state == to { + return true + } + if state != from { + return false + } + if atomic.CompareAndSwapUint64(&s.state, uint64(from), uint64(to)) { + return true + } + } } // markNeedCheck marks resolved store to be async resolve to check store addr change. func (s *Store) markNeedCheck(notifyCheckCh chan struct{}) { -retry: - oldState := s.getResolveState() - if oldState != resolved { - return - } - if !s.compareAndSwapState(oldState, needCheck) { - goto retry - } - select { - case notifyCheckCh <- struct{}{}: - default: + if s.changeResolveStateTo(resolved, needCheck) { + select { + case notifyCheckCh <- struct{}{}: + default: + } } } @@ -1940,10 +2079,20 @@ type livenessState uint32 var ( livenessSf singleflight.Group - // StoreLivenessTimeout is the max duration of resolving liveness of a TiKV instance. - StoreLivenessTimeout time.Duration + // storeLivenessTimeout is the max duration of resolving liveness of a TiKV instance. + storeLivenessTimeout time.Duration ) +// SetStoreLivenessTimeout sets storeLivenessTimeout to t. +func SetStoreLivenessTimeout(t time.Duration) { + storeLivenessTimeout = t +} + +// GetStoreLivenessTimeout returns storeLivenessTimeout. +func GetStoreLivenessTimeout() time.Duration { + return storeLivenessTimeout +} + const ( unknown livenessState = iota reachable @@ -1990,7 +2139,7 @@ func (s *Store) checkUntilHealth(c *RegionCache) { } } - bo := NewNoopBackoff(ctx) + bo := retry.NewNoopBackoff(ctx) l := s.requestLiveness(bo, c) if l == reachable { logutil.BgLogger().Info("[health check] store became reachable", zap.Uint64("storeID", s.storeID)) @@ -2006,7 +2155,7 @@ func (s *Store) requestLiveness(bo *Backoffer, c *RegionCache) (l livenessState) return c.testingKnobs.mockRequestLiveness(s, bo) } - if StoreLivenessTimeout == 0 { + if storeLivenessTimeout == 0 { return unreachable } @@ -2016,11 +2165,11 @@ func (s *Store) requestLiveness(bo *Backoffer, c *RegionCache) (l livenessState) } addr := s.addr rsCh := livenessSf.DoChan(addr, func() (interface{}, error) { - return invokeKVStatusAPI(addr, StoreLivenessTimeout), nil + return invokeKVStatusAPI(addr, storeLivenessTimeout), nil }) var ctx context.Context if bo != nil { - ctx = bo.ctx + ctx = bo.GetCtx() } else { ctx = context.Background() } @@ -2034,6 +2183,11 @@ func (s *Store) requestLiveness(bo *Backoffer, c *RegionCache) (l livenessState) return } +// GetAddr returns the address of the store +func (s *Store) GetAddr() string { + return s.addr +} + func invokeKVStatusAPI(addr string, timeout time.Duration) (l livenessState) { start := time.Now() defer func() { @@ -2107,8 +2261,8 @@ func createKVHealthClient(ctx context.Context, addr string) (*grpc.ClientConn, h ctx, addr, opt, - grpc.WithInitialWindowSize(grpcInitialWindowSize), - grpc.WithInitialConnWindowSize(grpcInitialConnWindowSize), + grpc.WithInitialWindowSize(client.GrpcInitialWindowSize), + grpc.WithInitialConnWindowSize(client.GrpcInitialConnWindowSize), grpc.WithConnectParams(grpc.ConnectParams{ Backoff: backoff.Config{ BaseDelay: 100 * time.Millisecond, // Default was 1s. @@ -2130,3 +2284,7 @@ func createKVHealthClient(ctx context.Context, addr string) (*grpc.ClientConn, h cli := healthpb.NewHealthClient(conn) return conn, cli, nil } + +func isSamePeer(lhs *metapb.Peer, rhs *metapb.Peer) bool { + return lhs == rhs || (lhs.GetId() == rhs.GetId() && lhs.GetStoreId() == rhs.GetStoreId()) +} diff --git a/store/tikv/region_cache_test.go b/store/tikv/region_cache_test.go index efb2ae9df73ab..ea42dbd79cda8 100644 --- a/store/tikv/region_cache_test.go +++ b/store/tikv/region_cache_test.go @@ -28,6 +28,8 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/mockstore/mocktikv" + "github.com/pingcap/tidb/store/tikv/retry" + "github.com/pingcap/tidb/store/tikv/tikvrpc" pd "github.com/tikv/pd/client" ) @@ -180,32 +182,176 @@ func (s *testRegionCacheSuite) TestSimple(c *C) { c.Assert(r, IsNil) } -func (s *testRegionCacheSuite) TestDropStore(c *C) { - bo := NewBackofferWithVars(context.Background(), 100, nil) +// TestResolveStateTransition verifies store's resolve state transition. For example, +// a newly added store is in unresolved state and will be resolved soon if it's an up store, +// or in tombstone state if it's a tombstone. +func (s *testRegionCacheSuite) TestResolveStateTransition(c *C) { + cache := s.cache + bo := retry.NewNoopBackoff(context.Background()) + + // Check resolving normal stores. The resolve state should be resolved. + for _, storeMeta := range s.cluster.GetAllStores() { + store := cache.getStoreByStoreID(storeMeta.GetId()) + c.Assert(store.getResolveState(), Equals, unresolved) + addr, err := store.initResolve(bo, cache) + c.Assert(err, IsNil) + c.Assert(addr, Equals, storeMeta.GetAddress()) + c.Assert(store.getResolveState(), Equals, resolved) + } + + waitResolve := func(s *Store) { + for i := 0; i < 10; i++ { + if s.getResolveState() != needCheck { + break + } + time.Sleep(50 * time.Millisecond) + } + } + + // Mark the store needCheck. The resolve state should be resolved soon. + store := cache.getStoreByStoreID(s.store1) + store.markNeedCheck(cache.notifyCheckCh) + waitResolve(store) + c.Assert(store.getResolveState(), Equals, resolved) + + // Mark the store needCheck and it becomes a tombstone. The resolve state should be tombstone. + s.cluster.MarkTombstone(s.store1) + store.markNeedCheck(cache.notifyCheckCh) + waitResolve(store) + c.Assert(store.getResolveState(), Equals, tombstone) + s.cluster.StartStore(s.store1) + + // Mark the store needCheck and it's deleted from PD. The resolve state should be tombstone. + cache.clear() + store = cache.getStoreByStoreID(s.store1) + store.initResolve(bo, cache) + c.Assert(store.getResolveState(), Equals, resolved) + storeMeta := s.cluster.GetStore(s.store1) s.cluster.RemoveStore(s.store1) - loc, err := s.cache.LocateKey(bo, []byte("a")) - c.Assert(err, IsNil) - ctx, err := s.cache.GetTiKVRPCContext(bo, loc.Region, kv.ReplicaReadLeader, 0) + store.markNeedCheck(cache.notifyCheckCh) + waitResolve(store) + c.Assert(store.getResolveState(), Equals, tombstone) + s.cluster.AddStore(storeMeta.GetId(), storeMeta.GetAddress(), storeMeta.GetLabels()...) + + // Mark the store needCheck and its address and labels are changed. + // The resolve state should be deleted and a new store is added to the cache. + cache.clear() + store = cache.getStoreByStoreID(s.store1) + store.initResolve(bo, cache) + c.Assert(store.getResolveState(), Equals, resolved) + s.cluster.UpdateStoreAddr(s.store1, store.addr+"0", &metapb.StoreLabel{Key: "k", Value: "v"}) + store.markNeedCheck(cache.notifyCheckCh) + waitResolve(store) + c.Assert(store.getResolveState(), Equals, deleted) + newStore := cache.getStoreByStoreID(s.store1) + c.Assert(newStore.getResolveState(), Equals, resolved) + c.Assert(newStore.addr, Equals, store.addr+"0") + c.Assert(newStore.labels, DeepEquals, []*metapb.StoreLabel{{Key: "k", Value: "v"}}) + + // Check initResolve()ing a tombstone store. The resolve state should be tombstone. + cache.clear() + s.cluster.MarkTombstone(s.store1) + store = cache.getStoreByStoreID(s.store1) + for i := 0; i < 2; i++ { + addr, err := store.initResolve(bo, cache) + c.Assert(err, IsNil) + c.Assert(addr, Equals, "") + c.Assert(store.getResolveState(), Equals, tombstone) + } + s.cluster.StartStore(s.store1) + cache.clear() + + // Check initResolve()ing a dropped store. The resolve state should be tombstone. + cache.clear() + storeMeta = s.cluster.GetStore(s.store1) + s.cluster.RemoveStore(s.store1) + store = cache.getStoreByStoreID(s.store1) + for i := 0; i < 2; i++ { + addr, err := store.initResolve(bo, cache) + c.Assert(err, IsNil) + c.Assert(addr, Equals, "") + c.Assert(store.getResolveState(), Equals, tombstone) + } + s.cluster.AddStore(storeMeta.GetId(), storeMeta.GetAddress(), storeMeta.GetLabels()...) +} + +// TestFilterDownPeersOrPeersOnTombstoneOrDroppedStore verifies the RegionCache filter +// region's down peers and peers on tombstone or dropped stores. RegionCache shouldn't +// report errors in such cases if there are available peers. +func (s *testRegionCacheSuite) TestFilterDownPeersOrPeersOnTombstoneOrDroppedStores(c *C) { + key := []byte("a") + bo := NewBackofferWithVars(context.Background(), 100, nil) + + verifyGetRPCCtx := func(meta *metapb.Region) { + loc, err := s.cache.LocateKey(bo, key) + c.Assert(loc, NotNil) + c.Assert(err, IsNil) + ctx, err := s.cache.GetTiKVRPCContext(bo, loc.Region, kv.ReplicaReadLeader, 0) + c.Assert(err, IsNil) + c.Assert(ctx, NotNil) + c.Assert(ctx.Meta, DeepEquals, meta) + ctx, err = s.cache.GetTiKVRPCContext(bo, loc.Region, kv.ReplicaReadFollower, rand.Uint32()) + c.Assert(err, IsNil) + c.Assert(ctx, NotNil) + c.Assert(ctx.Meta, DeepEquals, meta) + } + + // When all peers are normal, the cached region should contain all peers. + reg, err := s.cache.findRegionByKey(bo, key, false) + c.Assert(reg, NotNil) c.Assert(err, IsNil) - c.Assert(ctx, IsNil) - ctx, err = s.cache.GetTiKVRPCContext(bo, loc.Region, kv.ReplicaReadFollower, rand.Uint32()) + regInPD, _ := s.cluster.GetRegion(reg.GetID()) + c.Assert(reg.meta, DeepEquals, regInPD) + c.Assert(len(reg.meta.GetPeers()), Equals, len(reg.getStore().stores)) + verifyGetRPCCtx(reg.meta) + s.checkCache(c, 1) + s.cache.clear() + + // Shouldn't contain the peer on the tombstone store. + s.cluster.MarkTombstone(s.store1) + reg, err = s.cache.findRegionByKey(bo, key, false) + c.Assert(reg, NotNil) c.Assert(err, IsNil) - c.Assert(ctx, IsNil) - s.checkCache(c, 0) -} + c.Assert(len(reg.meta.GetPeers()), Equals, len(regInPD.GetPeers())-1) + c.Assert(len(reg.meta.GetPeers()), Equals, len(reg.getStore().stores)) + for _, peer := range reg.meta.GetPeers() { + c.Assert(peer.GetStoreId(), Not(Equals), s.store1) + } + for _, store := range reg.getStore().stores { + c.Assert(store.storeID, Not(Equals), s.store1) + } + verifyGetRPCCtx(reg.meta) + s.checkCache(c, 1) + s.cache.clear() + s.cluster.StartStore(s.store1) -func (s *testRegionCacheSuite) TestDropStoreRetry(c *C) { + // Shouldn't contain the peer on the dropped store. + store := s.cluster.GetStore(s.store1) s.cluster.RemoveStore(s.store1) - done := make(chan struct{}) - go func() { - time.Sleep(time.Millisecond * 10) - s.cluster.AddStore(s.store1, s.storeAddr(s.store1)) - close(done) - }() - loc, err := s.cache.LocateKey(s.bo, []byte("a")) + reg, err = s.cache.findRegionByKey(bo, key, false) + c.Assert(reg, NotNil) c.Assert(err, IsNil) - c.Assert(loc.Region.id, Equals, s.region1) - <-done + c.Assert(len(reg.meta.GetPeers()), Equals, len(regInPD.GetPeers())-1) + c.Assert(len(reg.meta.GetPeers()), Equals, len(reg.getStore().stores)) + for _, peer := range reg.meta.GetPeers() { + c.Assert(peer.GetStoreId(), Not(Equals), s.store1) + } + for _, store := range reg.getStore().stores { + c.Assert(store.storeID, Not(Equals), s.store1) + } + verifyGetRPCCtx(reg.meta) + s.checkCache(c, 1) + s.cache.clear() + s.cluster.AddStore(store.GetId(), store.GetAddress(), store.GetLabels()...) + + // Report an error when there's no available peers. + s.cluster.MarkTombstone(s.store1) + s.cluster.MarkTombstone(s.store2) + _, err = s.cache.findRegionByKey(bo, key, false) + c.Assert(err, NotNil) + c.Assert(err.Error(), Matches, ".*no available peers.*") + s.cluster.StartStore(s.store1) + s.cluster.StartStore(s.store2) } func (s *testRegionCacheSuite) TestUpdateLeader(c *C) { @@ -213,7 +359,7 @@ func (s *testRegionCacheSuite) TestUpdateLeader(c *C) { loc, err := s.cache.LocateKey(s.bo, []byte("a")) c.Assert(err, IsNil) // tikv-server reports `NotLeader` - s.cache.UpdateLeader(loc.Region, s.store2, 0) + s.cache.UpdateLeader(loc.Region, &metapb.Peer{Id: s.peer2, StoreId: s.store2}, 0) r := s.getRegion(c, []byte("a")) c.Assert(r, NotNil) @@ -238,7 +384,7 @@ func (s *testRegionCacheSuite) TestUpdateLeader2(c *C) { s.cluster.AddStore(store3, s.storeAddr(store3)) s.cluster.AddPeer(s.region1, store3, peer3) // tikv-server reports `NotLeader` - s.cache.UpdateLeader(loc.Region, store3, 0) + s.cache.UpdateLeader(loc.Region, &metapb.Peer{Id: peer3, StoreId: store3}, 0) // Store3 does not exist in cache, causes a reload from PD. r := s.getRegion(c, []byte("a")) @@ -262,7 +408,7 @@ func (s *testRegionCacheSuite) TestUpdateLeader2(c *C) { // tikv-server notifies new leader to pd-server. s.cluster.ChangeLeader(s.region1, peer3) // tikv-server reports `NotLeader` again. - s.cache.UpdateLeader(r.VerID(), store3, 0) + s.cache.UpdateLeader(r.VerID(), &metapb.Peer{Id: peer3, StoreId: store3}, 0) r = s.getRegion(c, []byte("a")) c.Assert(r, NotNil) c.Assert(r.GetID(), Equals, s.region1) @@ -297,7 +443,7 @@ func (s *testRegionCacheSuite) TestUpdateLeader3(c *C) { // tikv-server notifies new leader to pd-server. s.cluster.ChangeLeader(s.region1, peer3) // tikv-server reports `NotLeader`(store2 is the leader) - s.cache.UpdateLeader(loc.Region, s.store2, 0) + s.cache.UpdateLeader(loc.Region, &metapb.Peer{Id: s.peer2, StoreId: s.store2}, 0) // Store2 does not exist any more, causes a reload from PD. r := s.getRegion(c, []byte("a")) @@ -310,9 +456,9 @@ func (s *testRegionCacheSuite) TestUpdateLeader3(c *C) { ctx, err := s.cache.GetTiKVRPCContext(s.bo, loc.Region, kv.ReplicaReadLeader, seed) c.Assert(err, IsNil) c.Assert(ctx.Addr, Equals, "store2") - s.cache.OnSendFail(NewNoopBackoff(context.Background()), ctx, false, errors.New("send fail")) - s.cache.checkAndResolve(nil) - s.cache.UpdateLeader(loc.Region, s.store2, 0) + s.cache.OnSendFail(retry.NewNoopBackoff(context.Background()), ctx, false, errors.New("send fail")) + s.cache.checkAndResolve(nil, func(*Store) bool { return true }) + s.cache.UpdateLeader(loc.Region, &metapb.Peer{Id: s.peer2, StoreId: s.store2}, 0) addr := s.getAddr(c, []byte("a"), kv.ReplicaReadLeader, 0) c.Assert(addr, Equals, "") addr = s.getAddr(c, []byte("a"), kv.ReplicaReadLeader, 0) @@ -381,7 +527,7 @@ func (s *testRegionCacheSuite) TestSendFailedButLeaderNotChange(c *C) { c.Assert(ctxFollower1.Peer.Id, Not(Equals), ctxFollower2.Peer.Id) // access 1 it will return NotLeader, leader back to 2 again - s.cache.UpdateLeader(loc.Region, s.store2, ctx.AccessIdx) + s.cache.UpdateLeader(loc.Region, &metapb.Peer{Id: s.peer2, StoreId: s.store2}, ctx.AccessIdx) ctx, err = s.cache.GetTiKVRPCContext(s.bo, loc.Region, kv.ReplicaReadLeader, 0) c.Assert(err, IsNil) c.Assert(ctx.Peer.Id, Equals, s.peer2) @@ -462,7 +608,7 @@ func (s *testRegionCacheSuite) TestSendFailedInHibernateRegion(c *C) { c.Assert(ctxFollower1.Peer.Id, Not(Equals), ctxFollower2.Peer.Id) // access 2, it's in hibernate and return 0 leader, so switch to 3 - s.cache.UpdateLeader(loc.Region, 0, ctx.AccessIdx) + s.cache.UpdateLeader(loc.Region, nil, ctx.AccessIdx) ctx, err = s.cache.GetTiKVRPCContext(s.bo, loc.Region, kv.ReplicaReadLeader, 0) c.Assert(err, IsNil) c.Assert(ctx.Peer.Id, Equals, peer3) @@ -487,7 +633,7 @@ func (s *testRegionCacheSuite) TestSendFailedInHibernateRegion(c *C) { // again peer back to 1 ctx, err = s.cache.GetTiKVRPCContext(s.bo, loc.Region, kv.ReplicaReadLeader, 0) c.Assert(err, IsNil) - s.cache.UpdateLeader(loc.Region, 0, ctx.AccessIdx) + s.cache.UpdateLeader(loc.Region, nil, ctx.AccessIdx) ctx, err = s.cache.GetTiKVRPCContext(s.bo, loc.Region, kv.ReplicaReadLeader, 0) c.Assert(err, IsNil) c.Assert(ctx.Peer.Id, Equals, s.peer1) @@ -671,7 +817,7 @@ func (s *testRegionCacheSuite) TestSendFailedInMultipleNode(c *C) { c.Assert(ctxFollower1.Peer.Id, Equals, ctxFollower2.Peer.Id) // 3 can be access, so switch to 1 - s.cache.UpdateLeader(loc.Region, s.store1, ctx.AccessIdx) + s.cache.UpdateLeader(loc.Region, &metapb.Peer{Id: s.peer1, StoreId: s.store1}, ctx.AccessIdx) ctx, err = s.cache.GetTiKVRPCContext(s.bo, loc.Region, kv.ReplicaReadLeader, 0) c.Assert(err, IsNil) c.Assert(ctx.Peer.Id, Equals, s.peer1) @@ -866,11 +1012,11 @@ func (s *testRegionCacheSuite) TestRegionEpochAheadOfTiKV(c *C) { bo := NewBackofferWithVars(context.Background(), 2000000, nil) - err := cache.OnRegionEpochNotMatch(bo, &RPCContext{Region: region.VerID()}, []*metapb.Region{&r1}) + _, err := cache.OnRegionEpochNotMatch(bo, &RPCContext{Region: region.VerID()}, []*metapb.Region{&r1}) c.Assert(err, IsNil) - err = cache.OnRegionEpochNotMatch(bo, &RPCContext{Region: region.VerID()}, []*metapb.Region{&r2}) + _, err = cache.OnRegionEpochNotMatch(bo, &RPCContext{Region: region.VerID()}, []*metapb.Region{&r2}) c.Assert(err, IsNil) - c.Assert(len(bo.errors), Equals, 2) + c.Assert(bo.ErrorsNum(), Equals, 2) } func (s *testRegionCacheSuite) TestRegionEpochOnTiFlash(c *C) { @@ -898,7 +1044,7 @@ func (s *testRegionCacheSuite) TestRegionEpochOnTiFlash(c *C) { r := ctxTiFlash.Meta reqSend := NewRegionRequestSender(s.cache, nil) regionErr := &errorpb.Error{EpochNotMatch: &errorpb.EpochNotMatch{CurrentRegions: []*metapb.Region{r}}} - reqSend.onRegionError(s.bo, ctxTiFlash, nil, regionErr) + reqSend.onRegionError(s.bo, ctxTiFlash, nil, regionErr, nil) // check leader read should not go to tiflash lctx, err = s.cache.GetTiKVRPCContext(s.bo, loc1.Region, kv.ReplicaReadLeader, 0) @@ -931,123 +1077,6 @@ func loadRegionsToCache(cache *RegionCache, regionCnt int) { } } -func (s *testRegionCacheSuite) TestUpdateStoreAddr(c *C) { - mvccStore := mocktikv.MustNewMVCCStore() - defer mvccStore.Close() - - client := &RawKVClient{ - clusterID: 0, - regionCache: NewRegionCache(mocktikv.NewPDClient(s.cluster)), - rpcClient: mocktikv.NewRPCClient(s.cluster, mvccStore, nil), - } - defer client.Close() - testKey := []byte("test_key") - testValue := []byte("test_value") - err := client.Put(testKey, testValue) - c.Assert(err, IsNil) - // tikv-server reports `StoreNotMatch` And retry - store1Addr := s.storeAddr(s.store1) - s.cluster.UpdateStoreAddr(s.store1, s.storeAddr(s.store2)) - s.cluster.UpdateStoreAddr(s.store2, store1Addr) - - getVal, err := client.Get(testKey) - - c.Assert(err, IsNil) - c.Assert(getVal, BytesEquals, testValue) -} - -func (s *testRegionCacheSuite) TestReplaceAddrWithNewStore(c *C) { - mvccStore := mocktikv.MustNewMVCCStore() - defer mvccStore.Close() - - client := &RawKVClient{ - clusterID: 0, - regionCache: NewRegionCache(mocktikv.NewPDClient(s.cluster)), - rpcClient: mocktikv.NewRPCClient(s.cluster, mvccStore, nil), - } - defer client.Close() - testKey := []byte("test_key") - testValue := []byte("test_value") - err := client.Put(testKey, testValue) - c.Assert(err, IsNil) - - // make store2 using store1's addr and store1 offline - store1Addr := s.storeAddr(s.store1) - s.cluster.UpdateStoreAddr(s.store1, s.storeAddr(s.store2)) - s.cluster.UpdateStoreAddr(s.store2, store1Addr) - s.cluster.RemoveStore(s.store1) - s.cluster.ChangeLeader(s.region1, s.peer2) - s.cluster.RemovePeer(s.region1, s.peer1) - - getVal, err := client.Get(testKey) - - c.Assert(err, IsNil) - c.Assert(getVal, BytesEquals, testValue) -} - -func (s *testRegionCacheSuite) TestReplaceNewAddrAndOldOfflineImmediately(c *C) { - mvccStore := mocktikv.MustNewMVCCStore() - defer mvccStore.Close() - - client := &RawKVClient{ - clusterID: 0, - regionCache: NewRegionCache(mocktikv.NewPDClient(s.cluster)), - rpcClient: mocktikv.NewRPCClient(s.cluster, mvccStore, nil), - } - defer client.Close() - testKey := []byte("test_key") - testValue := []byte("test_value") - err := client.Put(testKey, testValue) - c.Assert(err, IsNil) - - // pre-load store2's address into cache via follower-read. - loc, err := client.regionCache.LocateKey(s.bo, testKey) - c.Assert(err, IsNil) - fctx, err := client.regionCache.GetTiKVRPCContext(s.bo, loc.Region, kv.ReplicaReadFollower, 0) - c.Assert(err, IsNil) - c.Assert(fctx.Store.storeID, Equals, s.store2) - c.Assert(fctx.Addr, Equals, "store2") - - // make store2 using store1's addr and store1 offline - store1Addr := s.storeAddr(s.store1) - s.cluster.UpdateStoreAddr(s.store1, s.storeAddr(s.store2)) - s.cluster.UpdateStoreAddr(s.store2, store1Addr) - s.cluster.RemoveStore(s.store1) - s.cluster.ChangeLeader(s.region1, s.peer2) - s.cluster.RemovePeer(s.region1, s.peer1) - - getVal, err := client.Get(testKey) - c.Assert(err, IsNil) - c.Assert(getVal, BytesEquals, testValue) -} - -func (s *testRegionCacheSuite) TestReplaceStore(c *C) { - mvccStore := mocktikv.MustNewMVCCStore() - defer mvccStore.Close() - - client := &RawKVClient{ - clusterID: 0, - regionCache: NewRegionCache(mocktikv.NewPDClient(s.cluster)), - rpcClient: mocktikv.NewRPCClient(s.cluster, mvccStore, nil), - } - defer client.Close() - testKey := []byte("test_key") - testValue := []byte("test_value") - err := client.Put(testKey, testValue) - c.Assert(err, IsNil) - - s.cluster.MarkTombstone(s.store1) - store3 := s.cluster.AllocID() - peer3 := s.cluster.AllocID() - s.cluster.AddStore(store3, s.storeAddr(s.store1)) - s.cluster.AddPeer(s.region1, store3, peer3) - s.cluster.RemovePeer(s.region1, s.peer1) - s.cluster.ChangeLeader(s.region1, peer3) - - err = client.Put(testKey, testValue) - c.Assert(err, IsNil) -} - func (s *testRegionCacheSuite) TestListRegionIDsInCache(c *C) { // ['' - 'm' - 'z'] region2 := s.cluster.AllocID() @@ -1261,11 +1290,11 @@ func (s *testRegionCacheSuite) TestFollowerMeetEpochNotMatch(c *C) { c.Assert(ctxFollower1.Store.storeID, Equals, s.store2) regionErr := &errorpb.Error{EpochNotMatch: &errorpb.EpochNotMatch{}} - reqSend.onRegionError(s.bo, ctxFollower1, &followReqSeed, regionErr) + reqSend.onRegionError(s.bo, ctxFollower1, &tikvrpc.Request{ReplicaReadSeed: &followReqSeed}, regionErr, nil) c.Assert(followReqSeed, Equals, uint32(1)) regionErr = &errorpb.Error{RegionNotFound: &errorpb.RegionNotFound{}} - reqSend.onRegionError(s.bo, ctxFollower1, &followReqSeed, regionErr) + reqSend.onRegionError(s.bo, ctxFollower1, &tikvrpc.Request{ReplicaReadSeed: &followReqSeed}, regionErr, nil) c.Assert(followReqSeed, Equals, uint32(2)) } @@ -1292,7 +1321,7 @@ func (s *testRegionCacheSuite) TestMixedMeetEpochNotMatch(c *C) { c.Assert(ctxFollower1.Store.storeID, Equals, s.store1) regionErr := &errorpb.Error{EpochNotMatch: &errorpb.EpochNotMatch{}} - reqSend.onRegionError(s.bo, ctxFollower1, &followReqSeed, regionErr) + reqSend.onRegionError(s.bo, ctxFollower1, &tikvrpc.Request{ReplicaReadSeed: &followReqSeed}, regionErr, nil) c.Assert(followReqSeed, Equals, uint32(1)) } @@ -1300,7 +1329,7 @@ func (s *testRegionCacheSuite) TestPeersLenChange(c *C) { // 2 peers [peer1, peer2] and let peer2 become leader loc, err := s.cache.LocateKey(s.bo, []byte("a")) c.Assert(err, IsNil) - s.cache.UpdateLeader(loc.Region, s.store2, 0) + s.cache.UpdateLeader(loc.Region, &metapb.Peer{Id: s.peer2, StoreId: s.store2}, 0) // current leader is peer2 in [peer1, peer2] loc, err = s.cache.LocateKey(s.bo, []byte("a")) @@ -1324,12 +1353,12 @@ func (s *testRegionCacheSuite) TestPeersLenChange(c *C) { } filterUnavailablePeers(cpRegion) region := &Region{meta: cpRegion.Meta} - err = region.init(s.cache) + err = region.init(s.bo, s.cache) c.Assert(err, IsNil) s.cache.insertRegionToCache(region) // OnSendFail should not panic - s.cache.OnSendFail(NewNoopBackoff(context.Background()), ctx, false, errors.New("send fail")) + s.cache.OnSendFail(retry.NewNoopBackoff(context.Background()), ctx, false, errors.New("send fail")) } func createSampleRegion(startKey, endKey []byte) *Region { @@ -1383,7 +1412,7 @@ func (s *testRegionCacheSuite) TestSwitchPeerWhenNoLeader(c *C) { c.Assert(ctx.Peer, Not(DeepEquals), prevCtx.Peer) } s.cache.InvalidateCachedRegionWithReason(loc.Region, NoLeader) - c.Assert(s.cache.getCachedRegionWithRLock(loc.Region).invalidReason, Equals, NoLeader) + c.Assert(s.cache.GetCachedRegionWithRLock(loc.Region).invalidReason, Equals, NoLeader) prevCtx = ctx } } @@ -1416,9 +1445,9 @@ func BenchmarkOnRequestFail(b *testing.B) { AccessIdx: accessIdx, Peer: peer, Store: store, - AccessMode: TiKVOnly, + AccessMode: tiKVOnly, } - r := cache.getCachedRegionWithRLock(rpcCtx.Region) + r := cache.GetCachedRegionWithRLock(rpcCtx.Region) if r != nil { r.getStore().switchNextTiKVPeer(r, rpcCtx.AccessIdx) } diff --git a/store/tikv/region_request.go b/store/tikv/region_request.go index cad0ed0379e96..247282476a717 100644 --- a/store/tikv/region_request.go +++ b/store/tikv/region_request.go @@ -32,18 +32,31 @@ import ( "github.com/pingcap/kvproto/pkg/coprocessor" "github.com/pingcap/kvproto/pkg/errorpb" "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/metapb" tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/pingcap/tidb/store/tikv/oracle" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/store/tikv/util" ) -// ShuttingDown is a flag to indicate tidb-server is exiting (Ctrl+C signal +// shuttingDown is a flag to indicate tidb-server is exiting (Ctrl+C signal // receved for example). If this flag is set, tikv client should not retry on // network error because tidb-server expect tikv client to exit as soon as possible. -var ShuttingDown uint32 +var shuttingDown uint32 + +// StoreShuttingDown atomically stores ShuttingDown into v. +func StoreShuttingDown(v uint32) { + atomic.StoreUint32(&shuttingDown, v) +} + +// LoadShuttingDown atomically loads ShuttingDown. +func LoadShuttingDown() uint32 { + return atomic.LoadUint32(&shuttingDown) +} // RegionRequestSender sends KV/Cop requests to tikv server. It handles network // errors and some region errors internally. @@ -57,16 +70,20 @@ var ShuttingDown uint32 // merge, or region balance, tikv server may not able to process request and // send back a RegionError. // RegionRequestSender takes care of errors that does not relevant to region -// range, such as 'I/O timeout', 'NotLeader', and 'ServerIsBusy'. For other -// errors, since region range have changed, the request may need to split, so we -// simply return the error to caller. +// range, such as 'I/O timeout', 'NotLeader', and 'ServerIsBusy'. If fails to +// send the request to all replicas, a fake rregion error may be returned. +// Caller which receives the error should retry the request. +// +// For other region errors, since region range have changed, the request may need to +// split, so we simply return the error to caller. type RegionRequestSender struct { - regionCache *RegionCache - client Client - storeAddr string - rpcError error - failStoreIDs map[uint64]struct{} - failProxyStoreIDs map[uint64]struct{} + regionCache *RegionCache + client Client + storeAddr string + rpcError error + leaderReplicaSelector *replicaSelector + failStoreIDs map[uint64]struct{} + failProxyStoreIDs map[uint64]struct{} RegionRequestRuntimeStats } @@ -181,12 +198,191 @@ func (s *RegionRequestSender) SetRPCError(err error) { s.rpcError = err } -// SendReq sends a request to tikv server. +// SendReq sends a request to tikv server. If fails to send the request to all replicas, +// a fake region error may be returned. Caller which receives the error should retry the request. func (s *RegionRequestSender) SendReq(bo *Backoffer, req *tikvrpc.Request, regionID RegionVerID, timeout time.Duration) (*tikvrpc.Response, error) { resp, _, err := s.SendReqCtx(bo, req, regionID, timeout, tikvrpc.TiKV) return resp, err } +type replica struct { + store *Store + peer *metapb.Peer + epoch uint32 + attempts int +} + +type replicaSelector struct { + regionCache *RegionCache + region *Region + // replicas contains all TiKV replicas for now and the leader is at the + // head of the slice. + replicas []*replica + // nextReplicaIdx points to the candidate for the next attempt. + nextReplicaIdx int +} + +func newReplicaSelector(regionCache *RegionCache, regionID RegionVerID) (*replicaSelector, error) { + cachedRegion := regionCache.GetCachedRegionWithRLock(regionID) + if cachedRegion == nil || !cachedRegion.isValid() { + return nil, nil + } + regionStore := cachedRegion.getStore() + replicas := make([]*replica, 0, regionStore.accessStoreNum(tiKVOnly)) + for _, storeIdx := range regionStore.accessIndex[tiKVOnly] { + replicas = append(replicas, &replica{ + store: regionStore.stores[storeIdx], + peer: cachedRegion.meta.Peers[storeIdx], + epoch: regionStore.storeEpochs[storeIdx], + attempts: 0, + }) + } + // Move the leader to the first slot. + replicas[regionStore.workTiKVIdx], replicas[0] = replicas[0], replicas[regionStore.workTiKVIdx] + return &replicaSelector{ + regionCache, + cachedRegion, + replicas, + 0, + }, nil +} + +// isExhausted returns true if runs out of all replicas. +func (s *replicaSelector) isExhausted() bool { + return s.nextReplicaIdx >= len(s.replicas) +} + +func (s *replicaSelector) nextReplica() *replica { + if s.isExhausted() { + return nil + } + return s.replicas[s.nextReplicaIdx] +} + +const maxReplicaAttempt = 10 + +// next creates the RPCContext of the current candidate replica. +// It returns a SendError if runs out of all replicas or the cached region is invalidated. +func (s *replicaSelector) next(bo *Backoffer) (*RPCContext, error) { + for { + if !s.region.isValid() { + metrics.TiKVReplicaSelectorFailureCounter.WithLabelValues("invalid").Inc() + return nil, nil + } + if s.isExhausted() { + metrics.TiKVReplicaSelectorFailureCounter.WithLabelValues("exhausted").Inc() + s.invalidateRegion() + return nil, nil + } + replica := s.replicas[s.nextReplicaIdx] + s.nextReplicaIdx++ + + // Limit the max attempts of each replica to prevent endless retry. + if replica.attempts >= maxReplicaAttempt { + continue + } + replica.attempts++ + + storeFailEpoch := atomic.LoadUint32(&replica.store.epoch) + if storeFailEpoch != replica.epoch { + // TODO(youjiali1995): Is it necessary to invalidate the region? + metrics.TiKVReplicaSelectorFailureCounter.WithLabelValues("stale_store").Inc() + s.invalidateRegion() + return nil, nil + } + addr, err := s.regionCache.getStoreAddr(bo, s.region, replica.store) + if err == nil && len(addr) != 0 { + return &RPCContext{ + Region: s.region.VerID(), + Meta: s.region.meta, + Peer: replica.peer, + Store: replica.store, + Addr: addr, + AccessMode: tiKVOnly, + TiKVNum: len(s.replicas), + }, nil + } + } +} + +func (s *replicaSelector) onSendFailure(bo *Backoffer, err error) { + metrics.RegionCacheCounterWithSendFail.Inc() + replica := s.replicas[s.nextReplicaIdx-1] + if replica.store.requestLiveness(bo, s.regionCache) == reachable { + s.rewind() + return + } + + store := replica.store + // invalidate regions in store. + if atomic.CompareAndSwapUint32(&store.epoch, replica.epoch, replica.epoch+1) { + logutil.BgLogger().Info("mark store's regions need be refill", zap.Uint64("id", store.storeID), zap.String("addr", store.addr), zap.Error(err)) + metrics.RegionCacheCounterWithInvalidateStoreRegionsOK.Inc() + // schedule a store addr resolve. + store.markNeedCheck(s.regionCache.notifyCheckCh) + } + // TODO(youjiali1995): It's not necessary, but some tests depend on it and it's not easy to fix. + if s.isExhausted() { + s.region.scheduleReload() + } +} + +// OnSendSuccess updates the leader of the cached region since the replicaSelector +// is only used for leader request. It's called when the request is sent to the +// replica successfully. +func (s *replicaSelector) OnSendSuccess() { + // The successful replica is not at the head of replicas which means it's not the + // leader in the cached region, so update leader. + if s.nextReplicaIdx-1 != 0 { + leader := s.replicas[s.nextReplicaIdx-1].peer + if !s.regionCache.switchWorkLeaderToPeer(s.region, leader) { + panic("the store must exist") + } + } +} + +func (s *replicaSelector) rewind() { + s.nextReplicaIdx-- +} + +// updateLeader updates the leader of the cached region. +// If the leader peer isn't found in the region, the region will be invalidated. +func (s *replicaSelector) updateLeader(leader *metapb.Peer) { + if leader == nil { + return + } + for i, replica := range s.replicas { + if isSamePeer(replica.peer, leader) { + if i < s.nextReplicaIdx { + s.nextReplicaIdx-- + } + // Move the leader replica to the front of candidates. + s.replicas[i], s.replicas[s.nextReplicaIdx] = s.replicas[s.nextReplicaIdx], s.replicas[i] + if s.replicas[s.nextReplicaIdx].attempts == maxReplicaAttempt { + // Give the replica one more chance and because the current replica is skipped, it + // won't result in infinite retry. + s.replicas[s.nextReplicaIdx].attempts = maxReplicaAttempt - 1 + } + // Update the workTiKVIdx so that following requests can be sent to the leader immediately. + if !s.regionCache.switchWorkLeaderToPeer(s.region, leader) { + panic("the store must exist") + } + logutil.BgLogger().Debug("switch region leader to specific leader due to kv return NotLeader", + zap.Uint64("regionID", s.region.GetID()), + zap.Uint64("leaderStoreID", leader.GetStoreId())) + return + } + } + // Invalidate the region since the new leader is not in the cached version. + s.region.invalidate(StoreNotFound) +} + +func (s *replicaSelector) invalidateRegion() { + if s.region != nil { + s.region.invalidate(Other) + } +} + func (s *RegionRequestSender) getRPCContext( bo *Backoffer, req *tikvrpc.Request, @@ -196,6 +392,20 @@ func (s *RegionRequestSender) getRPCContext( ) (*RPCContext, error) { switch et { case tikvrpc.TiKV: + // Now only requests sent to the replica leader will use the replica selector to get + // the RPC context. + // TODO(youjiali1995): make all requests use the replica selector. + if !s.regionCache.enableForwarding && req.ReplicaReadType == kv.ReplicaReadLeader { + if s.leaderReplicaSelector == nil { + selector, err := newReplicaSelector(s.regionCache, regionID) + if selector == nil || err != nil { + return nil, err + } + s.leaderReplicaSelector = selector + } + return s.leaderReplicaSelector.next(bo) + } + var seed uint32 if req.ReplicaReadSeed != nil { seed = *req.ReplicaReadSeed @@ -210,6 +420,16 @@ func (s *RegionRequestSender) getRPCContext( } } +func (s *RegionRequestSender) reset() { + s.leaderReplicaSelector = nil + s.failStoreIDs = nil + s.failProxyStoreIDs = nil +} + +func isFakeRegionError(err *errorpb.Error) bool { + return err != nil && err.GetEpochNotMatch() != nil && len(err.GetEpochNotMatch().CurrentRegions) == 0 +} + // SendReqCtx sends a request to tikv server and return response and RPCCtx of this RPC. func (s *RegionRequestSender) SendReqCtx( bo *Backoffer, @@ -223,12 +443,10 @@ func (s *RegionRequestSender) SendReqCtx( rpcCtx *RPCContext, err error, ) { - if span := opentracing.SpanFromContext(bo.ctx); span != nil && span.Tracer() != nil { + if span := opentracing.SpanFromContext(bo.GetCtx()); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("regionRequest.SendReqCtx", opentracing.ChildOf(span.Context())) defer span1.Finish() - // TODO(MyonKeminta): Make sure trace works without cloning the backoffer. - // bo = bo.Clone() - bo.ctx = opentracing.ContextWithSpan(bo.ctx, span1) + bo.SetCtx(opentracing.ContextWithSpan(bo.GetCtx(), span1)) } failpoint.Inject("tikvStoreSendReqResult", func(val failpoint.Value) { @@ -247,10 +465,6 @@ func (s *RegionRequestSender) SendReqCtx( Resp: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}}, }, nil, nil) } - case "callBackofferHook": - if bo.vars != nil && bo.vars.Hook != nil { - bo.vars.Hook("callBackofferHook", bo.vars) - } case "requestTiDBStoreError": if et == tikvrpc.TiDB { failpoint.Return(nil, nil, tikverr.ErrTiKVServerTimeout) @@ -262,37 +476,53 @@ func (s *RegionRequestSender) SendReqCtx( } }) + // If the MaxExecutionDurationMs is not set yet, we set it to be the RPC timeout duration + // so TiKV can give up the requests whose response TiDB cannot receive due to timeout. + if req.Context.MaxExecutionDurationMs == 0 { + req.Context.MaxExecutionDurationMs = uint64(timeout.Milliseconds()) + } + + s.reset() tryTimes := 0 + defer func() { + if tryTimes > 0 { + metrics.TiKVRequestRetryTimesHistogram.Observe(float64(tryTimes)) + } + }() for { - if (tryTimes > 0) && (tryTimes%1000 == 0) { - logutil.Logger(bo.ctx).Warn("retry get ", zap.Uint64("region = ", regionID.GetID()), zap.Int("times = ", tryTimes)) + if (tryTimes > 0) && (tryTimes%100 == 0) { + logutil.Logger(bo.GetCtx()).Warn("retry", zap.Uint64("region", regionID.GetID()), zap.Int("times", tryTimes)) } rpcCtx, err = s.getRPCContext(bo, req, regionID, et, opts...) if err != nil { return nil, nil, err } + if rpcCtx != nil { + rpcCtx.tryTimes = tryTimes + } failpoint.Inject("invalidCacheAndRetry", func() { // cooperate with github.com/pingcap/tidb/store/gcworker/setGcResolveMaxBackoff - if c := bo.ctx.Value("injectedBackoff"); c != nil { + if c := bo.GetCtx().Value("injectedBackoff"); c != nil { resp, err = tikvrpc.GenRegionErrorResp(req, &errorpb.Error{EpochNotMatch: &errorpb.EpochNotMatch{}}) failpoint.Return(resp, nil, err) } }) if rpcCtx == nil { + // TODO(youjiali1995): remove it when using the replica selector for all requests. // If the region is not found in cache, it must be out // of date and already be cleaned up. We can skip the // RPC by returning RegionError directly. // TODO: Change the returned error to something like "region missing in cache", // and handle this error like EpochNotMatch, which means to re-split the request and retry. - logutil.Logger(bo.ctx).Debug("throwing pseudo region error due to region not found in cache", zap.Stringer("region", ®ionID)) + logutil.Logger(bo.GetCtx()).Debug("throwing pseudo region error due to region not found in cache", zap.Stringer("region", ®ionID)) resp, err = tikvrpc.GenRegionErrorResp(req, &errorpb.Error{EpochNotMatch: &errorpb.EpochNotMatch{}}) return resp, nil, err } - logutil.Eventf(bo.ctx, "send %s request to region %d at %s", req.Type, regionID.id, rpcCtx.Addr) + logutil.Eventf(bo.GetCtx(), "send %s request to region %d at %s", req.Type, regionID.id, rpcCtx.Addr) s.storeAddr = rpcCtx.Addr var retry bool resp, retry, err = s.sendReqToRegion(bo, rpcCtx, req, timeout) @@ -301,7 +531,8 @@ func (s *RegionRequestSender) SendReqCtx( } // recheck whether the session/query is killed during the Next() - if bo.vars != nil && bo.vars.Killed != nil && atomic.LoadUint32(bo.vars.Killed) == 1 { + boVars := bo.GetVars() + if boVars != nil && boVars.Killed != nil && atomic.LoadUint32(boVars.Killed) == 1 { return nil, nil, tikverr.ErrQueryInterrupted } failpoint.Inject("mockRetrySendReqToRegion", func(val failpoint.Value) { @@ -319,8 +550,14 @@ func (s *RegionRequestSender) SendReqCtx( if err != nil { return nil, nil, errors.Trace(err) } + failpoint.Inject("mockDataIsNotReadyError", func(val failpoint.Value) { + regionErr = &errorpb.Error{} + if tryTimesLimit, ok := val.(int); ok && tryTimes <= tryTimesLimit { + regionErr.DataIsNotReady = &errorpb.DataIsNotReady{} + } + }) if regionErr != nil { - retry, err = s.onRegionError(bo, rpcCtx, req.ReplicaReadSeed, regionErr) + retry, err = s.onRegionError(bo, rpcCtx, req, regionErr, &opts) if err != nil { return nil, nil, errors.Trace(err) } @@ -328,6 +565,12 @@ func (s *RegionRequestSender) SendReqCtx( tryTimes++ continue } + } else { + // Clear the RPC Error since the request is evaluated successfully on a store. + s.rpcError = nil + if s.leaderReplicaSelector != nil { + s.leaderReplicaSelector.OnSendSuccess() + } } return resp, rpcCtx, nil } @@ -392,7 +635,7 @@ func (s *RegionRequestSender) sendReqToRegion(bo *Backoffer, rpcCtx *RPCContext, defer s.releaseStoreToken(rpcCtx.Store) } - ctx := bo.ctx + ctx := bo.GetCtx() if rawHook := ctx.Value(RPCCancellerCtxKey{}); rawHook != nil { var cancel context.CancelFunc ctx, cancel = rawHook.(*RPCCanceller).WithCancel(ctx) @@ -410,7 +653,7 @@ func (s *RegionRequestSender) sendReqToRegion(bo *Backoffer, rpcCtx *RPCContext, } var sessionID uint64 - if v := bo.ctx.Value(util.SessionID); v != nil { + if v := bo.GetCtx().Value(util.SessionID); v != nil { sessionID = v.(uint64) } @@ -443,7 +686,7 @@ func (s *RegionRequestSender) sendReqToRegion(bo *Backoffer, rpcCtx *RPCContext, RecordRegionRequestRuntimeStats(s.Stats, req.Type, time.Since(start)) failpoint.Inject("tikvStoreRespResult", func(val failpoint.Value) { if val.(bool) { - if req.Type == tikvrpc.CmdCop && bo.totalSleep == 0 { + if req.Type == tikvrpc.CmdCop && bo.GetTotalSleep() == 0 { failpoint.Return(&tikvrpc.Response{ Resp: &coprocessor.Response{RegionError: &errorpb.Error{EpochNotMatch: &errorpb.EpochNotMatch{}}}, }, false, nil) @@ -477,10 +720,7 @@ func (s *RegionRequestSender) sendReqToRegion(bo *Backoffer, rpcCtx *RPCContext, if val.(bool) { ctx1, cancel := context.WithCancel(context.Background()) cancel() - select { - case <-ctx1.Done(): - } - + <-ctx1.Done() ctx = ctx1 err = ctx.Err() resp = nil @@ -530,8 +770,7 @@ func (s *RegionRequestSender) getStoreToken(st *Store, limit int64) error { return nil } metrics.TiKVStoreLimitErrorCounter.WithLabelValues(st.addr, strconv.FormatUint(st.storeID, 10)).Inc() - return tikverr.ErrTokenLimit.GenWithStackByArgs(st.storeID) - + return &tikverr.ErrTokenLimit{StoreID: st.storeID} } func (s *RegionRequestSender) releaseStoreToken(st *Store) { @@ -545,22 +784,20 @@ func (s *RegionRequestSender) releaseStoreToken(st *Store) { } func (s *RegionRequestSender) onSendFail(bo *Backoffer, ctx *RPCContext, err error) error { - if span := opentracing.SpanFromContext(bo.ctx); span != nil && span.Tracer() != nil { + if span := opentracing.SpanFromContext(bo.GetCtx()); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("regionRequest.onSendFail", opentracing.ChildOf(span.Context())) defer span1.Finish() - // TODO(MyonKeminta): Make sure trace works without cloning the backoffer. - // bo = bo.Clone() - bo.ctx = opentracing.ContextWithSpan(bo.ctx, span1) + bo.SetCtx(opentracing.ContextWithSpan(bo.GetCtx(), span1)) } // If it failed because the context is cancelled by ourself, don't retry. if errors.Cause(err) == context.Canceled { return errors.Trace(err) - } else if atomic.LoadUint32(&ShuttingDown) > 0 { + } else if LoadShuttingDown() > 0 { return tikverr.ErrTiDBShuttingDown } if status.Code(errors.Cause(err)) == codes.Canceled { select { - case <-bo.ctx.Done(): + case <-bo.GetCtx().Done(): return errors.Trace(err) default: // If we don't cancel, but the error code is Canceled, it must be from grpc remote. @@ -571,7 +808,11 @@ func (s *RegionRequestSender) onSendFail(bo *Backoffer, ctx *RPCContext, err err } if ctx.Meta != nil { - s.regionCache.OnSendFail(bo, ctx, s.NeedReloadRegion(ctx), err) + if s.leaderReplicaSelector != nil { + s.leaderReplicaSelector.onSendFailure(bo, err) + } else { + s.regionCache.OnSendFail(bo, ctx, s.NeedReloadRegion(ctx), err) + } } // Retry on send request failure when it's not canceled. @@ -579,9 +820,9 @@ func (s *RegionRequestSender) onSendFail(bo *Backoffer, ctx *RPCContext, err err // TODO: the number of retry time should be limited:since region may be unavailable // when some unrecoverable disaster happened. if ctx.Store != nil && ctx.Store.storeType == tikvrpc.TiFlash { - err = bo.Backoff(BoTiFlashRPC, errors.Errorf("send tiflash request error: %v, ctx: %v, try next peer later", err, ctx)) + err = bo.Backoff(retry.BoTiFlashRPC, errors.Errorf("send tiflash request error: %v, ctx: %v, try next peer later", err, ctx)) } else { - err = bo.Backoff(BoTiKVRPC, errors.Errorf("send tikv request error: %v, ctx: %v, try next peer later", err, ctx)) + err = bo.Backoff(retry.BoTiKVRPC, errors.Errorf("send tikv request error: %v, ctx: %v, try next peer later", err, ctx)) } return errors.Trace(err) } @@ -599,9 +840,9 @@ func (s *RegionRequestSender) NeedReloadRegion(ctx *RPCContext) (need bool) { s.failProxyStoreIDs[ctx.ProxyStore.storeID] = struct{}{} } - if ctx.AccessMode == TiKVOnly && len(s.failStoreIDs)+len(s.failProxyStoreIDs) >= ctx.TiKVNum { + if ctx.AccessMode == tiKVOnly && len(s.failStoreIDs)+len(s.failProxyStoreIDs) >= ctx.TiKVNum { need = true - } else if ctx.AccessMode == TiFlashOnly && len(s.failStoreIDs) >= len(ctx.Meta.Peers)-ctx.TiKVNum { + } else if ctx.AccessMode == tiFlashOnly && len(s.failStoreIDs) >= len(ctx.Meta.Peers)-ctx.TiKVNum { need = true } else if len(s.failStoreIDs)+len(s.failProxyStoreIDs) >= len(ctx.Meta.Peers) { need = true @@ -633,47 +874,72 @@ func regionErrorToLabel(e *errorpb.Error) string { return "unknown" } -func (s *RegionRequestSender) onRegionError(bo *Backoffer, ctx *RPCContext, seed *uint32, regionErr *errorpb.Error) (retry bool, err error) { - if span := opentracing.SpanFromContext(bo.ctx); span != nil && span.Tracer() != nil { +func (s *RegionRequestSender) onRegionError(bo *Backoffer, ctx *RPCContext, req *tikvrpc.Request, regionErr *errorpb.Error, opts *[]StoreSelectorOption) (shouldRetry bool, err error) { + if span := opentracing.SpanFromContext(bo.GetCtx()); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("tikv.onRegionError", opentracing.ChildOf(span.Context())) defer span1.Finish() - // TODO(MyonKeminta): Make sure trace works without cloning the backoffer. - // bo = bo.Clone() - bo.ctx = opentracing.ContextWithSpan(bo.ctx, span1) + bo.SetCtx(opentracing.ContextWithSpan(bo.GetCtx(), span1)) + } + // Stale Read request will retry the leader or next peer on error, + // if txnScope is global, we will only retry the leader by using the WithLeaderOnly option, + // if txnScope is local, we will retry both other peers and the leader by the incresing seed. + if ctx.tryTimes < 1 && req != nil && req.TxnScope == oracle.GlobalTxnScope && req.GetStaleRead() { + *opts = append(*opts, WithLeaderOnly()) } + seed := req.GetReplicaReadSeed() + // NOTE: Please add the region error handler in the same order of errorpb.Error. metrics.TiKVRegionErrorCounter.WithLabelValues(regionErrorToLabel(regionErr)).Inc() + if notLeader := regionErr.GetNotLeader(); notLeader != nil { // Retry if error is `NotLeader`. logutil.BgLogger().Debug("tikv reports `NotLeader` retry later", zap.String("notLeader", notLeader.String()), zap.String("ctx", ctx.String())) - if notLeader.GetLeader() == nil { + if s.leaderReplicaSelector != nil { + leader := notLeader.GetLeader() + if leader == nil { + // The region may be during transferring leader. + if err = bo.Backoff(retry.BoRegionScheduling, errors.Errorf("no leader, ctx: %v", ctx)); err != nil { + return false, errors.Trace(err) + } + } else { + s.leaderReplicaSelector.updateLeader(notLeader.GetLeader()) + } + return true, nil + } else if notLeader.GetLeader() == nil { // The peer doesn't know who is the current leader. Generally it's because // the Raft group is in an election, but it's possible that the peer is // isolated and removed from the Raft group. So it's necessary to reload // the region from PD. s.regionCache.InvalidateCachedRegionWithReason(ctx.Region, NoLeader) - if err = bo.Backoff(BoRegionMiss, errors.Errorf("not leader: %v, ctx: %v", notLeader, ctx)); err != nil { + if err = bo.Backoff(retry.BoRegionScheduling, errors.Errorf("not leader: %v, ctx: %v", notLeader, ctx)); err != nil { return false, errors.Trace(err) } + return false, nil } else { // don't backoff if a new leader is returned. - s.regionCache.UpdateLeader(ctx.Region, notLeader.GetLeader().GetStoreId(), ctx.AccessIdx) + s.regionCache.UpdateLeader(ctx.Region, notLeader.GetLeader(), ctx.AccessIdx) + return true, nil } + } - return true, nil + // This peer is removed from the region. Invalidate the region since it's too stale. + if regionErr.GetRegionNotFound() != nil { + if seed != nil { + logutil.BgLogger().Debug("tikv reports `RegionNotFound` in follow-reader", + zap.Stringer("ctx", ctx), zap.Uint32("seed", *seed)) + *seed = *seed + 1 + } + s.regionCache.InvalidateCachedRegion(ctx.Region) + return false, nil } - if storeNotMatch := regionErr.GetStoreNotMatch(); storeNotMatch != nil { - // store not match - logutil.BgLogger().Debug("tikv reports `StoreNotMatch` retry later", - zap.Stringer("storeNotMatch", storeNotMatch), - zap.Stringer("ctx", ctx)) - ctx.Store.markNeedCheck(s.regionCache.notifyCheckCh) + if regionErr.GetKeyNotInRegion() != nil { + logutil.BgLogger().Debug("tikv reports `KeyNotInRegion`", zap.Stringer("ctx", ctx)) s.regionCache.InvalidateCachedRegion(ctx.Region) - return true, nil + return false, nil } if epochNotMatch := regionErr.GetEpochNotMatch(); epochNotMatch != nil { @@ -683,57 +949,151 @@ func (s *RegionRequestSender) onRegionError(bo *Backoffer, ctx *RPCContext, seed if seed != nil { *seed = *seed + 1 } - err = s.regionCache.OnRegionEpochNotMatch(bo, ctx, epochNotMatch.CurrentRegions) - return false, errors.Trace(err) + retry, err := s.regionCache.OnRegionEpochNotMatch(bo, ctx, epochNotMatch.CurrentRegions) + if !retry && s.leaderReplicaSelector != nil { + s.leaderReplicaSelector.invalidateRegion() + } + return retry, errors.Trace(err) } + if regionErr.GetServerIsBusy() != nil { logutil.BgLogger().Warn("tikv reports `ServerIsBusy` retry later", zap.String("reason", regionErr.GetServerIsBusy().GetReason()), zap.Stringer("ctx", ctx)) if ctx != nil && ctx.Store != nil && ctx.Store.storeType == tikvrpc.TiFlash { - err = bo.Backoff(boTiFlashServerBusy, errors.Errorf("server is busy, ctx: %v", ctx)) + err = bo.Backoff(retry.BoTiFlashServerBusy, errors.Errorf("server is busy, ctx: %v", ctx)) } else { - err = bo.Backoff(boTiKVServerBusy, errors.Errorf("server is busy, ctx: %v", ctx)) + err = bo.Backoff(retry.BoTiKVServerBusy, errors.Errorf("server is busy, ctx: %v", ctx)) } if err != nil { return false, errors.Trace(err) } + if s.leaderReplicaSelector != nil { + s.leaderReplicaSelector.rewind() + } return true, nil } + + // StaleCommand error indicates the request is sent to the old leader and its term is changed. + // We can't know whether the request is committed or not, so it's an undetermined error too, + // but we don't handle it now. if regionErr.GetStaleCommand() != nil { logutil.BgLogger().Debug("tikv reports `StaleCommand`", zap.Stringer("ctx", ctx)) - err = bo.Backoff(boStaleCmd, errors.Errorf("stale command, ctx: %v", ctx)) - if err != nil { - return false, errors.Trace(err) + if s.leaderReplicaSelector != nil { + // Needn't backoff because the new leader should be elected soon + // and the leaderReplicaSelector will try the next peer. + } else { + err = bo.Backoff(retry.BoStaleCmd, errors.Errorf("stale command, ctx: %v", ctx)) + if err != nil { + return false, errors.Trace(err) + } } return true, nil } + + if storeNotMatch := regionErr.GetStoreNotMatch(); storeNotMatch != nil { + // store not match + logutil.BgLogger().Debug("tikv reports `StoreNotMatch` retry later", + zap.Stringer("storeNotMatch", storeNotMatch), + zap.Stringer("ctx", ctx)) + ctx.Store.markNeedCheck(s.regionCache.notifyCheckCh) + s.regionCache.InvalidateCachedRegion(ctx.Region) + return false, nil + } + if regionErr.GetRaftEntryTooLarge() != nil { logutil.BgLogger().Warn("tikv reports `RaftEntryTooLarge`", zap.Stringer("ctx", ctx)) return false, errors.New(regionErr.String()) } - if regionErr.GetRegionNotFound() != nil && seed != nil { - logutil.BgLogger().Debug("tikv reports `RegionNotFound` in follow-reader", - zap.Stringer("ctx", ctx), zap.Uint32("seed", *seed)) - *seed = *seed + 1 - } + if regionErr.GetMaxTimestampNotSynced() != nil { - logutil.BgLogger().Warn("tikv reports `MaxTimestampNotSynced`", zap.Stringer("ctx", ctx)) - err = bo.Backoff(boMaxTsNotSynced, errors.Errorf("max timestamp not synced, ctx: %v", ctx)) + logutil.BgLogger().Debug("tikv reports `MaxTimestampNotSynced`", zap.Stringer("ctx", ctx)) + err = bo.Backoff(retry.BoMaxTsNotSynced, errors.Errorf("max timestamp not synced, ctx: %v", ctx)) if err != nil { return false, errors.Trace(err) } + if s.leaderReplicaSelector != nil { + s.leaderReplicaSelector.rewind() + } return true, nil } - // For other errors, we only drop cache here. - // Because caller may need to re-split the request. + + // A read request may be sent to a peer which has not been initialized yet, we should retry in this case. + if regionErr.GetRegionNotInitialized() != nil { + logutil.BgLogger().Debug("tikv reports `RegionNotInitialized` retry later", + zap.Uint64("store-id", ctx.Store.storeID), + zap.Uint64("region-id", regionErr.GetRegionNotInitialized().GetRegionId()), + zap.Stringer("ctx", ctx)) + if seed != nil { + *seed = *seed + 1 + } + return true, nil + } + + // The read-index can't be handled timely because the region is splitting or merging. + if regionErr.GetReadIndexNotReady() != nil { + logutil.BgLogger().Debug("tikv reports `ReadIndexNotReady` retry later", + zap.Uint64("store-id", ctx.Store.storeID), + zap.Uint64("region-id", regionErr.GetRegionNotInitialized().GetRegionId()), + zap.Stringer("ctx", ctx)) + if seed != nil { + *seed = *seed + 1 + } + // The region can't provide service until split or merge finished, so backoff. + err = bo.Backoff(retry.BoRegionScheduling, errors.Errorf("read index not ready, ctx: %v", ctx)) + if err != nil { + return false, errors.Trace(err) + } + if s.leaderReplicaSelector != nil { + s.leaderReplicaSelector.rewind() + } + return true, nil + } + + if regionErr.GetProposalInMergingMode() != nil { + logutil.BgLogger().Debug("tikv reports `ProposalInMergingMode`", zap.Stringer("ctx", ctx)) + // The region is merging and it can't provide service until merge finished, so backoff. + err = bo.Backoff(retry.BoRegionScheduling, errors.Errorf("region is merging, ctx: %v", ctx)) + if err != nil { + return false, errors.Trace(err) + } + if s.leaderReplicaSelector != nil { + s.leaderReplicaSelector.rewind() + } + return true, nil + } + + // A stale read request may be sent to a peer which the data is not ready yet, we should retry in this case. + // This error is specific to stale read and the target replica is randomly selected. If the request is sent + // to the leader, the data must be ready, so we don't backoff here. + if regionErr.GetDataIsNotReady() != nil { + logutil.BgLogger().Warn("tikv reports `DataIsNotReady` retry later", + zap.Uint64("store-id", ctx.Store.storeID), + zap.Uint64("peer-id", regionErr.GetDataIsNotReady().GetPeerId()), + zap.Uint64("region-id", regionErr.GetDataIsNotReady().GetRegionId()), + zap.Uint64("safe-ts", regionErr.GetDataIsNotReady().GetSafeTs()), + zap.Stringer("ctx", ctx)) + if seed != nil { + *seed = *seed + 1 + } + return true, nil + } + logutil.BgLogger().Debug("tikv reports region failed", zap.Stringer("regionErr", regionErr), zap.Stringer("ctx", ctx)) + + if s.leaderReplicaSelector != nil { + // Try the next replica. + return true, nil + } + // When the request is sent to TiDB, there is no region in the request, so the region id will be 0. // So when region id is 0, there is no business with region cache. if ctx.Region.id != 0 { s.regionCache.InvalidateCachedRegion(ctx.Region) } + // For other errors, we only drop cache here. + // Because caller may need to re-split the request. return false, nil } diff --git a/store/tikv/region_request_test.go b/store/tikv/region_request_test.go index 81e9cc4498a07..7492d43caf89b 100644 --- a/store/tikv/region_request_test.go +++ b/store/tikv/region_request_test.go @@ -24,14 +24,17 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/coprocessor" - "github.com/pingcap/kvproto/pkg/coprocessor_v2" "github.com/pingcap/kvproto/pkg/errorpb" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/mpp" "github.com/pingcap/kvproto/pkg/tikvpb" + tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/kv" + "github.com/pingcap/tidb/store/tikv/oracle" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/config" "github.com/pingcap/tidb/store/tikv/mockstore/mocktikv" @@ -71,7 +74,7 @@ func (s *testRegionRequestToSingleStoreSuite) SetUpTest(c *C) { s.store, s.peer, s.region = mocktikv.BootstrapWithSingleStore(s.cluster) pdCli := &CodecPDClient{mocktikv.NewPDClient(s.cluster)} s.cache = NewRegionCache(pdCli) - s.bo = NewNoopBackoff(context.Background()) + s.bo = retry.NewNoopBackoff(context.Background()) s.mvccStore = mocktikv.MustNewMVCCStore() client := mocktikv.NewRPCClient(s.cluster, s.mvccStore, nil) s.regionRequestSender = NewRegionRequestSender(s.cache, client) @@ -82,7 +85,7 @@ func (s *testRegionRequestToThreeStoresSuite) SetUpTest(c *C) { s.storeIDs, s.peerIDs, s.regionID, s.leaderPeer = mocktikv.BootstrapWithMultiStores(s.cluster, 3) pdCli := &CodecPDClient{mocktikv.NewPDClient(s.cluster)} s.cache = NewRegionCache(pdCli) - s.bo = NewNoopBackoff(context.Background()) + s.bo = retry.NewNoopBackoff(context.Background()) s.mvccStore = mocktikv.MustNewMVCCStore() client := mocktikv.NewRPCClient(s.cluster, s.mvccStore, nil) s.regionRequestSender = NewRegionRequestSender(s.cache, client) @@ -113,7 +116,7 @@ func (s *testRegionRequestToThreeStoresSuite) TestGetRPCContext(c *C) { _, err := s.cache.BatchLoadRegionsFromKey(s.bo, []byte{}, 1) c.Assert(err, IsNil) - var seed uint32 = 0 + var seed uint32 var regionID = RegionVerID{s.regionID, 0, 0} req := tikvrpc.NewReplicaReadRequest(tikvrpc.CmdGet, &kvrpcpb.GetRequest{}, kv.ReplicaReadLeader, &seed) @@ -160,8 +163,10 @@ func (s *testRegionRequestToSingleStoreSuite) TestOnRegionError(c *C) { }} bo := NewBackofferWithVars(context.Background(), 5, nil) resp, err := s.regionRequestSender.SendReq(bo, req, region.Region, time.Second) - c.Assert(err, NotNil) - c.Assert(resp, IsNil) + c.Assert(err, IsNil) + c.Assert(resp, NotNil) + regionErr, _ := resp.GetRegionError() + c.Assert(regionErr, NotNil) }() } @@ -177,10 +182,98 @@ func (s *testRegionRequestToThreeStoresSuite) TestStoreTokenLimit(c *C) { resp, err := s.regionRequestSender.SendReq(s.bo, req, region.Region, time.Second) c.Assert(err, NotNil) c.Assert(resp, IsNil) - c.Assert(err.Error(), Equals, "[tikv:9008]Store token is up to the limit, store id = 1") + e, ok := errors.Cause(err).(*tikverr.ErrTokenLimit) + c.Assert(ok, IsTrue) + c.Assert(e.StoreID, Equals, uint64(1)) kv.StoreLimit.Store(oldStoreLimit) } +// Test whether the Stale Read request will retry the leader or other peers on error. +func (s *testRegionRequestToThreeStoresSuite) TestStaleReadRetry(c *C) { + var seed uint32 + req := tikvrpc.NewReplicaReadRequest(tikvrpc.CmdGet, &kvrpcpb.GetRequest{}, kv.ReplicaReadMixed, &seed) + req.EnableStaleRead() + + // Test whether a global Stale Read request will only retry on the leader. + req.TxnScope = oracle.GlobalTxnScope + region, err := s.cache.LocateRegionByID(s.bo, s.regionID) + c.Assert(err, IsNil) + c.Assert(region, NotNil) + // Retry 1 time. + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/mockDataIsNotReadyError", `return(1)`), IsNil) + resp, ctx, err := s.regionRequestSender.SendReqCtx(s.bo, req, region.Region, time.Second, tikvrpc.TiKV) + c.Assert(err, IsNil) + c.Assert(resp.Resp, NotNil) + c.Assert(ctx, NotNil) + c.Assert(ctx.Peer.GetId(), Equals, s.leaderPeer) + + seed = 0 + region, err = s.cache.LocateRegionByID(s.bo, s.regionID) + c.Assert(err, IsNil) + c.Assert(region, NotNil) + // Retry 2 times. + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/mockDataIsNotReadyError", `return(2)`), IsNil) + resp, ctx, err = s.regionRequestSender.SendReqCtx(s.bo, req, region.Region, time.Second, tikvrpc.TiKV) + c.Assert(err, IsNil) + c.Assert(resp.Resp, NotNil) + c.Assert(ctx, NotNil) + c.Assert(ctx.Peer.GetId(), Equals, s.leaderPeer) + + seed = 0 + region, err = s.cache.LocateRegionByID(s.bo, s.regionID) + c.Assert(err, IsNil) + c.Assert(region, NotNil) + // Retry 3 times. + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/mockDataIsNotReadyError", `return(3)`), IsNil) + resp, ctx, err = s.regionRequestSender.SendReqCtx(s.bo, req, region.Region, time.Second, tikvrpc.TiKV) + c.Assert(err, IsNil) + c.Assert(resp.Resp, NotNil) + c.Assert(ctx, NotNil) + c.Assert(ctx.Peer.GetId(), Equals, s.leaderPeer) + + // Test whether a local Stale Read request will retry on the leader and other peers. + req.TxnScope = "local" + seed = 0 + region, err = s.cache.LocateRegionByID(s.bo, s.regionID) + c.Assert(err, IsNil) + c.Assert(region, NotNil) + // Retry 1 time. + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/mockDataIsNotReadyError", `return(1)`), IsNil) + resp, ctx, err = s.regionRequestSender.SendReqCtx(s.bo, req, region.Region, time.Second, tikvrpc.TiKV) + c.Assert(err, IsNil) + c.Assert(resp.Resp, NotNil) + c.Assert(ctx, NotNil) + peerID1 := ctx.Peer.GetId() + + seed = 0 + region, err = s.cache.LocateRegionByID(s.bo, s.regionID) + c.Assert(err, IsNil) + c.Assert(region, NotNil) + // Retry 2 times. + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/mockDataIsNotReadyError", `return(2)`), IsNil) + resp, ctx, err = s.regionRequestSender.SendReqCtx(s.bo, req, region.Region, time.Second, tikvrpc.TiKV) + c.Assert(err, IsNil) + c.Assert(resp.Resp, NotNil) + c.Assert(ctx, NotNil) + peerID2 := ctx.Peer.GetId() + c.Assert(peerID2, Not(Equals), peerID1) + + seed = 0 + region, err = s.cache.LocateRegionByID(s.bo, s.regionID) + c.Assert(err, IsNil) + c.Assert(region, NotNil) + // Retry 3 times. + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/mockDataIsNotReadyError", `return(3)`), IsNil) + resp, ctx, err = s.regionRequestSender.SendReqCtx(s.bo, req, region.Region, time.Second, tikvrpc.TiKV) + c.Assert(err, IsNil) + c.Assert(resp.Resp, NotNil) + c.Assert(ctx, NotNil) + peerID3 := ctx.Peer.GetId() + c.Assert(peerID3, Not(Equals), peerID1) + c.Assert(peerID3, Not(Equals), peerID2) + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/mockDataIsNotReadyError"), IsNil) +} + func (s *testRegionRequestToSingleStoreSuite) TestOnSendFailedWithStoreRestart(c *C) { req := tikvrpc.NewRequest(tikvrpc.CmdRawPut, &kvrpcpb.RawPutRequest{ Key: []byte("key"), @@ -192,11 +285,14 @@ func (s *testRegionRequestToSingleStoreSuite) TestOnSendFailedWithStoreRestart(c resp, err := s.regionRequestSender.SendReq(s.bo, req, region.Region, time.Second) c.Assert(err, IsNil) c.Assert(resp.Resp, NotNil) + c.Assert(s.regionRequestSender.rpcError, IsNil) // stop store. s.cluster.StopStore(s.store) _, err = s.regionRequestSender.SendReq(s.bo, req, region.Region, time.Second) c.Assert(err, NotNil) + // The RPC error shouldn't be nil since it failed to sent the request. + c.Assert(s.regionRequestSender.rpcError, NotNil) // start store. s.cluster.StartStore(s.store) @@ -206,9 +302,12 @@ func (s *testRegionRequestToSingleStoreSuite) TestOnSendFailedWithStoreRestart(c region, err = s.cache.LocateRegionByID(s.bo, s.region) c.Assert(err, IsNil) c.Assert(region, NotNil) + c.Assert(s.regionRequestSender.rpcError, NotNil) resp, err = s.regionRequestSender.SendReq(s.bo, req, region.Region, time.Second) c.Assert(err, IsNil) c.Assert(resp.Resp, NotNil) + // The RPC error should be nil since it's evaluated successfully. + c.Assert(s.regionRequestSender.rpcError, IsNil) } func (s *testRegionRequestToSingleStoreSuite) TestOnSendFailedWithCloseKnownStoreThenUseNewOne(c *C) { @@ -431,6 +530,9 @@ func (s *mockTikvGrpcServer) Coprocessor(context.Context, *coprocessor.Request) func (s *mockTikvGrpcServer) BatchCoprocessor(*coprocessor.BatchRequest, tikvpb.Tikv_BatchCoprocessorServer) error { return errors.New("unreachable") } +func (s *mockTikvGrpcServer) RawCoprocessor(context.Context, *kvrpcpb.RawCoprocessorRequest) (*kvrpcpb.RawCoprocessorResponse, error) { + return nil, errors.New("unreachable") +} func (s *mockTikvGrpcServer) DispatchMPPTask(context.Context, *mpp.DispatchTaskRequest) (*mpp.DispatchTaskResponse, error) { return nil, errors.New("unreachable") } @@ -483,7 +585,7 @@ func (s *mockTikvGrpcServer) RawCompareAndSwap(context.Context, *kvrpcpb.RawCASR return nil, errors.New("unreachable") } -func (s *mockTikvGrpcServer) CoprocessorV2(context.Context, *coprocessor_v2.RawCoprocessorRequest) (*coprocessor_v2.RawCoprocessorResponse, error) { +func (s *mockTikvGrpcServer) GetLockWaitInfo(context.Context, *kvrpcpb.GetLockWaitInfoRequest) (*kvrpcpb.GetLockWaitInfoResponse, error) { return nil, errors.New("unreachable") } @@ -592,9 +694,9 @@ func (s *testRegionRequestToThreeStoresSuite) TestSwitchPeerWhenNoLeader(c *C) { func (s *testRegionRequestToThreeStoresSuite) loadAndGetLeaderStore(c *C) (*Store, string) { region, err := s.regionRequestSender.regionCache.findRegionByKey(s.bo, []byte("a"), false) c.Assert(err, IsNil) - leaderStore, leaderPeer, _, leaderStoreIdx := region.WorkStorePeer(region.getStore()) + leaderStore, leaderPeer, _, _ := region.WorkStorePeer(region.getStore()) c.Assert(leaderPeer.Id, Equals, s.leaderPeer) - leaderAddr, err := s.regionRequestSender.regionCache.getStoreAddr(s.bo, region, leaderStore, leaderStoreIdx) + leaderAddr, err := s.regionRequestSender.regionCache.getStoreAddr(s.bo, region, leaderStore) c.Assert(err, IsNil) return leaderStore, leaderAddr } @@ -702,7 +804,7 @@ func (s *testRegionRequestToThreeStoresSuite) TestForwarding(c *C) { c.Assert(ctx, IsNil) c.Assert(len(s.regionRequestSender.failStoreIDs), Equals, 0) c.Assert(len(s.regionRequestSender.failProxyStoreIDs), Equals, 0) - region := s.regionRequestSender.regionCache.getCachedRegionWithRLock(loc.Region) + region := s.regionRequestSender.regionCache.GetCachedRegionWithRLock(loc.Region) c.Assert(region, NotNil) c.Assert(region.checkNeedReload(), IsTrue) @@ -760,3 +862,410 @@ func (s *testRegionRequestToSingleStoreSuite) TestGetRegionByIDFromCache(c *C) { c.Assert(region.Region.confVer, Equals, region.Region.confVer) c.Assert(region.Region.ver, Equals, v3) } + +func (s *testRegionRequestToThreeStoresSuite) TestReplicaSelector(c *C) { + regionLoc, err := s.cache.LocateRegionByID(s.bo, s.regionID) + c.Assert(err, IsNil) + c.Assert(regionLoc, NotNil) + region := s.cache.GetCachedRegionWithRLock(regionLoc.Region) + regionStore := region.getStore() + + // Create a fake region and change its leader to the last peer. + regionStore = regionStore.clone() + regionStore.workTiKVIdx = AccessIndex(len(regionStore.stores) - 1) + sidx, _ := regionStore.accessStore(tiKVOnly, regionStore.workTiKVIdx) + regionStore.stores[sidx].epoch++ + regionStore.storeEpochs[sidx]++ + // Add a TiFlash peer to the region. + peer := &metapb.Peer{Id: s.cluster.AllocID(), StoreId: s.cluster.AllocID()} + regionStore.accessIndex[tiFlashOnly] = append(regionStore.accessIndex[tiFlashOnly], len(regionStore.stores)) + regionStore.stores = append(regionStore.stores, &Store{storeID: peer.StoreId, storeType: tikvrpc.TiFlash}) + regionStore.storeEpochs = append(regionStore.storeEpochs, 0) + + region = &Region{ + meta: region.GetMeta(), + } + region.lastAccess = time.Now().Unix() + region.meta.Peers = append(region.meta.Peers, peer) + atomic.StorePointer(®ion.store, unsafe.Pointer(regionStore)) + + cache := NewRegionCache(s.cache.pdClient) + defer cache.Close() + cache.insertRegionToCache(region) + + // Verify creating the replicaSelector. + replicaSelector, err := newReplicaSelector(cache, regionLoc.Region) + c.Assert(replicaSelector, NotNil) + c.Assert(err, IsNil) + c.Assert(replicaSelector.region, Equals, region) + // Should only contains TiKV stores. + c.Assert(len(replicaSelector.replicas), Equals, regionStore.accessStoreNum(tiKVOnly)) + c.Assert(len(replicaSelector.replicas), Equals, len(regionStore.stores)-1) + c.Assert(replicaSelector.nextReplicaIdx == 0, IsTrue) + c.Assert(replicaSelector.isExhausted(), IsFalse) + + // Verify that the store matches the peer and epoch. + for _, replica := range replicaSelector.replicas { + c.Assert(replica.store.storeID, Equals, replica.peer.GetStoreId()) + c.Assert(replica.peer, Equals, region.getPeerOnStore(replica.store.storeID)) + c.Assert(replica.attempts == 0, IsTrue) + + for i, store := range regionStore.stores { + if replica.store == store { + c.Assert(replica.epoch, Equals, regionStore.storeEpochs[i]) + } + } + } + // Verify that the leader replica is at the head of replicas. + leaderStore, leaderPeer, _, _ := region.WorkStorePeer(regionStore) + leaderReplica := replicaSelector.replicas[0] + c.Assert(leaderReplica.store, Equals, leaderStore) + c.Assert(leaderReplica.peer, Equals, leaderPeer) + + assertRPCCtxEqual := func(rpcCtx *RPCContext, replica *replica) { + c.Assert(rpcCtx.Store, Equals, replicaSelector.replicas[replicaSelector.nextReplicaIdx-1].store) + c.Assert(rpcCtx.Peer, Equals, replicaSelector.replicas[replicaSelector.nextReplicaIdx-1].peer) + c.Assert(rpcCtx.Addr, Equals, replicaSelector.replicas[replicaSelector.nextReplicaIdx-1].store.addr) + c.Assert(rpcCtx.AccessMode, Equals, tiKVOnly) + } + + // Verify the correctness of next() + for i := 0; i < len(replicaSelector.replicas); i++ { + rpcCtx, err := replicaSelector.next(s.bo) + c.Assert(rpcCtx, NotNil) + c.Assert(err, IsNil) + c.Assert(rpcCtx.Region, Equals, regionLoc.Region) + c.Assert(rpcCtx.Meta, Equals, region.meta) + replica := replicaSelector.replicas[replicaSelector.nextReplicaIdx-1] + assertRPCCtxEqual(rpcCtx, replica) + c.Assert(replica.attempts, Equals, 1) + c.Assert(replicaSelector.nextReplicaIdx, Equals, i+1) + } + c.Assert(replicaSelector.isExhausted(), IsTrue) + rpcCtx, err := replicaSelector.next(s.bo) + c.Assert(rpcCtx, IsNil) + c.Assert(err, IsNil) + // The region should be invalidated if runs out of all replicas. + c.Assert(replicaSelector.region.isValid(), IsFalse) + + region.lastAccess = time.Now().Unix() + replicaSelector, err = newReplicaSelector(cache, regionLoc.Region) + c.Assert(replicaSelector, NotNil) + cache.testingKnobs.mockRequestLiveness = func(s *Store, bo *Backoffer) livenessState { + return reachable + } + for i := 0; i < maxReplicaAttempt; i++ { + rpcCtx, err := replicaSelector.next(s.bo) + c.Assert(rpcCtx, NotNil) + c.Assert(err, IsNil) + nextIdx := replicaSelector.nextReplicaIdx + // Verify that retry the same store if it's reachable. + replicaSelector.onSendFailure(s.bo, nil) + c.Assert(nextIdx, Equals, replicaSelector.nextReplicaIdx+1) + c.Assert(replicaSelector.nextReplica().attempts, Equals, i+1) + } + // Verify the maxReplicaAttempt limit for each replica. + rpcCtx, err = replicaSelector.next(s.bo) + c.Assert(err, IsNil) + assertRPCCtxEqual(rpcCtx, replicaSelector.replicas[1]) + c.Assert(replicaSelector.nextReplicaIdx, Equals, 2) + + // Verify updating leader. + replicaSelector, _ = newReplicaSelector(cache, regionLoc.Region) + replicaSelector.next(s.bo) + // The leader is the 3rd replica. After updating leader, it should be the next. + leader := replicaSelector.replicas[2] + replicaSelector.updateLeader(leader.peer) + c.Assert(replicaSelector.nextReplica(), Equals, leader) + c.Assert(replicaSelector.nextReplicaIdx, Equals, 1) + rpcCtx, _ = replicaSelector.next(s.bo) + assertRPCCtxEqual(rpcCtx, leader) + // Verify the regionStore is updated and the workTiKVIdx points to the leader. + regionStore = region.getStore() + leaderStore, leaderPeer, _, _ = region.WorkStorePeer(regionStore) + c.Assert(leaderStore, Equals, leader.store) + c.Assert(leaderPeer, Equals, leader.peer) + + replicaSelector, _ = newReplicaSelector(cache, regionLoc.Region) + replicaSelector.next(s.bo) + replicaSelector.next(s.bo) + replicaSelector.next(s.bo) + c.Assert(replicaSelector.isExhausted(), IsTrue) + // The leader is the 1st replica. After updating leader, it should be the next and + // the currnet replica is skipped. + leader = replicaSelector.replicas[0] + replicaSelector.updateLeader(leader.peer) + // The leader should be the next replica. + c.Assert(replicaSelector.nextReplica(), Equals, leader) + c.Assert(replicaSelector.nextReplicaIdx, Equals, 2) + rpcCtx, _ = replicaSelector.next(s.bo) + c.Assert(replicaSelector.isExhausted(), IsTrue) + assertRPCCtxEqual(rpcCtx, leader) + // Verify the regionStore is updated and the workTiKVIdx points to the leader. + regionStore = region.getStore() + leaderStore, leaderPeer, _, _ = region.WorkStorePeer(regionStore) + c.Assert(leaderStore, Equals, leader.store) + c.Assert(leaderPeer, Equals, leader.peer) + + // Give the leader one more chance even if it exceeds the maxReplicaAttempt. + replicaSelector, _ = newReplicaSelector(cache, regionLoc.Region) + leader = replicaSelector.replicas[0] + leader.attempts = maxReplicaAttempt + replicaSelector.updateLeader(leader.peer) + c.Assert(leader.attempts, Equals, maxReplicaAttempt-1) + rpcCtx, _ = replicaSelector.next(s.bo) + assertRPCCtxEqual(rpcCtx, leader) + c.Assert(leader.attempts, Equals, maxReplicaAttempt) + + // Invalidate the region if the leader is not in the region. + region.lastAccess = time.Now().Unix() + replicaSelector.updateLeader(&metapb.Peer{Id: s.cluster.AllocID(), StoreId: s.cluster.AllocID()}) + c.Assert(region.isValid(), IsFalse) + // Don't try next replica if the region is invalidated. + rpcCtx, err = replicaSelector.next(s.bo) + c.Assert(rpcCtx, IsNil) + c.Assert(err, IsNil) + + // Verify on send success. + region.lastAccess = time.Now().Unix() + replicaSelector, _ = newReplicaSelector(cache, regionLoc.Region) + replicaSelector.next(s.bo) + rpcCtx, err = replicaSelector.next(s.bo) + replicaSelector.OnSendSuccess() + // Verify the regionStore is updated and the workTiKVIdx points to the leader. + leaderStore, leaderPeer, _, _ = region.WorkStorePeer(region.getStore()) + c.Assert(leaderStore, Equals, rpcCtx.Store) + c.Assert(leaderPeer, Equals, rpcCtx.Peer) +} + +// TODO(youjiali1995): Remove duplicated tests. This test may be duplicated with other +// tests but it's a dedicated one to test sending requests with the replica selector. +func (s *testRegionRequestToThreeStoresSuite) TestSendReqWithReplicaSelector(c *C) { + req := tikvrpc.NewRequest(tikvrpc.CmdRawPut, &kvrpcpb.RawPutRequest{ + Key: []byte("key"), + Value: []byte("value"), + }) + region, err := s.cache.LocateRegionByID(s.bo, s.regionID) + c.Assert(err, IsNil) + c.Assert(region, NotNil) + + reloadRegion := func() { + s.regionRequestSender.leaderReplicaSelector.region.invalidate(Other) + region, _ = s.cache.LocateRegionByID(s.bo, s.regionID) + } + + hasFakeRegionError := func(resp *tikvrpc.Response) bool { + if resp == nil { + return false + } + regionErr, err := resp.GetRegionError() + if err != nil { + return false + } + return isFakeRegionError(regionErr) + } + + // Normal + bo := retry.NewBackoffer(context.Background(), -1) + sender := s.regionRequestSender + resp, err := sender.SendReq(bo, req, region.Region, time.Second) + c.Assert(err, IsNil) + c.Assert(resp, NotNil) + c.Assert(bo.GetTotalBackoffTimes() == 0, IsTrue) + + // Switch to the next Peer due to store failure and the leader is on the next peer. + bo = retry.NewBackoffer(context.Background(), -1) + s.cluster.ChangeLeader(s.regionID, s.peerIDs[1]) + s.cluster.StopStore(s.storeIDs[0]) + resp, err = sender.SendReq(bo, req, region.Region, time.Second) + c.Assert(err, IsNil) + c.Assert(resp, NotNil) + c.Assert(sender.leaderReplicaSelector.nextReplicaIdx, Equals, 2) + c.Assert(bo.GetTotalBackoffTimes() == 1, IsTrue) + s.cluster.StartStore(s.storeIDs[0]) + + // Leader is updated because of send success, so no backoff. + bo = retry.NewBackoffer(context.Background(), -1) + resp, err = sender.SendReq(bo, req, region.Region, time.Second) + c.Assert(err, IsNil) + c.Assert(resp, NotNil) + c.Assert(sender.leaderReplicaSelector.nextReplicaIdx, Equals, 1) + c.Assert(bo.GetTotalBackoffTimes() == 0, IsTrue) + + // Switch to the next peer due to leader failure but the new leader is not elected. + // Region will be invalidated due to store epoch changed. + reloadRegion() + s.cluster.StopStore(s.storeIDs[1]) + bo = retry.NewBackoffer(context.Background(), -1) + resp, err = sender.SendReq(bo, req, region.Region, time.Second) + c.Assert(err, IsNil) + c.Assert(hasFakeRegionError(resp), IsTrue) + c.Assert(sender.leaderReplicaSelector.isExhausted(), IsFalse) + c.Assert(bo.GetTotalBackoffTimes(), Equals, 1) + s.cluster.StartStore(s.storeIDs[1]) + + // Leader is changed. No backoff. + reloadRegion() + s.cluster.ChangeLeader(s.regionID, s.peerIDs[0]) + bo = retry.NewBackoffer(context.Background(), -1) + resp, err = sender.SendReq(bo, req, region.Region, time.Second) + c.Assert(err, IsNil) + c.Assert(resp, NotNil) + c.Assert(bo.GetTotalBackoffTimes(), Equals, 0) + + // No leader. Backoff for each replica and runs out all replicas. + s.cluster.GiveUpLeader(s.regionID) + bo = retry.NewBackoffer(context.Background(), -1) + resp, err = sender.SendReq(bo, req, region.Region, time.Second) + c.Assert(err, IsNil) + c.Assert(hasFakeRegionError(resp), IsTrue) + c.Assert(bo.GetTotalBackoffTimes(), Equals, 3) + c.Assert(sender.leaderReplicaSelector.isExhausted(), IsTrue) + c.Assert(sender.leaderReplicaSelector.region.isValid(), IsFalse) + s.cluster.ChangeLeader(s.regionID, s.peerIDs[0]) + + // The leader store is alive but can't provide service. + // Region will be invalidated due to running out of all replicas. + s.regionRequestSender.regionCache.testingKnobs.mockRequestLiveness = func(s *Store, bo *Backoffer) livenessState { + return reachable + } + reloadRegion() + s.cluster.StopStore(s.storeIDs[0]) + bo = retry.NewBackoffer(context.Background(), -1) + resp, err = sender.SendReq(bo, req, region.Region, time.Second) + c.Assert(err, IsNil) + c.Assert(hasFakeRegionError(resp), IsTrue) + c.Assert(sender.leaderReplicaSelector.isExhausted(), IsTrue) + c.Assert(sender.leaderReplicaSelector.region.isValid(), IsFalse) + c.Assert(bo.GetTotalBackoffTimes(), Equals, maxReplicaAttempt+2) + s.cluster.StartStore(s.storeIDs[0]) + + // Verify that retry the same replica when meets ServerIsBusy/MaxTimestampNotSynced/ReadIndexNotReady/ProposalInMergingMode. + for _, regionErr := range []*errorpb.Error{ + // ServerIsBusy takes too much time to test. + // {ServerIsBusy: &errorpb.ServerIsBusy{}}, + {MaxTimestampNotSynced: &errorpb.MaxTimestampNotSynced{}}, + {ReadIndexNotReady: &errorpb.ReadIndexNotReady{}}, + {ProposalInMergingMode: &errorpb.ProposalInMergingMode{}}, + } { + func() { + oc := sender.client + defer func() { + sender.client = oc + }() + s.regionRequestSender.client = &fnClient{func(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (response *tikvrpc.Response, err error) { + // Return the specific region error when accesses the leader. + if addr == s.cluster.GetStore(s.storeIDs[0]).Address { + return &tikvrpc.Response{Resp: &kvrpcpb.RawPutResponse{RegionError: regionErr}}, nil + } + // Return the not leader error when accesses followers. + return &tikvrpc.Response{Resp: &kvrpcpb.RawPutResponse{RegionError: &errorpb.Error{ + NotLeader: &errorpb.NotLeader{ + RegionId: region.Region.id, Leader: &metapb.Peer{Id: s.peerIDs[0], StoreId: s.storeIDs[0]}, + }}}}, nil + + }} + reloadRegion() + bo = retry.NewBackoffer(context.Background(), -1) + resp, err := sender.SendReq(bo, req, region.Region, time.Second) + c.Assert(err, IsNil) + c.Assert(hasFakeRegionError(resp), IsTrue) + c.Assert(sender.leaderReplicaSelector.isExhausted(), IsTrue) + c.Assert(sender.leaderReplicaSelector.region.isValid(), IsFalse) + c.Assert(bo.GetTotalBackoffTimes(), Equals, maxReplicaAttempt+2) + }() + } + + // Verify switch to the next peer immediately when meets StaleCommand. + reloadRegion() + func() { + oc := sender.client + defer func() { + sender.client = oc + }() + s.regionRequestSender.client = &fnClient{func(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (response *tikvrpc.Response, err error) { + return &tikvrpc.Response{Resp: &kvrpcpb.RawPutResponse{RegionError: &errorpb.Error{StaleCommand: &errorpb.StaleCommand{}}}}, nil + }} + reloadRegion() + bo = retry.NewBackoffer(context.Background(), -1) + resp, err := sender.SendReq(bo, req, region.Region, time.Second) + c.Assert(err, IsNil) + c.Assert(hasFakeRegionError(resp), IsTrue) + c.Assert(sender.leaderReplicaSelector.isExhausted(), IsTrue) + c.Assert(sender.leaderReplicaSelector.region.isValid(), IsFalse) + c.Assert(bo.GetTotalBackoffTimes(), Equals, 0) + }() + + // Verify don't invalidate region when meets unknown region errors. + reloadRegion() + func() { + oc := sender.client + defer func() { + sender.client = oc + }() + s.regionRequestSender.client = &fnClient{func(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (response *tikvrpc.Response, err error) { + return &tikvrpc.Response{Resp: &kvrpcpb.RawPutResponse{RegionError: &errorpb.Error{Message: ""}}}, nil + }} + reloadRegion() + bo = retry.NewBackoffer(context.Background(), -1) + resp, err := sender.SendReq(bo, req, region.Region, time.Second) + c.Assert(err, IsNil) + c.Assert(hasFakeRegionError(resp), IsTrue) + c.Assert(sender.leaderReplicaSelector.isExhausted(), IsTrue) + c.Assert(sender.leaderReplicaSelector.region.isValid(), IsFalse) + c.Assert(bo.GetTotalBackoffTimes(), Equals, 0) + }() + + // Verify invalidate region when meets StoreNotMatch/RegionNotFound/EpochNotMatch/NotLeader and can't find the leader in region. + for i, regionErr := range []*errorpb.Error{ + {StoreNotMatch: &errorpb.StoreNotMatch{}}, + {RegionNotFound: &errorpb.RegionNotFound{}}, + {EpochNotMatch: &errorpb.EpochNotMatch{}}, + {NotLeader: &errorpb.NotLeader{Leader: &metapb.Peer{}}}} { + func() { + oc := sender.client + defer func() { + sender.client = oc + }() + s.regionRequestSender.client = &fnClient{func(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (response *tikvrpc.Response, err error) { + return &tikvrpc.Response{Resp: &kvrpcpb.RawPutResponse{RegionError: regionErr}}, nil + + }} + reloadRegion() + bo = retry.NewBackoffer(context.Background(), -1) + resp, err := sender.SendReq(bo, req, region.Region, time.Second) + + // Return a sendError when meets NotLeader and can't find the leader in the region. + if i == 3 { + c.Assert(err, IsNil) + c.Assert(hasFakeRegionError(resp), IsTrue) + } else { + c.Assert(err, IsNil) + c.Assert(resp, NotNil) + regionErr, _ := resp.GetRegionError() + c.Assert(regionErr, NotNil) + } + c.Assert(sender.leaderReplicaSelector.isExhausted(), IsFalse) + c.Assert(sender.leaderReplicaSelector.region.isValid(), IsFalse) + c.Assert(bo.GetTotalBackoffTimes(), Equals, 0) + }() + } + + // Runs out of all replicas and then returns a send error. + s.regionRequestSender.regionCache.testingKnobs.mockRequestLiveness = func(s *Store, bo *Backoffer) livenessState { + return unreachable + } + reloadRegion() + for _, store := range s.storeIDs { + s.cluster.StopStore(store) + } + bo = retry.NewBackoffer(context.Background(), -1) + resp, err = sender.SendReq(bo, req, region.Region, time.Second) + c.Assert(err, IsNil) + c.Assert(hasFakeRegionError(resp), IsTrue) + c.Assert(bo.GetTotalBackoffTimes() == 3, IsTrue) + c.Assert(sender.leaderReplicaSelector.region.isValid(), IsFalse) + for _, store := range s.storeIDs { + s.cluster.StartStore(store) + } +} diff --git a/store/tikv/retry/backoff.go b/store/tikv/retry/backoff.go new file mode 100644 index 0000000000000..485a179ced6c9 --- /dev/null +++ b/store/tikv/retry/backoff.go @@ -0,0 +1,289 @@ +// Copyright 2016 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package retry + +import ( + "context" + "fmt" + "math" + "strings" + "sync/atomic" + "time" + + "github.com/opentracing/opentracing-go" + "github.com/pingcap/errors" + "github.com/pingcap/log" + tikverr "github.com/pingcap/tidb/store/tikv/error" + "github.com/pingcap/tidb/store/tikv/kv" + "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/util" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// Backoffer is a utility for retrying queries. +type Backoffer struct { + ctx context.Context + + fn map[string]backoffFn + maxSleep int + totalSleep int + + vars *kv.Variables + noop bool + + errors []error + configs []*Config + backoffSleepMS map[string]int + backoffTimes map[string]int + parent *Backoffer +} + +type txnStartCtxKeyType struct{} + +// TxnStartKey is a key for transaction start_ts info in context.Context. +var TxnStartKey interface{} = txnStartCtxKeyType{} + +// NewBackoffer (Deprecated) creates a Backoffer with maximum sleep time(in ms). +func NewBackoffer(ctx context.Context, maxSleep int) *Backoffer { + return &Backoffer{ + ctx: ctx, + maxSleep: maxSleep, + vars: kv.DefaultVars, + } +} + +// NewBackofferWithVars creates a Backoffer with maximum sleep time(in ms) and kv.Variables. +func NewBackofferWithVars(ctx context.Context, maxSleep int, vars *kv.Variables) *Backoffer { + return NewBackoffer(ctx, maxSleep).withVars(vars) +} + +// NewNoopBackoff create a Backoffer do nothing just return error directly +func NewNoopBackoff(ctx context.Context) *Backoffer { + return &Backoffer{ctx: ctx, noop: true} +} + +// withVars sets the kv.Variables to the Backoffer and return it. +func (b *Backoffer) withVars(vars *kv.Variables) *Backoffer { + if vars != nil { + b.vars = vars + } + // maxSleep is the max sleep time in millisecond. + // When it is multiplied by BackOffWeight, it should not be greater than MaxInt32. + if b.maxSleep > 0 && math.MaxInt32/b.vars.BackOffWeight >= b.maxSleep { + b.maxSleep *= b.vars.BackOffWeight + } + return b +} + +// Backoff sleeps a while base on the Config and records the error message. +// It returns a retryable error if total sleep time exceeds maxSleep. +func (b *Backoffer) Backoff(cfg *Config, err error) error { + if span := opentracing.SpanFromContext(b.ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan(fmt.Sprintf("tikv.backoff.%s", cfg), opentracing.ChildOf(span.Context())) + defer span1.Finish() + opentracing.ContextWithSpan(b.ctx, span1) + } + return b.BackoffWithCfgAndMaxSleep(cfg, -1, err) +} + +// BackoffWithMaxSleepTxnLockFast sleeps a while base on the MaxSleepTxnLock and records the error message +// and never sleep more than maxSleepMs for each sleep. +func (b *Backoffer) BackoffWithMaxSleepTxnLockFast(maxSleepMs int, err error) error { + cfg := BoTxnLockFast + return b.BackoffWithCfgAndMaxSleep(cfg, maxSleepMs, err) +} + +// BackoffWithCfgAndMaxSleep sleeps a while base on the Config and records the error message +// and never sleep more than maxSleepMs for each sleep. +func (b *Backoffer) BackoffWithCfgAndMaxSleep(cfg *Config, maxSleepMs int, err error) error { + if strings.Contains(err.Error(), tikverr.MismatchClusterID) { + logutil.BgLogger().Fatal("critical error", zap.Error(err)) + } + select { + case <-b.ctx.Done(): + return errors.Trace(err) + default: + } + + b.errors = append(b.errors, errors.Errorf("%s at %s", err.Error(), time.Now().Format(time.RFC3339Nano))) + b.configs = append(b.configs, cfg) + if b.noop || (b.maxSleep > 0 && b.totalSleep >= b.maxSleep) { + errMsg := fmt.Sprintf("%s backoffer.maxSleep %dms is exceeded, errors:", cfg.String(), b.maxSleep) + for i, err := range b.errors { + // Print only last 3 errors for non-DEBUG log levels. + if log.GetLevel() == zapcore.DebugLevel || i >= len(b.errors)-3 { + errMsg += "\n" + err.Error() + } + } + logutil.BgLogger().Warn(errMsg) + // Use the first backoff type to generate a MySQL error. + return b.configs[0].err + } + + // Lazy initialize. + if b.fn == nil { + b.fn = make(map[string]backoffFn) + } + f, ok := b.fn[cfg.name] + if !ok { + f = cfg.createBackoffFn(b.vars) + b.fn[cfg.name] = f + } + realSleep := f(b.ctx, maxSleepMs) + if cfg.metric != nil { + (*cfg.metric).Observe(float64(realSleep) / 1000) + } + b.totalSleep += realSleep + if b.backoffSleepMS == nil { + b.backoffSleepMS = make(map[string]int) + } + b.backoffSleepMS[cfg.name] += realSleep + if b.backoffTimes == nil { + b.backoffTimes = make(map[string]int) + } + b.backoffTimes[cfg.name]++ + + stmtExec := b.ctx.Value(util.ExecDetailsKey) + if stmtExec != nil { + detail := stmtExec.(*util.ExecDetails) + atomic.AddInt64(&detail.BackoffDuration, int64(realSleep)*int64(time.Millisecond)) + atomic.AddInt64(&detail.BackoffCount, 1) + } + + if b.vars != nil && b.vars.Killed != nil { + if atomic.LoadUint32(b.vars.Killed) == 1 { + return tikverr.ErrQueryInterrupted + } + } + + var startTs interface{} + if ts := b.ctx.Value(TxnStartKey); ts != nil { + startTs = ts + } + logutil.Logger(b.ctx).Debug("retry later", + zap.Error(err), + zap.Int("totalSleep", b.totalSleep), + zap.Int("maxSleep", b.maxSleep), + zap.Stringer("type", cfg), + zap.Reflect("txnStartTS", startTs)) + return nil +} + +func (b *Backoffer) String() string { + if b.totalSleep == 0 { + return "" + } + return fmt.Sprintf(" backoff(%dms %v)", b.totalSleep, b.configs) +} + +// Clone creates a new Backoffer which keeps current Backoffer's sleep time and errors, and shares +// current Backoffer's context. +func (b *Backoffer) Clone() *Backoffer { + return &Backoffer{ + ctx: b.ctx, + maxSleep: b.maxSleep, + totalSleep: b.totalSleep, + errors: b.errors, + vars: b.vars, + parent: b.parent, + } +} + +// Fork creates a new Backoffer which keeps current Backoffer's sleep time and errors, and holds +// a child context of current Backoffer's context. +func (b *Backoffer) Fork() (*Backoffer, context.CancelFunc) { + ctx, cancel := context.WithCancel(b.ctx) + return &Backoffer{ + ctx: ctx, + maxSleep: b.maxSleep, + totalSleep: b.totalSleep, + errors: b.errors, + vars: b.vars, + parent: b, + }, cancel +} + +// GetVars returns the binded vars. +func (b *Backoffer) GetVars() *kv.Variables { + return b.vars +} + +// GetTotalSleep returns total sleep time. +func (b *Backoffer) GetTotalSleep() int { + return b.totalSleep +} + +// GetTypes returns type list of this backoff and all its ancestors. +func (b *Backoffer) GetTypes() []string { + typs := make([]string, 0, len(b.configs)) + for b != nil { + for _, cfg := range b.configs { + typs = append(typs, cfg.String()) + } + b = b.parent + } + return typs +} + +// GetCtx returns the binded context. +func (b *Backoffer) GetCtx() context.Context { + return b.ctx +} + +// SetCtx sets the binded context to ctx. +func (b *Backoffer) SetCtx(ctx context.Context) { + b.ctx = ctx +} + +// GetBackoffTimes returns a map contains backoff time count by type. +func (b *Backoffer) GetBackoffTimes() map[string]int { + return b.backoffTimes +} + +// GetTotalBackoffTimes returns the total backoff times of the backoffer. +func (b *Backoffer) GetTotalBackoffTimes() int { + total := 0 + for _, time := range b.backoffTimes { + total += time + } + return total +} + +// GetBackoffSleepMS returns a map contains backoff sleep time by type. +func (b *Backoffer) GetBackoffSleepMS() map[string]int { + return b.backoffSleepMS +} + +// ErrorsNum returns the number of errors. +func (b *Backoffer) ErrorsNum() int { + return len(b.errors) +} + +// Reset resets the sleep state of the backoffer, so that following backoff +// can sleep shorter. The reason why we don't create a new backoffer is that +// backoffer is similar to context and it records some metrics that we +// want to record for an entire process which is composed of serveral stages. +func (b *Backoffer) Reset() { + b.fn = nil + b.totalSleep = 0 +} + +// ResetMaxSleep resets the sleep state and max sleep limit of the backoffer. +// It's used when switches to the next stage of the process. +func (b *Backoffer) ResetMaxSleep(maxSleep int) { + b.Reset() + b.maxSleep = maxSleep + b.withVars(b.vars) +} diff --git a/store/tikv/backoff_test.go b/store/tikv/retry/backoff_test.go similarity index 90% rename from store/tikv/backoff_test.go rename to store/tikv/retry/backoff_test.go index 11254937abd72..a0a566499b10f 100644 --- a/store/tikv/backoff_test.go +++ b/store/tikv/retry/backoff_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package tikv +package retry import ( "context" @@ -27,7 +27,7 @@ var _ = Suite(&testBackoffSuite{}) func (s *testBackoffSuite) TestBackoffWithMax(c *C) { b := NewBackofferWithVars(context.TODO(), 2000, nil) - err := b.BackoffWithMaxSleep(BoTxnLockFast, 30, errors.New("test")) + err := b.BackoffWithMaxSleepTxnLockFast(30, errors.New("test")) c.Assert(err, IsNil) c.Assert(b.totalSleep, Equals, 30) } diff --git a/store/tikv/retry/config.go b/store/tikv/retry/config.go new file mode 100644 index 0000000000000..7695fe046a2c8 --- /dev/null +++ b/store/tikv/retry/config.go @@ -0,0 +1,160 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package retry + +import ( + "context" + "math" + "math/rand" + "strings" + "time" + + tikverr "github.com/pingcap/tidb/store/tikv/error" + "github.com/pingcap/tidb/store/tikv/kv" + "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap" +) + +// Config is the configuration of the Backoff function. +type Config struct { + name string + metric *prometheus.Observer + fnCfg *BackoffFnCfg + err error +} + +// backoffFn is the backoff function which compute the sleep time and do sleep. +type backoffFn func(ctx context.Context, maxSleepMs int) int + +func (c *Config) createBackoffFn(vars *kv.Variables) backoffFn { + if strings.EqualFold(c.name, txnLockFastName) { + return newBackoffFn(vars.BackoffLockFast, c.fnCfg.cap, c.fnCfg.jitter) + } + return newBackoffFn(c.fnCfg.base, c.fnCfg.cap, c.fnCfg.jitter) +} + +// BackoffFnCfg is the configuration for the backoff func which implements exponential backoff with +// optional jitters. +// See http://www.awsarchitectureblog.com/2015/03/backoff.html +type BackoffFnCfg struct { + base int + cap int + jitter int +} + +// NewBackoffFnCfg creates the config for BackoffFn. +func NewBackoffFnCfg(base, cap, jitter int) *BackoffFnCfg { + return &BackoffFnCfg{ + base, + cap, + jitter, + } +} + +// NewConfig creates a new Config for the Backoff operation. +func NewConfig(name string, metric *prometheus.Observer, backoffFnCfg *BackoffFnCfg, err error) *Config { + return &Config{ + name: name, + metric: metric, + fnCfg: backoffFnCfg, + err: err, + } +} + +func (c *Config) String() string { + return c.name +} + +const txnLockFastName = "txnLockFast" + +// Backoff Config variables. +var ( + // TODO: distinguish tikv and tiflash in metrics + BoTiKVRPC = NewConfig("tikvRPC", &metrics.BackoffHistogramRPC, NewBackoffFnCfg(100, 2000, EqualJitter), tikverr.ErrTiKVServerTimeout) + BoTiFlashRPC = NewConfig("tiflashRPC", &metrics.BackoffHistogramRPC, NewBackoffFnCfg(100, 2000, EqualJitter), tikverr.ErrTiFlashServerTimeout) + BoTxnLock = NewConfig("txnLock", &metrics.BackoffHistogramLock, NewBackoffFnCfg(200, 3000, EqualJitter), tikverr.ErrResolveLockTimeout) + BoPDRPC = NewConfig("pdRPC", &metrics.BackoffHistogramPD, NewBackoffFnCfg(500, 3000, EqualJitter), tikverr.NewErrPDServerTimeout("")) + // change base time to 2ms, because it may recover soon. + BoRegionMiss = NewConfig("regionMiss", &metrics.BackoffHistogramRegionMiss, NewBackoffFnCfg(2, 500, NoJitter), tikverr.ErrRegionUnavailable) + BoRegionScheduling = NewConfig("regionScheduling", &metrics.BackoffHistogramRegionScheduling, NewBackoffFnCfg(2, 500, NoJitter), tikverr.ErrRegionUnavailable) + BoTiKVServerBusy = NewConfig("tikvServerBusy", &metrics.BackoffHistogramServerBusy, NewBackoffFnCfg(2000, 10000, EqualJitter), tikverr.ErrTiKVServerBusy) + BoTiFlashServerBusy = NewConfig("tiflashServerBusy", &metrics.BackoffHistogramServerBusy, NewBackoffFnCfg(2000, 10000, EqualJitter), tikverr.ErrTiFlashServerBusy) + BoTxnNotFound = NewConfig("txnNotFound", &metrics.BackoffHistogramEmpty, NewBackoffFnCfg(2, 500, NoJitter), tikverr.ErrResolveLockTimeout) + BoStaleCmd = NewConfig("staleCommand", &metrics.BackoffHistogramStaleCmd, NewBackoffFnCfg(2, 1000, NoJitter), tikverr.ErrTiKVStaleCommand) + BoMaxTsNotSynced = NewConfig("maxTsNotSynced", &metrics.BackoffHistogramEmpty, NewBackoffFnCfg(2, 500, NoJitter), tikverr.ErrTiKVMaxTimestampNotSynced) + // TxnLockFast's `base` load from vars.BackoffLockFast when create BackoffFn. + BoTxnLockFast = NewConfig(txnLockFastName, &metrics.BackoffHistogramLockFast, NewBackoffFnCfg(2, 3000, EqualJitter), tikverr.ErrResolveLockTimeout) +) + +const ( + // NoJitter makes the backoff sequence strict exponential. + NoJitter = 1 + iota + // FullJitter applies random factors to strict exponential. + FullJitter + // EqualJitter is also randomized, but prevents very short sleeps. + EqualJitter + // DecorrJitter increases the maximum jitter based on the last random value. + DecorrJitter +) + +// newBackoffFn creates a backoff func which implements exponential backoff with +// optional jitters. +// See http://www.awsarchitectureblog.com/2015/03/backoff.html +func newBackoffFn(base, cap, jitter int) backoffFn { + if base < 2 { + // Top prevent panic in 'rand.Intn'. + base = 2 + } + attempts := 0 + lastSleep := base + return func(ctx context.Context, maxSleepMs int) int { + var sleep int + switch jitter { + case NoJitter: + sleep = expo(base, cap, attempts) + case FullJitter: + v := expo(base, cap, attempts) + sleep = rand.Intn(v) + case EqualJitter: + v := expo(base, cap, attempts) + sleep = v/2 + rand.Intn(v/2) + case DecorrJitter: + sleep = int(math.Min(float64(cap), float64(base+rand.Intn(lastSleep*3-base)))) + } + logutil.BgLogger().Debug("backoff", + zap.Int("base", base), + zap.Int("sleep", sleep), + zap.Int("attempts", attempts)) + + realSleep := sleep + // when set maxSleepMs >= 0 in `tikv.BackoffWithMaxSleep` will force sleep maxSleepMs milliseconds. + if maxSleepMs >= 0 && realSleep > maxSleepMs { + realSleep = maxSleepMs + } + select { + case <-time.After(time.Duration(realSleep) * time.Millisecond): + attempts++ + lastSleep = sleep + return realSleep + case <-ctx.Done(): + return 0 + } + } +} + +func expo(base, cap, n int) int { + return int(math.Min(float64(cap), float64(base)*math.Pow(2.0, float64(n)))) +} diff --git a/store/tikv/scan.go b/store/tikv/scan.go index 19a14b3f73819..2f83c2db113be 100644 --- a/store/tikv/scan.go +++ b/store/tikv/scan.go @@ -18,10 +18,12 @@ import ( "context" "github.com/pingcap/errors" - pb "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/tidb/store/tikv/client" tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" "go.uber.org/zap" ) @@ -30,7 +32,7 @@ import ( type Scanner struct { snapshot *KVSnapshot batchSize int - cache []*pb.KvPair + cache []*kvrpcpb.KvPair idx int nextStartKey []byte endKey []byte @@ -85,9 +87,11 @@ func (s *Scanner) Value() []byte { return nil } +const scannerNextMaxBackoff = 600000 // 10 minutes + // Next return next element. func (s *Scanner) Next() error { - bo := NewBackofferWithVars(context.WithValue(context.Background(), TxnStartKey, s.snapshot.version), scannerNextMaxBackoff, s.snapshot.vars) + bo := retry.NewBackofferWithVars(context.WithValue(context.Background(), retry.TxnStartKey, s.snapshot.version), scannerNextMaxBackoff, s.snapshot.vars) if !s.valid { return errors.New("scanner iterator is invalid") } @@ -144,7 +148,7 @@ func (s *Scanner) startTS() uint64 { return s.snapshot.version } -func (s *Scanner) resolveCurrentLock(bo *Backoffer, current *pb.KvPair) error { +func (s *Scanner) resolveCurrentLock(bo *Backoffer, current *kvrpcpb.KvPair) error { ctx := context.Background() val, err := s.snapshot.get(ctx, bo, current.Key) if err != nil { @@ -161,7 +165,7 @@ func (s *Scanner) getData(bo *Backoffer) error { zap.String("nextEndKey", kv.StrKey(s.nextEndKey)), zap.Bool("reverse", s.reverse), zap.Uint64("txnStartTS", s.startTS())) - sender := NewRegionRequestSender(s.snapshot.store.regionCache, s.snapshot.store.client) + sender := NewRegionRequestSender(s.snapshot.store.regionCache, s.snapshot.store.GetTiKVClient()) var reqEndKey, reqStartKey []byte var loc *KeyLocation var err error @@ -187,11 +191,12 @@ func (s *Scanner) getData(bo *Backoffer) error { reqStartKey = loc.StartKey } } - sreq := &pb.ScanRequest{ - Context: &pb.Context{ - Priority: s.snapshot.priority.ToPB(), - NotFillCache: s.snapshot.notFillCache, - IsolationLevel: s.snapshot.isolationLevel.ToPB(), + sreq := &kvrpcpb.ScanRequest{ + Context: &kvrpcpb.Context{ + Priority: s.snapshot.priority.ToPB(), + NotFillCache: s.snapshot.notFillCache, + IsolationLevel: s.snapshot.isolationLevel.ToPB(), + ResourceGroupTag: s.snapshot.resourceGroupTag, }, StartKey: s.nextStartKey, EndKey: reqEndKey, @@ -206,13 +211,14 @@ func (s *Scanner) getData(bo *Backoffer) error { sreq.Reverse = true } s.snapshot.mu.RLock() - req := tikvrpc.NewReplicaReadRequest(tikvrpc.CmdScan, sreq, s.snapshot.mu.replicaRead, &s.snapshot.replicaReadSeed, pb.Context{ - Priority: s.snapshot.priority.ToPB(), - NotFillCache: s.snapshot.notFillCache, - TaskId: s.snapshot.mu.taskID, + req := tikvrpc.NewReplicaReadRequest(tikvrpc.CmdScan, sreq, s.snapshot.mu.replicaRead, &s.snapshot.replicaReadSeed, kvrpcpb.Context{ + Priority: s.snapshot.priority.ToPB(), + NotFillCache: s.snapshot.notFillCache, + TaskId: s.snapshot.mu.taskID, + ResourceGroupTag: s.snapshot.resourceGroupTag, }) s.snapshot.mu.RUnlock() - resp, err := sender.SendReq(bo, req, loc.Region, ReadTimeoutMedium) + resp, err := sender.SendReq(bo, req, loc.Region, client.ReadTimeoutMedium) if err != nil { return errors.Trace(err) } @@ -223,16 +229,21 @@ func (s *Scanner) getData(bo *Backoffer) error { if regionErr != nil { logutil.BgLogger().Debug("scanner getData failed", zap.Stringer("regionErr", regionErr)) - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) - if err != nil { - return errors.Trace(err) + // For other region error and the fake region error, backoff because + // there's something wrong. + // For the real EpochNotMatch error, don't backoff. + if regionErr.GetEpochNotMatch() == nil || isFakeRegionError(regionErr) { + err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) + if err != nil { + return errors.Trace(err) + } } continue } if resp.Resp == nil { return errors.Trace(tikverr.ErrBodyMissing) } - cmdScanResp := resp.Resp.(*pb.ScanResponse) + cmdScanResp := resp.Resp.(*kvrpcpb.ScanResponse) err = s.snapshot.store.CheckVisibility(s.startTS()) if err != nil { @@ -251,7 +262,7 @@ func (s *Scanner) getData(bo *Backoffer) error { return errors.Trace(err) } if msBeforeExpired > 0 { - err = bo.BackoffWithMaxSleep(BoTxnLockFast, int(msBeforeExpired), errors.Errorf("key is locked during scanning")) + err = bo.BackoffWithMaxSleepTxnLockFast(int(msBeforeExpired), errors.Errorf("key is locked during scanning")) if err != nil { return errors.Trace(err) } diff --git a/store/tikv/snapshot.go b/store/tikv/snapshot.go index 2b9926c7a2b9a..f08b6f906f537 100644 --- a/store/tikv/snapshot.go +++ b/store/tikv/snapshot.go @@ -27,15 +27,17 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/kvrpcpb" - pb "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/tidb/store/tikv/client" tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/store/tikv/unionstore" "github.com/pingcap/tidb/store/tikv/util" + "github.com/pingcap/tidb/util/sli" "go.uber.org/zap" ) @@ -103,10 +105,13 @@ type KVSnapshot struct { replicaRead kv.ReplicaReadType taskID uint64 isStaleness bool + txnScope string // MatchStoreLabels indicates the labels the store should be matched matchStoreLabels []*metapb.StoreLabel } sampleStep uint32 + // resourceGroupTag is use to set the kv request resource group tag. + resourceGroupTag []byte } // newTiKVSnapshot creates a snapshot of an TiKV store. @@ -126,6 +131,8 @@ func newTiKVSnapshot(store *KVStore, ts uint64, replicaReadSeed uint32) *KVSnaps } } +const batchGetMaxBackoff = 600000 // 10 minutes + // SetSnapshotTS resets the timestamp for reads. func (s *KVSnapshot) SetSnapshotTS(ts uint64) { // Sanity check for snapshot version. @@ -170,8 +177,8 @@ func (s *KVSnapshot) BatchGet(ctx context.Context, keys [][]byte) (map[string][] // We want [][]byte instead of []kv.Key, use some magic to save memory. bytesKeys := *(*[][]byte)(unsafe.Pointer(&keys)) - ctx = context.WithValue(ctx, TxnStartKey, s.version) - bo := NewBackofferWithVars(ctx, batchGetMaxBackoff, s.vars) + ctx = context.WithValue(ctx, retry.TxnStartKey, s.version) + bo := retry.NewBackofferWithVars(ctx, batchGetMaxBackoff, s.vars) // Create a map to collect key-values from region servers. var mu sync.Mutex @@ -228,6 +235,19 @@ type batchKeys struct { keys [][]byte } +func (b *batchKeys) relocate(bo *Backoffer, c *RegionCache) (bool, error) { + begin, end := b.keys[0], b.keys[len(b.keys)-1] + loc, err := c.LocateKey(bo, begin) + if err != nil { + return false, errors.Trace(err) + } + if !loc.Contains(end) { + return false, nil + } + b.region = loc.Region + return true, nil +} + // appendBatchKeysBySize appends keys to b. It may split the keys to make // sure each batch's size does not exceed the limit. func appendBatchKeysBySize(b []batchKeys, region RegionVerID, keys [][]byte, sizeFn func([]byte) int, limit int) []batchKeys { @@ -300,28 +320,29 @@ func (s *KVSnapshot) batchGetSingleRegion(bo *Backoffer, batch batchKeys, collec pending := batch.keys for { - isStaleness := false - var matchStoreLabels []*metapb.StoreLabel s.mu.RLock() - req := tikvrpc.NewReplicaReadRequest(tikvrpc.CmdBatchGet, &pb.BatchGetRequest{ + req := tikvrpc.NewReplicaReadRequest(tikvrpc.CmdBatchGet, &kvrpcpb.BatchGetRequest{ Keys: pending, Version: s.version, - }, s.mu.replicaRead, &s.replicaReadSeed, pb.Context{ - Priority: s.priority.ToPB(), - NotFillCache: s.notFillCache, - TaskId: s.mu.taskID, + }, s.mu.replicaRead, &s.replicaReadSeed, kvrpcpb.Context{ + Priority: s.priority.ToPB(), + NotFillCache: s.notFillCache, + TaskId: s.mu.taskID, + ResourceGroupTag: s.resourceGroupTag, }) - isStaleness = s.mu.isStaleness - matchStoreLabels = s.mu.matchStoreLabels + txnScope := s.mu.txnScope + isStaleness := s.mu.isStaleness + matchStoreLabels := s.mu.matchStoreLabels s.mu.RUnlock() - var ops []StoreSelectorOption + req.TxnScope = txnScope if isStaleness { req.EnableStaleRead() } + ops := make([]StoreSelectorOption, 0, 2) if len(matchStoreLabels) > 0 { ops = append(ops, WithMatchLabels(matchStoreLabels)) } - resp, _, _, err := cli.SendReqCtx(bo, req, batch.region, ReadTimeoutMedium, tikvrpc.TiKV, "", ops...) + resp, _, _, err := cli.SendReqCtx(bo, req, batch.region, client.ReadTimeoutMedium, tikvrpc.TiKV, "", ops...) if err != nil { return errors.Trace(err) @@ -331,17 +352,29 @@ func (s *KVSnapshot) batchGetSingleRegion(bo *Backoffer, batch batchKeys, collec return errors.Trace(err) } if regionErr != nil { - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + // For other region error and the fake region error, backoff because + // there's something wrong. + // For the real EpochNotMatch error, don't backoff. + if regionErr.GetEpochNotMatch() == nil || isFakeRegionError(regionErr) { + err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) + if err != nil { + return errors.Trace(err) + } + } + same, err := batch.relocate(bo, cli.regionCache) if err != nil { return errors.Trace(err) } + if same { + continue + } err = s.batchGetKeysByRegions(bo, pending, collectF) return errors.Trace(err) } if resp.Resp == nil { return errors.Trace(tikverr.ErrBodyMissing) } - batchGetResp := resp.Resp.(*pb.BatchGetResponse) + batchGetResp := resp.Resp.(*kvrpcpb.BatchGetResponse) var ( lockedKeys [][]byte locks []*Lock @@ -370,6 +403,9 @@ func (s *KVSnapshot) batchGetSingleRegion(bo *Backoffer, batch batchKeys, collec } } if batchGetResp.ExecDetailsV2 != nil { + readKeys := len(batchGetResp.Pairs) + readTime := float64(batchGetResp.ExecDetailsV2.GetTimeDetail().GetKvReadWallTimeMs() / 1000) + sli.ObserveReadSLI(uint64(readKeys), readTime) s.mergeExecDetail(batchGetResp.ExecDetailsV2) } if len(lockedKeys) > 0 { @@ -378,7 +414,7 @@ func (s *KVSnapshot) batchGetSingleRegion(bo *Backoffer, batch batchKeys, collec return errors.Trace(err) } if msBeforeExpired > 0 { - err = bo.BackoffWithMaxSleep(BoTxnLockFast, int(msBeforeExpired), errors.Errorf("batchGet lockedKeys: %d", len(lockedKeys))) + err = bo.BackoffWithMaxSleepTxnLockFast(int(msBeforeExpired), errors.Errorf("batchGet lockedKeys: %d", len(lockedKeys))) if err != nil { return errors.Trace(err) } @@ -394,6 +430,8 @@ func (s *KVSnapshot) batchGetSingleRegion(bo *Backoffer, batch batchKeys, collec } } +const getMaxBackoff = 600000 // 10 minutes + // Get gets the value for key k from snapshot. func (s *KVSnapshot) Get(ctx context.Context, k []byte) ([]byte, error) { @@ -401,8 +439,8 @@ func (s *KVSnapshot) Get(ctx context.Context, k []byte) ([]byte, error) { metrics.TxnCmdHistogramWithGet.Observe(time.Since(start).Seconds()) }(time.Now()) - ctx = context.WithValue(ctx, TxnStartKey, s.version) - bo := NewBackofferWithVars(ctx, getMaxBackoff, s.vars) + ctx = context.WithValue(ctx, retry.TxnStartKey, s.version) + bo := retry.NewBackofferWithVars(ctx, getMaxBackoff, s.vars) val, err := s.get(ctx, bo, k) s.recordBackoffInfo(bo) if err != nil { @@ -436,15 +474,13 @@ func (s *KVSnapshot) get(ctx context.Context, bo *Backoffer, k []byte) ([]byte, opentracing.ContextWithSpan(ctx, span1) } failpoint.Inject("snapshot-get-cache-fail", func(_ failpoint.Value) { - if bo.ctx.Value("TestSnapshotCache") != nil { + if bo.GetCtx().Value("TestSnapshotCache") != nil { panic("cache miss") } }) cli := NewClientHelper(s.store, s.resolvedLocks) - isStaleness := false - var matchStoreLabels []*metapb.StoreLabel s.mu.RLock() if s.mu.stats != nil { cli.Stats = make(map[tikvrpc.CmdType]*RPCRuntimeStats) @@ -453,16 +489,17 @@ func (s *KVSnapshot) get(ctx context.Context, bo *Backoffer, k []byte) ([]byte, }() } req := tikvrpc.NewReplicaReadRequest(tikvrpc.CmdGet, - &pb.GetRequest{ + &kvrpcpb.GetRequest{ Key: k, Version: s.version, - }, s.mu.replicaRead, &s.replicaReadSeed, pb.Context{ - Priority: s.priority.ToPB(), - NotFillCache: s.notFillCache, - TaskId: s.mu.taskID, + }, s.mu.replicaRead, &s.replicaReadSeed, kvrpcpb.Context{ + Priority: s.priority.ToPB(), + NotFillCache: s.notFillCache, + TaskId: s.mu.taskID, + ResourceGroupTag: s.resourceGroupTag, }) - isStaleness = s.mu.isStaleness - matchStoreLabels = s.mu.matchStoreLabels + isStaleness := s.mu.isStaleness + matchStoreLabels := s.mu.matchStoreLabels s.mu.RUnlock() var ops []StoreSelectorOption if isStaleness { @@ -479,7 +516,7 @@ func (s *KVSnapshot) get(ctx context.Context, bo *Backoffer, k []byte) ([]byte, if err != nil { return nil, errors.Trace(err) } - resp, _, _, err := cli.SendReqCtx(bo, req, loc.Region, ReadTimeoutShort, tikvrpc.TiKV, "", ops...) + resp, _, _, err := cli.SendReqCtx(bo, req, loc.Region, client.ReadTimeoutShort, tikvrpc.TiKV, "", ops...) if err != nil { return nil, errors.Trace(err) } @@ -488,17 +525,25 @@ func (s *KVSnapshot) get(ctx context.Context, bo *Backoffer, k []byte) ([]byte, return nil, errors.Trace(err) } if regionErr != nil { - err = bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) - if err != nil { - return nil, errors.Trace(err) + // For other region error and the fake region error, backoff because + // there's something wrong. + // For the real EpochNotMatch error, don't backoff. + if regionErr.GetEpochNotMatch() == nil || isFakeRegionError(regionErr) { + err = bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) + if err != nil { + return nil, errors.Trace(err) + } } continue } if resp.Resp == nil { return nil, errors.Trace(tikverr.ErrBodyMissing) } - cmdGetResp := resp.Resp.(*pb.GetResponse) + cmdGetResp := resp.Resp.(*kvrpcpb.GetResponse) if cmdGetResp.ExecDetailsV2 != nil { + readKeys := len(cmdGetResp.Value) + readTime := float64(cmdGetResp.ExecDetailsV2.GetTimeDetail().GetKvReadWallTimeMs() / 1000) + sli.ObserveReadSLI(uint64(readKeys), readTime) s.mergeExecDetail(cmdGetResp.ExecDetailsV2) } val := cmdGetResp.GetValue() @@ -522,7 +567,7 @@ func (s *KVSnapshot) get(ctx context.Context, bo *Backoffer, k []byte) ([]byte, return nil, errors.Trace(err) } if msBeforeExpired > 0 { - err = bo.BackoffWithMaxSleep(BoTxnLockFast, int(msBeforeExpired), errors.New(keyErr.String())) + err = bo.BackoffWithMaxSleepTxnLockFast(int(msBeforeExpired), errors.New(keyErr.String())) if err != nil { return nil, errors.Trace(err) } @@ -533,7 +578,7 @@ func (s *KVSnapshot) get(ctx context.Context, bo *Backoffer, k []byte) ([]byte, } } -func (s *KVSnapshot) mergeExecDetail(detail *pb.ExecDetailsV2) { +func (s *KVSnapshot) mergeExecDetail(detail *kvrpcpb.ExecDetailsV2) { s.mu.Lock() defer s.mu.Unlock() if detail == nil || s.mu.stats == nil { @@ -561,49 +606,6 @@ func (s *KVSnapshot) IterReverse(k []byte) (unionstore.Iterator, error) { return scanner, errors.Trace(err) } -// SetOption sets an option with a value, when val is nil, uses the default -// value of this option. Only ReplicaRead is supported for snapshot -func (s *KVSnapshot) SetOption(opt int, val interface{}) { - switch opt { - case kv.ReplicaRead: - s.mu.Lock() - s.mu.replicaRead = val.(kv.ReplicaReadType) - s.mu.Unlock() - case kv.TaskID: - s.mu.Lock() - s.mu.taskID = val.(uint64) - s.mu.Unlock() - case kv.CollectRuntimeStats: - s.mu.Lock() - s.mu.stats = val.(*SnapshotRuntimeStats) - s.mu.Unlock() - case kv.SampleStep: - s.sampleStep = val.(uint32) - case kv.IsStalenessReadOnly: - s.mu.Lock() - s.mu.isStaleness = val.(bool) - s.mu.Unlock() - case kv.MatchStoreLabels: - s.mu.Lock() - s.mu.matchStoreLabels = val.([]*metapb.StoreLabel) - s.mu.Unlock() - } -} - -// DelOption deletes an option. -func (s *KVSnapshot) DelOption(opt int) { - switch opt { - case kv.ReplicaRead: - s.mu.Lock() - s.mu.replicaRead = kv.ReplicaReadLeader - s.mu.Unlock() - case kv.CollectRuntimeStats: - s.mu.Lock() - s.mu.stats = nil - s.mu.Unlock() - } -} - // SetNotFillCache indicates whether tikv should skip filling cache when // loading data. func (s *KVSnapshot) SetNotFillCache(b bool) { @@ -615,16 +617,70 @@ func (s *KVSnapshot) SetKeyOnly(b bool) { s.keyOnly = b } +// SetReplicaRead sets up the replica read type. +func (s *KVSnapshot) SetReplicaRead(readType kv.ReplicaReadType) { + s.mu.Lock() + defer s.mu.Unlock() + s.mu.replicaRead = readType +} + // SetIsolationLevel sets the isolation level used to scan data from tikv. func (s *KVSnapshot) SetIsolationLevel(level IsoLevel) { s.isolationLevel = level } +// SetSampleStep skips 'step - 1' number of keys after each returned key. +func (s *KVSnapshot) SetSampleStep(step uint32) { + s.sampleStep = step +} + // SetPriority sets the priority for tikv to execute commands. func (s *KVSnapshot) SetPriority(pri Priority) { s.priority = pri } +// SetTaskID marks current task's unique ID to allow TiKV to schedule +// tasks more fairly. +func (s *KVSnapshot) SetTaskID(id uint64) { + s.mu.Lock() + defer s.mu.Unlock() + s.mu.taskID = id +} + +// SetRuntimeStats sets the stats to collect runtime statistics. +// Set it to nil to clear stored stats. +func (s *KVSnapshot) SetRuntimeStats(stats *SnapshotRuntimeStats) { + s.mu.Lock() + defer s.mu.Unlock() + s.mu.stats = stats +} + +// SetTxnScope sets up the txn scope. +func (s *KVSnapshot) SetTxnScope(txnScope string) { + s.mu.Lock() + defer s.mu.Unlock() + s.mu.txnScope = txnScope +} + +// SetIsStatenessReadOnly indicates whether the transaction is staleness read only transaction +func (s *KVSnapshot) SetIsStatenessReadOnly(b bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.mu.isStaleness = b +} + +// SetMatchStoreLabels sets up labels to filter target stores. +func (s *KVSnapshot) SetMatchStoreLabels(labels []*metapb.StoreLabel) { + s.mu.Lock() + defer s.mu.Unlock() + s.mu.matchStoreLabels = labels +} + +// SetResourceGroupTag sets resource group of the kv request. +func (s *KVSnapshot) SetResourceGroupTag(tag []byte) { + s.resourceGroupTag = tag +} + // SnapCacheHitCount gets the snapshot cache hit count. Only for test. func (s *KVSnapshot) SnapCacheHitCount() int { return int(atomic.LoadInt64(&s.mu.hitCnt)) @@ -637,14 +693,14 @@ func (s *KVSnapshot) SnapCacheSize() int { return len(s.mu.cached) } -func extractLockFromKeyErr(keyErr *pb.KeyError) (*Lock, error) { +func extractLockFromKeyErr(keyErr *kvrpcpb.KeyError) (*Lock, error) { if locked := keyErr.GetLocked(); locked != nil { return NewLock(locked), nil } return nil, extractKeyErr(keyErr) } -func extractKeyErr(keyErr *pb.KeyError) error { +func extractKeyErr(keyErr *kvrpcpb.KeyError) error { if val, err := util.MockRetryableErrorResp.Eval(); err == nil { if val.(bool) { keyErr.Conflict = nil @@ -679,7 +735,7 @@ func extractKeyErr(keyErr *pb.KeyError) error { func (s *KVSnapshot) recordBackoffInfo(bo *Backoffer) { s.mu.RLock() - if s.mu.stats == nil || bo.totalSleep == 0 { + if s.mu.stats == nil || bo.GetTotalSleep() == 0 { s.mu.RUnlock() return } @@ -690,14 +746,14 @@ func (s *KVSnapshot) recordBackoffInfo(bo *Backoffer) { return } if s.mu.stats.backoffSleepMS == nil { - s.mu.stats.backoffSleepMS = bo.backoffSleepMS - s.mu.stats.backoffTimes = bo.backoffTimes + s.mu.stats.backoffSleepMS = bo.GetBackoffSleepMS() + s.mu.stats.backoffTimes = bo.GetBackoffTimes() return } - for k, v := range bo.backoffSleepMS { + for k, v := range bo.GetBackoffSleepMS() { s.mu.stats.backoffSleepMS[k] += v } - for k, v := range bo.backoffTimes { + for k, v := range bo.GetBackoffTimes() { s.mu.stats.backoffTimes[k] += v } } @@ -726,8 +782,8 @@ func (s *KVSnapshot) mergeRegionRequestStats(stats map[tikvrpc.CmdType]*RPCRunti // SnapshotRuntimeStats records the runtime stats of snapshot. type SnapshotRuntimeStats struct { rpcStats RegionRequestRuntimeStats - backoffSleepMS map[BackoffType]int - backoffTimes map[BackoffType]int + backoffSleepMS map[string]int + backoffTimes map[string]int scanDetail *util.ScanDetail timeDetail *util.TimeDetail } @@ -741,8 +797,8 @@ func (rs *SnapshotRuntimeStats) Clone() *SnapshotRuntimeStats { } } if len(rs.backoffSleepMS) > 0 { - newRs.backoffSleepMS = make(map[BackoffType]int) - newRs.backoffTimes = make(map[BackoffType]int) + newRs.backoffSleepMS = make(map[string]int) + newRs.backoffTimes = make(map[string]int) for k, v := range rs.backoffSleepMS { newRs.backoffSleepMS[k] += v } @@ -763,10 +819,10 @@ func (rs *SnapshotRuntimeStats) Merge(other *SnapshotRuntimeStats) { } if len(other.backoffSleepMS) > 0 { if rs.backoffSleepMS == nil { - rs.backoffSleepMS = make(map[BackoffType]int) + rs.backoffSleepMS = make(map[string]int) } if rs.backoffTimes == nil { - rs.backoffTimes = make(map[BackoffType]int) + rs.backoffTimes = make(map[string]int) } for k, v := range other.backoffSleepMS { rs.backoffSleepMS[k] += v @@ -787,7 +843,7 @@ func (rs *SnapshotRuntimeStats) String() string { } ms := rs.backoffSleepMS[k] d := time.Duration(ms) * time.Millisecond - buf.WriteString(fmt.Sprintf("%s_backoff:{num:%d, total_time:%s}", k.String(), v, util.FormatDuration(d))) + buf.WriteString(fmt.Sprintf("%s_backoff:{num:%d, total_time:%s}", k, v, util.FormatDuration(d))) } timeDetail := rs.timeDetail.String() if timeDetail != "" { diff --git a/store/tikv/split_region.go b/store/tikv/split_region.go index 5839aa4d73c96..045b4ccb9560d 100644 --- a/store/tikv/split_region.go +++ b/store/tikv/split_region.go @@ -24,9 +24,11 @@ import ( "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/tidb/store/tikv/client" tikverr "github.com/pingcap/tidb/store/tikv/error" "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/store/tikv/util" pd "github.com/tikv/pd/client" @@ -56,7 +58,7 @@ func (s *KVStore) splitBatchRegionsReq(bo *Backoffer, keys [][]byte, scatter boo return nil, nil } // The first time it enters this function. - if bo.totalSleep == 0 { + if bo.GetTotalSleep() == 0 { logutil.BgLogger().Info("split batch regions request", zap.Int("split key count", len(keys)), zap.Int("batch count", len(batches)), @@ -76,8 +78,8 @@ func (s *KVStore) splitBatchRegionsReq(bo *Backoffer, keys [][]byte, scatter boo util.WithRecovery(func() { select { case ch <- s.batchSendSingleRegion(backoffer, b, scatter, tableID): - case <-bo.ctx.Done(): - ch <- singleBatchResp{err: bo.ctx.Err()} + case <-bo.GetCtx().Done(): + ch <- singleBatchResp{err: bo.GetCtx().Err()} } }, func(r interface{}) { if r != nil { @@ -110,8 +112,8 @@ func (s *KVStore) splitBatchRegionsReq(bo *Backoffer, keys [][]byte, scatter boo func (s *KVStore) batchSendSingleRegion(bo *Backoffer, batch batch, scatter bool, tableID *int64) singleBatchResp { if val, err := util.MockSplitRegionTimeout.Eval(); err == nil { if val.(bool) { - if _, ok := bo.ctx.Deadline(); ok { - <-bo.ctx.Done() + if _, ok := bo.GetCtx().Deadline(); ok { + <-bo.GetCtx().Done() } } } @@ -122,8 +124,8 @@ func (s *KVStore) batchSendSingleRegion(bo *Backoffer, batch batch, scatter bool Priority: kvrpcpb.CommandPri_Normal, }) - sender := NewRegionRequestSender(s.regionCache, s.client) - resp, err := sender.SendReq(bo, req, batch.regionID, ReadTimeoutShort) + sender := NewRegionRequestSender(s.regionCache, s.GetTiKVClient()) + resp, err := sender.SendReq(bo, req, batch.regionID, client.ReadTimeoutShort) batchResp := singleBatchResp{resp: resp} if err != nil { @@ -136,7 +138,7 @@ func (s *KVStore) batchSendSingleRegion(bo *Backoffer, batch batch, scatter bool return batchResp } if regionErr != nil { - err := bo.Backoff(BoRegionMiss, errors.New(regionErr.String())) + err := bo.Backoff(retry.BoRegionMiss, errors.New(regionErr.String())) if err != nil { batchResp.err = errors.Trace(err) return batchResp @@ -192,9 +194,14 @@ func (s *KVStore) batchSendSingleRegion(bo *Backoffer, batch batch, scatter bool return batchResp } +const ( + splitRegionBackoff = 20000 + maxSplitRegionsBackoff = 120000 +) + // SplitRegions splits regions by splitKeys. func (s *KVStore) SplitRegions(ctx context.Context, splitKeys [][]byte, scatter bool, tableID *int64) (regionIDs []uint64, err error) { - bo := NewBackofferWithVars(ctx, int(math.Min(float64(len(splitKeys))*splitRegionBackoff, maxSplitRegionsBackoff)), nil) + bo := retry.NewBackofferWithVars(ctx, int(math.Min(float64(len(splitKeys))*splitRegionBackoff, maxSplitRegionsBackoff)), nil) resp, err := s.splitBatchRegionsReq(bo, splitKeys, scatter, tableID) regionIDs = make([]uint64, 0, len(splitKeys)) if resp != nil && resp.Resp != nil { @@ -215,7 +222,7 @@ func (s *KVStore) scatterRegion(bo *Backoffer, regionID uint64, tableID *int64) if tableID != nil { opts = append(opts, pd.WithGroup(fmt.Sprintf("%v", *tableID))) } - _, err := s.pdClient.ScatterRegions(bo.ctx, []uint64{regionID}, opts...) + _, err := s.pdClient.ScatterRegions(bo.GetCtx(), []uint64{regionID}, opts...) if val, err2 := util.MockScatterRegionTimeout.Eval(); err2 == nil { if val.(bool) { @@ -226,7 +233,7 @@ func (s *KVStore) scatterRegion(bo *Backoffer, regionID uint64, tableID *int64) if err == nil { break } - err = bo.Backoff(BoPDRPC, errors.New(err.Error())) + err = bo.Backoff(retry.BoPDRPC, errors.New(err.Error())) if err != nil { return errors.Trace(err) } @@ -273,6 +280,8 @@ func (s *KVStore) preSplitRegion(ctx context.Context, group groupedMutations) bo return true } +const waitScatterRegionFinishBackoff = 120000 + // WaitScatterRegionFinish implements SplittableStore interface. // backOff is the back off time of the wait scatter region.(Milliseconds) // if backOff <= 0, the default wait scatter back off time will be used. @@ -283,7 +292,7 @@ func (s *KVStore) WaitScatterRegionFinish(ctx context.Context, regionID uint64, logutil.BgLogger().Info("wait scatter region", zap.Uint64("regionID", regionID), zap.Int("backoff(ms)", backOff)) - bo := NewBackofferWithVars(ctx, backOff, nil) + bo := retry.NewBackofferWithVars(ctx, backOff, nil) logFreq := 0 for { resp, err := s.pdClient.GetOperator(ctx, regionID) @@ -310,9 +319,9 @@ func (s *KVStore) WaitScatterRegionFinish(ctx context.Context, regionID uint64, logFreq++ } if err != nil { - err = bo.Backoff(BoRegionMiss, errors.New(err.Error())) + err = bo.Backoff(retry.BoRegionMiss, errors.New(err.Error())) } else { - err = bo.Backoff(BoRegionMiss, errors.New("wait scatter region timeout")) + err = bo.Backoff(retry.BoRegionMiss, errors.New("wait scatter region timeout")) } if err != nil { return errors.Trace(err) @@ -322,7 +331,7 @@ func (s *KVStore) WaitScatterRegionFinish(ctx context.Context, regionID uint64, // CheckRegionInScattering uses to check whether scatter region finished. func (s *KVStore) CheckRegionInScattering(regionID uint64) (bool, error) { - bo := NewBackofferWithVars(context.Background(), locateRegionMaxBackoff, nil) + bo := retry.NewBackofferWithVars(context.Background(), locateRegionMaxBackoff, nil) for { resp, err := s.pdClient.GetOperator(context.Background(), regionID) if err == nil && resp != nil { @@ -331,7 +340,7 @@ func (s *KVStore) CheckRegionInScattering(regionID uint64) (bool, error) { } } if err != nil { - err = bo.Backoff(BoRegionMiss, errors.New(err.Error())) + err = bo.Backoff(retry.BoRegionMiss, errors.New(err.Error())) } else { return true, nil } diff --git a/store/tikv/store_type.go b/store/tikv/store_type.go index e77a5580d6447..893885d929955 100644 --- a/store/tikv/store_type.go +++ b/store/tikv/store_type.go @@ -20,42 +20,30 @@ import ( "github.com/pingcap/tidb/store/tikv/tikvrpc" ) -// AccessMode uses to index stores for different region cache access requirements. -type AccessMode int +// accessMode uses to index stores for different region cache access requirements. +type accessMode int const ( - // TiKVOnly indicates stores list that use for TiKv access(include both leader request and follower read). - TiKVOnly AccessMode = iota - // TiFlashOnly indicates stores list that use for TiFlash request. - TiFlashOnly - // NumAccessMode reserved to keep max access mode value. - NumAccessMode + // tiKVOnly indicates stores list that use for TiKv access(include both leader request and follower read). + tiKVOnly accessMode = iota + // tiFlashOnly indicates stores list that use for TiFlash request. + tiFlashOnly + // numAccessMode reserved to keep max access mode value. + numAccessMode ) -func (a AccessMode) String() string { +func (a accessMode) String() string { switch a { - case TiKVOnly: + case tiKVOnly: return "TiKvOnly" - case TiFlashOnly: + case tiFlashOnly: return "TiFlashOnly" default: return fmt.Sprintf("%d", a) } } -// Constants to determine engine type. -// They should be synced with PD. -const ( - engineLabelKey = "engine" - engineLabelTiFlash = "tiflash" -) - // GetStoreTypeByMeta gets store type by store meta pb. func GetStoreTypeByMeta(store *metapb.Store) tikvrpc.EndpointType { - for _, label := range store.Labels { - if label.Key == engineLabelKey && label.Value == engineLabelTiFlash { - return tikvrpc.TiFlash - } - } - return tikvrpc.TiKV + return tikvrpc.GetStoreTypeByMeta(store) } diff --git a/store/tikv/test_probe.go b/store/tikv/test_probe.go index a6a4f8d826655..eb813335a6dd7 100644 --- a/store/tikv/test_probe.go +++ b/store/tikv/test_probe.go @@ -21,7 +21,8 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/kvrpcpb" - pb "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/store/tikv/unionstore" pd "github.com/tikv/pd/client" @@ -66,8 +67,9 @@ func (s StoreProbe) ClearTxnLatches() { // SendTxnHeartbeat renews a txn's ttl. func (s StoreProbe) SendTxnHeartbeat(ctx context.Context, key []byte, startTS uint64, ttl uint64) (uint64, error) { - bo := NewBackofferWithVars(ctx, PrewriteMaxBackoff, nil) - return sendTxnHeartBeat(bo, s.KVStore, key, startTS, ttl) + bo := retry.NewBackofferWithVars(ctx, PrewriteMaxBackoff, nil) + newTTL, _, err := sendTxnHeartBeat(bo, s.KVStore, key, startTS, ttl) + return newTTL, err } // LoadSafePoint from safepoint kv. @@ -80,6 +82,23 @@ func (s StoreProbe) SaveSafePoint(v uint64) error { return saveSafePoint(s.GetSafePointKV(), v) } +// SetRegionCacheStore is used to set a store in region cache, for testing only +func (s StoreProbe) SetRegionCacheStore(id uint64, storeType tikvrpc.EndpointType, state uint64, labels []*metapb.StoreLabel) { + s.regionCache.storeMu.Lock() + defer s.regionCache.storeMu.Unlock() + s.regionCache.storeMu.stores[id] = &Store{ + storeID: id, + storeType: storeType, + state: state, + labels: labels, + } +} + +// SetSafeTS is used to set safeTS for the store with `storeID` +func (s StoreProbe) SetSafeTS(storeID, safeTS uint64) { + s.setSafeTS(storeID, safeTS) +} + // TxnProbe wraps a txn and exports internal states for testing purpose. type TxnProbe struct { *KVTxn @@ -266,12 +285,12 @@ func (c CommitterProbe) PrewriteAllMutations(ctx context.Context) error { // PrewriteMutations performs the first phase of commit for given keys. func (c CommitterProbe) PrewriteMutations(ctx context.Context, mutations CommitterMutations) error { - return c.prewriteMutations(NewBackofferWithVars(ctx, PrewriteMaxBackoff, nil), mutations) + return c.prewriteMutations(retry.NewBackofferWithVars(ctx, PrewriteMaxBackoff, nil), mutations) } // CommitMutations performs the second phase of commit. func (c CommitterProbe) CommitMutations(ctx context.Context) error { - return c.commitMutations(NewBackofferWithVars(ctx, int(atomic.LoadUint64(&CommitMaxBackoff)), nil), c.mutationsOfKeys([][]byte{c.primaryKey})) + return c.commitMutations(retry.NewBackofferWithVars(ctx, int(atomic.LoadUint64(&CommitMaxBackoff)), nil), c.mutationsOfKeys([][]byte{c.primaryKey})) } // MutationsOfKeys returns mutations match the keys. @@ -281,7 +300,7 @@ func (c CommitterProbe) MutationsOfKeys(keys [][]byte) CommitterMutations { // PessimisticRollbackMutations rolls mutations back. func (c CommitterProbe) PessimisticRollbackMutations(ctx context.Context, muts CommitterMutations) error { - return c.pessimisticRollbackMutations(NewBackofferWithVars(ctx, pessimisticRollbackMaxBackoff, nil), muts) + return c.pessimisticRollbackMutations(retry.NewBackofferWithVars(ctx, pessimisticRollbackMaxBackoff, nil), muts) } // Cleanup cleans dirty data of a committer. @@ -366,7 +385,7 @@ func (c CommitterProbe) SetPrimaryKeyBlocker(ac, bk chan struct{}) { // CleanupMutations performs the clean up phase. func (c CommitterProbe) CleanupMutations(ctx context.Context) error { - bo := NewBackofferWithVars(ctx, cleanupMaxBackoff, nil) + bo := retry.NewBackofferWithVars(ctx, cleanupMaxBackoff, nil) return c.cleanupMutations(bo, c.mutations) } @@ -386,7 +405,7 @@ func (s SnapshotProbe) RecordBackoffInfo(bo *Backoffer) { } // MergeExecDetail merges exec stats into snapshot's stats. -func (s SnapshotProbe) MergeExecDetail(detail *pb.ExecDetailsV2) { +func (s SnapshotProbe) MergeExecDetail(detail *kvrpcpb.ExecDetailsV2) { s.mergeExecDetail(detail) } @@ -402,7 +421,7 @@ type LockProbe struct { } // ExtractLockFromKeyErr makes a Lock based on a key error. -func (l LockProbe) ExtractLockFromKeyErr(err *pb.KeyError) (*Lock, error) { +func (l LockProbe) ExtractLockFromKeyErr(err *kvrpcpb.KeyError) (*Lock, error) { return extractLockFromKeyErr(err) } @@ -434,13 +453,13 @@ func (l LockResolverProbe) ResolveLockAsync(bo *Backoffer, lock *Lock, status Tx // ResolveLock resolves single lock. func (l LockResolverProbe) ResolveLock(ctx context.Context, lock *Lock) error { - bo := NewBackofferWithVars(ctx, pessimisticLockMaxBackoff, nil) + bo := retry.NewBackofferWithVars(ctx, pessimisticLockMaxBackoff, nil) return l.resolveLock(bo, lock, TxnStatus{}, false, make(map[RegionVerID]struct{})) } // ResolvePessimisticLock resolves single pessimistic lock. func (l LockResolverProbe) ResolvePessimisticLock(ctx context.Context, lock *Lock) error { - bo := NewBackofferWithVars(ctx, pessimisticLockMaxBackoff, nil) + bo := retry.NewBackofferWithVars(ctx, pessimisticLockMaxBackoff, nil) return l.resolvePessimisticLock(bo, lock, make(map[RegionVerID]struct{})) } diff --git a/store/tikv/tests/1pc_test.go b/store/tikv/tests/1pc_test.go index 6ff57700f4e3c..2b7e5501f1600 100644 --- a/store/tikv/tests/1pc_test.go +++ b/store/tikv/tests/1pc_test.go @@ -19,6 +19,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/pingcap/tidb/store/tikv/mockstore" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/store/tikv/util" ) @@ -179,7 +180,7 @@ func (s *testOnePCSuite) Test1PCIsolation(c *C) { func (s *testOnePCSuite) Test1PCDisallowMultiRegion(c *C) { // This test doesn't support tikv mode. - if *WithTiKV { + if *mockstore.WithTiKV { return } @@ -247,7 +248,7 @@ func (s *testOnePCSuite) Test1PCLinearizability(c *C) { func (s *testOnePCSuite) Test1PCWithMultiDC(c *C) { // It requires setting placement rules to run with TiKV - if *WithTiKV { + if *mockstore.WithTiKV { return } diff --git a/store/tikv/tests/2pc_fail_test.go b/store/tikv/tests/2pc_fail_test.go index d8b35e4b28c53..8b00dba08996e 100644 --- a/store/tikv/tests/2pc_fail_test.go +++ b/store/tikv/tests/2pc_fail_test.go @@ -93,6 +93,22 @@ func (s *testCommitterSuite) TestFailCommitPrimaryKeyError(c *C) { c.Assert(terror.ErrorNotEqual(err, terror.ErrResultUndetermined), IsTrue) } +// TestFailCommitPrimaryRPCErrorThenKeyError tests KeyError overwrites the undeterminedErr. +func (s *testCommitterSuite) TestFailCommitPrimaryRPCErrorThenKeyError(c *C) { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/mockstore/mocktikv/rpcCommitResult", `1*return("timeout")->return("keyError")`), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/mockstore/mocktikv/rpcCommitResult"), IsNil) + }() + // Ensure it returns the original error without wrapped to ErrResultUndetermined + // if it meets KeyError. + t3 := s.begin(c) + err := t3.Set([]byte("c"), []byte("c1")) + c.Assert(err, IsNil) + err = t3.Commit(context.Background()) + c.Assert(err, NotNil) + c.Assert(terror.ErrorEqual(err, terror.ErrResultUndetermined), IsFalse) +} + func (s *testCommitterSuite) TestFailCommitTimeout(c *C) { c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/mockstore/mocktikv/rpcCommitTimeout", `return(true)`), IsNil) defer func() { diff --git a/store/tikv/tests/2pc_test.go b/store/tikv/tests/2pc_test.go index 6d7d7e89d1a8e..2ecb6267b0f50 100644 --- a/store/tikv/tests/2pc_test.go +++ b/store/tikv/tests/2pc_test.go @@ -28,7 +28,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/kvrpcpb" - pb "github.com/pingcap/kvproto/pkg/kvrpcpb" drivertxn "github.com/pingcap/tidb/store/driver/txn" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/config" @@ -57,6 +56,7 @@ func (s *testCommitterSuite) SetUpSuite(c *C) { atomic.StoreUint64(&tikv.ManagedLockTTL, 3000) // 3s s.OneByOneSuite.SetUpSuite(c) atomic.StoreUint64(&tikv.CommitMaxBackoff, 1000) + atomic.StoreUint64(&tikv.VeryLongMaxBackoff, 1000) } func (s *testCommitterSuite) SetUpTest(c *C) { @@ -91,6 +91,7 @@ func (s *testCommitterSuite) SetUpTest(c *C) { func (s *testCommitterSuite) TearDownSuite(c *C) { atomic.StoreUint64(&tikv.CommitMaxBackoff, 20000) + atomic.StoreUint64(&tikv.VeryLongMaxBackoff, 600000) s.store.Close() s.OneByOneSuite.TearDownSuite(c) } @@ -104,7 +105,7 @@ func (s *testCommitterSuite) begin(c *C) tikv.TxnProbe { func (s *testCommitterSuite) beginAsyncCommit(c *C) tikv.TxnProbe { txn, err := s.store.Begin() c.Assert(err, IsNil) - txn.SetOption(kv.EnableAsyncCommit, true) + txn.SetEnableAsyncCommit(true) return txn } @@ -602,12 +603,12 @@ func (s *testCommitterSuite) TestRejectCommitTS(c *C) { // Use max.Uint64 to read the data and success. // That means the final commitTS > startTS+2, it's not the one we provide. // So we cover the rety commitTS logic. - txn1, err := s.store.BeginWithStartTS(oracle.GlobalTxnScope, committer.GetStartTS()+2) + txn1, err := s.store.BeginWithOption(tikv.DefaultStartTSOption().SetStartTS(committer.GetStartTS() + 2)) c.Assert(err, IsNil) _, err = txn1.Get(bo.GetCtx(), []byte("x")) c.Assert(tikverr.IsErrNotFound(err), IsTrue) - txn2, err := s.store.BeginWithStartTS(oracle.GlobalTxnScope, math.MaxUint64) + txn2, err := s.store.BeginWithOption(tikv.DefaultStartTSOption().SetStartTS(math.MaxUint64)) c.Assert(err, IsNil) val, err := txn2.Get(bo.GetCtx(), []byte("x")) c.Assert(err, IsNil) @@ -712,8 +713,7 @@ func (s *testCommitterSuite) TestPessimisticLockReturnValues(c *C) { txn = s.begin(c) txn.SetPessimistic(true) lockCtx := &kv.LockCtx{ForUpdateTS: txn.StartTS(), WaitStartTime: time.Now()} - lockCtx.ReturnValues = true - lockCtx.Values = map[string]kv.ReturnedValue{} + lockCtx.InitReturnValues(2) c.Assert(txn.LockKeys(context.Background(), lockCtx, key, key2), IsNil) c.Assert(lockCtx.Values, HasLen, 2) c.Assert(lockCtx.Values[string(key)].Value, BytesEquals, key) @@ -724,7 +724,7 @@ func (s *testCommitterSuite) TestPessimisticLockReturnValues(c *C) { func (s *testCommitterSuite) TestElapsedTTL(c *C) { key := []byte("key") txn := s.begin(c) - txn.SetStartTS(oracle.ComposeTS(oracle.GetPhysical(time.Now().Add(time.Second*10)), 1)) + txn.SetStartTS(oracle.GoTimeToTS(time.Now().Add(time.Second*10)) + 1) txn.SetPessimistic(true) time.Sleep(time.Millisecond * 100) lockCtx := &kv.LockCtx{ @@ -1024,7 +1024,7 @@ func (s *testCommitterSuite) TestPessimisticLockPrimary(c *C) { c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/txnNotFoundRetTTL"), IsNil) c.Assert(err, IsNil) waitErr := <-doneCh - c.Assert(tikverr.ErrLockWaitTimeout.Equal(waitErr), IsTrue) + c.Assert(tikverr.ErrLockWaitTimeout, Equals, waitErr) } func (s *testCommitterSuite) TestResolvePessimisticLock(c *C) { @@ -1042,10 +1042,10 @@ func (s *testCommitterSuite) TestResolvePessimisticLock(c *C) { c.Assert(err, IsNil) mutation := commit.MutationsOfKeys([][]byte{untouchedIndexKey, noValueIndexKey}) c.Assert(mutation.Len(), Equals, 2) - c.Assert(mutation.GetOp(0), Equals, pb.Op_Lock) + c.Assert(mutation.GetOp(0), Equals, kvrpcpb.Op_Lock) c.Assert(mutation.GetKey(0), BytesEquals, untouchedIndexKey) c.Assert(mutation.GetValue(0), BytesEquals, untouchedIndexValue) - c.Assert(mutation.GetOp(1), Equals, pb.Op_Lock) + c.Assert(mutation.GetOp(1), Equals, kvrpcpb.Op_Lock) c.Assert(mutation.GetKey(1), BytesEquals, noValueIndexKey) c.Assert(mutation.GetValue(1), BytesEquals, []byte{}) } diff --git a/store/tikv/tests/async_commit_fail_test.go b/store/tikv/tests/async_commit_fail_test.go index a791f16c54e86..e6d15f847bbe7 100644 --- a/store/tikv/tests/async_commit_fail_test.go +++ b/store/tikv/tests/async_commit_fail_test.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/store/tikv" tikverr "github.com/pingcap/tidb/store/tikv/error" + "github.com/pingcap/tidb/store/tikv/mockstore" "github.com/pingcap/tidb/store/tikv/util" ) @@ -43,7 +44,7 @@ func (s *testAsyncCommitFailSuite) SetUpTest(c *C) { // committing primary region task. func (s *testAsyncCommitFailSuite) TestFailAsyncCommitPrewriteRpcErrors(c *C) { // This test doesn't support tikv mode because it needs setting failpoint in unistore. - if *WithTiKV { + if *mockstore.WithTiKV { return } @@ -75,7 +76,7 @@ func (s *testAsyncCommitFailSuite) TestFailAsyncCommitPrewriteRpcErrors(c *C) { func (s *testAsyncCommitFailSuite) TestAsyncCommitPrewriteCancelled(c *C) { // This test doesn't support tikv mode because it needs setting failpoint in unistore. - if *WithTiKV { + if *mockstore.WithTiKV { return } @@ -135,7 +136,7 @@ func (s *testAsyncCommitFailSuite) TestPointGetWithAsyncCommit(c *C) { func (s *testAsyncCommitFailSuite) TestSecondaryListInPrimaryLock(c *C) { // This test doesn't support tikv mode. - if *WithTiKV { + if *mockstore.WithTiKV { return } @@ -233,3 +234,51 @@ func (s *testAsyncCommitFailSuite) TestAsyncCommitContextCancelCausingUndetermin c.Assert(err, NotNil) c.Assert(txn.GetCommitter().GetUndeterminedErr(), NotNil) } + +// TestAsyncCommitRPCErrorThenWriteConflict verifies that the determined failure error overwrites undetermined error. +func (s *testAsyncCommitFailSuite) TestAsyncCommitRPCErrorThenWriteConflict(c *C) { + // This test doesn't support tikv mode because it needs setting failpoint in unistore. + if *mockstore.WithTiKV { + return + } + + txn := s.beginAsyncCommit(c) + err := txn.Set([]byte("a"), []byte("va")) + c.Assert(err, IsNil) + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/unistore/rpcPrewriteResult", `1*return("timeout")->return("writeConflict")`), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/unistore/rpcPrewriteResult"), IsNil) + }() + + ctx := context.WithValue(context.Background(), util.SessionID, uint64(1)) + err = txn.Commit(ctx) + c.Assert(err, NotNil) + c.Assert(txn.GetCommitter().GetUndeterminedErr(), IsNil) +} + +// TestAsyncCommitRPCErrorThenWriteConflictInChild verifies that the determined failure error in a child recursion +// overwrites the undetermined error in the parent. +func (s *testAsyncCommitFailSuite) TestAsyncCommitRPCErrorThenWriteConflictInChild(c *C) { + // This test doesn't support tikv mode because it needs setting failpoint in unistore. + if *mockstore.WithTiKV { + return + } + + txn := s.beginAsyncCommit(c) + err := txn.Set([]byte("a"), []byte("va")) + c.Assert(err, IsNil) + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/unistore/rpcPrewriteResult", `1*return("timeout")->return("writeConflict")`), IsNil) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/forceRecursion", `return`), IsNil) + + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/unistore/rpcPrewriteResult"), IsNil) + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/forceRecursion"), IsNil) + }() + + ctx := context.WithValue(context.Background(), util.SessionID, uint64(1)) + err = txn.Commit(ctx) + c.Assert(err, NotNil) + c.Assert(txn.GetCommitter().GetUndeterminedErr(), IsNil) +} diff --git a/store/tikv/tests/async_commit_test.go b/store/tikv/tests/async_commit_test.go index 0f4985fa7ab86..4afb548204c13 100644 --- a/store/tikv/tests/async_commit_test.go +++ b/store/tikv/tests/async_commit_test.go @@ -28,7 +28,7 @@ import ( "github.com/pingcap/tidb/store/mockstore/unistore" "github.com/pingcap/tidb/store/tikv" tikverr "github.com/pingcap/tidb/store/tikv/error" - "github.com/pingcap/tidb/store/tikv/kv" + "github.com/pingcap/tidb/store/tikv/mockstore" "github.com/pingcap/tidb/store/tikv/mockstore/cluster" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/store/tikv/tikvrpc" @@ -48,7 +48,7 @@ type testAsyncCommitCommon struct { } func (s *testAsyncCommitCommon) setUpTest(c *C) { - if *WithTiKV { + if *mockstore.WithTiKV { s.store = NewTestStore(c) return } @@ -127,14 +127,14 @@ func (s *testAsyncCommitCommon) mustGetNoneFromSnapshot(c *C, version uint64, ke func (s *testAsyncCommitCommon) beginAsyncCommitWithLinearizability(c *C) tikv.TxnProbe { txn := s.beginAsyncCommit(c) - txn.SetOption(kv.GuaranteeLinearizability, true) + txn.SetCausalConsistency(false) return txn } func (s *testAsyncCommitCommon) beginAsyncCommit(c *C) tikv.TxnProbe { txn, err := s.store.Begin() c.Assert(err, IsNil) - txn.SetOption(kv.EnableAsyncCommit, true) + txn.SetEnableAsyncCommit(true) return tikv.TxnProbe{KVTxn: txn} } @@ -160,7 +160,7 @@ func (s *testAsyncCommitSuite) SetUpTest(c *C) { func (s *testAsyncCommitSuite) lockKeysWithAsyncCommit(c *C, keys, values [][]byte, primaryKey, primaryValue []byte, commitPrimary bool) (uint64, uint64) { txn, err := s.store.Begin() c.Assert(err, IsNil) - txn.SetOption(kv.EnableAsyncCommit, true) + txn.SetEnableAsyncCommit(true) for i, k := range keys { if len(values[i]) > 0 { err = txn.Set(k, values[i]) @@ -196,7 +196,7 @@ func (s *testAsyncCommitSuite) lockKeysWithAsyncCommit(c *C, keys, values [][]by func (s *testAsyncCommitSuite) TestCheckSecondaries(c *C) { // This test doesn't support tikv mode. - if *WithTiKV { + if *mockstore.WithTiKV { return } @@ -403,7 +403,7 @@ func (s *testAsyncCommitSuite) TestAsyncCommitLinearizability(c *C) { // TestAsyncCommitWithMultiDC tests that async commit can only be enabled in global transactions func (s *testAsyncCommitSuite) TestAsyncCommitWithMultiDC(c *C) { // It requires setting placement rules to run with TiKV - if *WithTiKV { + if *mockstore.WithTiKV { return } diff --git a/store/tikv/tests/lock_test.go b/store/tikv/tests/lock_test.go index 3c6c652d96041..d52329c50cfa5 100644 --- a/store/tikv/tests/lock_test.go +++ b/store/tikv/tests/lock_test.go @@ -19,13 +19,17 @@ import ( "fmt" "math" "runtime" + "sync" "time" . "github.com/pingcap/check" + "github.com/pingcap/errors" "github.com/pingcap/failpoint" + deadlockpb "github.com/pingcap/kvproto/pkg/deadlock" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/store/tikv" tikverr "github.com/pingcap/tidb/store/tikv/error" + "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/store/tikv/tikvrpc" ) @@ -493,7 +497,7 @@ func (s *testLockSuite) TestBatchResolveLocks(c *C) { c.Assert(msBeforeLockExpired, Greater, int64(0)) lr := s.store.NewLockResolver() - bo := tikv.NewBackofferWithVars(context.Background(), tikv.GcResolveLockMaxBackoff, nil) + bo := tikv.NewGcResolveLockMaxBackoffer(context.Background()) loc, err := s.store.GetRegionCache().LocateKey(bo, locks[0].Primary) c.Assert(err, IsNil) // Check BatchResolveLocks resolve the lock even the ttl is not expired. @@ -640,3 +644,131 @@ func (s *testLockSuite) TestBatchResolveTxnFallenBackFromAsyncCommit(c *C) { _, err = t3.Get(context.Background(), []byte("fb2")) c.Assert(tikverr.IsErrNotFound(err), IsTrue) } + +func (s *testLockSuite) TestDeadlockReportWaitChain(c *C) { + // Utilities to make the test logic clear and simple. + type txnWrapper struct { + tikv.TxnProbe + wg sync.WaitGroup + } + + makeLockCtx := func(txn *txnWrapper, resourceGroupTag string) *kv.LockCtx { + return &kv.LockCtx{ + ForUpdateTS: txn.StartTS(), + WaitStartTime: time.Now(), + LockWaitTime: 1000, + ResourceGroupTag: []byte(resourceGroupTag), + } + } + + // Prepares several transactions and each locks a key. + prepareTxns := func(num int) []*txnWrapper { + res := make([]*txnWrapper, 0, num) + for i := 0; i < num; i++ { + txnProbe, err := s.store.Begin() + c.Assert(err, IsNil) + txn := &txnWrapper{TxnProbe: txnProbe} + txn.SetPessimistic(true) + tag := fmt.Sprintf("tag-init%v", i) + key := []byte{'k', byte(i)} + err = txn.LockKeys(context.Background(), makeLockCtx(txn, tag), key) + c.Assert(err, IsNil) + + res = append(res, txn) + } + return res + } + + // Let the i-th trnasaction lock the key that has been locked by j-th transaction + tryLock := func(txns []*txnWrapper, i int, j int) error { + c.Logf("txn %v try locking %v", i, j) + txn := txns[i] + tag := fmt.Sprintf("tag-%v-%v", i, j) + key := []byte{'k', byte(j)} + return txn.LockKeys(context.Background(), makeLockCtx(txn, tag), key) + } + + // Asserts the i-th transaction waits for the j-th transaction. + makeWaitFor := func(txns []*txnWrapper, i int, j int) { + txns[i].wg.Add(1) + go func() { + defer txns[i].wg.Done() + err := tryLock(txns, i, j) + // After the lock being waited for is released, the transaction returns a WriteConflict error + // unconditionally, which is by design. + c.Assert(err, NotNil) + c.Logf("txn %v wait for %v finished, err: %s", i, j, err.Error()) + _, ok := errors.Cause(err).(*tikverr.ErrWriteConflict) + c.Assert(ok, IsTrue) + }() + } + + waitAndRollback := func(txns []*txnWrapper, i int) { + // It's expected that each transaction should be rolled back after its blocker, so that `Rollback` will not + // run when there's concurrent `LockKeys` running. + // If it's blocked on the `Wait` forever, it means the transaction's blocker is not rolled back. + c.Logf("rollback txn %v", i) + txns[i].wg.Wait() + err := txns[i].Rollback() + c.Assert(err, IsNil) + } + + // Check the given WaitForEntry is caused by txn[i] waiting for txn[j]. + checkWaitChainEntry := func(txns []*txnWrapper, entry *deadlockpb.WaitForEntry, i, j int) { + c.Assert(entry.Txn, Equals, txns[i].StartTS()) + c.Assert(entry.WaitForTxn, Equals, txns[j].StartTS()) + c.Assert(entry.Key, BytesEquals, []byte{'k', byte(j)}) + c.Assert(string(entry.ResourceGroupTag), Equals, fmt.Sprintf("tag-%v-%v", i, j)) + } + + c.Log("test case 1: 1->0->1") + + txns := prepareTxns(2) + + makeWaitFor(txns, 0, 1) + // Sleep for a while to make sure it has been blocked. + time.Sleep(time.Millisecond * 100) + + // txn2 tries locking k1 and encounters deadlock error. + err := tryLock(txns, 1, 0) + c.Assert(err, NotNil) + dl, ok := errors.Cause(err).(*tikverr.ErrDeadlock) + c.Assert(ok, IsTrue) + + waitChain := dl.GetWaitChain() + c.Assert(len(waitChain), Equals, 2) + checkWaitChainEntry(txns, waitChain[0], 0, 1) + checkWaitChainEntry(txns, waitChain[1], 1, 0) + + // Each transaction should be rolled back after its blocker being rolled back + waitAndRollback(txns, 1) + waitAndRollback(txns, 0) + + c.Log("test case 2: 3->2->0->1->3") + txns = prepareTxns(4) + + makeWaitFor(txns, 0, 1) + makeWaitFor(txns, 2, 0) + makeWaitFor(txns, 1, 3) + // Sleep for a while to make sure it has been blocked. + time.Sleep(time.Millisecond * 100) + + err = tryLock(txns, 3, 2) + c.Assert(err, NotNil) + dl, ok = errors.Cause(err).(*tikverr.ErrDeadlock) + c.Assert(ok, IsTrue) + + waitChain = dl.GetWaitChain() + c.Assert(len(waitChain), Equals, 4) + c.Logf("wait chain: \n** %v\n**%v\n**%v\n**%v\n", waitChain[0], waitChain[1], waitChain[2], waitChain[3]) + checkWaitChainEntry(txns, waitChain[0], 2, 0) + checkWaitChainEntry(txns, waitChain[1], 0, 1) + checkWaitChainEntry(txns, waitChain[2], 1, 3) + checkWaitChainEntry(txns, waitChain[3], 3, 2) + + // Each transaction should be rolled back after its blocker being rolled back + waitAndRollback(txns, 3) + waitAndRollback(txns, 1) + waitAndRollback(txns, 0) + waitAndRollback(txns, 2) +} diff --git a/store/tikv/tests/prewrite_test.go b/store/tikv/tests/prewrite_test.go index 6f75959b4afe4..1b80d05abf373 100644 --- a/store/tikv/tests/prewrite_test.go +++ b/store/tikv/tests/prewrite_test.go @@ -15,7 +15,7 @@ package tikv_test import ( . "github.com/pingcap/check" - pb "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/store/mockstore/unistore" "github.com/pingcap/tidb/store/tikv" ) @@ -45,9 +45,9 @@ func (s *testPrewriteSuite) TestSetMinCommitTSInAsyncCommit(c *C) { c.Assert(err, IsNil) committer.SetUseAsyncCommit() - buildRequest := func() *pb.PrewriteRequest { + buildRequest := func() *kvrpcpb.PrewriteRequest { req := committer.BuildPrewriteRequest(1, 1, 1, committer.GetMutations(), 1) - return req.Req.(*pb.PrewriteRequest) + return req.Req.(*kvrpcpb.PrewriteRequest) } // no forUpdateTS diff --git a/store/tikv/tests/snapshot_fail_test.go b/store/tikv/tests/snapshot_fail_test.go index 1360841bd743a..6740dafdc523b 100644 --- a/store/tikv/tests/snapshot_fail_test.go +++ b/store/tikv/tests/snapshot_fail_test.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/tidb/store/mockstore/unistore" "github.com/pingcap/tidb/store/tikv" tikverr "github.com/pingcap/tidb/store/tikv/error" - "github.com/pingcap/tidb/store/tikv/kv" + "github.com/pingcap/tidb/store/tikv/mockstore" ) type testSnapshotFailSuite struct { @@ -63,7 +63,7 @@ func (s *testSnapshotFailSuite) cleanup(c *C) { func (s *testSnapshotFailSuite) TestBatchGetResponseKeyError(c *C) { // Meaningless to test with tikv because it has a mock key error - if *WithTiKV { + if *mockstore.WithTiKV { return } defer s.cleanup(c) @@ -92,7 +92,7 @@ func (s *testSnapshotFailSuite) TestBatchGetResponseKeyError(c *C) { func (s *testSnapshotFailSuite) TestScanResponseKeyError(c *C) { // Meaningless to test with tikv because it has a mock key error - if *WithTiKV { + if *mockstore.WithTiKV { return } defer s.cleanup(c) @@ -151,7 +151,7 @@ func (s *testSnapshotFailSuite) TestRetryMaxTsPointGetSkipLock(c *C) { c.Assert(err, IsNil) err = txn.Set([]byte("k2"), []byte("v2")) c.Assert(err, IsNil) - txn.SetOption(kv.EnableAsyncCommit, true) + txn.SetEnableAsyncCommit(true) c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/asyncCommitDoNothing", "return"), IsNil) c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/twoPCShortLockTTL", "return"), IsNil) @@ -181,7 +181,7 @@ func (s *testSnapshotFailSuite) TestRetryMaxTsPointGetSkipLock(c *C) { // Prewrite k1 and k2 again without committing them txn, err = s.store.Begin() c.Assert(err, IsNil) - txn.SetOption(kv.EnableAsyncCommit, true) + txn.SetEnableAsyncCommit(true) err = txn.Set([]byte("k1"), []byte("v3")) c.Assert(err, IsNil) err = txn.Set([]byte("k2"), []byte("v4")) @@ -210,9 +210,9 @@ func (s *testSnapshotFailSuite) TestRetryPointGetResolveTS(c *C) { c.Assert(txn.Set([]byte("k1"), []byte("v1")), IsNil) err = txn.Set([]byte("k2"), []byte("v2")) c.Assert(err, IsNil) - txn.SetOption(kv.EnableAsyncCommit, false) + txn.SetEnableAsyncCommit(false) txn.SetEnable1PC(false) - txn.SetOption(kv.GuaranteeLinearizability, false) + txn.SetCausalConsistency(true) // Prewrite the lock without committing it c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/beforeCommit", `pause`), IsNil) diff --git a/store/tikv/tests/snapshot_test.go b/store/tikv/tests/snapshot_test.go index fa1ccdd5735bd..1ae3f303740ff 100644 --- a/store/tikv/tests/snapshot_test.go +++ b/store/tikv/tests/snapshot_test.go @@ -23,10 +23,9 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/errors" "github.com/pingcap/failpoint" - pb "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/store/tikv" tikverr "github.com/pingcap/tidb/store/tikv/error" - "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/tikvrpc" "go.uber.org/zap" @@ -270,22 +269,22 @@ func (s *testSnapshotSuite) TestSnapshotRuntimeStats(c *C) { tikv.RecordRegionRequestRuntimeStats(reqStats.Stats, tikvrpc.CmdGet, time.Second) tikv.RecordRegionRequestRuntimeStats(reqStats.Stats, tikvrpc.CmdGet, time.Millisecond) snapshot := s.store.GetSnapshot(0) - snapshot.SetOption(kv.CollectRuntimeStats, &tikv.SnapshotRuntimeStats{}) + snapshot.SetRuntimeStats(&tikv.SnapshotRuntimeStats{}) snapshot.MergeRegionRequestStats(reqStats.Stats) snapshot.MergeRegionRequestStats(reqStats.Stats) bo := tikv.NewBackofferWithVars(context.Background(), 2000, nil) - err := bo.BackoffWithMaxSleep(tikv.BoTxnLockFast, 30, errors.New("test")) + err := bo.BackoffWithMaxSleepTxnLockFast(30, errors.New("test")) c.Assert(err, IsNil) snapshot.RecordBackoffInfo(bo) snapshot.RecordBackoffInfo(bo) expect := "Get:{num_rpc:4, total_time:2s},txnLockFast_backoff:{num:2, total_time:60ms}" c.Assert(snapshot.FormatStats(), Equals, expect) - detail := &pb.ExecDetailsV2{ - TimeDetail: &pb.TimeDetail{ + detail := &kvrpcpb.ExecDetailsV2{ + TimeDetail: &kvrpcpb.TimeDetail{ WaitWallTimeMs: 100, ProcessWallTimeMs: 100, }, - ScanDetailV2: &pb.ScanDetailV2{ + ScanDetailV2: &kvrpcpb.ScanDetailV2{ ProcessedVersions: 10, TotalVersions: 15, RocksdbBlockReadCount: 20, diff --git a/store/tikv/tests/store_test.go b/store/tikv/tests/store_test.go index 659dc6ea8f226..6a989965c1dc8 100644 --- a/store/tikv/tests/store_test.go +++ b/store/tikv/tests/store_test.go @@ -19,8 +19,7 @@ import ( "time" . "github.com/pingcap/check" - "github.com/pingcap/failpoint" - pb "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/store/tikv/oracle/oracles" @@ -96,15 +95,15 @@ func (s *testStoreSuite) TestOracle(c *C) { type checkRequestClient struct { tikv.Client - priority pb.CommandPri + priority kvrpcpb.CommandPri } func (c *checkRequestClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) { resp, err := c.Client.SendRequest(ctx, addr, req, timeout) if c.priority != req.Priority { if resp.Resp != nil { - if getResp, ok := resp.Resp.(*pb.GetResponse); ok { - getResp.Error = &pb.KeyError{ + if getResp, ok := resp.Resp.(*kvrpcpb.GetResponse); ok { + getResp.Error = &kvrpcpb.KeyError{ Abort: "request check error", } } @@ -122,7 +121,7 @@ func (s *testStoreSuite) TestRequestPriority(c *C) { // Cover 2PC commit. txn, err := s.store.Begin() c.Assert(err, IsNil) - client.priority = pb.CommandPri_High + client.priority = kvrpcpb.CommandPri_High txn.SetPriority(tikv.PriorityHigh) err = txn.Set([]byte("key"), []byte("value")) c.Assert(err, IsNil) @@ -132,20 +131,20 @@ func (s *testStoreSuite) TestRequestPriority(c *C) { // Cover the basic Get request. txn, err = s.store.Begin() c.Assert(err, IsNil) - client.priority = pb.CommandPri_Low + client.priority = kvrpcpb.CommandPri_Low txn.SetPriority(tikv.PriorityLow) _, err = txn.Get(context.TODO(), []byte("key")) c.Assert(err, IsNil) // A counter example. - client.priority = pb.CommandPri_Low + client.priority = kvrpcpb.CommandPri_Low txn.SetPriority(tikv.PriorityNormal) _, err = txn.Get(context.TODO(), []byte("key")) // err is translated to "try again later" by backoffer, so doesn't check error value here. c.Assert(err, NotNil) // Cover Seek request. - client.priority = pb.CommandPri_High + client.priority = kvrpcpb.CommandPri_High txn.SetPriority(tikv.PriorityHigh) iter, err := txn.Iter([]byte("key"), nil) c.Assert(err, IsNil) @@ -154,20 +153,3 @@ func (s *testStoreSuite) TestRequestPriority(c *C) { } iter.Close() } - -func (s *testStoreSerialSuite) TestOracleChangeByFailpoint(c *C) { - defer func() { - failpoint.Disable("github.com/pingcap/tidb/store/tikv/oracle/changeTSFromPD") - }() - c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/oracle/changeTSFromPD", - "return(10000)"), IsNil) - o := &oracles.MockOracle{} - s.store.SetOracle(o) - ctx := context.Background() - t1, err := s.store.GetTimestampWithRetry(tikv.NewBackofferWithVars(ctx, 100, nil), oracle.GlobalTxnScope) - c.Assert(err, IsNil) - c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/oracle/changeTSFromPD"), IsNil) - t2, err := s.store.GetTimestampWithRetry(tikv.NewBackofferWithVars(ctx, 100, nil), oracle.GlobalTxnScope) - c.Assert(err, IsNil) - c.Assert(t1, Greater, t2) -} diff --git a/store/tikv/tests/ticlient_slow_test.go b/store/tikv/tests/ticlient_slow_test.go index 61f6748874d14..97f0edcfda397 100644 --- a/store/tikv/tests/ticlient_slow_test.go +++ b/store/tikv/tests/ticlient_slow_test.go @@ -21,10 +21,11 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/kv" + "github.com/pingcap/tidb/store/tikv/mockstore" ) func (s *testTiclientSuite) TestSplitRegionIn2PC(c *C) { - if *WithTiKV { + if *mockstore.WithTiKV { c.Skip("scatter will timeout with single node TiKV") } config := tikv.ConfigProbe{} diff --git a/store/tikv/tests/util_test.go b/store/tikv/tests/util_test.go index b39584393f12b..2fb841b059716 100644 --- a/store/tikv/tests/util_test.go +++ b/store/tikv/tests/util_test.go @@ -18,7 +18,6 @@ import ( "flag" "fmt" "strings" - "sync" "unsafe" . "github.com/pingcap/check" @@ -28,14 +27,13 @@ import ( "github.com/pingcap/tidb/store/mockstore/unistore" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/config" + "github.com/pingcap/tidb/store/tikv/mockstore" "github.com/pingcap/tidb/store/tikv/util/codec" pd "github.com/tikv/pd/client" ) var ( - withTiKVGlobalLock sync.RWMutex - WithTiKV = flag.Bool("with-tikv", false, "run tests with TiKV cluster started. (not use the mock server)") - pdAddrs = flag.String("pd-addrs", "127.0.0.1:2379", "pd addrs") + pdAddrs = flag.String("pd-addrs", "127.0.0.1:2379", "pd addrs") ) // NewTestStore creates a KVStore for testing purpose. @@ -44,7 +42,7 @@ func NewTestStore(c *C) *tikv.KVStore { flag.Parse() } - if *WithTiKV { + if *mockstore.WithTiKV { addrs := strings.Split(*pdAddrs, ",") pdClient, err := pd.NewClient(addrs, pd.SecurityOption{}) c.Assert(err, IsNil) @@ -86,23 +84,7 @@ func clearStorage(store *tikv.KVStore) error { } // OneByOneSuite is a suite, When with-tikv flag is true, there is only one storage, so the test suite have to run one by one. -type OneByOneSuite struct{} - -func (s *OneByOneSuite) SetUpSuite(c *C) { - if *WithTiKV { - withTiKVGlobalLock.Lock() - } else { - withTiKVGlobalLock.RLock() - } -} - -func (s *OneByOneSuite) TearDownSuite(c *C) { - if *WithTiKV { - withTiKVGlobalLock.Unlock() - } else { - withTiKVGlobalLock.RUnlock() - } -} +type OneByOneSuite = mockstore.OneByOneSuite func encodeKey(prefix, s string) []byte { return codec.EncodeBytes(nil, []byte(fmt.Sprintf("%s_%s", prefix, s))) diff --git a/store/tikv/tikv_test.go b/store/tikv/tikv_test.go index a5b703ee3ff7a..8d25938cf0df2 100644 --- a/store/tikv/tikv_test.go +++ b/store/tikv/tikv_test.go @@ -14,40 +14,22 @@ package tikv import ( - "flag" - "sync" + "testing" . "github.com/pingcap/check" + "github.com/pingcap/tidb/store/tikv/mockstore" ) -var ( - withTiKVGlobalLock sync.RWMutex - WithTiKV = flag.Bool("with-tikv", false, "run tests with TiKV cluster started. (not use the mock server)") -) - -// OneByOneSuite is a suite, When with-tikv flag is true, there is only one storage, so the test suite have to run one by one. -type OneByOneSuite struct{} - -func (s *OneByOneSuite) SetUpSuite(c *C) { - if *WithTiKV { - withTiKVGlobalLock.Lock() - } else { - withTiKVGlobalLock.RLock() - } -} - -func (s *OneByOneSuite) TearDownSuite(c *C) { - if *WithTiKV { - withTiKVGlobalLock.Unlock() - } else { - withTiKVGlobalLock.RUnlock() - } -} - +type OneByOneSuite = mockstore.OneByOneSuite type testTiKVSuite struct { OneByOneSuite } +func TestT(t *testing.T) { + CustomVerboseFlag = true + TestingT(t) +} + var _ = Suite(&testTiKVSuite{}) func (s *testTiKVSuite) TestBasicFunc(c *C) { diff --git a/store/tikv/tikvrpc/endpoint.go b/store/tikv/tikvrpc/endpoint.go index 8eb881c81efb2..3cd6fabbea794 100644 --- a/store/tikv/tikvrpc/endpoint.go +++ b/store/tikv/tikvrpc/endpoint.go @@ -13,6 +13,8 @@ package tikvrpc +import "github.com/pingcap/kvproto/pkg/metapb" + // EndpointType represents the type of a remote endpoint.. type EndpointType uint8 @@ -35,3 +37,20 @@ func (t EndpointType) Name() string { } return "unspecified" } + +// Constants to determine engine type. +// They should be synced with PD. +const ( + engineLabelKey = "engine" + engineLabelTiFlash = "tiflash" +) + +// GetStoreTypeByMeta gets store type by store meta pb. +func GetStoreTypeByMeta(store *metapb.Store) EndpointType { + for _, label := range store.Labels { + if label.Key == engineLabelKey && label.Value == engineLabelTiFlash { + return TiFlash + } + } + return TiKV +} diff --git a/store/tikv/tikvrpc/tikvrpc.go b/store/tikv/tikvrpc/tikvrpc.go index ac213e0da8239..ae450b6019799 100644 --- a/store/tikv/tikvrpc/tikvrpc.go +++ b/store/tikv/tikvrpc/tikvrpc.go @@ -69,6 +69,7 @@ const ( CmdPhysicalScanLock CmdStoreSafeTS + CmdLockWaitInfo CmdCop CmdType = 512 + iota CmdCopStream @@ -168,6 +169,8 @@ func (t CmdType) String() string { return "TxnHeartBeat" case CmdStoreSafeTS: return "StoreSafeTS" + case CmdLockWaitInfo: + return "LockWaitInfo" } return "Unknown" } @@ -177,6 +180,7 @@ type Request struct { Type CmdType Req interface{} kvrpcpb.Context + TxnScope string ReplicaReadType kv.ReplicaReadType // different from `kvrpcpb.Context.ReplicaRead` ReplicaReadSeed *uint32 // pointer to follower read seed in snapshot/coprocessor StoreTp EndpointType @@ -211,6 +215,14 @@ func NewReplicaReadRequest(typ CmdType, pointer interface{}, replicaReadType kv. return req } +// GetReplicaReadSeed returns ReplicaReadSeed pointer. +func (req *Request) GetReplicaReadSeed() *uint32 { + if req != nil { + return req.ReplicaReadSeed + } + return nil +} + // EnableStaleRead enables stale read func (req *Request) EnableStaleRead() { req.StaleRead = true @@ -427,6 +439,11 @@ func (req *Request) StoreSafeTS() *kvrpcpb.StoreSafeTSRequest { return req.Req.(*kvrpcpb.StoreSafeTSRequest) } +// LockWaitInfo returns GetLockWaitInfoRequest in request. +func (req *Request) LockWaitInfo() *kvrpcpb.GetLockWaitInfoRequest { + return req.Req.(*kvrpcpb.GetLockWaitInfoRequest) +} + // ToBatchCommandsRequest converts the request to an entry in BatchCommands request. func (req *Request) ToBatchCommandsRequest() *tikvpb.BatchCommandsRequest_Request { switch req.Type { @@ -924,6 +941,8 @@ func CallRPC(ctx context.Context, client tikvpb.TikvClient, req *Request) (*Resp resp.Resp, err = client.KvTxnHeartBeat(ctx, req.TxnHeartBeat()) case CmdStoreSafeTS: resp.Resp, err = client.GetStoreSafeTS(ctx, req.StoreSafeTS()) + case CmdLockWaitInfo: + resp.Resp, err = client.GetLockWaitInfo(ctx, req.LockWaitInfo()) default: return nil, errors.Errorf("invalid request type: %v", req.Type) } diff --git a/store/tikv/txn.go b/store/tikv/txn.go index 4e462653c415c..3cde92561d8ee 100644 --- a/store/tikv/txn.go +++ b/store/tikv/txn.go @@ -31,9 +31,11 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/kvrpcpb" tikverr "github.com/pingcap/tidb/store/tikv/error" - "github.com/pingcap/tidb/store/tikv/kv" + tikv "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/store/tikv/logutil" "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/pingcap/tidb/store/tikv/oracle" + "github.com/pingcap/tidb/store/tikv/retry" "github.com/pingcap/tidb/store/tikv/unionstore" "github.com/pingcap/tidb/store/tikv/util" "go.uber.org/zap" @@ -50,6 +52,31 @@ type SchemaAmender interface { AmendTxn(ctx context.Context, startInfoSchema SchemaVer, change *RelatedSchemaChange, mutations CommitterMutations) (CommitterMutations, error) } +// StartTSOption indicates the option when beginning a transaction +// `TxnScope` must be set for each object +// Every other fields are optional, but currently at most one of them can be set +type StartTSOption struct { + TxnScope string + StartTS *uint64 +} + +// DefaultStartTSOption creates a default StartTSOption, ie. Work in GlobalTxnScope and get start ts when got used +func DefaultStartTSOption() StartTSOption { + return StartTSOption{TxnScope: oracle.GlobalTxnScope} +} + +// SetStartTS returns a new StartTSOption with StartTS set to the given startTS +func (to StartTSOption) SetStartTS(startTS uint64) StartTSOption { + to.StartTS = &startTS + return to +} + +// SetTxnScope returns a new StartTSOption with TxnScope set to txnScope +func (to StartTSOption) SetTxnScope(txnScope string) StartTSOption { + to.TxnScope = txnScope + return to +} + // KVTxn contains methods to interact with a TiKV transaction. type KVTxn struct { snapshot *KVSnapshot @@ -60,7 +87,7 @@ type KVTxn struct { commitTS uint64 mu sync.Mutex // For thread-safe LockKeys function. setCnt int64 - vars *kv.Variables + vars *tikv.Variables committer *twoPhaseCommitter lockedCnt int @@ -78,49 +105,50 @@ type KVTxn struct { syncLog bool priority Priority isPessimistic bool + enableAsyncCommit bool enable1PC bool + causalConsistency bool scope string kvFilter KVFilter + resourceGroupTag []byte } -func newTiKVTxn(store *KVStore, txnScope string) (*KVTxn, error) { +// ExtractStartTS use `option` to get the proper startTS for a transaction. +func ExtractStartTS(store *KVStore, option StartTSOption) (uint64, error) { + if option.StartTS != nil { + return *option.StartTS, nil + } bo := NewBackofferWithVars(context.Background(), tsoMaxBackoff, nil) - startTS, err := store.getTimestampWithRetry(bo, txnScope) + return store.getTimestampWithRetry(bo, option.TxnScope) +} + +func newTiKVTxnWithOptions(store *KVStore, options StartTSOption) (*KVTxn, error) { + if options.TxnScope == "" { + options.TxnScope = oracle.GlobalTxnScope + } + startTS, err := ExtractStartTS(store, options) if err != nil { return nil, errors.Trace(err) } - return newTiKVTxnWithStartTS(store, txnScope, startTS, store.nextReplicaReadSeed()) -} - -// newTiKVTxnWithStartTS creates a txn with startTS. -func newTiKVTxnWithStartTS(store *KVStore, txnScope string, startTS uint64, replicaReadSeed uint32) (*KVTxn, error) { - snapshot := newTiKVSnapshot(store, startTS, replicaReadSeed) - return &KVTxn{ + snapshot := newTiKVSnapshot(store, startTS, store.nextReplicaReadSeed()) + newTiKVTxn := &KVTxn{ snapshot: snapshot, us: unionstore.NewUnionStore(snapshot), store: store, startTS: startTS, startTime: time.Now(), valid: true, - vars: kv.DefaultVars, - scope: txnScope, - }, nil -} - -func newTiKVTxnWithExactStaleness(store *KVStore, txnScope string, prevSec uint64) (*KVTxn, error) { - bo := NewBackofferWithVars(context.Background(), tsoMaxBackoff, nil) - startTS, err := store.getStalenessTimestamp(bo, txnScope, prevSec) - if err != nil { - return nil, errors.Trace(err) + vars: tikv.DefaultVars, + scope: options.TxnScope, } - return newTiKVTxnWithStartTS(store, txnScope, startTS, store.nextReplicaReadSeed()) + return newTiKVTxn, nil } // SetSuccess is used to probe if kv variables are set or not. It is ONLY used in test cases. var SetSuccess = false // SetVars sets variables to the transaction. -func (txn *KVTxn) SetVars(vars *kv.Variables) { +func (txn *KVTxn) SetVars(vars *tikv.Variables) { txn.vars = vars txn.snapshot.vars = vars failpoint.Inject("probeSetVars", func(val failpoint.Value) { @@ -131,7 +159,7 @@ func (txn *KVTxn) SetVars(vars *kv.Variables) { } // GetVars gets variables from the transaction. -func (txn *KVTxn) GetVars() *kv.Variables { +func (txn *KVTxn) GetVars() *tikv.Variables { return txn.vars } @@ -178,27 +206,6 @@ func (txn *KVTxn) Delete(k []byte) error { return txn.us.GetMemBuffer().Delete(k) } -// SetOption sets an option with a value, when val is nil, uses the default -// value of this option. -func (txn *KVTxn) SetOption(opt int, val interface{}) { - txn.us.SetOption(opt, val) - txn.snapshot.SetOption(opt, val) - switch opt { - case kv.SchemaAmender: - txn.schemaAmender = val.(SchemaAmender) - } -} - -// GetOption returns the option -func (txn *KVTxn) GetOption(opt int) interface{} { - return txn.us.GetOption(opt) -} - -// DelOption deletes an option. -func (txn *KVTxn) DelOption(opt int) { - txn.us.DelOption(opt) -} - // SetSchemaLeaseChecker sets a hook to check schema version. func (txn *KVTxn) SetSchemaLeaseChecker(checker SchemaLeaseChecker) { txn.schemaLeaseChecker = checker @@ -225,17 +232,40 @@ func (txn *KVTxn) SetPriority(pri Priority) { txn.GetSnapshot().SetPriority(pri) } +// SetResourceGroupTag sets the resource tag for both write and read. +func (txn *KVTxn) SetResourceGroupTag(tag []byte) { + txn.resourceGroupTag = tag + txn.GetSnapshot().SetResourceGroupTag(tag) +} + +// SetSchemaAmender sets an amender to update mutations after schema change. +func (txn *KVTxn) SetSchemaAmender(sa SchemaAmender) { + txn.schemaAmender = sa +} + // SetCommitCallback sets up a function that will be called when the transaction // is finished. func (txn *KVTxn) SetCommitCallback(f func(string, error)) { txn.commitCallback = f } +// SetEnableAsyncCommit indicates if the transaction will try to use async commit. +func (txn *KVTxn) SetEnableAsyncCommit(b bool) { + txn.enableAsyncCommit = b +} + // SetEnable1PC indicates if the transaction will try to use 1 phase commit. func (txn *KVTxn) SetEnable1PC(b bool) { txn.enable1PC = b } +// SetCausalConsistency indicates if the transaction does not need to +// guarantee linearizability. Default value is false which means +// linearizability is guaranteed. +func (txn *KVTxn) SetCausalConsistency(b bool) { + txn.causalConsistency = b +} + // SetScope sets the geographical scope of the transaction. func (txn *KVTxn) SetScope(scope string) { txn.scope = scope @@ -251,6 +281,12 @@ func (txn *KVTxn) IsPessimistic() bool { return txn.isPessimistic } +// IsCasualConsistency returns if the transaction allows linearizability +// inconsistency. +func (txn *KVTxn) IsCasualConsistency() bool { + return txn.causalConsistency +} + // GetScope returns the geographical scope of the transaction. func (txn *KVTxn) GetScope() string { return txn.scope @@ -310,13 +346,19 @@ func (txn *KVTxn) Commit(ctx context.Context) error { } defer func() { + detail := committer.getDetail() + detail.Mu.Lock() + metrics.TiKVTxnCommitBackoffSeconds.Observe(float64(detail.Mu.CommitBackoffTime) / float64(time.Second)) + metrics.TiKVTxnCommitBackoffCount.Observe(float64(len(detail.Mu.BackoffTypes))) + detail.Mu.Unlock() + ctxValue := ctx.Value(util.CommitDetailCtxKey) if ctxValue != nil { commitDetail := ctxValue.(**util.CommitDetails) if *commitDetail != nil { (*commitDetail).TxnRetry++ } else { - *commitDetail = committer.getDetail() + *commitDetail = detail } } }() @@ -383,7 +425,7 @@ func (txn *KVTxn) rollbackPessimisticLocks() error { if txn.lockedCnt == 0 { return nil } - bo := NewBackofferWithVars(context.Background(), cleanupMaxBackoff, txn.vars) + bo := retry.NewBackofferWithVars(context.Background(), cleanupMaxBackoff, txn.vars) keys := txn.collectLockedKeys() return txn.committer.pessimisticRollbackMutations(bo, &PlainMutations{keys: keys}) } @@ -442,8 +484,8 @@ func (txn *KVTxn) onCommitted(err error) { } // LockKeys tries to lock the entries with the keys in KV store. -// lockWaitTime in ms, except that tidbkv.LockAlwaysWait(0) means always wait lock, tidbkv.LockNowait(-1) means nowait lock -func (txn *KVTxn) LockKeys(ctx context.Context, lockCtx *kv.LockCtx, keysInput ...[]byte) error { +// lockWaitTime in ms, except that kv.LockAlwaysWait(0) means always wait lock, kv.LockNowait(-1) means nowait lock +func (txn *KVTxn) LockKeys(ctx context.Context, lockCtx *tikv.LockCtx, keysInput ...[]byte) error { // Exclude keys that are already locked. var err error keys := make([][]byte, 0, len(keysInput)) @@ -494,7 +536,7 @@ func (txn *KVTxn) LockKeys(ctx context.Context, lockCtx *kv.LockCtx, keysInput . if lockCtx.ReturnValues && locked { // An already locked key can not return values, we add an entry to let the caller get the value // in other ways. - lockCtx.Values[string(key)] = kv.ReturnedValue{AlreadyLocked: true} + lockCtx.Values[string(key)] = tikv.ReturnedValue{AlreadyLocked: true} } } if len(keys) == 0 { @@ -524,16 +566,16 @@ func (txn *KVTxn) LockKeys(ctx context.Context, lockCtx *kv.LockCtx, keysInput . lockCtx.Stats = &util.LockKeysDetails{ LockKeys: int32(len(keys)), } - bo := NewBackofferWithVars(ctx, pessimisticLockMaxBackoff, txn.vars) + bo := retry.NewBackofferWithVars(ctx, pessimisticLockMaxBackoff, txn.vars) txn.committer.forUpdateTS = lockCtx.ForUpdateTS // If the number of keys greater than 1, it can be on different region, // concurrently execute on multiple regions may lead to deadlock. txn.committer.isFirstLock = txn.lockedCnt == 0 && len(keys) == 1 err = txn.committer.pessimisticLockMutations(bo, lockCtx, &PlainMutations{keys: keys}) - if bo.totalSleep > 0 { - atomic.AddInt64(&lockCtx.Stats.BackoffTime, int64(bo.totalSleep)*int64(time.Millisecond)) + if bo.GetTotalSleep() > 0 { + atomic.AddInt64(&lockCtx.Stats.BackoffTime, int64(bo.GetTotalSleep())*int64(time.Millisecond)) lockCtx.Stats.Mu.Lock() - lockCtx.Stats.Mu.BackoffTypes = append(lockCtx.Stats.Mu.BackoffTypes, bo.types...) + lockCtx.Stats.Mu.BackoffTypes = append(lockCtx.Stats.Mu.BackoffTypes, bo.GetTypes()...) lockCtx.Stats.Mu.Unlock() } if lockCtx.Killed != nil { @@ -551,16 +593,24 @@ func (txn *KVTxn) LockKeys(ctx context.Context, lockCtx *kv.LockCtx, keysInput . keyMayBeLocked := !(tikverr.IsErrWriteConflict(err) || tikverr.IsErrKeyExist(err)) // If there is only 1 key and lock fails, no need to do pessimistic rollback. if len(keys) > 1 || keyMayBeLocked { + dl, ok := errors.Cause(err).(*tikverr.ErrDeadlock) + if ok && lockCtx.OnDeadlock != nil { + // Call OnDeadlock before pessimistic rollback. + lockCtx.OnDeadlock(dl) + } wg := txn.asyncPessimisticRollback(ctx, keys) - if dl, ok := errors.Cause(err).(*tikverr.ErrDeadlock); ok && hashInKeys(dl.DeadlockKeyHash, keys) { - dl.IsRetryable = true - // Wait for the pessimistic rollback to finish before we retry the statement. - wg.Wait() - // Sleep a little, wait for the other transaction that blocked by this transaction to acquire the lock. - time.Sleep(time.Millisecond * 5) - failpoint.Inject("SingleStmtDeadLockRetrySleep", func() { - time.Sleep(300 * time.Millisecond) - }) + if ok { + logutil.Logger(ctx).Debug("deadlock error received", zap.Uint64("startTS", txn.startTS), zap.Stringer("deadlockInfo", dl)) + if hashInKeys(dl.DeadlockKeyHash, keys) { + dl.IsRetryable = true + // Wait for the pessimistic rollback to finish before we retry the statement. + wg.Wait() + // Sleep a little, wait for the other transaction that blocked by this transaction to acquire the lock. + time.Sleep(time.Millisecond * 5) + failpoint.Inject("SingleStmtDeadLockRetrySleep", func() { + time.Sleep(300 * time.Millisecond) + }) + } } } if assignedPrimaryKey { @@ -574,16 +624,16 @@ func (txn *KVTxn) LockKeys(ctx context.Context, lockCtx *kv.LockCtx, keysInput . } } for _, key := range keys { - valExists := kv.SetKeyLockedValueExists + valExists := tikv.SetKeyLockedValueExists // PointGet and BatchPointGet will return value in pessimistic lock response, the value may not exist. // For other lock modes, the locked key values always exist. if lockCtx.ReturnValues { - val, _ := lockCtx.Values[string(key)] + val := lockCtx.Values[string(key)] if len(val.Value) == 0 { - valExists = kv.SetKeyLockedValueNotExists + valExists = tikv.SetKeyLockedValueNotExists } } - memBuf.UpdateFlags(key, kv.SetKeyLocked, kv.DelNeedCheckExists, valExists) + memBuf.UpdateFlags(key, tikv.SetKeyLocked, tikv.DelNeedCheckExists, valExists) } txn.lockedCnt += len(keys) return nil @@ -603,6 +653,8 @@ func deduplicateKeys(keys [][]byte) [][]byte { return deduped } +const pessimisticRollbackMaxBackoff = 20000 + func (txn *KVTxn) asyncPessimisticRollback(ctx context.Context, keys [][]byte) *sync.WaitGroup { // Clone a new committer for execute in background. committer := &twoPhaseCommitter{ @@ -631,7 +683,7 @@ func (txn *KVTxn) asyncPessimisticRollback(ctx context.Context, keys [][]byte) * } }) - err := committer.pessimisticRollbackMutations(NewBackofferWithVars(ctx, pessimisticRollbackMaxBackoff, txn.vars), &PlainMutations{keys: keys}) + err := committer.pessimisticRollbackMutations(retry.NewBackofferWithVars(ctx, pessimisticRollbackMaxBackoff, txn.vars), &PlainMutations{keys: keys}) if err != nil { logutil.Logger(ctx).Warn("[kv] pessimisticRollback failed.", zap.Error(err)) } diff --git a/store/tikv/unionstore/mock.go b/store/tikv/unionstore/mock.go index 8a459a2170966..e62b1a4108147 100644 --- a/store/tikv/unionstore/mock.go +++ b/store/tikv/unionstore/mock.go @@ -55,4 +55,3 @@ func (s *mockSnapshot) IterReverse(k []byte) (Iterator, error) { } func (s *mockSnapshot) SetOption(opt int, val interface{}) {} -func (s *mockSnapshot) DelOption(opt int) {} diff --git a/store/tikv/unionstore/union_store.go b/store/tikv/unionstore/union_store.go index f9a077d1c1352..08354975e38c5 100644 --- a/store/tikv/unionstore/union_store.go +++ b/store/tikv/unionstore/union_store.go @@ -59,7 +59,6 @@ type uSnapshot interface { type KVUnionStore struct { memBuffer *MemDB snapshot uSnapshot - opts options } // NewUnionStore builds a new unionStore. @@ -67,7 +66,6 @@ func NewUnionStore(snapshot uSnapshot) *KVUnionStore { return &KVUnionStore{ snapshot: snapshot, memBuffer: newMemDB(), - opts: make(map[int]interface{}), } } @@ -131,30 +129,8 @@ func (us *KVUnionStore) UnmarkPresumeKeyNotExists(k []byte) { us.memBuffer.UpdateFlags(k, kv.DelPresumeKeyNotExists) } -// SetOption implements the unionStore SetOption interface. -func (us *KVUnionStore) SetOption(opt int, val interface{}) { - us.opts[opt] = val -} - -// DelOption implements the unionStore DelOption interface. -func (us *KVUnionStore) DelOption(opt int) { - delete(us.opts, opt) -} - -// GetOption implements the unionStore GetOption interface. -func (us *KVUnionStore) GetOption(opt int) interface{} { - return us.opts[opt] -} - // SetEntrySizeLimit sets the size limit for each entry and total buffer. func (us *KVUnionStore) SetEntrySizeLimit(entryLimit, bufferLimit uint64) { us.memBuffer.entrySizeLimit = entryLimit us.memBuffer.bufferSizeLimit = bufferLimit } - -type options map[int]interface{} - -func (opts options) Get(opt int) (interface{}, bool) { - v, ok := opts[opt] - return v, ok -} diff --git a/store/tikv/util/execdetails.go b/store/tikv/util/execdetails.go index eeaaf92da6b27..0f2dab372ff16 100644 --- a/store/tikv/util/execdetails.go +++ b/store/tikv/util/execdetails.go @@ -15,7 +15,6 @@ package util import ( "bytes" - "fmt" "math" "strconv" "sync" @@ -47,10 +46,10 @@ type CommitDetails struct { WaitPrewriteBinlogTime time.Duration CommitTime time.Duration LocalLatchTime time.Duration - CommitBackoffTime int64 Mu struct { sync.Mutex - BackoffTypes []fmt.Stringer + CommitBackoffTime int64 + BackoffTypes []string } ResolveLockTime int64 WriteKeys int @@ -66,12 +65,12 @@ func (cd *CommitDetails) Merge(other *CommitDetails) { cd.WaitPrewriteBinlogTime += other.WaitPrewriteBinlogTime cd.CommitTime += other.CommitTime cd.LocalLatchTime += other.LocalLatchTime - cd.CommitBackoffTime += other.CommitBackoffTime cd.ResolveLockTime += other.ResolveLockTime cd.WriteKeys += other.WriteKeys cd.WriteSize += other.WriteSize cd.PrewriteRegionNum += other.PrewriteRegionNum cd.TxnRetry += other.TxnRetry + cd.Mu.CommitBackoffTime += other.Mu.CommitBackoffTime cd.Mu.BackoffTypes = append(cd.Mu.BackoffTypes, other.Mu.BackoffTypes...) } @@ -83,14 +82,14 @@ func (cd *CommitDetails) Clone() *CommitDetails { WaitPrewriteBinlogTime: cd.WaitPrewriteBinlogTime, CommitTime: cd.CommitTime, LocalLatchTime: cd.LocalLatchTime, - CommitBackoffTime: cd.CommitBackoffTime, ResolveLockTime: cd.ResolveLockTime, WriteKeys: cd.WriteKeys, WriteSize: cd.WriteSize, PrewriteRegionNum: cd.PrewriteRegionNum, TxnRetry: cd.TxnRetry, } - commit.Mu.BackoffTypes = append([]fmt.Stringer{}, cd.Mu.BackoffTypes...) + commit.Mu.BackoffTypes = append([]string{}, cd.Mu.BackoffTypes...) + commit.Mu.CommitBackoffTime = cd.Mu.CommitBackoffTime return commit } @@ -103,7 +102,7 @@ type LockKeysDetails struct { BackoffTime int64 Mu struct { sync.Mutex - BackoffTypes []fmt.Stringer + BackoffTypes []string } LockRPCTime int64 LockRPCCount int64 @@ -135,7 +134,7 @@ func (ld *LockKeysDetails) Clone() *LockKeysDetails { LockRPCCount: ld.LockRPCCount, RetryCount: ld.RetryCount, } - lock.Mu.BackoffTypes = append([]fmt.Stringer{}, ld.Mu.BackoffTypes...) + lock.Mu.BackoffTypes = append([]string{}, ld.Mu.BackoffTypes...) return lock } @@ -271,6 +270,8 @@ type TimeDetail struct { // cannot be excluded for now, like Mutex wait time, which is included in this field, so that // this field is called wall time instead of CPU time. WaitTime time.Duration + // KvReadWallTimeMs is the time used in KV Scan/Get. + KvReadWallTimeMs time.Duration } // String implements the fmt.Stringer interface. @@ -298,5 +299,6 @@ func (td *TimeDetail) MergeFromTimeDetail(timeDetail *kvrpcpb.TimeDetail) { if timeDetail != nil { td.WaitTime += time.Duration(timeDetail.WaitWallTimeMs) * time.Millisecond td.ProcessTime += time.Duration(timeDetail.ProcessWallTimeMs) * time.Millisecond + td.KvReadWallTimeMs += time.Duration(timeDetail.KvReadWallTimeMs) * time.Millisecond } } diff --git a/table/index.go b/table/index.go index 5a9f32fbbfd3f..336efb7f574c2 100644 --- a/table/index.go +++ b/table/index.go @@ -66,11 +66,11 @@ type Index interface { // Create supports insert into statement. Create(ctx sessionctx.Context, txn kv.Transaction, indexedValues []types.Datum, h kv.Handle, handleRestoreData []types.Datum, opts ...CreateIdxOptFunc) (kv.Handle, error) // Delete supports delete from statement. - Delete(sc *stmtctx.StatementContext, us kv.UnionStore, indexedValues []types.Datum, h kv.Handle) error + Delete(sc *stmtctx.StatementContext, txn kv.Transaction, indexedValues []types.Datum, h kv.Handle) error // Drop supports drop table, drop index statements. - Drop(us kv.UnionStore) error + Drop(txn kv.Transaction) error // Exist supports check index exists or not. - Exist(sc *stmtctx.StatementContext, us kv.UnionStore, indexedValues []types.Datum, h kv.Handle) (bool, kv.Handle, error) + Exist(sc *stmtctx.StatementContext, txn kv.Transaction, indexedValues []types.Datum, h kv.Handle) (bool, kv.Handle, error) // GenIndexKey generates an index key. GenIndexKey(sc *stmtctx.StatementContext, indexedValues []types.Datum, h kv.Handle, buf []byte) (key []byte, distinct bool, err error) // Seek supports where clause. diff --git a/table/table.go b/table/table.go index b39304adcaaa6..f76210a9bdd42 100644 --- a/table/table.go +++ b/table/table.go @@ -97,6 +97,8 @@ var ( ErrSequenceHasRunOut = dbterror.ClassTable.NewStd(mysql.ErrSequenceRunOut) // ErrRowDoesNotMatchGivenPartitionSet returns when the destination partition conflict with the partition selection. ErrRowDoesNotMatchGivenPartitionSet = dbterror.ClassTable.NewStd(mysql.ErrRowDoesNotMatchGivenPartitionSet) + // ErrTempTableFull returns a table is full error, it's used by temporary table now. + ErrTempTableFull = dbterror.ClassTable.NewStd(mysql.ErrRecordFileFull) ) // RecordIterFunc is used for low-level record iteration. diff --git a/table/tables/index.go b/table/tables/index.go index 8b4630d47f70d..aef03d0590aaa 100644 --- a/table/tables/index.go +++ b/table/tables/index.go @@ -26,7 +26,6 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/stmtctx" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" @@ -184,9 +183,8 @@ func (c *index) Create(sctx sessionctx.Context, txn kv.Transaction, indexedValue return nil, err } - us := txn.GetUnionStore() if !distinct || skipCheck || opt.Untouched { - err = us.GetMemBuffer().Set(key, idxVal) + err = txn.GetMemBuffer().Set(key, idxVal) return nil, err } @@ -202,18 +200,18 @@ func (c *index) Create(sctx sessionctx.Context, txn kv.Transaction, indexedValue var value []byte if sctx.GetSessionVars().LazyCheckKeyNotExists() { - value, err = us.GetMemBuffer().Get(ctx, key) + value, err = txn.GetMemBuffer().Get(ctx, key) } else { - value, err = us.Get(ctx, key) + value, err = txn.Get(ctx, key) } if err != nil && !kv.IsErrNotFound(err) { return nil, err } if err != nil || len(value) == 0 { if sctx.GetSessionVars().LazyCheckKeyNotExists() && err != nil { - err = us.GetMemBuffer().SetWithFlags(key, idxVal, tikvstore.SetPresumeKeyNotExists) + err = txn.GetMemBuffer().SetWithFlags(key, idxVal, kv.SetPresumeKeyNotExists) } else { - err = us.GetMemBuffer().Set(key, idxVal) + err = txn.GetMemBuffer().Set(key, idxVal) } return nil, err } @@ -226,22 +224,22 @@ func (c *index) Create(sctx sessionctx.Context, txn kv.Transaction, indexedValue } // Delete removes the entry for handle h and indexedValues from KV index. -func (c *index) Delete(sc *stmtctx.StatementContext, us kv.UnionStore, indexedValues []types.Datum, h kv.Handle) error { +func (c *index) Delete(sc *stmtctx.StatementContext, txn kv.Transaction, indexedValues []types.Datum, h kv.Handle) error { key, distinct, err := c.GenIndexKey(sc, indexedValues, h, nil) if err != nil { return err } if distinct { - err = us.GetMemBuffer().DeleteWithFlags(key, tikvstore.SetNeedLocked) + err = txn.GetMemBuffer().DeleteWithFlags(key, kv.SetNeedLocked) } else { - err = us.GetMemBuffer().Delete(key) + err = txn.GetMemBuffer().Delete(key) } return err } // Drop removes the KV index from store. -func (c *index) Drop(us kv.UnionStore) error { - it, err := us.Iter(c.prefix, c.prefix.PrefixNext()) +func (c *index) Drop(txn kv.Transaction) error { + it, err := txn.Iter(c.prefix, c.prefix.PrefixNext()) if err != nil { return err } @@ -252,7 +250,7 @@ func (c *index) Drop(us kv.UnionStore) error { if !it.Key().HasPrefix(c.prefix) { break } - err := us.GetMemBuffer().Delete(it.Key()) + err := txn.GetMemBuffer().Delete(it.Key()) if err != nil { return err } @@ -298,13 +296,13 @@ func (c *index) SeekFirst(r kv.Retriever) (iter table.IndexIterator, err error) return &indexIter{it: it, idx: c, prefix: c.prefix, colInfos: colInfos, tps: tps}, nil } -func (c *index) Exist(sc *stmtctx.StatementContext, us kv.UnionStore, indexedValues []types.Datum, h kv.Handle) (bool, kv.Handle, error) { +func (c *index) Exist(sc *stmtctx.StatementContext, txn kv.Transaction, indexedValues []types.Datum, h kv.Handle) (bool, kv.Handle, error) { key, distinct, err := c.GenIndexKey(sc, indexedValues, h, nil) if err != nil { return false, nil, err } - value, err := us.Get(context.TODO(), key) + value, err := txn.Get(context.TODO(), key) if kv.IsErrNotFound(err) { return false, nil, nil } diff --git a/table/tables/index_test.go b/table/tables/index_test.go index 9345e86bab185..2c0a417746d42 100644 --- a/table/tables/index_test.go +++ b/table/tables/index_test.go @@ -104,15 +104,15 @@ func (s *testIndexSuite) TestIndex(c *C) { c.Assert(h.IntValue(), Equals, int64(1)) it.Close() sc := &stmtctx.StatementContext{TimeZone: time.Local} - exist, _, err := index.Exist(sc, txn.GetUnionStore(), values, kv.IntHandle(100)) + exist, _, err := index.Exist(sc, txn, values, kv.IntHandle(100)) c.Assert(err, IsNil) c.Assert(exist, IsFalse) - exist, _, err = index.Exist(sc, txn.GetUnionStore(), values, kv.IntHandle(1)) + exist, _, err = index.Exist(sc, txn, values, kv.IntHandle(1)) c.Assert(err, IsNil) c.Assert(exist, IsTrue) - err = index.Delete(sc, txn.GetUnionStore(), values, kv.IntHandle(1)) + err = index.Delete(sc, txn, values, kv.IntHandle(1)) c.Assert(err, IsNil) it, err = index.SeekFirst(txn) @@ -132,7 +132,7 @@ func (s *testIndexSuite) TestIndex(c *C) { c.Assert(err, IsNil) c.Assert(hit, IsFalse) - err = index.Drop(txn.GetUnionStore()) + err = index.Drop(txn) c.Assert(err, IsNil) it, hit, err = index.Seek(sc, txn, values) @@ -194,12 +194,12 @@ func (s *testIndexSuite) TestIndex(c *C) { c.Assert(h.IntValue(), Equals, int64(1)) it.Close() - exist, h, err = index.Exist(sc, txn.GetUnionStore(), values, kv.IntHandle(1)) + exist, h, err = index.Exist(sc, txn, values, kv.IntHandle(1)) c.Assert(err, IsNil) c.Assert(h.IntValue(), Equals, int64(1)) c.Assert(exist, IsTrue) - exist, h, err = index.Exist(sc, txn.GetUnionStore(), values, kv.IntHandle(2)) + exist, h, err = index.Exist(sc, txn, values, kv.IntHandle(2)) c.Assert(err, NotNil) c.Assert(h.IntValue(), Equals, int64(1)) c.Assert(exist, IsTrue) diff --git a/table/tables/partition.go b/table/tables/partition.go index 7f27785931e28..165f188866550 100644 --- a/table/tables/partition.go +++ b/table/tables/partition.go @@ -998,7 +998,7 @@ func (t *partitionedTable) locateRangePartition(ctx sessionctx.Context, pi *mode if isNull { return true } - return ranges.compare(i, ret, unsigned) > 0 + return ranges.Compare(i, ret, unsigned) > 0 }) if isNull { pos = 0 @@ -1081,6 +1081,18 @@ func (t *partitionedTable) GetPartitionByRow(ctx sessionctx.Context, r []types.D return t.partitions[pid], nil } +// GetPartitionByRow returns a Table, which is actually a Partition. +func (t *partitionTableWithGivenSets) GetPartitionByRow(ctx sessionctx.Context, r []types.Datum) (table.PhysicalTable, error) { + pid, err := t.locatePartition(ctx, t.Meta().GetPartitionInfo(), r) + if err != nil { + return nil, errors.Trace(err) + } + if _, ok := t.givenSetPartitions[pid]; !ok { + return nil, errors.WithStack(table.ErrRowDoesNotMatchGivenPartitionSet) + } + return t.partitions[pid], nil +} + // AddRecord implements the AddRecord method for the table.Table interface. func (t *partitionedTable) AddRecord(ctx sessionctx.Context, r []types.Datum, opts ...table.AddRecordOption) (recordID kv.Handle, err error) { return partitionedTableAddRecord(ctx, t, r, nil, opts) @@ -1107,15 +1119,15 @@ func partitionedTableAddRecord(ctx sessionctx.Context, t *partitionedTable, r [] // checks the given partition set for AddRecord/UpdateRecord operations. type partitionTableWithGivenSets struct { *partitionedTable - partitions map[int64]struct{} + givenSetPartitions map[int64]struct{} } -// NewPartitionTableithGivenSets creates a new partition table from a partition table. -func NewPartitionTableithGivenSets(tbl table.PartitionedTable, partitions map[int64]struct{}) table.PartitionedTable { +// NewPartitionTableWithGivenSets creates a new partition table from a partition table. +func NewPartitionTableWithGivenSets(tbl table.PartitionedTable, partitions map[int64]struct{}) table.PartitionedTable { if raw, ok := tbl.(*partitionedTable); ok { return &partitionTableWithGivenSets{ - partitionedTable: raw, - partitions: partitions, + partitionedTable: raw, + givenSetPartitions: partitions, } } return tbl @@ -1123,12 +1135,12 @@ func NewPartitionTableithGivenSets(tbl table.PartitionedTable, partitions map[in // AddRecord implements the AddRecord method for the table.Table interface. func (t *partitionTableWithGivenSets) AddRecord(ctx sessionctx.Context, r []types.Datum, opts ...table.AddRecordOption) (recordID kv.Handle, err error) { - return partitionedTableAddRecord(ctx, t.partitionedTable, r, t.partitions, opts) + return partitionedTableAddRecord(ctx, t.partitionedTable, r, t.givenSetPartitions, opts) } func (t *partitionTableWithGivenSets) GetAllPartitionIDs() []int64 { ptIDs := make([]int64, 0, len(t.partitions)) - for id := range t.partitions { + for id := range t.givenSetPartitions { ptIDs = append(ptIDs, id) } return ptIDs @@ -1162,7 +1174,7 @@ func (t *partitionedTable) UpdateRecord(ctx context.Context, sctx sessionctx.Con } func (t *partitionTableWithGivenSets) UpdateRecord(ctx context.Context, sctx sessionctx.Context, h kv.Handle, currData, newData []types.Datum, touched []bool) error { - return partitionedTableUpdateRecord(ctx, sctx, t.partitionedTable, h, currData, newData, touched, t.partitions) + return partitionedTableUpdateRecord(ctx, sctx, t.partitionedTable, h, currData, newData, touched, t.givenSetPartitions) } func partitionedTableUpdateRecord(gctx context.Context, ctx sessionctx.Context, t *partitionedTable, h kv.Handle, currData, newData []types.Datum, touched []bool, partitionSelection map[int64]struct{}) error { @@ -1179,6 +1191,10 @@ func partitionedTableUpdateRecord(gctx context.Context, ctx sessionctx.Context, if _, ok := partitionSelection[to]; !ok { return errors.WithStack(table.ErrRowDoesNotMatchGivenPartitionSet) } + // Should not have been read from this partition! Checked already in GetPartitionByRow() + if _, ok := partitionSelection[from]; !ok { + return errors.WithStack(table.ErrRowDoesNotMatchGivenPartitionSet) + } } // The old and new data locate in different partitions. @@ -1243,7 +1259,8 @@ func compareUnsigned(v1, v2 int64) int { return -1 } -func (lt *ForRangePruning) compare(ith int, v int64, unsigned bool) int { +// Compare is to be used in the binary search to locate partition +func (lt *ForRangePruning) Compare(ith int, v int64, unsigned bool) int { if ith == len(lt.LessThan)-1 { if lt.MaxValue { return 1 diff --git a/table/tables/partition_test.go b/table/tables/partition_test.go index a453c3b2ee967..dc7d9956f0c60 100644 --- a/table/tables/partition_test.go +++ b/table/tables/partition_test.go @@ -530,3 +530,19 @@ func (ts *testSuite) TestIssue21574(c *C) { tk.MustExec("drop table t_21574") tk.MustExec("create table t_21574 (`key` int, `table` int) partition by list columns (`key`,`table`) (partition p0 values in ((1,1)));") } + +func (ts *testSuite) TestIssue24746(c *C) { + tk := testkit.NewTestKitWithInit(c, ts.store) + tk.MustExec("use test") + tk.MustExec("drop tables if exists t_24746") + tk.MustExec("create table t_24746 (a int, b varchar(60), c int, primary key(a)) partition by range(a) (partition p0 values less than (5),partition p1 values less than (10), partition p2 values less than maxvalue)") + defer tk.MustExec("drop table t_24746") + err := tk.ExecToErr("insert into t_24746 partition (p1) values(4,'ERROR, not matching partition p1',4)") + c.Assert(table.ErrRowDoesNotMatchGivenPartitionSet.Equal(err), IsTrue) + tk.MustExec("insert into t_24746 partition (p0) values(4,'OK, first row in correct partition',4)") + err = tk.ExecToErr("insert into t_24746 partition (p0) values(4,'DUPLICATE, in p0',4) on duplicate key update a = a + 1, b = 'ERROR, not allowed to write to p1'") + c.Assert(table.ErrRowDoesNotMatchGivenPartitionSet.Equal(err), IsTrue) + // Actual bug, before the fix this was updating the row in p0 (deleting it in p0 and inserting in p1): + err = tk.ExecToErr("insert into t_24746 partition (p1) values(4,'ERROR, not allowed to read from partition p0',4) on duplicate key update a = a + 1, b = 'ERROR, not allowed to read from p0!'") + c.Assert(table.ErrRowDoesNotMatchGivenPartitionSet.Equal(err), IsTrue) +} diff --git a/table/tables/tables.go b/table/tables/tables.go index 8fd3cca9e2657..c78d1909cf135 100644 --- a/table/tables/tables.go +++ b/table/tables/tables.go @@ -36,7 +36,7 @@ import ( "github.com/pingcap/tidb/sessionctx/binloginfo" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx/variable" - tikvstore "github.com/pingcap/tidb/store/tikv/kv" + "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" @@ -46,6 +46,7 @@ import ( "github.com/pingcap/tidb/util/generatedexpr" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/stringutil" + "github.com/pingcap/tidb/util/tableutil" "github.com/pingcap/tipb/go-binlog" "github.com/pingcap/tipb/go-tipb" "go.uber.org/zap" @@ -322,8 +323,13 @@ func (t *TableCommon) UpdateRecord(ctx context.Context, sctx sessionctx.Context, sh := memBuffer.Staging() defer memBuffer.Cleanup(sh) - if meta := t.Meta(); meta.TempTableType == model.TempTableGlobal { - addTemporaryTableID(sctx, meta.ID) + if m := t.Meta(); m.TempTableType == model.TempTableGlobal { + if tmpTable := addTemporaryTable(sctx, m); tmpTable != nil { + if tmpTable.GetSize() > sctx.GetSessionVars().TMPTableSize { + return table.ErrTempTableFull.GenWithStackByArgs(m.Name.O) + } + defer handleTempTableSize(tmpTable, txn.Size(), txn) + } } var colIDs, binlogColIDs []int64 @@ -588,12 +594,20 @@ func TryGetCommonPkColumns(tbl table.Table) []*table.Column { return pkCols } -func addTemporaryTableID(sctx sessionctx.Context, id int64) { - txnCtx := sctx.GetSessionVars().TxnCtx - if txnCtx.GlobalTemporaryTables == nil { - txnCtx.GlobalTemporaryTables = make(map[int64]struct{}) - } - txnCtx.GlobalTemporaryTables[id] = struct{}{} +func addTemporaryTable(sctx sessionctx.Context, tblInfo *model.TableInfo) tableutil.TempTable { + tempTable := sctx.GetSessionVars().GetTemporaryTable(tblInfo) + tempTable.SetModified(true) + return tempTable +} + +// The size of a temporary table is calculated by accumulating the transaction size delta. +func handleTempTableSize(t tableutil.TempTable, txnSizeBefore int, txn kv.Transaction) { + txnSizeNow := txn.Size() + delta := txnSizeNow - txnSizeBefore + + oldSize := t.GetSize() + newSize := oldSize + int64(delta) + t.SetSize(newSize) } // AddRecord implements table.Table AddRecord interface. @@ -608,8 +622,13 @@ func (t *TableCommon) AddRecord(sctx sessionctx.Context, r []types.Datum, opts . fn.ApplyOn(&opt) } - if meta := t.Meta(); meta.TempTableType == model.TempTableGlobal { - addTemporaryTableID(sctx, meta.ID) + if m := t.Meta(); m.TempTableType == model.TempTableGlobal { + if tmpTable := addTemporaryTable(sctx, m); tmpTable != nil { + if tmpTable.GetSize() > sctx.GetSessionVars().TMPTableSize { + return nil, table.ErrTempTableFull.GenWithStackByArgs(m.Name.O) + } + defer handleTempTableSize(tmpTable, txn.Size(), txn) + } } var ctx context.Context @@ -771,7 +790,7 @@ func (t *TableCommon) AddRecord(sctx sessionctx.Context, r []types.Datum, opts . } if setPresume { - err = memBuffer.SetWithFlags(key, value, tikvstore.SetPresumeKeyNotExists) + err = memBuffer.SetWithFlags(key, value, kv.SetPresumeKeyNotExists) } else { err = memBuffer.Set(key, value) } @@ -1010,8 +1029,17 @@ func (t *TableCommon) RemoveRecord(ctx sessionctx.Context, h kv.Handle, r []type return err } - if meta := t.Meta(); meta.TempTableType == model.TempTableGlobal { - addTemporaryTableID(ctx, meta.ID) + txn, err := ctx.Txn(true) + if err != nil { + return err + } + if m := t.Meta(); m.TempTableType == model.TempTableGlobal { + if tmpTable := addTemporaryTable(ctx, m); tmpTable != nil { + if tmpTable.GetSize() > ctx.GetSessionVars().TMPTableSize { + return table.ErrTempTableFull.GenWithStackByArgs(m.Name.O) + } + defer handleTempTableSize(tmpTable, txn.Size(), txn) + } } // The table has non-public column and this column is doing the operation of "modify/change column". @@ -1153,7 +1181,7 @@ func (t *TableCommon) removeRowIndices(ctx sessionctx.Context, h kv.Handle, rec logutil.BgLogger().Info("remove row index failed", zap.Any("index", v.Meta()), zap.Uint64("txnStartTS", txn.StartTS()), zap.String("handle", h.String()), zap.Any("record", rec), zap.Error(err)) return err } - if err = v.Delete(ctx.GetSessionVars().StmtCtx, txn.GetUnionStore(), vals, h); err != nil { + if err = v.Delete(ctx.GetSessionVars().StmtCtx, txn, vals, h); err != nil { if v.Meta().State != model.StatePublic && kv.ErrNotExist.Equal(err) { // If the index is not in public state, we may have not created the index, // or already deleted the index, so skip ErrNotExist error. @@ -1168,7 +1196,7 @@ func (t *TableCommon) removeRowIndices(ctx sessionctx.Context, h kv.Handle, rec // removeRowIndex implements table.Table RemoveRowIndex interface. func (t *TableCommon) removeRowIndex(sc *stmtctx.StatementContext, h kv.Handle, vals []types.Datum, idx table.Index, txn kv.Transaction) error { - return idx.Delete(sc, txn.GetUnionStore(), vals, h) + return idx.Delete(sc, txn, vals, h) } // buildIndexForRow implements table.Table BuildIndexForRow interface. @@ -1370,7 +1398,17 @@ func OverflowShardBits(recordID int64, shardRowIDBits uint64, typeBitsLength uin // Allocators implements table.Table Allocators interface. func (t *TableCommon) Allocators(ctx sessionctx.Context) autoid.Allocators { - if ctx == nil || ctx.GetSessionVars().IDAllocator == nil { + if ctx == nil { + return t.allocs + } else if ctx.GetSessionVars().IDAllocator == nil { + // Use an independent allocator for global temporary tables. + if t.meta.TempTableType == model.TempTableGlobal { + if alloc := ctx.GetSessionVars().GetTemporaryTable(t.meta).GetAutoIDAllocator(); alloc != nil { + return autoid.Allocators{alloc} + } + // If the session is not in a txn, for example, in "show create table", use the original allocator. + // Otherwise the would be a nil pointer dereference. + } return t.allocs } @@ -1408,6 +1446,9 @@ func shouldWriteBinlog(ctx sessionctx.Context, tblInfo *model.TableInfo) bool { if ctx.GetSessionVars().BinlogClient == nil { return false } + if tblInfo.TempTableType != model.TempTableNone { + return false + } return !ctx.GetSessionVars().InRestrictedSQL } @@ -1498,6 +1539,7 @@ func getDuplicateErrorHandleString(t table.Table, handle kv.Handle, row []types. func init() { table.TableFromMeta = TableFromMeta table.MockTableFromMeta = MockTableFromMeta + tableutil.TempTableFromMeta = TempTableFromMeta } // sequenceCommon cache the sequence value. @@ -1763,3 +1805,55 @@ func BuildTableScanFromInfos(tableInfo *model.TableInfo, columnInfos []*model.Co } return tsExec } + +// TemporaryTable is used to store transaction-specific or session-specific information for global / local temporary tables. +// For example, stats and autoID should have their own copies of data, instead of being shared by all sessions. +type TemporaryTable struct { + // Whether it's modified in this transaction. + modified bool + // The stats of this table. So far it's always pseudo stats. + stats *statistics.Table + // The autoID allocator of this table. + autoIDAllocator autoid.Allocator + // Table size. + size int64 +} + +// TempTableFromMeta builds a TempTable from model.TableInfo. +func TempTableFromMeta(tblInfo *model.TableInfo) tableutil.TempTable { + return &TemporaryTable{ + modified: false, + stats: statistics.PseudoTable(tblInfo), + autoIDAllocator: autoid.NewAllocatorFromTempTblInfo(tblInfo), + } +} + +// GetAutoIDAllocator is implemented from TempTable.GetAutoIDAllocator. +func (t *TemporaryTable) GetAutoIDAllocator() autoid.Allocator { + return t.autoIDAllocator +} + +// SetModified is implemented from TempTable.SetModified. +func (t *TemporaryTable) SetModified(modified bool) { + t.modified = modified +} + +// GetModified is implemented from TempTable.GetModified. +func (t *TemporaryTable) GetModified() bool { + return t.modified +} + +// GetStats is implemented from TempTable.GetStats. +func (t *TemporaryTable) GetStats() interface{} { + return t.stats +} + +// GetSize gets the table size. +func (t *TemporaryTable) GetSize() int64 { + return t.size +} + +// SetSize sets the table size. +func (t *TemporaryTable) SetSize(v int64) { + t.size = v +} diff --git a/tablecodec/tablecodec.go b/tablecodec/tablecodec.go index 215f8d05c27fa..de766831bc245 100644 --- a/tablecodec/tablecodec.go +++ b/tablecodec/tablecodec.go @@ -1292,6 +1292,7 @@ func TruncateIndexValue(v *types.Datum, idxCol *model.IndexColumn, tblCol *model if notStringType { return } + originalKind := v.Kind() isUTF8Charset := tblCol.Charset == charset.CharsetUTF8 || tblCol.Charset == charset.CharsetUTF8MB4 if isUTF8Charset && utf8.RuneCount(v.GetBytes()) > idxCol.Length { rs := bytes.Runes(v.GetBytes()) @@ -1303,7 +1304,7 @@ func TruncateIndexValue(v *types.Datum, idxCol *model.IndexColumn, tblCol *model } } else if !isUTF8Charset && len(v.GetBytes()) > idxCol.Length { v.SetBytes(v.GetBytes()[:idxCol.Length]) - if v.Kind() == types.KindString { + if originalKind == types.KindString { v.SetString(v.GetString(), tblCol.Collate) } } diff --git a/telemetry/data.go b/telemetry/data.go index 1cf0e3d66aa25..fa1de90efe092 100644 --- a/telemetry/data.go +++ b/telemetry/data.go @@ -27,6 +27,7 @@ type telemetryData struct { TrackingID string `json:"trackingId"` FeatureUsage *featureUsage `json:"featureUsage"` WindowedStats []*windowData `json:"windowedStats"` + SlowQueryStats *slowQueryStats `json:"slowQueryStats"` } func generateTelemetryData(ctx sessionctx.Context, trackingID string) telemetryData { @@ -43,6 +44,10 @@ func generateTelemetryData(ctx sessionctx.Context, trackingID string) telemetryD if f, err := getFeatureUsage(ctx); err == nil { r.FeatureUsage = f } + if s, err := getSlowQueryStats(ctx); err == nil { + r.SlowQueryStats = s + } + r.WindowedStats = getWindowData() r.TelemetryHostExtra = getTelemetryHostExtraInfo() return r @@ -50,4 +55,6 @@ func generateTelemetryData(ctx sessionctx.Context, trackingID string) telemetryD func postReportTelemetryData() { postReportTxnUsage() + postReportCTEUsage() + postReportSlowQueryStats() } diff --git a/telemetry/data_feature_usage.go b/telemetry/data_feature_usage.go index ff3e3c672cff0..508bd70e8c23c 100644 --- a/telemetry/data_feature_usage.go +++ b/telemetry/data_feature_usage.go @@ -15,31 +15,70 @@ package telemetry import ( "context" + "errors" + "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/infoschema" + m "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/sqlexec" ) type featureUsage struct { - Txn *TxnUsage `json:"txn"` - ClusterIndex map[string]bool `json:"clusterIndex"` + // transaction usage information + Txn *TxnUsage `json:"txn"` + // cluster index usage information + // key is the first 6 characters of sha2(TABLE_NAME, 256) + ClusterIndex *ClusterIndexUsage `json:"clusterIndex"` + TemporaryTable bool `json:"temporaryTable"` + CTE *m.CTEUsageCounter `json:"cte"` } func getFeatureUsage(ctx sessionctx.Context) (*featureUsage, error) { - // init - usageInfo := featureUsage{ - ClusterIndex: make(map[string]bool), + + clusterIdxUsage, err := GetClusterIndexUsageInfo(ctx) + if err != nil { + logutil.BgLogger().Info(err.Error()) + return nil, err } - // cluster index + // transaction related feature + txnUsage := GetTxnUsageInfo(ctx) + + // Avoid the circle dependency. + temporaryTable := ctx.(TemporaryTableFeatureChecker).TemporaryTableExists() + + cteUsage := GetCTEUsageInfo(ctx) + + return &featureUsage{txnUsage, clusterIdxUsage, temporaryTable, cteUsage}, nil +} + +// ClusterIndexUsage records the usage info of all the tables, no more than 10k tables +type ClusterIndexUsage map[string]TableClusteredInfo + +// TableClusteredInfo records the usage info of clusterindex of each table +// CLUSTERED, NON_CLUSTERED, NA +type TableClusteredInfo struct { + IsClustered bool `json:"isClustered"` // True means CLUSTERED, False means NON_CLUSTERED + ClusterPKType string `json:"clusterPKType"` // INT means clustered PK type is int + // NON_INT means clustered PK type is not int + // NA means this field is no meaningful information +} + +// GetClusterIndexUsageInfo gets the ClusterIndex usage information. It's exported for future test. +func GetClusterIndexUsageInfo(ctx sessionctx.Context) (cu *ClusterIndexUsage, err error) { + usage := make(ClusterIndexUsage) exec := ctx.(sqlexec.RestrictedSQLExecutor) + + // query INFORMATION_SCHEMA.tables to get the latest table information about ClusterIndex stmt, err := exec.ParseWithParams(context.TODO(), ` - SELECT left(sha2(TABLE_NAME, 256), 6) name, TIDB_PK_TYPE + SELECT left(sha2(TABLE_NAME, 256), 6) table_name_hash, TIDB_PK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.tables WHERE table_schema not in ('INFORMATION_SCHEMA', 'METRICS_SCHEMA', 'PERFORMANCE_SCHEMA', 'mysql') - ORDER BY name + ORDER BY table_name_hash limit 10000`) if err != nil { return nil, err @@ -48,21 +87,61 @@ func getFeatureUsage(ctx sessionctx.Context) (*featureUsage, error) { if err != nil { return nil, err } + + defer func() { + if r := recover(); r != nil { + switch x := r.(type) { + case string: + err = errors.New(x) + case error: + err = x + default: + err = errors.New("unknown failure") + } + } + }() + + err = ctx.RefreshTxnCtx(context.TODO()) + if err != nil { + return nil, err + } + infoSchema := ctx.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema) + + // check ClusterIndex information for each table + // row: 0 = table_name_hash, 1 = TIDB_PK_TYPE, 2 = TABLE_SCHEMA (db), 3 = TABLE_NAME + for _, row := range rows { - if row.Len() < 2 { + if row.Len() < 4 { continue } - isClustered := false + tblClusteredInfo := TableClusteredInfo{false, "NA"} if row.GetString(1) == "CLUSTERED" { - isClustered = true + tblClusteredInfo.IsClustered = true + table, err := infoSchema.TableByName(model.NewCIStr(row.GetString(2)), model.NewCIStr(row.GetString(3))) + if err != nil { + continue + } + tableInfo := table.Meta() + if tableInfo.PKIsHandle { + tblClusteredInfo.ClusterPKType = "INT" + } else if tableInfo.IsCommonHandle { + tblClusteredInfo.ClusterPKType = "NON_INT" + } else { + // if both CLUSTERED IS TURE and CLUSTERPKTYPE IS NA met, this else is hit + // it means the status of INFORMATION_SCHEMA.tables if not consistent with session.Context + // WE SHOULD treat this issue SERIOUSLY + } } - usageInfo.ClusterIndex[row.GetString(0)] = isClustered + usage[row.GetString(0)] = tblClusteredInfo } - // transaction related feature - usageInfo.Txn = GetTxnUsageInfo(ctx) + return &usage, nil +} - return &usageInfo, nil +// TemporaryTableFeatureChecker is defined to avoid package circle dependency. +// The session struct implements this interface. +type TemporaryTableFeatureChecker interface { + TemporaryTableExists() bool } // TxnUsage records the usage info of transaction related features, including @@ -74,6 +153,7 @@ type TxnUsage struct { } var initialTxnCommitCounter metrics.TxnCommitCounter +var initialCTECounter m.CTEUsageCounter // GetTxnUsageInfo gets the usage info of transaction related features. It's exported for tests. func GetTxnUsageInfo(ctx sessionctx.Context) *TxnUsage { @@ -93,3 +173,15 @@ func GetTxnUsageInfo(ctx sessionctx.Context) *TxnUsage { func postReportTxnUsage() { initialTxnCommitCounter = metrics.GetTxnCommitCounter() } + +// ResetCTEUsage resets CTE usages. +func postReportCTEUsage() { + initialCTECounter = m.GetCTECounter() +} + +// GetCTEUsageInfo gets the CTE usages. +func GetCTEUsageInfo(ctx sessionctx.Context) *m.CTEUsageCounter { + curr := m.GetCTECounter() + diff := curr.Sub(initialCTECounter) + return &diff +} diff --git a/telemetry/data_feature_usage_test.go b/telemetry/data_feature_usage_test.go index 9164591e09224..2d69d093e5360 100644 --- a/telemetry/data_feature_usage_test.go +++ b/telemetry/data_feature_usage_test.go @@ -90,3 +90,17 @@ func (s *testFeatureInfoSuite) TestTxnUsageInfo(c *C) { c.Assert(txnUsage.TxnCommitCounter.OnePC, Greater, int64(0)) c.Assert(txnUsage.TxnCommitCounter.TwoPC, Greater, int64(0)) } + +func (s *testFeatureInfoSuite) TestTemporaryTable(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("set tidb_enable_global_temporary_table=true") + tk.MustExec("use test") + usage, err := telemetry.GetFeatureUsage(tk.Se) + c.Assert(err, IsNil) + c.Assert(usage.TemporaryTable, IsFalse) + + tk.MustExec("create global temporary table t (id int) on commit delete rows") + usage, err = telemetry.GetFeatureUsage(tk.Se) + c.Assert(err, IsNil) + c.Assert(usage.TemporaryTable, IsTrue) +} diff --git a/telemetry/data_slow_query.go b/telemetry/data_slow_query.go new file mode 100644 index 0000000000000..b2408b3223a05 --- /dev/null +++ b/telemetry/data_slow_query.go @@ -0,0 +1,155 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package telemetry + +import ( + "context" + "errors" + "fmt" + "strconv" + "sync" + "time" + + pingcapErrors "github.com/pingcap/errors" + "github.com/pingcap/tidb/domain/infosync" + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/util/logutil" + pmodel "github.com/prometheus/common/model" + "go.uber.org/zap" +) + +type slowQueryStats struct { + // Slow Query statistic buckets + SQBInfo *SlowQueryBucket `json:"slowQueryBucket"` +} + +// SlowQueryBucket records the statistic information of slow query buckets +// Buckets: prometheus.ExponentialBuckets(0.001, 2, 28), // 1ms ~ 1.5days // defined in metrics/server.go +type SlowQueryBucket map[string]int + +func (bucketMap SlowQueryBucket) String() string { + if bucketMap == nil { + return "nil" + } + var retStr string = "{" + for k, v := range bucketMap { + retStr += k + ":" + strconv.Itoa(v) + "," + } + retStr = retStr[:len(retStr)-1] + return retStr +} + +const slowQueryBucketNum = 29 //prometheus.ExponentialBuckets(0.001, 2, 28), and 1 more +Inf + +var ( + // lastSQBInfo records last statistic information of slow query buckets + lastSQBInfo SlowQueryBucket + // currentSQBInfo records current statitic information of slow query buckets + currentSQBInfo SlowQueryBucket + slowQueryLock sync.Mutex +) + +func getSlowQueryStats(ctx sessionctx.Context) (*slowQueryStats, error) { + slowQueryBucket, err := getSlowQueryBucket(ctx) + if err != nil { + logutil.BgLogger().Info(err.Error()) + return nil, err + } + + return &slowQueryStats{slowQueryBucket}, nil +} + +// getSlowQueryBucket genenrates the delta SlowQueryBucket to report +func getSlowQueryBucket(ctx sessionctx.Context) (*SlowQueryBucket, error) { + // update currentSQBInfo first, then gen delta + if err := updateCurrentSQB(ctx); err != nil { + return nil, err + } + delta := calculateDeltaSQB() + return delta, nil +} + +// updateCurrentSQB records current slow query buckets +func updateCurrentSQB(ctx sessionctx.Context) (err error) { + defer func() { + if r := recover(); r != nil { + err = pingcapErrors.Errorf(fmt.Sprintln(r)) + } + }() + + pQueryCtx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + pQueryTs := time.Now().Add(-time.Minute) + promQL := "tidb_server_slow_query_process_duration_seconds_bucket{sql_type=\"general\"}" + value, err := querySQLMetric(pQueryCtx, pQueryTs, promQL) + + if err != nil && err != infosync.ErrPrometheusAddrIsNotSet { + logutil.BgLogger().Info("querySlowQueryMetric got error") + return err + } + if value == nil { + return + } + if value.Type() != pmodel.ValVector { + return errors.New("Prom vector expected, got " + value.Type().String()) + } + promVec := value.(pmodel.Vector) + slowQueryLock.Lock() + for _, sample := range promVec { + metric := sample.Metric + bucketName := metric["le"] //hardcode bucket upper bound + currentSQBInfo[string(bucketName)] = int(sample.Value) + } + slowQueryLock.Unlock() + return nil +} + +// calculateDeltaSQB calculate the delta between current slow query bucket and last slow query bucket +func calculateDeltaSQB() *SlowQueryBucket { + deltaMap := make(SlowQueryBucket) + slowQueryLock.Lock() + for key, value := range currentSQBInfo { + deltaMap[key] = value - (lastSQBInfo)[key] + } + slowQueryLock.Unlock() + return &deltaMap +} + +// init Init lastSQBInfo, follow the definition of metrics/server.go +// Buckets: prometheus.ExponentialBuckets(0.001, 2, 28), // 1ms ~ 1.5days +func init() { + lastSQBInfo = make(SlowQueryBucket) + currentSQBInfo = make(SlowQueryBucket) + + bucketBase := 0.001 // From 0.001 to 134217.728, total 28 float number; the 29th is +Inf + for i := 0; i < slowQueryBucketNum-1; i++ { + lastSQBInfo[strconv.FormatFloat(bucketBase, 'f', 3, 32)] = 0 + currentSQBInfo[strconv.FormatFloat(bucketBase, 'f', 3, 32)] = 0 + bucketBase += bucketBase + } + lastSQBInfo["+Inf"] = 0 + currentSQBInfo["+Inf"] = 0 + + logutil.BgLogger().Info("Telemetry slow query stats initialized", zap.String("currentSQBInfo", currentSQBInfo.String()), zap.String("lastSQBInfo", lastSQBInfo.String())) +} + +// postReportSlowQueryStats copy currentSQBInfo to lastSQBInfo to be ready for next report +// this function is designed for being compatible with preview telemetry +func postReportSlowQueryStats() { + slowQueryLock.Lock() + lastSQBInfo = currentSQBInfo + currentSQBInfo = make(SlowQueryBucket) + slowQueryLock.Unlock() + logutil.BgLogger().Info("Telemetry slow query stats, postReportSlowQueryStats finished") +} diff --git a/telemetry/data_window.go b/telemetry/data_window.go index 5dd1f6e750bba..cb86f23bc09a2 100644 --- a/telemetry/data_window.go +++ b/telemetry/data_window.go @@ -14,10 +14,18 @@ package telemetry import ( + "context" "sync" "time" + "github.com/pingcap/errors" + "github.com/pingcap/tidb/domain/infosync" + "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/prometheus/client_golang/api" + promv1 "github.com/prometheus/client_golang/api/prometheus/v1" + pmodel "github.com/prometheus/common/model" "go.uber.org/atomic" + "go.uber.org/zap" ) var ( @@ -51,6 +59,7 @@ const ( maxSubWindowLength = int(ReportInterval / SubWindowSize) // TODO: Ceiling? maxSubWindowLengthInWindow = int(WindowSize / SubWindowSize) // TODO: Ceiling? + promReadTimeout = time.Second * 30 ) type windowData struct { @@ -58,6 +67,14 @@ type windowData struct { ExecuteCount uint64 `json:"executeCount"` TiFlashUsage tiFlashUsageData `json:"tiFlashUsage"` CoprCacheUsage coprCacheUsageData `json:"coprCacheUsage"` + SQLUsage sqlUsageData `json:"SQLUsage"` +} + +type sqlType map[string]int64 + +type sqlUsageData struct { + SQLTotal int64 `json:"total"` + SQLType sqlType `json:"type"` } type coprCacheUsageData struct { @@ -80,6 +97,75 @@ var ( subWindowsLock = sync.RWMutex{} ) +func getSQLSum(sqlTypeData *sqlType) int64 { + result := int64(0) + for _, v := range *sqlTypeData { + result += v + } + return result +} + +func readSQLMetric(timepoint time.Time, SQLResult *sqlUsageData) error { + ctx := context.TODO() + promQL := "sum(tidb_executor_statement_total{}) by (instance,type)" + result, err := querySQLMetric(ctx, timepoint, promQL) + if err != nil { + if err1, ok := err.(*promv1.Error); ok { + return errors.Errorf("query metric error, msg: %v, detail: %v", err1.Msg, err1.Detail) + } + return errors.Errorf("query metric error: %v", err.Error()) + } + + anylisSQLUsage(result, SQLResult) + return nil +} + +func querySQLMetric(ctx context.Context, queryTime time.Time, promQL string) (result pmodel.Value, err error) { + // Add retry to avoid network error. + var prometheusAddr string + for i := 0; i < 5; i++ { + //TODO: the prometheus will be Integrated into the PD, then we need to query the prometheus in PD directly, which need change the quire API + prometheusAddr, err = infosync.GetPrometheusAddr() + if err == nil || err == infosync.ErrPrometheusAddrIsNotSet { + break + } + time.Sleep(100 * time.Millisecond) + } + if err != nil { + return nil, err + } + promClient, err := api.NewClient(api.Config{ + Address: prometheusAddr, + }) + if err != nil { + return nil, err + } + promQLAPI := promv1.NewAPI(promClient) + ctx, cancel := context.WithTimeout(ctx, promReadTimeout) + defer cancel() + // Add retry to avoid network error. + for i := 0; i < 5; i++ { + result, _, err = promQLAPI.Query(ctx, promQL, queryTime) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + return result, err +} + +func anylisSQLUsage(promResult pmodel.Value, SQLResult *sqlUsageData) { + switch promResult.Type() { + case pmodel.ValVector: + matrix := promResult.(pmodel.Vector) + for _, m := range matrix { + v := m.Value + promLable := string(m.Metric[pmodel.LabelName("type")]) + SQLResult.SQLType[promLable] = int64(float64(v)) + } + } +} + // RotateSubWindow rotates the telemetry sub window. func RotateSubWindow() { thisSubWindow := windowData{ @@ -98,7 +184,18 @@ func RotateSubWindow() { GTE80: CurrentCoprCacheHitRatioGTE80Count.Swap(0), GTE100: CurrentCoprCacheHitRatioGTE100Count.Swap(0), }, + SQLUsage: sqlUsageData{ + SQLTotal: 0, + SQLType: make(sqlType), + }, } + + if err := readSQLMetric(time.Now(), &thisSubWindow.SQLUsage); err != nil { + logutil.BgLogger().Error("Error exists when calling prometheus", zap.Error(err)) + + } + thisSubWindow.SQLUsage.SQLTotal = getSQLSum(&thisSubWindow.SQLUsage.SQLType) + subWindowsLock.Lock() rotatedSubWindows = append(rotatedSubWindows, &thisSubWindow) if len(rotatedSubWindows) > maxSubWindowLength { @@ -108,6 +205,14 @@ func RotateSubWindow() { subWindowsLock.Unlock() } +func calDeltaSQLTypeMap(cur sqlType, last sqlType) sqlType { + deltaMap := make(sqlType) + for key, value := range cur { + deltaMap[key] = value - (last)[key] + } + return deltaMap +} + // getWindowData returns data aggregated by window size. func getWindowData() []*windowData { results := make([]*windowData, 0) @@ -117,6 +222,12 @@ func getWindowData() []*windowData { i := 0 for i < len(rotatedSubWindows) { thisWindow := *rotatedSubWindows[i] + var startWindow windowData + if i == 0 { + startWindow = thisWindow + } else { + startWindow = *rotatedSubWindows[i-1] + } aggregatedSubWindows := 1 // Aggregate later sub windows i++ @@ -131,6 +242,8 @@ func getWindowData() []*windowData { thisWindow.CoprCacheUsage.GTE40 += rotatedSubWindows[i].CoprCacheUsage.GTE40 thisWindow.CoprCacheUsage.GTE80 += rotatedSubWindows[i].CoprCacheUsage.GTE80 thisWindow.CoprCacheUsage.GTE100 += rotatedSubWindows[i].CoprCacheUsage.GTE100 + thisWindow.SQLUsage.SQLTotal = rotatedSubWindows[i].SQLUsage.SQLTotal - startWindow.SQLUsage.SQLTotal + thisWindow.SQLUsage.SQLType = calDeltaSQLTypeMap(rotatedSubWindows[i].SQLUsage.SQLType, startWindow.SQLUsage.SQLType) aggregatedSubWindows++ i++ } diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 55d6dfb215d34..85e6554ab5296 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -170,8 +170,6 @@ func InitialRun(ctx sessionctx.Context, etcdClient *clientv3.Client) error { if err != nil { return err } - logutil.BgLogger().Info("Telemetry configuration", zap.String("endpoint", apiEndpoint), zap.Duration("report_interval", ReportInterval), zap.Bool("enabled", enabled)) - return ReportUsageData(ctx, etcdClient) } diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go index fab7f82ade9fe..ecb6aa00768b6 100644 --- a/telemetry/telemetry_test.go +++ b/telemetry/telemetry_test.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/telemetry" + "github.com/pingcap/tidb/util/testkit" "go.etcd.io/etcd/integration" ) @@ -145,3 +146,38 @@ func (s *testSuite) Test03Report(c *C) { c.Assert(jsonParsed.Path("error_msg").Data().(string), Equals, "telemetry is disabled") c.Assert(jsonParsed.Path("is_request_sent").Data().(bool), Equals, false) } + +func (s *testSuite) TestCTEPreviewAndReport(c *C) { + config.GetGlobalConfig().EnableTelemetry = true + + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("with cte as (select 1) select * from cte") + tk.MustExec("with recursive cte as (select 1) select * from cte") + tk.MustExec("with recursive cte(n) as (select 1 union select * from cte where n < 5) select * from cte") + tk.MustExec("select 1") + + res, err := telemetry.PreviewUsageData(s.se, s.etcdCluster.RandClient()) + c.Assert(err, IsNil) + + jsonParsed, err := gabs.ParseJSON([]byte(res)) + c.Assert(err, IsNil) + + c.Assert(int(jsonParsed.Path("featureUsage.cte.nonRecursiveCTEUsed").Data().(float64)), Equals, 2) + c.Assert(int(jsonParsed.Path("featureUsage.cte.recursiveUsed").Data().(float64)), Equals, 1) + // TODO: Fix this case. If run this test singly, the result is 2. But if run the whole test, the result is 4. + //c.Assert(int(jsonParsed.Path("featureUsage.cte.nonCTEUsed").Data().(float64)), Equals, 2) + + err = telemetry.ReportUsageData(s.se, s.etcdCluster.RandClient()) + c.Assert(err, IsNil) + + res, err = telemetry.PreviewUsageData(s.se, s.etcdCluster.RandClient()) + c.Assert(err, IsNil) + + jsonParsed, err = gabs.ParseJSON([]byte(res)) + c.Assert(err, IsNil) + + c.Assert(int(jsonParsed.Path("featureUsage.cte.nonRecursiveCTEUsed").Data().(float64)), Equals, 0) + c.Assert(int(jsonParsed.Path("featureUsage.cte.recursiveUsed").Data().(float64)), Equals, 0) + c.Assert(int(jsonParsed.Path("featureUsage.cte.nonCTEUsed").Data().(float64)), Equals, 0) +} diff --git a/telemetry/util_test.go b/telemetry/util_test.go index 035a8f9a3b2f8..f480bef6de62c 100644 --- a/telemetry/util_test.go +++ b/telemetry/util_test.go @@ -17,6 +17,7 @@ import ( "testing" . "github.com/pingcap/check" + "github.com/pingcap/tidb/sessionctx" ) var _ = Suite(&testUtilSuite{}) @@ -62,3 +63,6 @@ func (s *testUtilSuite) TestParseAddress(c *C) { c.Assert(port, Equals, tt.expectedPort) } } + +// GetFeatureUsage exports getFeatureUsage for testing. +var GetFeatureUsage func(ctx sessionctx.Context) (*featureUsage, error) = getFeatureUsage diff --git a/tests/globalkilltest/global_kill_test.go b/tests/globalkilltest/global_kill_test.go index f424a4b52d167..164e5fbf02525 100644 --- a/tests/globalkilltest/global_kill_test.go +++ b/tests/globalkilltest/global_kill_test.go @@ -535,6 +535,7 @@ func (s *TestGlobalKillSuite) TestMultipleTiDB(c *C) { } func (s *TestGlobalKillSuite) TestLostConnection(c *C) { + c.Skip("unstable, skip race test") c.Assert(s.pdErr, IsNil, Commentf(msgErrConnectPD, s.pdErr)) // tidb1 diff --git a/tests/globalkilltest/go.mod b/tests/globalkilltest/go.mod index f5919d7a2df4b..0fccf3572c3fe 100644 --- a/tests/globalkilltest/go.mod +++ b/tests/globalkilltest/go.mod @@ -1,6 +1,6 @@ module github.com/pingcap/tests/globalkilltest -go 1.13 +go 1.16 require ( github.com/go-sql-driver/mysql v1.5.0 diff --git a/tidb-server/main.go b/tidb-server/main.go index f070d2eeec48d..aecae4e924192 100644 --- a/tidb-server/main.go +++ b/tidb-server/main.go @@ -17,7 +17,6 @@ import ( "context" "flag" "fmt" - "io/ioutil" "os" "runtime" "strconv" @@ -52,6 +51,7 @@ import ( "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/util" + "github.com/pingcap/tidb/util/deadlockhistory" "github.com/pingcap/tidb/util/disk" "github.com/pingcap/tidb/util/domainutil" "github.com/pingcap/tidb/util/kvcache" @@ -64,12 +64,12 @@ import ( "github.com/pingcap/tidb/util/sys/linux" storageSys "github.com/pingcap/tidb/util/sys/storage" "github.com/pingcap/tidb/util/systimemon" + "github.com/pingcap/tidb/util/topsql" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/push" pd "github.com/tikv/pd/client" "go.uber.org/automaxprocs/maxprocs" "go.uber.org/zap" - "google.golang.org/grpc/grpclog" ) // Flag Names @@ -178,6 +178,7 @@ func main() { printInfo() setupBinlogClient() setupMetrics() + topsql.SetupTopSQL() storage, dom := createStoreAndDomain() svr := createServer(storage, dom) @@ -528,15 +529,18 @@ func setGlobalVars() { variable.SetSysVar(variable.TiDBForcePriority, mysql.Priority2Str[priority]) variable.SetSysVar(variable.TiDBOptDistinctAggPushDown, variable.BoolToOnOff(cfg.Performance.DistinctAggPushDown)) - variable.SetSysVar(variable.TIDBMemQuotaQuery, strconv.FormatInt(cfg.MemQuotaQuery, 10)) + variable.SetSysVar(variable.TiDBMemQuotaQuery, strconv.FormatInt(cfg.MemQuotaQuery, 10)) variable.SetSysVar(variable.LowerCaseTableNames, strconv.Itoa(cfg.LowerCaseTableNames)) variable.SetSysVar(variable.LogBin, variable.BoolToOnOff(cfg.Binlog.Enable)) variable.SetSysVar(variable.Port, fmt.Sprintf("%d", cfg.Port)) variable.SetSysVar(variable.Socket, cfg.Socket) variable.SetSysVar(variable.DataDir, cfg.Path) variable.SetSysVar(variable.TiDBSlowQueryFile, cfg.Log.SlowQueryFile) - variable.SetSysVar(variable.TiDBIsolationReadEngines, strings.Join(cfg.IsolationRead.Engines, ", ")) + variable.SetSysVar(variable.TiDBIsolationReadEngines, strings.Join(cfg.IsolationRead.Engines, ",")) variable.MemoryUsageAlarmRatio.Store(cfg.Performance.MemoryUsageAlarmRatio) + if hostname, err := os.Hostname(); err != nil { + variable.SetSysVar(variable.Hostname, hostname) + } if cfg.Security.EnableSEM { sem.Enable() @@ -559,7 +563,7 @@ func setGlobalVars() { } atomic.StoreUint64(&tikv.CommitMaxBackoff, uint64(parseDuration(cfg.TiKVClient.CommitTimeout).Seconds()*1000)) - tikv.RegionCacheTTLSec = int64(cfg.TiKVClient.RegionCacheTTL) + tikv.SetRegionCacheTTLSec(int64(cfg.TiKVClient.RegionCacheTTL)) domainutil.RepairInfo.SetRepairMode(cfg.RepairMode) domainutil.RepairInfo.SetRepairTableList(cfg.RepairTableList) executor.GlobalDiskUsageTracker.SetBytesLimit(cfg.TempStorageQuota) @@ -572,12 +576,13 @@ func setGlobalVars() { kvcache.GlobalLRUMemUsageTracker.AttachToGlobalTracker(executor.GlobalMemoryUsageTracker) t, err := time.ParseDuration(cfg.TiKVClient.StoreLivenessTimeout) - if err != nil { + if err != nil || t < 0 { logutil.BgLogger().Fatal("invalid duration value for store-liveness-timeout", zap.String("currentValue", cfg.TiKVClient.StoreLivenessTimeout)) } - tikv.StoreLivenessTimeout = t + tikv.SetStoreLivenessTimeout(t) parsertypes.TiDBStrictIntegerDisplayWidth = cfg.DeprecateIntegerDisplayWidth + deadlockhistory.GlobalDeadlockHistory.Resize(cfg.PessimisticTxn.DeadlockHistoryCapacity) } func setupLog() { @@ -585,14 +590,6 @@ func setupLog() { err := logutil.InitZapLogger(cfg.Log.ToLogConfig()) terror.MustNil(err) - err = logutil.InitLogger(cfg.Log.ToLogConfig()) - terror.MustNil(err) - - if len(os.Getenv("GRPC_DEBUG")) > 0 { - grpclog.SetLoggerV2(grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 999)) - } else { - grpclog.SetLoggerV2(grpclog.NewLoggerV2(ioutil.Discard, ioutil.Discard, os.Stderr)) - } // trigger internal http(s) client init. util.InternalHTTPClient() } @@ -629,7 +626,7 @@ func setupMetrics() { metrics.TimeJumpBackCounter.Inc() } callBackCount := 0 - sucessCallBack := func() { + successCallBack := func() { callBackCount++ // It is callback by monitor per second, we increase metrics.KeepAliveCounter per 5s. if callBackCount >= 5 { @@ -637,7 +634,7 @@ func setupMetrics() { metrics.KeepAliveCounter.Inc() } } - go systimemon.StartMonitor(time.Now, systimeErrHandler, sucessCallBack) + go systimemon.StartMonitor(time.Now, systimeErrHandler, successCallBack) pushMetric(cfg.Status.MetricsAddr, time.Duration(cfg.Status.MetricsInterval)*time.Second) } @@ -654,7 +651,7 @@ func setupTracing() { } func closeDomainAndStorage(storage kv.Storage, dom *domain.Domain) { - atomic.StoreUint32(&tikv.ShuttingDown, 1) + tikv.StoreShuttingDown(1) dom.Close() err := storage.Close() terror.Log(errors.Trace(err)) @@ -669,6 +666,7 @@ func cleanup(svr *server.Server, storage kv.Storage, dom *domain.Domain, gracefu plugin.Shutdown(context.Background()) closeDomainAndStorage(storage, dom) disk.CleanUp() + topsql.Close() } func stringToList(repairString string) []string { diff --git a/tidb-server/main_test.go b/tidb-server/main_test.go index b36166c6ed0a4..f504e9bd47656 100644 --- a/tidb-server/main_test.go +++ b/tidb-server/main_test.go @@ -40,14 +40,14 @@ var _ = Suite(&testMainSuite{}) type testMainSuite struct{} func (t *testMainSuite) TestSetGlobalVars(c *C) { - c.Assert(variable.GetSysVar(variable.TiDBIsolationReadEngines).Value, Equals, "tikv, tiflash, tidb") - c.Assert(variable.GetSysVar(variable.TIDBMemQuotaQuery).Value, Equals, "1073741824") + c.Assert(variable.GetSysVar(variable.TiDBIsolationReadEngines).Value, Equals, "tikv,tiflash,tidb") + c.Assert(variable.GetSysVar(variable.TiDBMemQuotaQuery).Value, Equals, "1073741824") config.UpdateGlobal(func(conf *config.Config) { conf.IsolationRead.Engines = []string{"tikv", "tidb"} conf.MemQuotaQuery = 9999999 }) setGlobalVars() - c.Assert(variable.GetSysVar(variable.TiDBIsolationReadEngines).Value, Equals, "tikv, tidb") - c.Assert(variable.GetSysVar(variable.TIDBMemQuotaQuery).Value, Equals, "9999999") + c.Assert(variable.GetSysVar(variable.TiDBIsolationReadEngines).Value, Equals, "tikv,tidb") + c.Assert(variable.GetSysVar(variable.TiDBMemQuotaQuery).Value, Equals, "9999999") } diff --git a/tools/check/go.mod b/tools/check/go.mod index 171d04af71830..90b34e7a296c4 100644 --- a/tools/check/go.mod +++ b/tools/check/go.mod @@ -11,13 +11,14 @@ require ( github.com/mgechev/revive v0.0.0-20181210140514-b4cc152955fb github.com/nicksnyder/go-i18n v1.10.0 // indirect github.com/pelletier/go-toml v1.2.0 // indirect + github.com/pingcap/errors v0.11.5-0.20210513014640-40f9a1999b3b // indirect + github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce // indirect github.com/securego/gosec v0.0.0-20181211171558-12400f9a1ca7 gopkg.in/alecthomas/gometalinter.v2 v2.0.12 // indirect gopkg.in/alecthomas/gometalinter.v3 v3.0.0 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20170321130658-9670b87a702e // indirect - gopkg.in/yaml.v2 v2.2.2 // indirect honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3 ) -go 1.13 +go 1.16 diff --git a/tools/check/go.sum b/tools/check/go.sum index b4f8676c60fe9..7558a1677d777 100644 --- a/tools/check/go.sum +++ b/tools/check/go.sum @@ -5,6 +5,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZq github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/chzchzchz/goword v0.0.0-20170907005317-a9744cb52b03 h1:0wUHjDfbCAROEAZ96zAJGwcNMkPIheFaIjtQyv3QqfM= github.com/chzchzchz/goword v0.0.0-20170907005317-a9744cb52b03/go.mod h1:uFE9hX+zXEwvyUThZ4gDb9vkAwc5DoHUnRSEpH0VrOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnephin/govet v0.0.0-20171012192244-4a96d43e39d3/go.mod h1:pPTX0MEEoAnfbrAGFj4nSVNhl6YbugRj6eardUZdtGo= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= @@ -44,17 +45,33 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.5-0.20210513014640-40f9a1999b3b h1:j9MP8ma4e75tckq11+n4EhB2xq0xwYNoxL8w9JTZRhs= +github.com/pingcap/errors v0.11.5-0.20210513014640-40f9a1999b3b/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMtVcOkjUcuQKh+YrluSo7+7YMCQSzy30= +github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735 h1:7YvPJVmEeFHR1Tj9sZEYsmarJEQfMVYpd/Vyy/A8dqE= github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/securego/gosec v0.0.0-20181211171558-12400f9a1ca7 h1:Ca7U7/rZ+caxjW2na7wbmgmaPsoSCIlpc6sm0aWtFg0= github.com/securego/gosec v0.0.0-20181211171558-12400f9a1ca7/go.mod h1:m3KbCTwh9vLhm6AKBjE+ALesKilKcQHezI1uVOti0Ks= +github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc= +github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -72,8 +89,10 @@ golang.org/x/tools v0.0.0-20180911133044-677d2ff680c1/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563 h1:NIou6eNFigscvKJmsbyez16S2cIS6idossORlFtSt2E= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190925020647-22afafe3322a h1:3GxqzBPBt1O2dIiPnzldQ5d25CAMWJFBZTpqxLPfjs8= golang.org/x/tools v0.0.0-20190925020647-22afafe3322a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200225230052-807dcd883420 h1:4RJNOV+2rLxMEfr6QIpC7GEv9MjD6ApGXTCLrNF9+eA= golang.org/x/tools v0.0.0-20200225230052-807dcd883420/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -90,4 +109,4 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= \ No newline at end of file +honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/types/datum.go b/types/datum.go index 195f1df36211d..95e20e1d1b2f3 100644 --- a/types/datum.go +++ b/types/datum.go @@ -958,8 +958,21 @@ func (d *Datum) convertToString(sc *stmtctx.StatementContext, target *FieldType) s = d.GetMysqlEnum().String() case KindMysqlSet: s = d.GetMysqlSet().String() - case KindBinaryLiteral, KindMysqlBit: + case KindBinaryLiteral: s = d.GetBinaryLiteral().ToString() + case KindMysqlBit: + // issue #25037 + // bit to binary/varbinary. should consider transferring to uint first. + if target.Tp == mysql.TypeString || (target.Tp == mysql.TypeVarchar && target.Collate == charset.CollationBin) { + val, err := d.GetBinaryLiteral().ToInt(sc) + if err != nil { + s = d.GetBinaryLiteral().ToString() + } else { + s = strconv.FormatUint(val, 10) + } + } else { + s = d.GetBinaryLiteral().ToString() + } case KindMysqlJSON: s = d.GetMysqlJSON().String() default: @@ -1182,6 +1195,14 @@ func (d *Datum) convertToMysqlTime(sc *stmtctx.StatementContext, target *FieldTy t, err = ParseTime(sc, d.GetString(), tp, fsp) case KindInt64: t, err = ParseTimeFromNum(sc, d.GetInt64(), tp, fsp) + case KindUint64: + intOverflow64 := d.GetInt64() < 0 + if intOverflow64 { + uNum := strconv.FormatUint(d.GetUint64(), 10) + t, err = ZeroDate, ErrWrongValue.GenWithStackByArgs(TimeStr, uNum) + } else { + t, err = ParseTimeFromNum(sc, d.GetInt64(), tp, fsp) + } case KindMysqlJSON: j := d.GetMysqlJSON() var s string @@ -1457,7 +1478,7 @@ func (d *Datum) convertToMysqlBit(sc *stmtctx.StatementContext, target *FieldTyp return Datum{}, errors.Trace(ErrDataTooLong.GenWithStack("Data Too Long, field len %d", target.Flen)) } if target.Flen < 64 && uintValue >= 1<<(uint64(target.Flen)) { - uintValue &= (1 << (uint64(target.Flen))) - 1 + uintValue = (1 << (uint64(target.Flen))) - 1 err = ErrDataTooLong.GenWithStack("Data Too Long, field len %d", target.Flen) } byteSize := (target.Flen + 7) >> 3 diff --git a/types/json/binary_functions_test.go b/types/json/binary_functions_test.go index 8191638f7b2f6..12a9c8ece8ae2 100644 --- a/types/json/binary_functions_test.go +++ b/types/json/binary_functions_test.go @@ -35,6 +35,6 @@ func (s *testJSONFuncSuite) TestdecodeEscapedUnicode(c *C) { func BenchmarkDecodeEscapedUnicode(b *testing.B) { for i := 0; i < b.N; i++ { in := "597d" - decodeEscapedUnicode([]byte(in)) + _, _, _ = decodeEscapedUnicode([]byte(in)) } } diff --git a/types/time.go b/types/time.go index cae6123dd1f24..28a34b0284833 100644 --- a/types/time.go +++ b/types/time.go @@ -1799,7 +1799,7 @@ func splitDuration(t gotime.Duration) (int, int, int, int, int) { var maxDaysInMonth = []int{31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} -func getTime(sc *stmtctx.StatementContext, num int64, tp byte) (Time, error) { +func getTime(sc *stmtctx.StatementContext, num, originNum int64, tp byte) (Time, error) { s1 := num / 1000000 s2 := num - s1*1000000 @@ -1815,7 +1815,8 @@ func getTime(sc *stmtctx.StatementContext, num int64, tp byte) (Time, error) { ct, ok := FromDateChecked(year, month, day, hour, minute, second, 0) if !ok { - return ZeroDatetime, errors.Trace(ErrWrongValue.GenWithStackByArgs(TimeStr, "")) + numStr := strconv.FormatInt(originNum, 10) + return ZeroDatetime, errors.Trace(ErrWrongValue.GenWithStackByArgs(TimeStr, numStr)) } t := NewTime(ct, tp, DefaultFsp) err := t.check(sc) @@ -1831,11 +1832,12 @@ func parseDateTimeFromNum(sc *stmtctx.StatementContext, num int64) (Time, error) if num == 0 { return t, nil } + originNum := num // Check datetime type. if num >= 10000101000000 { t.SetType(mysql.TypeDatetime) - return getTime(sc, num, t.Type()) + return getTime(sc, num, originNum, t.Type()) } // Check MMDD. @@ -1847,7 +1849,7 @@ func parseDateTimeFromNum(sc *stmtctx.StatementContext, num int64) (Time, error) // YYMMDD, year: 2000-2069 if num <= (70-1)*10000+1231 { num = (num + 20000000) * 1000000 - return getTime(sc, num, t.Type()) + return getTime(sc, num, originNum, t.Type()) } // Check YYMMDD. @@ -1859,13 +1861,13 @@ func parseDateTimeFromNum(sc *stmtctx.StatementContext, num int64) (Time, error) // YYMMDD, year: 1970-1999 if num <= 991231 { num = (num + 19000000) * 1000000 - return getTime(sc, num, t.Type()) + return getTime(sc, num, originNum, t.Type()) } // Adjust hour/min/second. if num <= 99991231 { num = num * 1000000 - return getTime(sc, num, t.Type()) + return getTime(sc, num, originNum, t.Type()) } // Check MMDDHHMMSS. @@ -1880,7 +1882,7 @@ func parseDateTimeFromNum(sc *stmtctx.StatementContext, num int64) (Time, error) // YYMMDDHHMMSS, 2000-2069 if num <= 69*10000000000+1231235959 { num = num + 20000000000000 - return getTime(sc, num, t.Type()) + return getTime(sc, num, originNum, t.Type()) } // Check YYYYMMDDHHMMSS. @@ -1892,10 +1894,10 @@ func parseDateTimeFromNum(sc *stmtctx.StatementContext, num int64) (Time, error) // YYMMDDHHMMSS, 1970-1999 if num <= 991231235959 { num = num + 19000000000000 - return getTime(sc, num, t.Type()) + return getTime(sc, num, originNum, t.Type()) } - return getTime(sc, num, t.Type()) + return getTime(sc, num, originNum, t.Type()) } // ParseTime parses a formatted string with type tp and specific fsp. diff --git a/util/admin/admin.go b/util/admin/admin.go index 20217a53c1b6d..84481d006abb8 100644 --- a/util/admin/admin.go +++ b/util/admin/admin.go @@ -389,18 +389,18 @@ func CheckRecordAndIndex(sessCtx sessionctx.Context, txn kv.Transaction, t table vals1[i] = colDefVal } } - isExist, h2, err := idx.Exist(sc, txn.GetUnionStore(), vals1, h1) + isExist, h2, err := idx.Exist(sc, txn, vals1, h1) if kv.ErrKeyExists.Equal(err) { record1 := &RecordData{Handle: h1, Values: vals1} record2 := &RecordData{Handle: h2, Values: vals1} - return false, ErrDataInConsistent.GenWithStack("index:%#v != record:%#v", record2, record1) + return false, ErrDataInConsistent.GenWithStackByArgs(record2, record1) } if err != nil { return false, errors.Trace(err) } if !isExist { record := &RecordData{Handle: h1, Values: vals1} - return false, ErrDataInConsistent.GenWithStack("index:%#v != record:%#v", nil, record) + return false, ErrDataInConsistent.GenWithStackByArgs(nil, record) } return true, nil diff --git a/util/checksum/checksum.go b/util/checksum/checksum.go index 843440547100c..ef90f44e7cb00 100644 --- a/util/checksum/checksum.go +++ b/util/checksum/checksum.go @@ -42,11 +42,12 @@ var checksumReaderBufPool = sync.Pool{ // | -- 4B -- | -- 1020B -- || -- 4B -- | -- 1020B -- || -- 4B -- | -- 60B -- | // | -- checksum -- | -- payload -- || -- checksum -- | -- payload -- || -- checksum -- | -- payload -- | type Writer struct { - err error - w io.WriteCloser - buf []byte - payload []byte - payloadUsed int + err error + w io.WriteCloser + buf []byte + payload []byte + payloadUsed int + flushedUserDataCnt int64 } // NewWriter returns a new Writer which calculates and stores a CRC-32 checksum for the payload before @@ -104,10 +105,21 @@ func (w *Writer) Flush() error { w.err = err return err } + w.flushedUserDataCnt += int64(w.payloadUsed) w.payloadUsed = 0 return nil } +// GetCache returns the byte slice that holds the data not flushed to disk. +func (w *Writer) GetCache() []byte { + return w.payload[:w.payloadUsed] +} + +// GetCacheDataOffset return the user data offset in cache. +func (w *Writer) GetCacheDataOffset() int64 { + return w.flushedUserDataCnt +} + // Close implements the io.Closer interface. func (w *Writer) Close() (err error) { err = w.Flush() diff --git a/util/checksum/checksum_test.go b/util/checksum/checksum_test.go index b0de5b90586c9..1473903fbe080 100644 --- a/util/checksum/checksum_test.go +++ b/util/checksum/checksum_test.go @@ -651,3 +651,75 @@ func (s *testChecksumSuite) testTiCase3651and3652(c *check.C, encrypt bool) { assertReadAt(0, make([]byte, 10200), nil, 10200, strings.Repeat("0123456789", 1020), f1) assertReadAt(0, make([]byte, 10200), nil, 10200, strings.Repeat("0123456789", 1020), f2) } + +var checkFlushedData = func(c *check.C, f io.ReaderAt, off int64, readBufLen int, assertN int, assertErr interface{}, assertRes []byte) { + readBuf := make([]byte, readBufLen) + r := NewReader(f) + n, err := r.ReadAt(readBuf, off) + c.Assert(err, check.Equals, assertErr) + c.Assert(n, check.Equals, assertN) + c.Assert(bytes.Compare(readBuf, assertRes), check.Equals, 0) +} + +func (s *testChecksumSuite) TestChecksumWriter(c *check.C) { + path := "checksum_TestChecksumWriter" + f, err := os.Create(path) + c.Assert(err, check.IsNil) + defer func() { + err = f.Close() + c.Assert(err, check.IsNil) + err = os.Remove(path) + c.Assert(err, check.IsNil) + }() + + buf := bytes.NewBuffer(nil) + testData := "0123456789" + for i := 0; i < 100; i++ { + buf.WriteString(testData) + } + + // Write 1000 bytes and flush. + w := NewWriter(f) + n, err := w.Write(buf.Bytes()) + c.Assert(err, check.IsNil) + c.Assert(n, check.Equals, 1000) + + err = w.Flush() + c.Assert(err, check.IsNil) + checkFlushedData(c, f, 0, 1000, 1000, nil, buf.Bytes()) + + // All data flushed, so no data in cache. + cacheOff := w.GetCacheDataOffset() + c.Assert(cacheOff, check.Equals, int64(1000)) +} + +func (s *testChecksumSuite) TestChecksumWriterAutoFlush(c *check.C) { + path := "checksum_TestChecksumWriterAutoFlush" + f, err := os.Create(path) + c.Assert(err, check.IsNil) + defer func() { + err = f.Close() + c.Assert(err, check.IsNil) + err = os.Remove(path) + c.Assert(err, check.IsNil) + }() + + w := NewWriter(f) + + buf := bytes.NewBuffer(nil) + testData := "0123456789" + for i := 0; i < 102; i++ { + buf.WriteString(testData) + } + n, err := w.Write(buf.Bytes()) + c.Assert(err, check.IsNil) + c.Assert(n, check.Equals, len(buf.Bytes())) + + // This write will trigger flush. + n, err = w.Write([]byte("0")) + c.Assert(err, check.IsNil) + c.Assert(n, check.Equals, 1) + checkFlushedData(c, f, 0, 1020, 1020, nil, buf.Bytes()) + cacheOff := w.GetCacheDataOffset() + c.Assert(cacheOff, check.Equals, int64(len(buf.Bytes()))) +} diff --git a/util/chunk/chunk.go b/util/chunk/chunk.go index a4350bd9628e3..a15279e1a7ada 100644 --- a/util/chunk/chunk.go +++ b/util/chunk/chunk.go @@ -291,6 +291,21 @@ func (c *Chunk) CopyConstruct() *Chunk { return newChk } +// CopyConstructSel is just like CopyConstruct, +// but ignore the rows that was not selected. +func (c *Chunk) CopyConstructSel() *Chunk { + if c.sel == nil { + return c.CopyConstruct() + } + newChk := renewWithCapacity(c, c.capacity, c.requiredRows) + for colIdx, dstCol := range newChk.columns { + for _, rowIdx := range c.sel { + appendCellByCell(dstCol, c.columns[colIdx], rowIdx) + } + } + return newChk +} + // GrowAndReset resets the Chunk and doubles the capacity of the Chunk. // The doubled capacity should not be larger than maxChunkSize. // TODO: this method will be used in following PR. @@ -505,8 +520,10 @@ func (c *Chunk) Append(other *Chunk, begin, end int) { } else { beginOffset, endOffset := src.offsets[begin], src.offsets[end] dst.data = append(dst.data, src.data[beginOffset:endOffset]...) + lastOffset := dst.offsets[len(dst.offsets)-1] for i := begin; i < end; i++ { - dst.offsets = append(dst.offsets, dst.offsets[len(dst.offsets)-1]+src.offsets[i+1]-src.offsets[i]) + lastOffset += src.offsets[i+1] - src.offsets[i] + dst.offsets = append(dst.offsets, lastOffset) } } for i := begin; i < end; i++ { diff --git a/util/chunk/chunk_test.go b/util/chunk/chunk_test.go index 67222328794db..8c9eec60f99aa 100644 --- a/util/chunk/chunk_test.go +++ b/util/chunk/chunk_test.go @@ -16,7 +16,6 @@ package chunk import ( "bytes" "fmt" - "io/ioutil" "math" "os" "strconv" @@ -36,7 +35,7 @@ import ( ) func TestT(t *testing.T) { - path, _ := ioutil.TempDir("", "oom-use-tmp-storage") + path, _ := os.MkdirTemp("", "oom-use-tmp-storage") config.UpdateGlobal(func(conf *config.Config) { conf.TempStoragePath = path }) @@ -1179,3 +1178,80 @@ func BenchmarkBatchAppendRows(b *testing.B) { }) } } + +func BenchmarkAppendRows(b *testing.B) { + b.ReportAllocs() + rowChk := newChunk(8, 8, 0, 0) + + for i := 0; i < 4096; i++ { + rowChk.AppendNull(0) + rowChk.AppendInt64(1, 1) + rowChk.AppendString(2, "abcd") + rowChk.AppendBytes(3, []byte("abcd")) + } + + type testCaseConf struct { + batchSize int + } + testCaseConfs := []testCaseConf{ + {batchSize: 2}, + {batchSize: 8}, + {batchSize: 16}, + {batchSize: 100}, + {batchSize: 1000}, + {batchSize: 4000}, + } + + chk := newChunk(8, 8, 0, 0) + for _, conf := range testCaseConfs { + b.ResetTimer() + b.Run(fmt.Sprintf("row-%d", conf.batchSize), func(b *testing.B) { + for i := 0; i < b.N; i++ { + chk.Reset() + for j := 0; j < conf.batchSize; j++ { + chk.AppendRow(rowChk.GetRow(j)) + } + } + }) + b.ResetTimer() + b.Run(fmt.Sprintf("column-%d", conf.batchSize), func(b *testing.B) { + for i := 0; i < b.N; i++ { + chk.Reset() + chk.Append(rowChk, 0, conf.batchSize) + } + }) + } +} + +func BenchmarkAppend(b *testing.B) { + b.ReportAllocs() + rowChk := newChunk(0, 0) + + for i := 0; i < 4096; i++ { + rowChk.AppendString(0, "abcd") + rowChk.AppendBytes(1, []byte("abcd")) + } + + type testCaseConf struct { + batchSize int + } + testCaseConfs := []testCaseConf{ + {batchSize: 2}, + {batchSize: 8}, + {batchSize: 16}, + {batchSize: 100}, + {batchSize: 1000}, + {batchSize: 4000}, + } + + chk := newChunk(0, 0) + for _, conf := range testCaseConfs { + b.ResetTimer() + b.Run(fmt.Sprintf("column-%d", conf.batchSize), func(b *testing.B) { + for i := 0; i < b.N; i++ { + chk.Reset() + chk.Append(rowChk, 0, conf.batchSize) + } + }) + } +} diff --git a/util/chunk/disk.go b/util/chunk/disk.go index c7962c9aa9e9d..a9f7eecec3641 100644 --- a/util/chunk/disk.go +++ b/util/chunk/disk.go @@ -14,9 +14,7 @@ package chunk import ( - "errors" "io" - "io/ioutil" "os" "strconv" "sync" @@ -46,6 +44,9 @@ type ListInDisk struct { diskTracker *disk.Tracker // track disk usage. numRowsInDisk int + checksumWriter *checksum.Writer + cipherWriter *encrypt.Writer + // ctrCipher stores the key and nonce using by aes encrypt io layer ctrCipher *encrypt.CtrCipher } @@ -67,7 +68,7 @@ func (l *ListInDisk) initDiskFile() (err error) { if err != nil { return } - l.disk, err = ioutil.TempFile(config.GetGlobalConfig().TempStoragePath, defaultChunkListInDiskPath+strconv.Itoa(l.diskTracker.Label())) + l.disk, err = os.CreateTemp(config.GetGlobalConfig().TempStoragePath, defaultChunkListInDiskPath+strconv.Itoa(l.diskTracker.Label())) if err != nil { return errors2.Trace(err) } @@ -78,9 +79,11 @@ func (l *ListInDisk) initDiskFile() (err error) { if err != nil { return } - underlying = encrypt.NewWriter(l.disk, l.ctrCipher) + l.cipherWriter = encrypt.NewWriter(l.disk, l.ctrCipher) + underlying = l.cipherWriter } - l.w = checksum.NewWriter(underlying) + l.checksumWriter = checksum.NewWriter(underlying) + l.w = l.checksumWriter l.bufFlushMutex = sync.RWMutex{} return } @@ -128,7 +131,7 @@ func (l *ListInDisk) flush() (err error) { // Warning: do not mix Add and GetRow (always use GetRow after you have added all the chunks), and do not use Add concurrently. func (l *ListInDisk) Add(chk *Chunk) (err error) { if chk.NumRows() == 0 { - return errors.New("chunk appended to List should have at least 1 row") + return errors2.New("chunk appended to List should have at least 1 row") } if l.disk == nil { err = l.initDiskFile() @@ -164,16 +167,16 @@ func (l *ListInDisk) GetChunk(chkIdx int) (*Chunk, error) { // GetRow gets a Row from the ListInDisk by RowPtr. func (l *ListInDisk) GetRow(ptr RowPtr) (row Row, err error) { - err = l.flush() if err != nil { return } off := l.offsets[ptr.ChkIdx][ptr.RowIdx] var underlying io.ReaderAt = l.disk if l.ctrCipher != nil { - underlying = encrypt.NewReader(l.disk, l.ctrCipher) + underlying = NewReaderWithCache(encrypt.NewReader(l.disk, l.ctrCipher), l.cipherWriter.GetCache(), l.cipherWriter.GetCacheDataOffset()) } - r := io.NewSectionReader(checksum.NewReader(underlying), off, l.offWrite-off) + checksumReader := NewReaderWithCache(checksum.NewReader(underlying), l.checksumWriter.GetCache(), l.checksumWriter.GetCacheDataOffset()) + r := io.NewSectionReader(checksumReader, off, l.offWrite-off) format := rowInDisk{numCol: len(l.fieldTypes)} _, err = format.ReadFrom(r) if err != nil { @@ -367,3 +370,51 @@ func (format *diskFormatRow) toMutRow(fields []*types.FieldType) MutRow { } return MutRow{c: chk} } + +// ReaderWithCache helps to read data that has not be flushed to underlying layer. +// By using ReaderWithCache, user can still write data into ListInDisk even after reading. +type ReaderWithCache struct { + r io.ReaderAt + cacheOff int64 + cache []byte +} + +// NewReaderWithCache returns a ReaderWithCache. +func NewReaderWithCache(r io.ReaderAt, cache []byte, cacheOff int64) *ReaderWithCache { + return &ReaderWithCache{ + r: r, + cacheOff: cacheOff, + cache: cache, + } +} + +// ReadAt implements the ReadAt interface. +func (r *ReaderWithCache) ReadAt(p []byte, off int64) (readCnt int, err error) { + readCnt, err = r.r.ReadAt(p, off) + if err != io.EOF { + return readCnt, err + } + + if len(p) == readCnt { + return readCnt, err + } else if len(p) < readCnt { + return readCnt, errors2.Trace(errors2.Errorf("cannot read more data than user requested"+ + "(readCnt: %v, len(p): %v", readCnt, len(p))) + } + + // When got here, user input is not filled fully, so we need read data from cache. + err = nil + p = p[readCnt:] + beg := off - r.cacheOff + if beg < 0 { + // This happens when only partial data of user requested resides in r.cache. + beg = 0 + } + end := int(beg) + len(p) + if end > len(r.cache) { + err = io.EOF + end = len(r.cache) + } + readCnt += copy(p, r.cache[beg:end]) + return readCnt, err +} diff --git a/util/chunk/disk_test.go b/util/chunk/disk_test.go index 86461de5659c7..a3cd929a4886f 100644 --- a/util/chunk/disk_test.go +++ b/util/chunk/disk_test.go @@ -14,12 +14,13 @@ package chunk import ( + "bytes" "fmt" "io" - "io/ioutil" "math/rand" "os" "path/filepath" + "reflect" "strconv" "strings" "testing" @@ -30,6 +31,8 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/types/json" + "github.com/pingcap/tidb/util/checksum" + "github.com/pingcap/tidb/util/encrypt" ) func initChunks(numChk, numRow int) ([]*Chunk, []*types.FieldType) { @@ -142,7 +145,7 @@ type listInDiskWriteDisk struct { func newListInDiskWriteDisk(fieldTypes []*types.FieldType) (*listInDiskWriteDisk, error) { l := listInDiskWriteDisk{*NewListInDisk(fieldTypes)} - disk, err := ioutil.TempFile(config.GetGlobalConfig().TempStoragePath, strconv.Itoa(l.diskTracker.Label())) + disk, err := os.CreateTemp(config.GetGlobalConfig().TempStoragePath, strconv.Itoa(l.diskTracker.Label())) if err != nil { return nil, err } @@ -219,6 +222,8 @@ func (s *testChunkSuite) TestListInDiskWithChecksum(c *check.C) { }) testListInDisk(c) + testReaderWithCache(c) + testReaderWithCacheNoFlush(c) } func (s *testChunkSuite) TestListInDiskWithChecksumAndEncrypt(c *check.C) { @@ -227,4 +232,129 @@ func (s *testChunkSuite) TestListInDiskWithChecksumAndEncrypt(c *check.C) { conf.Security.SpilledFileEncryptionMethod = config.SpilledFileEncryptionMethodAES128CTR }) testListInDisk(c) + + testReaderWithCache(c) + testReaderWithCacheNoFlush(c) +} + +// Following diagram describes the testdata we use to test: +// 4 B: checksum of this segment. +// 8 B: all columns' length, in the following example, we will only have one column. +// 1012 B: data in file. because max length of each segment is 1024, so we only have 1020B for user payload. +// +// Data in File Data in mem cache +// +------+------------------------------------------+ +-----------------------------+ +// | | 1020B payload | | | +// |4Bytes| +---------+----------------------------+ | | | +// |checksum|8B collen| 1012B user data | | | 12B remained user data | +// | | +---------+----------------------------+ | | | +// | | | | | +// +------+------------------------------------------+ +-----------------------------+ +func testReaderWithCache(c *check.C) { + testData := "0123456789" + buf := bytes.NewBuffer(nil) + for i := 0; i < 102; i++ { + buf.WriteString(testData) + } + buf.WriteString("0123") + + field := []*types.FieldType{types.NewFieldType(mysql.TypeString)} + chk := NewChunkWithCapacity(field, 1) + chk.AppendString(0, buf.String()) + l := NewListInDisk(field) + err := l.Add(chk) + c.Assert(err, check.IsNil) + + // Basic test for GetRow(). + row, err := l.GetRow(RowPtr{0, 0}) + c.Assert(err, check.IsNil) + c.Assert(row.GetDatumRow(field), check.DeepEquals, chk.GetRow(0).GetDatumRow(field)) + + var underlying io.ReaderAt = l.disk + if l.ctrCipher != nil { + underlying = NewReaderWithCache(encrypt.NewReader(l.disk, l.ctrCipher), l.cipherWriter.GetCache(), l.cipherWriter.GetCacheDataOffset()) + } + checksumReader := NewReaderWithCache(checksum.NewReader(underlying), l.checksumWriter.GetCache(), l.checksumWriter.GetCacheDataOffset()) + + // Read all data. + data := make([]byte, 1024) + // Offset is 8, because we want to ignore col length. + readCnt, err := checksumReader.ReadAt(data, 8) + c.Assert(err, check.IsNil) + c.Assert(readCnt, check.Equals, 1024) + c.Assert(reflect.DeepEqual(data, buf.Bytes()), check.IsTrue) + + // Only read data of mem cache. + data = make([]byte, 1024) + readCnt, err = checksumReader.ReadAt(data, 1020) + c.Assert(err, check.Equals, io.EOF) + c.Assert(readCnt, check.Equals, 12) + c.Assert(reflect.DeepEqual(data[:12], buf.Bytes()[1012:]), check.IsTrue) + + // Read partial data of mem cache. + data = make([]byte, 1024) + readCnt, err = checksumReader.ReadAt(data, 1025) + c.Assert(err, check.Equals, io.EOF) + c.Assert(readCnt, check.Equals, 7) + c.Assert(reflect.DeepEqual(data[:7], buf.Bytes()[1017:]), check.IsTrue) + + // Read partial data from both file and mem cache. + data = make([]byte, 1024) + readCnt, err = checksumReader.ReadAt(data, 1010) + c.Assert(err, check.Equals, io.EOF) + c.Assert(readCnt, check.Equals, 22) + c.Assert(reflect.DeepEqual(data[:22], buf.Bytes()[1002:]), check.IsTrue) + + // Offset is too large, so no data is read. + data = make([]byte, 1024) + readCnt, err = checksumReader.ReadAt(data, 1032) + c.Assert(err, check.Equals, io.EOF) + c.Assert(readCnt, check.Equals, 0) + c.Assert(reflect.DeepEqual(data, make([]byte, 1024)), check.IsTrue) + + // Only read 1 byte from mem cache. + data = make([]byte, 1024) + readCnt, err = checksumReader.ReadAt(data, 1031) + c.Assert(err, check.Equals, io.EOF) + c.Assert(readCnt, check.Equals, 1) + c.Assert(reflect.DeepEqual(data[:1], buf.Bytes()[1023:]), check.IsTrue) + + // Test user requested data is small. + // Only request 10 bytes. + data = make([]byte, 10) + readCnt, err = checksumReader.ReadAt(data, 1010) + c.Assert(err, check.IsNil) + c.Assert(readCnt, check.Equals, 10) + c.Assert(reflect.DeepEqual(data, buf.Bytes()[1002:1012]), check.IsTrue) +} + +// Here we test situations where size of data is small, so no data is flushed to disk. +func testReaderWithCacheNoFlush(c *check.C) { + testData := "0123456789" + + field := []*types.FieldType{types.NewFieldType(mysql.TypeString)} + chk := NewChunkWithCapacity(field, 1) + chk.AppendString(0, testData) + l := NewListInDisk(field) + err := l.Add(chk) + c.Assert(err, check.IsNil) + + // Basic test for GetRow(). + row, err := l.GetRow(RowPtr{0, 0}) + c.Assert(err, check.IsNil) + c.Assert(row.GetDatumRow(field), check.DeepEquals, chk.GetRow(0).GetDatumRow(field)) + + var underlying io.ReaderAt = l.disk + if l.ctrCipher != nil { + underlying = NewReaderWithCache(encrypt.NewReader(l.disk, l.ctrCipher), l.cipherWriter.GetCache(), l.cipherWriter.GetCacheDataOffset()) + } + checksumReader := NewReaderWithCache(checksum.NewReader(underlying), l.checksumWriter.GetCache(), l.checksumWriter.GetCacheDataOffset()) + + // Read all data. + data := make([]byte, 1024) + // Offset is 8, because we want to ignore col length. + readCnt, err := checksumReader.ReadAt(data, 8) + c.Assert(err, check.Equals, io.EOF) + c.Assert(readCnt, check.Equals, len(testData)) + c.Assert(reflect.DeepEqual(data[:10], []byte(testData)), check.IsTrue) } diff --git a/util/chunk/row_container.go b/util/chunk/row_container.go index f67cbb36b76da..0ef0b573e1bb2 100644 --- a/util/chunk/row_container.go +++ b/util/chunk/row_container.go @@ -90,7 +90,6 @@ func (c *RowContainer) SpillToDisk() { } } c.m.records.Clear() - return } // Reset resets RowContainer. @@ -447,7 +446,6 @@ func (c *SortedRowContainer) Sort() { func (c *SortedRowContainer) sortAndSpillToDisk() { c.Sort() c.RowContainer.SpillToDisk() - return } // Add appends a chunk into the SortedRowContainer. diff --git a/util/chunk/row_container_test.go b/util/chunk/row_container_test.go index feed2290f38b6..a39346e34ff80 100644 --- a/util/chunk/row_container_test.go +++ b/util/chunk/row_container_test.go @@ -113,6 +113,28 @@ func (r *rowContainerTestSuite) TestSpillAction(c *check.C) { rc.actionSpill.WaitForTest() c.Assert(err, check.IsNil) c.Assert(rc.AlreadySpilledSafeForTest(), check.Equals, true) + + // Read + resChk, err := rc.GetChunk(0) + c.Assert(err, check.IsNil) + c.Assert(resChk.NumRows(), check.Equals, chk.NumRows()) + for rowIdx := 0; rowIdx < resChk.NumRows(); rowIdx++ { + c.Assert(resChk.GetRow(rowIdx).GetDatumRow(fields), check.DeepEquals, chk.GetRow(rowIdx).GetDatumRow(fields)) + } + // Write again + err = rc.Add(chk) + rc.actionSpill.WaitForTest() + c.Assert(err, check.IsNil) + c.Assert(rc.AlreadySpilledSafeForTest(), check.Equals, true) + + // Read + resChk, err = rc.GetChunk(2) + c.Assert(err, check.IsNil) + c.Assert(resChk.NumRows(), check.Equals, chk.NumRows()) + for rowIdx := 0; rowIdx < resChk.NumRows(); rowIdx++ { + c.Assert(resChk.GetRow(rowIdx).GetDatumRow(fields), check.DeepEquals, chk.GetRow(rowIdx).GetDatumRow(fields)) + } + err = rc.Reset() c.Assert(err, check.IsNil) } diff --git a/util/cteutil/storage.go b/util/cteutil/storage.go new file mode 100644 index 0000000000000..d2607892db62c --- /dev/null +++ b/util/cteutil/storage.go @@ -0,0 +1,280 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package cteutil + +import ( + "sync" + + "github.com/pingcap/errors" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/disk" + "github.com/pingcap/tidb/util/memory" +) + +var _ Storage = &StorageRC{} + +// Storage is a temporary storage to store the intermidate data of CTE. +// +// Common usage as follows: +// +// storage.Lock() +// if !storage.Done() { +// fill all data into storage +// } +// storage.UnLock() +// read data from storage +type Storage interface { + // If is first called, will open underlying storage. Otherwise will add ref count by one. + OpenAndRef() error + + // Minus ref count by one, if ref count is zero, close underlying storage. + DerefAndClose() (err error) + + // SwapData swaps data of two storage. + // Other metainfo is not touched, such ref count/done flag etc. + SwapData(other Storage) error + + // Reopen reset storage and related info. + // So the status of Storage is like a new created one. + Reopen() error + + // Add chunk into underlying storage. + // Should return directly if chk is empty. + Add(chk *chunk.Chunk) error + + // Get Chunk by index. + GetChunk(chkIdx int) (*chunk.Chunk, error) + + // Get row by RowPtr. + GetRow(ptr chunk.RowPtr) (chunk.Row, error) + + // NumChunks return chunk number of the underlying storage. + NumChunks() int + + // NumRows return row number of the underlying storage. + NumRows() int + + // Storage is not thread-safe. + // By using Lock(), users can achieve the purpose of ensuring thread safety. + Lock() + Unlock() + + // Usually, Storage is filled first, then user can read it. + // User can check whether Storage is filled first, if not, they can fill it. + Done() bool + SetDone() + + // Readers use iter information to determine + // whether they need to read data from the beginning. + SetIter(iter int) + GetIter() int + + // We use this channel to notify reader that Storage is ready to read. + // It exists only to solve the special implementation of IndexLookUpJoin. + // We will find a better way and remove this later. + GetBegCh() chan struct{} + + GetMemTracker() *memory.Tracker + GetDiskTracker() *disk.Tracker + ActionSpill() *chunk.SpillDiskAction +} + +// StorageRC implements Storage interface using RowContainer. +type StorageRC struct { + mu sync.Mutex + refCnt int + tp []*types.FieldType + chkSize int + + begCh chan struct{} + done bool + iter int + + rc *chunk.RowContainer +} + +// NewStorageRowContainer create a new StorageRC. +func NewStorageRowContainer(tp []*types.FieldType, chkSize int) *StorageRC { + return &StorageRC{tp: tp, chkSize: chkSize} +} + +// OpenAndRef impls Storage OpenAndRef interface. +func (s *StorageRC) OpenAndRef() (err error) { + if !s.valid() { + s.rc = chunk.NewRowContainer(s.tp, s.chkSize) + s.refCnt = 1 + s.begCh = make(chan struct{}) + s.iter = 0 + } else { + s.refCnt += 1 + } + return nil +} + +// DerefAndClose impls Storage DerefAndClose interface. +func (s *StorageRC) DerefAndClose() (err error) { + if !s.valid() { + return errors.New("Storage not opend yet") + } + s.refCnt -= 1 + if s.refCnt < 0 { + return errors.New("Storage ref count is less than zero") + } else if s.refCnt == 0 { + // TODO: unreg memtracker + if err = s.rc.Close(); err != nil { + return err + } + if err = s.resetAll(); err != nil { + return err + } + } + return nil +} + +// SwapData impls Storage Swap interface. +func (s *StorageRC) SwapData(other Storage) (err error) { + otherRC, ok := other.(*StorageRC) + if !ok { + return errors.New("cannot swap if underlying storages are different") + } + s.tp, otherRC.tp = otherRC.tp, s.tp + s.chkSize, otherRC.chkSize = otherRC.chkSize, s.chkSize + + s.rc, otherRC.rc = otherRC.rc, s.rc + return nil +} + +// Reopen impls Storage Reopen interface. +func (s *StorageRC) Reopen() (err error) { + if err = s.rc.Reset(); err != nil { + return err + } + s.iter = 0 + s.begCh = make(chan struct{}) + s.done = false + // Create a new RowContainer. + // Because some meta infos in old RowContainer are not resetted. + // Such as memTracker/actionSpill etc. So we just use a new one. + s.rc = chunk.NewRowContainer(s.tp, s.chkSize) + return nil +} + +// Add impls Storage Add interface. +func (s *StorageRC) Add(chk *chunk.Chunk) (err error) { + if !s.valid() { + return errors.New("Storage is not valid") + } + if chk.NumRows() == 0 { + return nil + } + return s.rc.Add(chk) +} + +// GetChunk impls Storage GetChunk interface. +func (s *StorageRC) GetChunk(chkIdx int) (*chunk.Chunk, error) { + if !s.valid() { + return nil, errors.New("Storage is not valid") + } + return s.rc.GetChunk(chkIdx) +} + +// GetRow impls Storage GetRow interface. +func (s *StorageRC) GetRow(ptr chunk.RowPtr) (chunk.Row, error) { + if !s.valid() { + return chunk.Row{}, errors.New("Storage is not valid") + } + return s.rc.GetRow(ptr) +} + +// NumChunks impls Storage NumChunks interface. +func (s *StorageRC) NumChunks() int { + return s.rc.NumChunks() +} + +// NumRows impls Storage NumRows interface. +func (s *StorageRC) NumRows() int { + return s.rc.NumRow() +} + +// Lock impls Storage Lock interface. +func (s *StorageRC) Lock() { + s.mu.Lock() +} + +// Unlock impls Storage Unlock interface. +func (s *StorageRC) Unlock() { + s.mu.Unlock() +} + +// Done impls Storage Done interface. +func (s *StorageRC) Done() bool { + return s.done +} + +// SetDone impls Storage SetDone interface. +func (s *StorageRC) SetDone() { + s.done = true +} + +// SetIter impls Storage SetIter interface. +func (s *StorageRC) SetIter(iter int) { + s.iter = iter +} + +// GetIter impls Storage GetIter interface. +func (s *StorageRC) GetIter() int { + return s.iter +} + +// GetBegCh impls Storage GetBegCh interface. +func (s *StorageRC) GetBegCh() chan struct{} { + return s.begCh +} + +// GetMemTracker impls Storage GetMemTracker interface. +func (s *StorageRC) GetMemTracker() *memory.Tracker { + return s.rc.GetMemTracker() +} + +// GetDiskTracker impls Storage GetDiskTracker interface. +func (s *StorageRC) GetDiskTracker() *memory.Tracker { + return s.rc.GetDiskTracker() +} + +// ActionSpill impls Storage ActionSpill interface. +func (s *StorageRC) ActionSpill() *chunk.SpillDiskAction { + return s.rc.ActionSpill() +} + +// ActionSpillForTest is for test. +func (s *StorageRC) ActionSpillForTest() *chunk.SpillDiskAction { + return s.rc.ActionSpillForTest() +} + +func (s *StorageRC) resetAll() error { + s.refCnt = -1 + s.begCh = nil + s.done = false + s.iter = 0 + if err := s.rc.Reset(); err != nil { + return err + } + s.rc = nil + return nil +} + +func (s *StorageRC) valid() bool { + return s.refCnt > 0 && s.rc != nil +} diff --git a/util/cteutil/storage_test.go b/util/cteutil/storage_test.go new file mode 100644 index 0000000000000..0e494978f2f84 --- /dev/null +++ b/util/cteutil/storage_test.go @@ -0,0 +1,262 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package cteutil + +import ( + "reflect" + "strconv" + "testing" + + "github.com/pingcap/check" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/chunk" +) + +func TestT(t *testing.T) { + check.TestingT(t) +} + +var _ = check.Suite(&StorageRCTestSuite{}) + +type StorageRCTestSuite struct{} + +func (test *StorageRCTestSuite) TestStorageBasic(c *check.C) { + fields := []*types.FieldType{types.NewFieldType(mysql.TypeLong)} + chkSize := 1 + storage := NewStorageRowContainer(fields, chkSize) + c.Assert(storage, check.NotNil) + + // Close before open. + err := storage.DerefAndClose() + c.Assert(err, check.NotNil) + + err = storage.OpenAndRef() + c.Assert(err, check.IsNil) + + err = storage.DerefAndClose() + c.Assert(err, check.IsNil) + + err = storage.DerefAndClose() + c.Assert(err, check.NotNil) + + // Open twice. + err = storage.OpenAndRef() + c.Assert(err, check.IsNil) + err = storage.OpenAndRef() + c.Assert(err, check.IsNil) + err = storage.DerefAndClose() + c.Assert(err, check.IsNil) + err = storage.DerefAndClose() + c.Assert(err, check.IsNil) + err = storage.DerefAndClose() + c.Assert(err, check.NotNil) +} + +func (test *StorageRCTestSuite) TestOpenAndClose(c *check.C) { + fields := []*types.FieldType{types.NewFieldType(mysql.TypeLong)} + chkSize := 1 + storage := NewStorageRowContainer(fields, chkSize) + + for i := 0; i < 10; i++ { + err := storage.OpenAndRef() + c.Assert(err, check.IsNil) + } + + for i := 0; i < 9; i++ { + err := storage.DerefAndClose() + c.Assert(err, check.IsNil) + } + err := storage.DerefAndClose() + c.Assert(err, check.IsNil) + + err = storage.DerefAndClose() + c.Assert(err, check.NotNil) +} + +func (test *StorageRCTestSuite) TestAddAndGetChunk(c *check.C) { + fields := []*types.FieldType{types.NewFieldType(mysql.TypeLong)} + chkSize := 10 + + storage := NewStorageRowContainer(fields, chkSize) + + inChk := chunk.NewChunkWithCapacity(fields, chkSize) + for i := 0; i < chkSize; i++ { + inChk.AppendInt64(0, int64(i)) + } + + err := storage.Add(inChk) + c.Assert(err, check.NotNil) + + err = storage.OpenAndRef() + c.Assert(err, check.IsNil) + + err = storage.Add(inChk) + c.Assert(err, check.IsNil) + + outChk, err1 := storage.GetChunk(0) + c.Assert(err1, check.IsNil) + + in64s := inChk.Column(0).Int64s() + out64s := outChk.Column(0).Int64s() + + c.Assert(reflect.DeepEqual(in64s, out64s), check.IsTrue) +} + +func (test *StorageRCTestSuite) TestSpillToDisk(c *check.C) { + fields := []*types.FieldType{types.NewFieldType(mysql.TypeLong)} + chkSize := 10 + storage := NewStorageRowContainer(fields, chkSize) + var tmp interface{} = storage + + inChk := chunk.NewChunkWithCapacity(fields, chkSize) + for i := 0; i < chkSize; i++ { + inChk.AppendInt64(0, int64(i)) + } + + err := storage.OpenAndRef() + c.Assert(err, check.IsNil) + + memTracker := storage.GetMemTracker() + memTracker.SetBytesLimit(inChk.MemoryUsage() + 1) + action := tmp.(*StorageRC).ActionSpillForTest() + memTracker.FallbackOldAndSetNewAction(action) + diskTracker := storage.GetDiskTracker() + + // All data is in memory. + err = storage.Add(inChk) + c.Assert(err, check.IsNil) + outChk, err1 := storage.GetChunk(0) + c.Assert(err1, check.IsNil) + in64s := inChk.Column(0).Int64s() + out64s := outChk.Column(0).Int64s() + c.Assert(reflect.DeepEqual(in64s, out64s), check.IsTrue) + + c.Assert(memTracker.BytesConsumed(), check.Greater, int64(0)) + c.Assert(memTracker.MaxConsumed(), check.Greater, int64(0)) + c.Assert(diskTracker.BytesConsumed(), check.Equals, int64(0)) + c.Assert(diskTracker.MaxConsumed(), check.Equals, int64(0)) + + // Add again, and will trigger spill to disk. + err = storage.Add(inChk) + c.Assert(err, check.IsNil) + action.WaitForTest() + c.Assert(memTracker.BytesConsumed(), check.Equals, int64(0)) + c.Assert(memTracker.MaxConsumed(), check.Greater, int64(0)) + c.Assert(diskTracker.BytesConsumed(), check.Greater, int64(0)) + c.Assert(diskTracker.MaxConsumed(), check.Greater, int64(0)) + + outChk, err = storage.GetChunk(0) + c.Assert(err, check.IsNil) + out64s = outChk.Column(0).Int64s() + c.Assert(reflect.DeepEqual(in64s, out64s), check.IsTrue) + + outChk, err = storage.GetChunk(1) + c.Assert(err, check.IsNil) + out64s = outChk.Column(0).Int64s() + c.Assert(reflect.DeepEqual(in64s, out64s), check.IsTrue) +} + +func (test *StorageRCTestSuite) TestReopen(c *check.C) { + fields := []*types.FieldType{types.NewFieldType(mysql.TypeLong)} + chkSize := 10 + storage := NewStorageRowContainer(fields, chkSize) + err := storage.OpenAndRef() + c.Assert(err, check.IsNil) + + inChk := chunk.NewChunkWithCapacity(fields, chkSize) + for i := 0; i < chkSize; i++ { + inChk.AppendInt64(0, int64(i)) + } + err = storage.Add(inChk) + c.Assert(err, check.IsNil) + c.Assert(storage.NumChunks(), check.Equals, 1) + + err = storage.Reopen() + c.Assert(err, check.IsNil) + c.Assert(storage.NumChunks(), check.Equals, 0) + + err = storage.Add(inChk) + c.Assert(err, check.IsNil) + c.Assert(storage.NumChunks(), check.Equals, 1) + + outChk, err := storage.GetChunk(0) + c.Assert(err, check.IsNil) + in64s := inChk.Column(0).Int64s() + out64s := outChk.Column(0).Int64s() + c.Assert(reflect.DeepEqual(in64s, out64s), check.IsTrue) + + // Reopen multiple times. + for i := 0; i < 100; i++ { + err = storage.Reopen() + c.Assert(err, check.IsNil) + } + err = storage.Add(inChk) + c.Assert(err, check.IsNil) + c.Assert(storage.NumChunks(), check.Equals, 1) + + outChk, err = storage.GetChunk(0) + c.Assert(err, check.IsNil) + in64s = inChk.Column(0).Int64s() + out64s = outChk.Column(0).Int64s() + c.Assert(reflect.DeepEqual(in64s, out64s), check.IsTrue) +} + +func (test *StorageRCTestSuite) TestSwapData(c *check.C) { + tp1 := []*types.FieldType{types.NewFieldType(mysql.TypeLong)} + chkSize := 10 + storage1 := NewStorageRowContainer(tp1, chkSize) + err := storage1.OpenAndRef() + c.Assert(err, check.IsNil) + inChk1 := chunk.NewChunkWithCapacity(tp1, chkSize) + for i := 0; i < chkSize; i++ { + inChk1.AppendInt64(0, int64(i)) + } + in1 := inChk1.Column(0).Int64s() + err = storage1.Add(inChk1) + c.Assert(err, check.IsNil) + + tp2 := []*types.FieldType{types.NewFieldType(mysql.TypeVarString)} + storage2 := NewStorageRowContainer(tp2, chkSize) + err = storage2.OpenAndRef() + c.Assert(err, check.IsNil) + + inChk2 := chunk.NewChunkWithCapacity(tp2, chkSize) + for i := 0; i < chkSize; i++ { + inChk2.AppendString(0, strconv.FormatInt(int64(i), 10)) + } + var in2 []string + for i := 0; i < inChk2.NumRows(); i++ { + in2 = append(in2, inChk2.Column(0).GetString(i)) + } + err = storage2.Add(inChk2) + c.Assert(err, check.IsNil) + + err = storage1.SwapData(storage2) + c.Assert(err, check.IsNil) + + outChk1, err := storage1.GetChunk(0) + c.Assert(err, check.IsNil) + outChk2, err := storage2.GetChunk(0) + c.Assert(err, check.IsNil) + + var out1 []string + for i := 0; i < outChk1.NumRows(); i++ { + out1 = append(out1, outChk1.Column(0).GetString(i)) + } + out2 := outChk2.Column(0).Int64s() + + c.Assert(reflect.DeepEqual(in1, out2), check.IsTrue) + c.Assert(reflect.DeepEqual(in2, out1), check.IsTrue) +} diff --git a/util/deadlockhistory/deadlock_history.go b/util/deadlockhistory/deadlock_history.go new file mode 100644 index 0000000000000..68bf99beaaadf --- /dev/null +++ b/util/deadlockhistory/deadlock_history.go @@ -0,0 +1,220 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package deadlockhistory + +import ( + "encoding/hex" + "strings" + "sync" + "time" + + "github.com/pingcap/parser/mysql" + tikverr "github.com/pingcap/tidb/store/tikv/error" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/logutil" + "github.com/pingcap/tidb/util/resourcegrouptag" + "go.uber.org/zap" +) + +// WaitChainItem represents an entry in a deadlock's wait chain. +type WaitChainItem struct { + TryLockTxn uint64 + SQLDigest string + Key []byte + AllSQLDigests []string + TxnHoldingLock uint64 +} + +// DeadlockRecord represents a deadlock events, and contains multiple transactions' information. +type DeadlockRecord struct { + // The ID doesn't need to be set manually and it's set when it's added into the DeadlockHistory by invoking its Push + // method. + ID uint64 + OccurTime time.Time + IsRetryable bool + WaitChain []WaitChainItem +} + +// DeadlockHistory is a collection for maintaining recent several deadlock events. All its public APIs are thread safe. +type DeadlockHistory struct { + sync.RWMutex + + deadlocks []*DeadlockRecord + + // The `head` and `size` makes the `deadlocks` array behaves like a deque. The valid elements are + // deadlocks[head:head+size], or deadlocks[head:] + deadlocks[:head+size-len] if `head+size` exceeds the array's + // length. + head int + size int + + // currentID is used to allocate IDs for deadlock records pushed to the queue that's unique in the deadlock + // history queue instance. + currentID uint64 +} + +// NewDeadlockHistory creates an instance of DeadlockHistory +func NewDeadlockHistory(capacity uint) *DeadlockHistory { + return &DeadlockHistory{ + deadlocks: make([]*DeadlockRecord, capacity), + currentID: 1, + } +} + +// GlobalDeadlockHistory is the global instance of DeadlockHistory, which is used to maintain recent several recent +// deadlock events globally. +// The real size of the deadlock history table should be initialized with `Resize` +// in `setGlobalVars` in tidb-server/main.go +var GlobalDeadlockHistory = NewDeadlockHistory(0) + +// Resize update the DeadlockHistory's table max capacity to newCapacity +func (d *DeadlockHistory) Resize(newCapacity uint) { + d.Lock() + defer d.Unlock() + if newCapacity != uint(len(d.deadlocks)) { + current := d.getAll() + d.head = 0 + if uint(len(current)) < newCapacity { + // extend deadlocks + d.deadlocks = make([]*DeadlockRecord, newCapacity) + copy(d.deadlocks, current) + } else { + // shrink deadlocks, keep the last len(current)-newCapacity items + // use append here to force golang to realloc the underlying array to save memory + d.deadlocks = append([]*DeadlockRecord{}, current[uint(len(current))-newCapacity:]...) + d.size = int(newCapacity) + } + } +} + +// Push pushes an element into the queue. It will set the `ID` field of the record, and add the pointer directly to +// the collection. Be aware that do not modify the record's content after pushing. +func (d *DeadlockHistory) Push(record *DeadlockRecord) { + d.Lock() + defer d.Unlock() + + capacity := len(d.deadlocks) + if capacity == 0 { + return + } + + record.ID = d.currentID + d.currentID++ + + if d.size == capacity { + // The current head is popped and it's cell becomes the latest pushed item. + d.deadlocks[d.head] = record + d.head = (d.head + 1) % capacity + } else if d.size < capacity { + d.deadlocks[(d.head+d.size)%capacity] = record + d.size++ + } else { + panic("unreachable") + } +} + +// GetAll gets all collected deadlock events. +func (d *DeadlockHistory) GetAll() []*DeadlockRecord { + d.RLock() + defer d.RUnlock() + return d.getAll() +} + +// getAll is a thread unsafe version of GetAll() for internal use +func (d *DeadlockHistory) getAll() []*DeadlockRecord { + res := make([]*DeadlockRecord, 0, d.size) + capacity := len(d.deadlocks) + if d.head+d.size <= capacity { + res = append(res, d.deadlocks[d.head:d.head+d.size]...) + } else { + res = append(res, d.deadlocks[d.head:]...) + res = append(res, d.deadlocks[:(d.head+d.size)%capacity]...) + } + return res +} + +// GetAllDatum gets all collected deadlock events, and make it into datum that matches the definition of the table +// `INFORMATION_SCHEMA.DEADLOCKS`. +func (d *DeadlockHistory) GetAllDatum() [][]types.Datum { + records := d.GetAll() + rowsCount := 0 + for _, rec := range records { + rowsCount += len(rec.WaitChain) + } + + rows := make([][]types.Datum, 0, rowsCount) + + row := make([]interface{}, 7) + for _, rec := range records { + row[0] = rec.ID + row[1] = types.NewTime(types.FromGoTime(rec.OccurTime), mysql.TypeTimestamp, types.MaxFsp) + row[2] = rec.IsRetryable + + for _, item := range rec.WaitChain { + row[3] = item.TryLockTxn + + row[4] = nil + if len(item.SQLDigest) > 0 { + row[4] = item.SQLDigest + } + + row[5] = nil + if len(item.Key) > 0 { + row[5] = strings.ToUpper(hex.EncodeToString(item.Key)) + } + + row[6] = item.TxnHoldingLock + + // TODO: Implement the ALL_SQL_DIGESTS column for the deadlock table. + + rows = append(rows, types.MakeDatums(row...)) + } + } + + return rows +} + +// Clear clears content from deadlock histories +func (d *DeadlockHistory) Clear() { + d.Lock() + defer d.Unlock() + for i := 0; i < len(d.deadlocks); i++ { + d.deadlocks[i] = nil + } + d.head = 0 + d.size = 0 +} + +// ErrDeadlockToDeadlockRecord generates a DeadlockRecord from the information in a `tikverr.ErrDeadlock` error. +func ErrDeadlockToDeadlockRecord(dl *tikverr.ErrDeadlock) *DeadlockRecord { + waitChain := make([]WaitChainItem, 0, len(dl.WaitChain)) + for _, rawItem := range dl.WaitChain { + sqlDigest, err := resourcegrouptag.DecodeResourceGroupTag(rawItem.ResourceGroupTag) + if err != nil { + logutil.BgLogger().Warn("decoding resource group tag encounters error", zap.Error(err)) + } + waitChain = append(waitChain, WaitChainItem{ + TryLockTxn: rawItem.Txn, + SQLDigest: hex.EncodeToString(sqlDigest), + Key: rawItem.Key, + AllSQLDigests: nil, + TxnHoldingLock: rawItem.WaitForTxn, + }) + } + rec := &DeadlockRecord{ + OccurTime: time.Now(), + IsRetryable: dl.IsRetryable, + WaitChain: waitChain, + } + return rec +} diff --git a/util/deadlockhistory/deadlock_history_test.go b/util/deadlockhistory/deadlock_history_test.go new file mode 100644 index 0000000000000..f5f6f7285e1d6 --- /dev/null +++ b/util/deadlockhistory/deadlock_history_test.go @@ -0,0 +1,325 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package deadlockhistory + +import ( + "testing" + "time" + + . "github.com/pingcap/check" + "github.com/pingcap/kvproto/pkg/deadlock" + "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/parser" + tikverr "github.com/pingcap/tidb/store/tikv/error" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tipb/go-tipb" +) + +type testDeadlockHistorySuite struct{} + +var _ = Suite(&testDeadlockHistorySuite{}) + +func TestT(t *testing.T) { + TestingT(t) +} + +func (s *testDeadlockHistorySuite) TestDeadlockHistoryCollection(c *C) { + h := NewDeadlockHistory(1) + c.Assert(len(h.GetAll()), Equals, 0) + c.Assert(h.head, Equals, 0) + c.Assert(h.size, Equals, 0) + + rec1 := &DeadlockRecord{ + OccurTime: time.Now(), + } + h.Push(rec1) + res := h.GetAll() + c.Assert(len(res), Equals, 1) + c.Assert(res[0], Equals, rec1) // Checking pointer equals is ok. + c.Assert(res[0].ID, Equals, uint64(1)) + c.Assert(h.head, Equals, 0) + c.Assert(h.size, Equals, 1) + + rec2 := &DeadlockRecord{ + OccurTime: time.Now(), + } + h.Push(rec2) + res = h.GetAll() + c.Assert(len(res), Equals, 1) + c.Assert(res[0], Equals, rec2) + c.Assert(res[0].ID, Equals, uint64(2)) + c.Assert(h.head, Equals, 0) + c.Assert(h.size, Equals, 1) + + h.Clear() + c.Assert(len(h.GetAll()), Equals, 0) + + h = NewDeadlockHistory(3) + rec1 = &DeadlockRecord{ + OccurTime: time.Now(), + } + h.Push(rec1) + res = h.GetAll() + c.Assert(len(res), Equals, 1) + c.Assert(res[0], Equals, rec1) // Checking pointer equals is ok. + c.Assert(res[0].ID, Equals, uint64(1)) + c.Assert(h.head, Equals, 0) + c.Assert(h.size, Equals, 1) + + rec2 = &DeadlockRecord{ + OccurTime: time.Now(), + } + h.Push(rec2) + res = h.GetAll() + c.Assert(len(res), Equals, 2) + c.Assert(res[0], Equals, rec1) + c.Assert(res[0].ID, Equals, uint64(1)) + c.Assert(res[1], Equals, rec2) + c.Assert(res[1].ID, Equals, uint64(2)) + c.Assert(h.head, Equals, 0) + c.Assert(h.size, Equals, 2) + + rec3 := &DeadlockRecord{ + OccurTime: time.Now(), + } + h.Push(rec3) + res = h.GetAll() + c.Assert(len(res), Equals, 3) + c.Assert(res[0], Equals, rec1) + c.Assert(res[0].ID, Equals, uint64(1)) + c.Assert(res[1], Equals, rec2) + c.Assert(res[1].ID, Equals, uint64(2)) + c.Assert(res[2], Equals, rec3) + c.Assert(res[2].ID, Equals, uint64(3)) + c.Assert(h.head, Equals, 0) + c.Assert(h.size, Equals, 3) + + // Continuously pushing items to check the correctness of the deque + expectedItems := []*DeadlockRecord{rec1, rec2, rec3} + expectedIDs := []uint64{1, 2, 3} + expectedDequeHead := 0 + for i := 0; i < 6; i++ { + newRec := &DeadlockRecord{ + OccurTime: time.Now(), + } + h.Push(newRec) + + expectedItems = append(expectedItems[1:], newRec) + for idx := range expectedIDs { + expectedIDs[idx]++ + } + expectedDequeHead = (expectedDequeHead + 1) % 3 + + res = h.GetAll() + c.Assert(len(res), Equals, 3) + for idx, item := range res { + c.Assert(item, Equals, expectedItems[idx]) + c.Assert(item.ID, Equals, expectedIDs[idx]) + } + c.Assert(h.head, Equals, expectedDequeHead) + c.Assert(h.size, Equals, 3) + } + + h.Clear() + c.Assert(len(h.GetAll()), Equals, 0) +} + +func (s *testDeadlockHistorySuite) TestGetDatum(c *C) { + time1 := time.Date(2021, 05, 14, 15, 28, 30, 123456000, time.UTC) + time2 := time.Date(2022, 06, 15, 16, 29, 31, 123457000, time.UTC) + + h := NewDeadlockHistory(10) + h.Push(&DeadlockRecord{ + OccurTime: time1, + IsRetryable: false, + WaitChain: []WaitChainItem{ + { + TryLockTxn: 101, + SQLDigest: "sql1", + Key: []byte("k1"), + AllSQLDigests: []string{"sql1", "sql2"}, + TxnHoldingLock: 102, + }, + // It should work even some information are missing. + { + TryLockTxn: 102, + TxnHoldingLock: 101, + }, + }, + }) + h.Push(&DeadlockRecord{ + OccurTime: time2, + IsRetryable: true, + WaitChain: []WaitChainItem{ + { + TryLockTxn: 201, + AllSQLDigests: []string{}, + TxnHoldingLock: 202, + }, + { + TryLockTxn: 202, + AllSQLDigests: []string{"sql1"}, + TxnHoldingLock: 201, + }, + }, + }) + // A deadlock error without wait chain shows nothing in the query result. + h.Push(&DeadlockRecord{ + OccurTime: time.Now(), + IsRetryable: false, + WaitChain: nil, + }) + + res := h.GetAllDatum() + c.Assert(len(res), Equals, 4) + for _, row := range res { + c.Assert(len(row), Equals, 7) + } + + toGoTime := func(d types.Datum) time.Time { + v, ok := d.GetValue().(types.Time) + c.Assert(ok, IsTrue) + t, err := v.GoTime(time.UTC) + c.Assert(err, IsNil) + return t + } + + c.Assert(res[0][0].GetValue(), Equals, uint64(1)) // ID + c.Assert(toGoTime(res[0][1]), Equals, time1) // OCCUR_TIME + c.Assert(res[0][2].GetValue(), Equals, int64(0)) // RETRYABLE + c.Assert(res[0][3].GetValue(), Equals, uint64(101)) // TRY_LOCK_TRX_ID + c.Assert(res[0][4].GetValue(), Equals, "sql1") // SQL_DIGEST + c.Assert(res[0][5].GetValue(), Equals, "6B31") // KEY + c.Assert(res[0][6].GetValue(), Equals, uint64(102)) // TRX_HOLDING_LOCK + + c.Assert(res[1][0].GetValue(), Equals, uint64(1)) // ID + c.Assert(toGoTime(res[1][1]), Equals, time1) // OCCUR_TIME + c.Assert(res[1][2].GetValue(), Equals, int64(0)) // RETRYABLE + c.Assert(res[1][3].GetValue(), Equals, uint64(102)) // TRY_LOCK_TRX_ID + c.Assert(res[1][4].GetValue(), Equals, nil) // SQL_DIGEST + c.Assert(res[1][5].GetValue(), Equals, nil) // KEY + c.Assert(res[1][6].GetValue(), Equals, uint64(101)) // TRX_HOLDING_LOCK + + c.Assert(res[2][0].GetValue(), Equals, uint64(2)) // ID + c.Assert(toGoTime(res[2][1]), Equals, time2) // OCCUR_TIME + c.Assert(res[2][2].GetValue(), Equals, int64(1)) // RETRYABLE + c.Assert(res[2][3].GetValue(), Equals, uint64(201)) // TRY_LOCK_TRX_ID + c.Assert(res[2][6].GetValue(), Equals, uint64(202)) // TRX_HOLDING_LOCK + + c.Assert(res[3][0].GetValue(), Equals, uint64(2)) // ID + c.Assert(toGoTime(res[3][1]), Equals, time2) // OCCUR_TIME + c.Assert(res[3][2].GetValue(), Equals, int64(1)) // RETRYABLE + c.Assert(res[3][3].GetValue(), Equals, uint64(202)) // TRY_LOCK_TRX_ID + c.Assert(res[3][6].GetValue(), Equals, uint64(201)) // TRX_HOLDING_LOCK +} + +func (s *testDeadlockHistorySuite) TestErrDeadlockToDeadlockRecord(c *C) { + digest1, digest2 := parser.NewDigest([]byte("aabbccdd")), parser.NewDigest([]byte("ddccbbaa")) + tag1 := tipb.ResourceGroupTag{SqlDigest: digest1.Bytes()} + tag2 := tipb.ResourceGroupTag{SqlDigest: digest2.Bytes()} + tag1Data, _ := tag1.Marshal() + tag2Data, _ := tag2.Marshal() + err := &tikverr.ErrDeadlock{ + Deadlock: &kvrpcpb.Deadlock{ + LockTs: 101, + LockKey: []byte("k1"), + DeadlockKeyHash: 1234567, + WaitChain: []*deadlock.WaitForEntry{ + { + Txn: 100, + WaitForTxn: 101, + Key: []byte("k2"), + ResourceGroupTag: tag1Data, + }, + { + Txn: 101, + WaitForTxn: 100, + Key: []byte("k1"), + ResourceGroupTag: tag2Data, + }, + }, + }, + IsRetryable: true, + } + + expectedRecord := &DeadlockRecord{ + IsRetryable: true, + WaitChain: []WaitChainItem{ + { + TryLockTxn: 100, + SQLDigest: digest1.String(), + Key: []byte("k2"), + TxnHoldingLock: 101, + }, + { + TryLockTxn: 101, + SQLDigest: digest2.String(), + Key: []byte("k1"), + TxnHoldingLock: 100, + }, + }, + } + + record := ErrDeadlockToDeadlockRecord(err) + // The OccurTime is set to time.Now + c.Assert(time.Since(record.OccurTime), Less, time.Millisecond*5) + expectedRecord.OccurTime = record.OccurTime + c.Assert(record, DeepEquals, expectedRecord) +} + +func dummyRecord() *DeadlockRecord { + return &DeadlockRecord{} +} + +func (s *testDeadlockHistorySuite) TestResize(c *C) { + h := NewDeadlockHistory(2) + h.Push(dummyRecord()) // id=1 inserted + h.Push(dummyRecord()) // id=2 inserted, + h.Push(dummyRecord()) // id=3 inserted, id=1 is removed + c.Assert(h.head, Equals, 1) + c.Assert(h.size, Equals, 2) + c.Assert(len(h.GetAll()), Equals, 2) + c.Assert(h.GetAll()[0].ID, Equals, uint64(2)) + c.Assert(h.GetAll()[1].ID, Equals, uint64(3)) + + h.Resize(3) + c.Assert(h.head, Equals, 0) + c.Assert(h.size, Equals, 2) + h.Push(dummyRecord()) // id=4 inserted + c.Assert(h.head, Equals, 0) + c.Assert(h.size, Equals, 3) + c.Assert(len(h.GetAll()), Equals, 3) + c.Assert(h.GetAll()[0].ID, Equals, uint64(2)) + c.Assert(h.GetAll()[1].ID, Equals, uint64(3)) + c.Assert(h.GetAll()[2].ID, Equals, uint64(4)) + + h.Resize(2) // id=2 removed + c.Assert(h.head, Equals, 0) + c.Assert(h.size, Equals, 2) + c.Assert(len(h.GetAll()), Equals, 2) + c.Assert(h.GetAll()[0].ID, Equals, uint64(3)) + c.Assert(h.GetAll()[1].ID, Equals, uint64(4)) + + h.Resize(0) // all removed + c.Assert(h.head, Equals, 0) + c.Assert(h.size, Equals, 0) + c.Assert(len(h.GetAll()), Equals, 0) + + h.Resize(2) + c.Assert(h.head, Equals, 0) + c.Assert(h.size, Equals, 0) + h.Push(dummyRecord()) // id=5 inserted + c.Assert(h.head, Equals, 0) + c.Assert(h.size, Equals, 1) +} diff --git a/util/disk/tempDir.go b/util/disk/tempDir.go index 2d42d21d82d39..7c0a1a9d45239 100644 --- a/util/disk/tempDir.go +++ b/util/disk/tempDir.go @@ -14,7 +14,6 @@ package disk import ( - "io/ioutil" "os" "path/filepath" @@ -81,7 +80,7 @@ func InitializeTempDir() error { return err } - subDirs, err := ioutil.ReadDir(tempDir) + subDirs, err := os.ReadDir(tempDir) if err != nil { return err } diff --git a/util/disk/tempDir_test.go b/util/disk/tempDir_test.go index 019f122f22da8..436f53a4c4a7f 100644 --- a/util/disk/tempDir_test.go +++ b/util/disk/tempDir_test.go @@ -14,7 +14,6 @@ package disk import ( - "io/ioutil" "os" "sync" "testing" @@ -24,7 +23,7 @@ import ( ) func TestT(t *testing.T) { - path, _ := ioutil.TempDir("", "tmp-storage-disk-pkg") + path, _ := os.MkdirTemp("", "tmp-storage-disk-pkg") config.UpdateGlobal(func(conf *config.Config) { conf.TempStoragePath = path }) diff --git a/util/encrypt/ase_layer.go b/util/encrypt/aes_layer.go similarity index 91% rename from util/encrypt/ase_layer.go rename to util/encrypt/aes_layer.go index 2bcea4373073f..a27d23da90fa6 100644 --- a/util/encrypt/ase_layer.go +++ b/util/encrypt/aes_layer.go @@ -71,11 +71,12 @@ func (ctr *CtrCipher) stream(counter uint64) cipher.Stream { // Writer implements an io.WriteCloser, it encrypt data using AES before writing to the underlying object. type Writer struct { - err error - w io.WriteCloser - n int - buf []byte - cipherStream cipher.Stream + err error + w io.WriteCloser + n int + buf []byte + cipherStream cipher.Stream + flushedUserDataCnt int64 } // NewWriter returns a new Writer which encrypt data using AES before writing to the underlying object. @@ -123,6 +124,7 @@ func (w *Writer) Flush() error { } w.cipherStream.XORKeyStream(w.buf[:w.n], w.buf[:w.n]) n, err := w.w.Write(w.buf[:w.n]) + w.flushedUserDataCnt += int64(n) if n < w.n && err == nil { err = io.ErrShortWrite } @@ -134,6 +136,16 @@ func (w *Writer) Flush() error { return nil } +// GetCache returns the byte slice that holds the data not flushed to disk. +func (w *Writer) GetCache() []byte { + return w.buf[:w.n] +} + +// GetCacheDataOffset return the user data offset in cache. +func (w *Writer) GetCacheDataOffset() int64 { + return w.flushedUserDataCnt +} + // Close implements the io.Closer interface. func (w *Writer) Close() (err error) { err = w.Flush() diff --git a/util/execdetails/execdetails.go b/util/execdetails/execdetails.go index ca045352dbd33..ac16e1fa13f09 100644 --- a/util/execdetails/execdetails.go +++ b/util/execdetails/execdetails.go @@ -142,11 +142,11 @@ func (d ExecDetails) String() string { if commitDetails.GetCommitTsTime > 0 { parts = append(parts, GetCommitTSTimeStr+": "+strconv.FormatFloat(commitDetails.GetCommitTsTime.Seconds(), 'f', -1, 64)) } - commitBackoffTime := atomic.LoadInt64(&commitDetails.CommitBackoffTime) + commitDetails.Mu.Lock() + commitBackoffTime := commitDetails.Mu.CommitBackoffTime if commitBackoffTime > 0 { parts = append(parts, CommitBackoffTimeStr+": "+strconv.FormatFloat(time.Duration(commitBackoffTime).Seconds(), 'f', -1, 64)) } - commitDetails.Mu.Lock() if len(commitDetails.Mu.BackoffTypes) > 0 { parts = append(parts, BackoffTypesStr+": "+fmt.Sprintf("%v", commitDetails.Mu.BackoffTypes)) } @@ -234,11 +234,11 @@ func (d ExecDetails) ToZapFields() (fields []zap.Field) { if commitDetails.GetCommitTsTime > 0 { fields = append(fields, zap.String("get_commit_ts_time", fmt.Sprintf("%v", strconv.FormatFloat(commitDetails.GetCommitTsTime.Seconds(), 'f', -1, 64)+"s"))) } - commitBackoffTime := atomic.LoadInt64(&commitDetails.CommitBackoffTime) + commitDetails.Mu.Lock() + commitBackoffTime := commitDetails.Mu.CommitBackoffTime if commitBackoffTime > 0 { fields = append(fields, zap.String("commit_backoff_time", fmt.Sprintf("%v", strconv.FormatFloat(time.Duration(commitBackoffTime).Seconds(), 'f', -1, 64)+"s"))) } - commitDetails.Mu.Lock() if len(commitDetails.Mu.BackoffTypes) > 0 { fields = append(fields, zap.String("backoff_types", fmt.Sprintf("%v", commitDetails.Mu.BackoffTypes))) } @@ -640,7 +640,7 @@ func getPlanIDFromExecutionSummary(summary *tipb.ExecutorExecutionSummary) (int, } // RecordOneCopTask records a specific cop tasks's execution detail. -func (e *RuntimeStatsColl) RecordOneCopTask(planID int, storeType string, address string, summary *tipb.ExecutorExecutionSummary) { +func (e *RuntimeStatsColl) RecordOneCopTask(planID int, storeType string, address string, summary *tipb.ExecutorExecutionSummary) int { // for TiFlash cop response, ExecutorExecutionSummary contains executor id, so if there is a valid executor id in // summary, use it overwrite the planID if id, valid := getPlanIDFromExecutionSummary(summary); valid { @@ -648,6 +648,7 @@ func (e *RuntimeStatsColl) RecordOneCopTask(planID int, storeType string, addres } copStats := e.GetOrCreateCopStats(planID, storeType) copStats.RecordOneCopTask(address, summary) + return planID } // RecordScanDetail records a specific cop tasks's cop detail. @@ -703,9 +704,7 @@ func (e *RuntimeStatsWithConcurrencyInfo) SetConcurrencyInfo(infos ...*Concurren e.Lock() defer e.Unlock() e.concurrency = e.concurrency[:0] - for _, info := range infos { - e.concurrency = append(e.concurrency, info) - } + e.concurrency = append(e.concurrency, infos...) } // Clone implements the RuntimeStats interface. @@ -736,12 +735,7 @@ func (e *RuntimeStatsWithConcurrencyInfo) String() string { } // Merge implements the RuntimeStats interface. -func (e *RuntimeStatsWithConcurrencyInfo) Merge(rs RuntimeStats) { - tmp, ok := rs.(*RuntimeStatsWithConcurrencyInfo) - if !ok { - return - } - e.concurrency = append(e.concurrency, tmp.concurrency...) +func (e *RuntimeStatsWithConcurrencyInfo) Merge(_ RuntimeStats) { } // RuntimeStatsWithCommit is the RuntimeStats with commit detail. @@ -809,18 +803,18 @@ func (e *RuntimeStatsWithCommit) String() string { buf.WriteString(", commit:") buf.WriteString(FormatDuration(e.Commit.CommitTime)) } - commitBackoffTime := atomic.LoadInt64(&e.Commit.CommitBackoffTime) + e.Commit.Mu.Lock() + commitBackoffTime := e.Commit.Mu.CommitBackoffTime if commitBackoffTime > 0 { buf.WriteString(", backoff: {time: ") buf.WriteString(FormatDuration(time.Duration(commitBackoffTime))) - e.Commit.Mu.Lock() if len(e.Commit.Mu.BackoffTypes) > 0 { buf.WriteString(", type: ") buf.WriteString(e.formatBackoff(e.Commit.Mu.BackoffTypes)) } - e.Commit.Mu.Unlock() buf.WriteString("}") } + e.Commit.Mu.Unlock() if e.Commit.ResolveLockTime > 0 { buf.WriteString(", resolve_lock: ") buf.WriteString(FormatDuration(time.Duration(e.Commit.ResolveLockTime))) @@ -894,14 +888,13 @@ func (e *RuntimeStatsWithCommit) String() string { return buf.String() } -func (e *RuntimeStatsWithCommit) formatBackoff(backoffTypes []fmt.Stringer) string { +func (e *RuntimeStatsWithCommit) formatBackoff(backoffTypes []string) string { if len(backoffTypes) == 0 { return "" } tpMap := make(map[string]struct{}) tpArray := []string{} - for _, tp := range backoffTypes { - tpStr := tp.String() + for _, tpStr := range backoffTypes { _, ok := tpMap[tpStr] if ok { continue diff --git a/util/execdetails/execdetails_test.go b/util/execdetails/execdetails_test.go index 371d06006051f..461814e1b7d3e 100644 --- a/util/execdetails/execdetails_test.go +++ b/util/execdetails/execdetails_test.go @@ -14,7 +14,6 @@ package execdetails import ( - "fmt" "strconv" "sync" "testing" @@ -22,7 +21,6 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/tidb/store/tikv/util" - "github.com/pingcap/tidb/util/stringutil" "github.com/pingcap/tipb/go-tipb" ) @@ -36,22 +34,22 @@ func TestString(t *testing.T) { BackoffTime: time.Second, RequestCount: 1, CommitDetail: &util.CommitDetails{ - GetCommitTsTime: time.Second, - PrewriteTime: time.Second, - CommitTime: time.Second, - LocalLatchTime: time.Second, - CommitBackoffTime: int64(time.Second), + GetCommitTsTime: time.Second, + PrewriteTime: time.Second, + CommitTime: time.Second, + LocalLatchTime: time.Second, + Mu: struct { sync.Mutex - BackoffTypes []fmt.Stringer - }{BackoffTypes: []fmt.Stringer{ - stringutil.MemoizeStr(func() string { - return "backoff1" - }), - stringutil.MemoizeStr(func() string { - return "backoff2" - }), - }}, + CommitBackoffTime int64 + BackoffTypes []string + }{ + CommitBackoffTime: int64(time.Second), + BackoffTypes: []string{ + "backoff1", + "backoff2", + }, + }, ResolveLockTime: 1000000000, // 10^9 ns = 1s WriteKeys: 1, WriteSize: 1, @@ -206,24 +204,17 @@ func TestCopRuntimeStatsForTiFlash(t *testing.T) { } func TestRuntimeStatsWithCommit(t *testing.T) { commitDetail := &util.CommitDetails{ - GetCommitTsTime: time.Second, - PrewriteTime: time.Second, - CommitTime: time.Second, - CommitBackoffTime: int64(time.Second), + GetCommitTsTime: time.Second, + PrewriteTime: time.Second, + CommitTime: time.Second, Mu: struct { sync.Mutex - BackoffTypes []fmt.Stringer - }{BackoffTypes: []fmt.Stringer{ - stringutil.MemoizeStr(func() string { - return "backoff1" - }), - stringutil.MemoizeStr(func() string { - return "backoff2" - }), - stringutil.MemoizeStr(func() string { - return "backoff1" - }), - }}, + CommitBackoffTime int64 + BackoffTypes []string + }{ + CommitBackoffTime: int64(time.Second), + BackoffTypes: []string{"backoff1", "backoff2", "backoff1"}, + }, ResolveLockTime: int64(time.Second), WriteKeys: 3, WriteSize: 66, @@ -245,17 +236,11 @@ func TestRuntimeStatsWithCommit(t *testing.T) { BackoffTime: int64(time.Second * 3), Mu: struct { sync.Mutex - BackoffTypes []fmt.Stringer - }{BackoffTypes: []fmt.Stringer{ - stringutil.MemoizeStr(func() string { - return "backoff4" - }), - stringutil.MemoizeStr(func() string { - return "backoff5" - }), - stringutil.MemoizeStr(func() string { - return "backoff5" - }), + BackoffTypes []string + }{BackoffTypes: []string{ + "backoff4", + "backoff5", + "backoff5", }}, LockRPCTime: int64(time.Second * 5), LockRPCCount: 50, @@ -294,11 +279,8 @@ func TestRootRuntimeStats(t *testing.T) { stmtStats.RegisterStats(pid, &RuntimeStatsWithCommit{ Commit: commitDetail, }) - concurrency = &RuntimeStatsWithConcurrencyInfo{} - concurrency.SetConcurrencyInfo(NewConcurrencyInfo("concurrent", 0)) - stmtStats.RegisterStats(pid, concurrency) stats := stmtStats.GetRootStats(1) - expect := "time:3s, loops:2, worker:15, concurrent:OFF, commit_txn: {prewrite:1s, get_commit_ts:1s, commit:1s, region_num:5, write_keys:3, write_byte:66, txn_retry:2}" + expect := "time:3s, loops:2, worker:15, commit_txn: {prewrite:1s, get_commit_ts:1s, commit:1s, region_num:5, write_keys:3, write_byte:66, txn_retry:2}" if stats.String() != expect { t.Fatalf("%v != %v", stats.String(), expect) } diff --git a/util/expensivequery/memory_usage_alarm.go b/util/expensivequery/memory_usage_alarm.go index fbe9b6c5ff438..dbe13ccb973bc 100644 --- a/util/expensivequery/memory_usage_alarm.go +++ b/util/expensivequery/memory_usage_alarm.go @@ -15,7 +15,6 @@ package expensivequery import ( "fmt" - "io/ioutil" "os" "path/filepath" "runtime" @@ -65,7 +64,7 @@ func (record *memoryUsageAlarm) initMemoryUsageAlarmRecord() { } record.lastProfileFileName = make([][]string, 2) // Read last records - files, err := ioutil.ReadDir(record.tmpDir) + files, err := os.ReadDir(record.tmpDir) if err != nil { record.err = err return @@ -83,7 +82,6 @@ func (record *memoryUsageAlarm) initMemoryUsageAlarmRecord() { } } record.initialized = true - return } // If Performance.ServerMemoryQuota is set, use `ServerMemoryQuota * MemoryUsageAlarmRatio` to check oom risk. diff --git a/util/filesort/filesort_test.go b/util/filesort/filesort_test.go index fbbb48204c747..d518f28e9ac1d 100644 --- a/util/filesort/filesort_test.go +++ b/util/filesort/filesort_test.go @@ -14,7 +14,6 @@ package filesort import ( - "io/ioutil" "math/rand" "os" "testing" @@ -127,7 +126,7 @@ func (s *testFileSortSuite) TestInMemory(c *C) { ret bool ) - tmpDir, err = ioutil.TempDir("", "util_filesort_test") + tmpDir, err = os.MkdirTemp("", "util_filesort_test") c.Assert(err, IsNil) fsBuilder := new(Builder) @@ -177,7 +176,7 @@ func (s *testFileSortSuite) TestMultipleFiles(c *C) { ret bool ) - tmpDir, err = ioutil.TempDir("", "util_filesort_test") + tmpDir, err = os.MkdirTemp("", "util_filesort_test") c.Assert(err, IsNil) fsBuilder := new(Builder) @@ -245,7 +244,7 @@ func (s *testFileSortSuite) TestMultipleWorkers(c *C) { ret bool ) - tmpDir, err = ioutil.TempDir("", "util_filesort_test") + tmpDir, err = os.MkdirTemp("", "util_filesort_test") c.Assert(err, IsNil) fsBuilder := new(Builder) @@ -294,13 +293,13 @@ func (s *testFileSortSuite) TestClose(c *C) { // Prepare two FileSorter instances for tests fsBuilder := new(Builder) - tmpDir0, err = ioutil.TempDir("", "util_filesort_test") + tmpDir0, err = os.MkdirTemp("", "util_filesort_test") c.Assert(err, IsNil) fs0, err = fsBuilder.SetSC(sc).SetSchema(keySize, valSize).SetBuf(bufSize).SetWorkers(1).SetDesc(byDesc).SetDir(tmpDir0).Build() c.Assert(err, IsNil) defer fs0.Close() - tmpDir1, err = ioutil.TempDir("", "util_filesort_test") + tmpDir1, err = os.MkdirTemp("", "util_filesort_test") c.Assert(err, IsNil) fs1, err = fsBuilder.SetSC(sc).SetSchema(keySize, valSize).SetBuf(bufSize).SetWorkers(1).SetDesc(byDesc).SetDir(tmpDir1).Build() c.Assert(err, IsNil) @@ -373,13 +372,13 @@ func (s *testFileSortSuite) TestMismatchedUsage(c *C) { // Prepare two FileSorter instances for tests fsBuilder := new(Builder) - tmpDir, err = ioutil.TempDir("", "util_filesort_test") + tmpDir, err = os.MkdirTemp("", "util_filesort_test") c.Assert(err, IsNil) fs0, err = fsBuilder.SetSC(sc).SetSchema(keySize, valSize).SetBuf(bufSize).SetWorkers(1).SetDesc(byDesc).SetDir(tmpDir).Build() c.Assert(err, IsNil) defer fs0.Close() - tmpDir, err = ioutil.TempDir("", "util_filesort_test") + tmpDir, err = os.MkdirTemp("", "util_filesort_test") c.Assert(err, IsNil) fs1, err = fsBuilder.SetSC(sc).SetSchema(keySize, valSize).SetBuf(bufSize).SetWorkers(1).SetDesc(byDesc).SetDir(tmpDir).Build() c.Assert(err, IsNil) diff --git a/util/format/format_test.go b/util/format/format_test.go index 6dd9b4f8d730f..5ce2159f48207 100644 --- a/util/format/format_test.go +++ b/util/format/format_test.go @@ -15,7 +15,7 @@ package format import ( "bytes" - "io/ioutil" + "io" "testing" . "github.com/pingcap/check" @@ -35,7 +35,7 @@ type testFormatSuite struct { func checkFormat(c *C, f Formatter, buf *bytes.Buffer, str, expect string) { _, err := f.Format(str, 3) c.Assert(err, IsNil) - b, err := ioutil.ReadAll(buf) + b, err := io.ReadAll(buf) c.Assert(err, IsNil) c.Assert(string(b), Equals, expect) } diff --git a/util/kvcache/simple_lru.go b/util/kvcache/simple_lru.go index cee4b0363f39a..aec402adb09ed 100644 --- a/util/kvcache/simple_lru.go +++ b/util/kvcache/simple_lru.go @@ -118,6 +118,9 @@ func (l *SimpleLRUCache) Put(key Key, value Value) { if l.size > l.capacity { lru := l.cache.Back() l.cache.Remove(lru) + if l.onEvict != nil { + l.onEvict(lru.Value.(*cacheEntry).key, lru.Value.(*cacheEntry).value) + } delete(l.elements, string(lru.Value.(*cacheEntry).key.Hash())) l.size-- } diff --git a/util/kvcache/simple_lru_test.go b/util/kvcache/simple_lru_test.go index 9040bf60e7765..d8b38ad12fb3f 100644 --- a/util/kvcache/simple_lru_test.go +++ b/util/kvcache/simple_lru_test.go @@ -58,35 +58,47 @@ func (s *testLRUCacheSuite) TestPut(c *C) { maxMem, err := memory.MemTotal() c.Assert(err, IsNil) - lru := NewSimpleLRUCache(3, 0, maxMem) - c.Assert(lru.capacity, Equals, uint(3)) + lruMaxMem := NewSimpleLRUCache(3, 0, maxMem) + lruZeroQuota := NewSimpleLRUCache(3, 0, 0) + c.Assert(lruMaxMem.capacity, Equals, uint(3)) + c.Assert(lruZeroQuota.capacity, Equals, uint(3)) keys := make([]*mockCacheKey, 5) vals := make([]int64, 5) - droppedKv := make(map[Key]Value) + maxMemDroppedKv := make(map[Key]Value) + zeroQuotaDroppedKv := make(map[Key]Value) - lru.SetOnEvict(func(key Key, value Value) { - droppedKv[key] = value + // test onEvict function + lruMaxMem.SetOnEvict(func(key Key, value Value) { + maxMemDroppedKv[key] = value + }) + // test onEvict function on 0 value of quota + lruZeroQuota.SetOnEvict(func(key Key, value Value) { + zeroQuotaDroppedKv[key] = value }) for i := 0; i < 5; i++ { keys[i] = newMockHashKey(int64(i)) vals[i] = int64(i) - lru.Put(keys[i], vals[i]) + lruMaxMem.Put(keys[i], vals[i]) + lruZeroQuota.Put(keys[i], vals[i]) } - c.Assert(lru.size, Equals, lru.capacity) - c.Assert(lru.size, Equals, uint(3)) + c.Assert(lruMaxMem.size, Equals, lruMaxMem.capacity) + c.Assert(lruZeroQuota.size, Equals, lruZeroQuota.capacity) + c.Assert(lruMaxMem.size, Equals, uint(3)) + c.Assert(lruZeroQuota.size, Equals, lruMaxMem.size) // test for non-existent elements - c.Assert(len(droppedKv), Equals, 2) + c.Assert(len(maxMemDroppedKv), Equals, 2) for i := 0; i < 2; i++ { - element, exists := lru.elements[string(keys[i].Hash())] + element, exists := lruMaxMem.elements[string(keys[i].Hash())] c.Assert(exists, IsFalse) c.Assert(element, IsNil) - c.Assert(droppedKv[keys[i]], Equals, vals[i]) + c.Assert(maxMemDroppedKv[keys[i]], Equals, vals[i]) + c.Assert(zeroQuotaDroppedKv[keys[i]], Equals, vals[i]) } // test for existent elements - root := lru.cache.Front() + root := lruMaxMem.cache.Front() c.Assert(root, NotNil) for i := 4; i >= 2; i-- { entry, ok := root.Value.(*cacheEntry) @@ -98,7 +110,7 @@ func (s *testLRUCacheSuite) TestPut(c *C) { c.Assert(key, NotNil) c.Assert(key, Equals, keys[i]) - element, exists := lru.elements[string(keys[i].Hash())] + element, exists := lruMaxMem.elements[string(keys[i].Hash())] c.Assert(exists, IsTrue) c.Assert(element, NotNil) c.Assert(element, Equals, root) diff --git a/util/logutil/log.go b/util/logutil/log.go index 48088318b2d6b..b62484658495f 100644 --- a/util/logutil/log.go +++ b/util/logutil/log.go @@ -14,35 +14,26 @@ package logutil import ( - "bytes" "context" "fmt" "os" - "path/filepath" - "runtime" "runtime/trace" - "sort" - "strings" "time" + gzap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" "github.com/opentracing/opentracing-go" tlog "github.com/opentracing/opentracing-go/log" "github.com/pingcap/errors" - zaplog "github.com/pingcap/log" - tikvlog "github.com/pingcap/tidb/store/tikv/logutil" - log "github.com/sirupsen/logrus" + "github.com/pingcap/log" "go.uber.org/zap" "go.uber.org/zap/zapcore" - "gopkg.in/natefinch/lumberjack.v2" ) const ( - defaultLogTimeFormat = "2006/01/02 15:04:05.000" // DefaultLogMaxSize is the default size of log files. DefaultLogMaxSize = 300 // MB // DefaultLogFormat is the default format of the log. DefaultLogFormat = "text" - defaultLogLevel = log.InfoLevel // DefaultSlowThreshold is the default slow log threshold in millisecond. DefaultSlowThreshold = 300 // DefaultQueryLogMaxLen is the default max length of the query in the log. @@ -51,6 +42,8 @@ const ( DefaultRecordPlanInSlowLog = 1 // DefaultTiDBEnableSlowLog enables TiDB to log slow queries. DefaultTiDBEnableSlowLog = true + // GRPCLogDebugVerbosity enables max verbosity when debugging grpc code. + GRPCLogDebugVerbosity = 99 ) // EmptyFileLogConfig is an empty FileLogConfig. @@ -58,12 +51,12 @@ var EmptyFileLogConfig = FileLogConfig{} // FileLogConfig serializes file log related config in toml/json. type FileLogConfig struct { - zaplog.FileLogConfig + log.FileLogConfig } // NewFileLogConfig creates a FileLogConfig. func NewFileLogConfig(maxSize uint) FileLogConfig { - return FileLogConfig{FileLogConfig: zaplog.FileLogConfig{ + return FileLogConfig{FileLogConfig: log.FileLogConfig{ MaxSize: int(maxSize), }, } @@ -71,16 +64,16 @@ func NewFileLogConfig(maxSize uint) FileLogConfig { // LogConfig serializes log related config in toml/json. type LogConfig struct { - zaplog.Config + log.Config // SlowQueryFile filename, default to File log config on empty. SlowQueryFile string } // NewLogConfig creates a LogConfig. -func NewLogConfig(level, format, slowQueryFile string, fileCfg FileLogConfig, disableTimestamp bool, opts ...func(*zaplog.Config)) *LogConfig { +func NewLogConfig(level, format, slowQueryFile string, fileCfg FileLogConfig, disableTimestamp bool, opts ...func(*log.Config)) *LogConfig { c := &LogConfig{ - Config: zaplog.Config{ + Config: log.Config{ Level: level, Format: format, DisableTimestamp: disableTimestamp, @@ -94,102 +87,6 @@ func NewLogConfig(level, format, slowQueryFile string, fileCfg FileLogConfig, di return c } -// isSKippedPackageName tests wether path name is on log library calling stack. -func isSkippedPackageName(name string) bool { - return strings.Contains(name, "github.com/sirupsen/logrus") || - strings.Contains(name, "github.com/coreos/pkg/capnslog") -} - -// modifyHook injects file name and line pos into log entry. -type contextHook struct{} - -// Fire implements logrus.Hook interface -// https://github.com/sirupsen/logrus/issues/63 -func (hook *contextHook) Fire(entry *log.Entry) error { - pc := make([]uintptr, 4) - cnt := runtime.Callers(8, pc) - - for i := 0; i < cnt; i++ { - fu := runtime.FuncForPC(pc[i] - 1) - name := fu.Name() - if !isSkippedPackageName(name) { - file, line := fu.FileLine(pc[i] - 1) - entry.Data["file"] = filepath.Base(file) - entry.Data["line"] = line - break - } - } - return nil -} - -// Levels implements logrus.Hook interface. -func (hook *contextHook) Levels() []log.Level { - return log.AllLevels -} - -func stringToLogLevel(level string) log.Level { - switch strings.ToLower(level) { - case "fatal": - return log.FatalLevel - case "error": - return log.ErrorLevel - case "warn", "warning": - return log.WarnLevel - case "debug": - return log.DebugLevel - case "info": - return log.InfoLevel - } - return defaultLogLevel -} - -// textFormatter is for compatibility with ngaut/log -type textFormatter struct { - DisableTimestamp bool - EnableEntryOrder bool -} - -// Format implements logrus.Formatter -func (f *textFormatter) Format(entry *log.Entry) ([]byte, error) { - var b *bytes.Buffer - if entry.Buffer != nil { - b = entry.Buffer - } else { - b = &bytes.Buffer{} - } - - if !f.DisableTimestamp { - fmt.Fprintf(b, "%s ", entry.Time.Format(defaultLogTimeFormat)) - } - if file, ok := entry.Data["file"]; ok { - fmt.Fprintf(b, "%s:%v:", file, entry.Data["line"]) - } - fmt.Fprintf(b, " [%s] %s", entry.Level.String(), entry.Message) - - if f.EnableEntryOrder { - keys := make([]string, 0, len(entry.Data)) - for k := range entry.Data { - if k != "file" && k != "line" { - keys = append(keys, k) - } - } - sort.Strings(keys) - for _, k := range keys { - fmt.Fprintf(b, " %v=%v", k, entry.Data[k]) - } - } else { - for k, v := range entry.Data { - if k != "file" && k != "line" { - fmt.Fprintf(b, " %v=%v", k, v) - } - } - } - - b.WriteByte('\n') - - return b.Bytes(), nil -} - const ( // SlowLogTimeFormat is the time format for slow log. SlowLogTimeFormat = time.RFC3339Nano @@ -197,125 +94,34 @@ const ( OldSlowLogTimeFormat = "2006-01-02-15:04:05.999999999 -0700" ) -type slowLogFormatter struct{} - -func (f *slowLogFormatter) Format(entry *log.Entry) ([]byte, error) { - var b *bytes.Buffer - if entry.Buffer != nil { - b = entry.Buffer - } else { - b = &bytes.Buffer{} - } +// SlowQueryLogger is used to log slow query, InitZapLogger will modify it according to config file. +var SlowQueryLogger = log.L() - fmt.Fprintf(b, "# Time: %s\n", entry.Time.Format(SlowLogTimeFormat)) - fmt.Fprintf(b, "%s\n", entry.Message) - return b.Bytes(), nil -} - -func stringToLogFormatter(format string, disableTimestamp bool) log.Formatter { - switch strings.ToLower(format) { - case "text": - return &textFormatter{ - DisableTimestamp: disableTimestamp, - } - default: - return &textFormatter{} - } -} - -// initFileLog initializes file based logging options. -func initFileLog(cfg *zaplog.FileLogConfig, logger *log.Logger) error { - if st, err := os.Stat(cfg.Filename); err == nil { - if st.IsDir() { - return errors.New("can't use directory as log file name") - } - } - if cfg.MaxSize == 0 { - cfg.MaxSize = DefaultLogMaxSize - } - - // use lumberjack to logrotate - output := &lumberjack.Logger{ - Filename: cfg.Filename, - MaxSize: cfg.MaxSize, - MaxBackups: cfg.MaxBackups, - MaxAge: cfg.MaxDays, - LocalTime: true, - } - - if logger == nil { - log.SetOutput(output) - } else { - logger.Out = output - } - return nil -} - -// SlowQueryLogger is used to log slow query, InitLogger will modify it according to config file. -var SlowQueryLogger = log.StandardLogger() - -// SlowQueryZapLogger is used to log slow query, InitZapLogger will modify it according to config file. -var SlowQueryZapLogger = zaplog.L() - -// InitLogger initializes PD's logger. +// InitLogger delegates to InitZapLogger. Keeping it here for historical reason. func InitLogger(cfg *LogConfig) error { - log.SetLevel(stringToLogLevel(cfg.Level)) - log.AddHook(&contextHook{}) - - if cfg.Format == "" { - cfg.Format = DefaultLogFormat - } - formatter := stringToLogFormatter(cfg.Format, cfg.DisableTimestamp) - log.SetFormatter(formatter) - - if len(cfg.File.Filename) != 0 { - if err := initFileLog(&cfg.File, nil); err != nil { - return errors.Trace(err) - } - } - - if len(cfg.SlowQueryFile) != 0 { - SlowQueryLogger = log.New() - tmp := cfg.File - tmp.Filename = cfg.SlowQueryFile - if err := initFileLog(&tmp, SlowQueryLogger); err != nil { - return errors.Trace(err) - } - SlowQueryLogger.Formatter = &slowLogFormatter{} - } - - // Setup log key for tikv client. - tikvlog.CtxLogKey = ctxLogKey - - return nil + return InitZapLogger(cfg) } // InitZapLogger initializes a zap logger with cfg. func InitZapLogger(cfg *LogConfig) error { - gl, props, err := zaplog.InitLogger(&cfg.Config, zap.AddStacktrace(zapcore.FatalLevel)) + gl, props, err := log.InitLogger(&cfg.Config, zap.AddStacktrace(zapcore.FatalLevel)) if err != nil { return errors.Trace(err) } - zaplog.ReplaceGlobals(gl, props) - - if len(cfg.SlowQueryFile) != 0 { - sqfCfg := zaplog.FileLogConfig{ - MaxSize: cfg.File.MaxSize, - Filename: cfg.SlowQueryFile, - } - sqCfg := &zaplog.Config{ - Level: cfg.Level, - Format: cfg.Format, - DisableTimestamp: cfg.DisableTimestamp, - File: sqfCfg, - } - sqLogger, _, err := zaplog.InitLogger(sqCfg) - if err != nil { - return errors.Trace(err) - } - SlowQueryZapLogger = sqLogger + log.ReplaceGlobals(gl, props) + + // init dedicated logger for slow query log + SlowQueryLogger, err = newSlowQueryLogger(cfg) + if err != nil { + return errors.Trace(err) + } + + // init logger for grpc debugging + if len(os.Getenv("GRPC_DEBUG")) > 0 { + // more information for verbosity: https://github.com/google/glog#verbose-logging + gzap.ReplaceGrpcLoggerV2WithVerbosity(gl, GRPCLogDebugVerbosity) } else { - SlowQueryZapLogger = gl + gzap.ReplaceGrpcLoggerV2(gl) } return nil @@ -327,7 +133,7 @@ func SetLevel(level string) error { if err := l.UnmarshalText([]byte(level)); err != nil { return errors.Trace(err) } - zaplog.SetLevel(l.Level()) + log.SetLevel(l.Level()) return nil } @@ -341,12 +147,12 @@ func Logger(ctx context.Context) *zap.Logger { if ctxlogger, ok := ctx.Value(ctxLogKey).(*zap.Logger); ok { return ctxlogger } - return zaplog.L() + return log.L() } // BgLogger is alias of `logutil.BgLogger()` func BgLogger() *zap.Logger { - return zaplog.L() + return log.L() } // WithConnID attaches connId to context. @@ -355,7 +161,7 @@ func WithConnID(ctx context.Context, connID uint64) context.Context { if ctxLogger, ok := ctx.Value(ctxLogKey).(*zap.Logger); ok { logger = ctxLogger } else { - logger = zaplog.L() + logger = log.L() } return context.WithValue(ctx, ctxLogKey, logger.With(zap.Uint64("conn", connID))) } @@ -366,7 +172,7 @@ func WithTraceLogger(ctx context.Context, connID uint64) context.Context { if ctxLogger, ok := ctx.Value(ctxLogKey).(*zap.Logger); ok { logger = ctxLogger } else { - logger = zaplog.L() + logger = log.L() } return context.WithValue(ctx, ctxLogKey, wrapTraceLogger(ctx, connID, logger)) } @@ -374,7 +180,7 @@ func WithTraceLogger(ctx context.Context, connID uint64) context.Context { func wrapTraceLogger(ctx context.Context, connID uint64, logger *zap.Logger) *zap.Logger { return logger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { tl := &traceLog{ctx: ctx} - traceCore := zaplog.NewTextCore(zaplog.NewTextEncoder(&zaplog.Config{}), tl, tl). + traceCore := log.NewTextCore(log.NewTextEncoder(&log.Config{}), tl, tl). With([]zapcore.Field{zap.Uint64("conn", connID)}) return zapcore.NewTee(traceCore, core) })) @@ -403,7 +209,7 @@ func WithKeyValue(ctx context.Context, key, value string) context.Context { if ctxLogger, ok := ctx.Value(ctxLogKey).(*zap.Logger); ok { logger = ctxLogger } else { - logger = zaplog.L() + logger = log.L() } return context.WithValue(ctx, ctxLogKey, logger.With(zap.String(key, value))) } diff --git a/util/logutil/log_test.go b/util/logutil/log_test.go index 4961aa28e98f1..ecc9fb9d03679 100644 --- a/util/logutil/log_test.go +++ b/util/logutil/log_test.go @@ -17,7 +17,6 @@ import ( "bufio" "bytes" "context" - "fmt" "io" "os" "runtime" @@ -25,19 +24,15 @@ import ( "testing" . "github.com/pingcap/check" - zaplog "github.com/pingcap/log" - log "github.com/sirupsen/logrus" + "github.com/pingcap/log" "go.uber.org/zap" ) const ( - logPattern = `\d\d\d\d/\d\d/\d\d \d\d:\d\d:\d\d\.\d\d\d ([\w_%!$@.,+~-]+|\\.)+:\d+: \[(fatal|error|warning|info|debug)\] .*?\n` // zapLogPatern is used to match the zap log format, such as the following log: - // [2019/02/13 15:56:05.385 +08:00] [INFO] [log_test.go:167] ["info message"] ["str key"=val] ["int key"=123] - zapLogPattern = `\[\d\d\d\d/\d\d/\d\d \d\d:\d\d:\d\d.\d\d\d\ (\+|-)\d\d:\d\d\] \[(FATAL|ERROR|WARN|INFO|DEBUG)\] \[([\w_%!$@.,+~-]+|\\.)+:\d+\] \[.*\] (\[.*=.*\]).*\n` - // [2019/02/13 15:56:05.385 +08:00] [INFO] [log_test.go:167] ["info message"] ["str key"=val] ["int key"=123] + // [2019/02/13 15:56:05.385 +08:00] [INFO] [log_test.go:167] ["info message"] [conn=conn1] ["str key"=val] ["int key"=123] zapLogWithConnIDPattern = `\[\d\d\d\d/\d\d/\d\d \d\d:\d\d:\d\d.\d\d\d\ (\+|-)\d\d:\d\d\] \[(FATAL|ERROR|WARN|INFO|DEBUG)\] \[([\w_%!$@.,+~-]+|\\.)+:\d+\] \[.*\] \[conn=.*\] (\[.*=.*\]).*\n` - // [2019/02/13 15:56:05.385 +08:00] [INFO] [log_test.go:167] ["info message"] ["str key"=val] ["int key"=123] + // [2019/02/13 15:56:05.385 +08:00] [INFO] [log_test.go:167] ["info message"] [ctxKey=ctxKey1] ["str key"=val] ["int key"=123] zapLogWithKeyValPattern = `\[\d\d\d\d/\d\d/\d\d \d\d:\d\d:\d\d.\d\d\d\ (\+|-)\d\d:\d\d\] \[(FATAL|ERROR|WARN|INFO|DEBUG)\] \[([\w_%!$@.,+~-]+|\\.)+:\d+\] \[.*\] \[ctxKey=.*\] (\[.*=.*\]).*\n` ) @@ -53,156 +48,14 @@ type testLogSuite struct { buf *bytes.Buffer } -func (s *testLogSuite) SetUpSuite(c *C) { +func (s *testLogSuite) SetUpSuite(_ *C) { s.buf = &bytes.Buffer{} } -func (s *testLogSuite) SetUpTest(c *C) { +func (s *testLogSuite) SetUpTest(_ *C) { s.buf = &bytes.Buffer{} } -func (s *testLogSuite) TestStringToLogLevel(c *C) { - c.Assert(stringToLogLevel("fatal"), Equals, log.FatalLevel) - c.Assert(stringToLogLevel("ERROR"), Equals, log.ErrorLevel) - c.Assert(stringToLogLevel("warn"), Equals, log.WarnLevel) - c.Assert(stringToLogLevel("warning"), Equals, log.WarnLevel) - c.Assert(stringToLogLevel("debug"), Equals, log.DebugLevel) - c.Assert(stringToLogLevel("info"), Equals, log.InfoLevel) - c.Assert(stringToLogLevel("whatever"), Equals, log.InfoLevel) -} - -// TestLogging assure log format and log redirection works. -func (s *testLogSuite) TestLogging(c *C) { - conf := NewLogConfig("warn", DefaultLogFormat, "", NewFileLogConfig(0), false) - conf.File.Filename = "log_file" - c.Assert(InitLogger(conf), IsNil) - - log.SetOutput(s.buf) - - log.Infof("[this message should not be sent to buf]") - c.Assert(s.buf.Len(), Equals, 0) - - log.Warningf("[this message should be sent to buf]") - entry, err := s.buf.ReadString('\n') - c.Assert(err, IsNil) - c.Assert(entry, Matches, logPattern) - - log.Warnf("this message comes from logrus") - entry, err = s.buf.ReadString('\n') - c.Assert(err, IsNil) - c.Assert(entry, Matches, logPattern) - c.Assert(strings.Contains(entry, "log_test.go"), IsTrue) -} - -func (s *testLogSuite) TestSlowQueryLogger(c *C) { - fileName := "slow_query" - os.Remove(fileName) - conf := NewLogConfig("info", DefaultLogFormat, fileName, NewFileLogConfig(DefaultLogMaxSize), false) - c.Assert(conf.File.MaxSize, Equals, DefaultLogMaxSize) - err := InitLogger(conf) - c.Assert(err, IsNil) - defer os.Remove(fileName) - - SlowQueryLogger.Debug("debug message") - SlowQueryLogger.Info("info message") - SlowQueryLogger.Warn("warn message") - SlowQueryLogger.Error("error message") - c.Assert(s.buf.Len(), Equals, 0) - - f, err := os.Open(fileName) - c.Assert(err, IsNil) - defer f.Close() - - r := bufio.NewReader(f) - for { - var str string - str, err = r.ReadString('\n') - if err != nil { - break - } - if strings.HasPrefix(str, "# ") { - c.Assert(str, Matches, `# Time: .*?\n`) - } else { - c.Assert(str, Matches, `.*? message\n`) - } - } - c.Assert(err, Equals, io.EOF) -} - -func (s *testLogSuite) TestLoggerKeepOrder(c *C) { - conf := NewLogConfig("warn", DefaultLogFormat, "", EmptyFileLogConfig, true) - c.Assert(InitLogger(conf), IsNil) - logger := log.StandardLogger() - ft, ok := logger.Formatter.(*textFormatter) - c.Assert(ok, IsTrue) - ft.EnableEntryOrder = true - logger.Out = s.buf - logEntry := log.NewEntry(logger) - logEntry.Data = log.Fields{ - "connectionId": 1, - "costTime": "1", - "database": "test", - "sql": "select 1", - "txnStartTS": 1, - } - - _, _, line, _ := runtime.Caller(0) - logEntry.WithField("type", "slow-query").WithField("succ", true).Warnf("slow-query") - expectMsg := fmt.Sprintf("log_test.go:%v: [warning] slow-query connectionId=1 costTime=1 database=test sql=select 1 succ=true txnStartTS=1 type=slow-query\n", line+1) - c.Assert(s.buf.String(), Equals, expectMsg) - - s.buf.Reset() - logEntry.Data = log.Fields{ - "a": "a", - "d": "d", - "e": "e", - "b": "b", - "f": "f", - "c": "c", - } - - _, _, line, _ = runtime.Caller(0) - logEntry.Warnf("slow-query") - expectMsg = fmt.Sprintf("log_test.go:%v: [warning] slow-query a=a b=b c=c d=d e=e f=f\n", line+1) - c.Assert(s.buf.String(), Equals, expectMsg) -} - -func (s *testLogSuite) TestSlowQueryZapLogger(c *C) { - if runtime.GOOS == "windows" { - // Skip this test on windows for two reasons: - // 1. The pattern match fails somehow. It seems windows treat \n as slash and character n. - // 2. Remove file doesn't work as long as the log instance hold the file. - c.Skip("skip on windows") - } - - fileName := "slow_query" - conf := NewLogConfig("info", DefaultLogFormat, fileName, EmptyFileLogConfig, false) - err := InitZapLogger(conf) - c.Assert(err, IsNil) - defer os.Remove(fileName) - - SlowQueryZapLogger.Debug("debug message", zap.String("str key", "val")) - SlowQueryZapLogger.Info("info message", zap.String("str key", "val")) - SlowQueryZapLogger.Warn("warn", zap.Int("int key", 123)) - SlowQueryZapLogger.Error("error message", zap.Bool("bool key", true)) - - f, err := os.Open(fileName) - c.Assert(err, IsNil) - defer f.Close() - - r := bufio.NewReader(f) - for { - var str string - str, err = r.ReadString('\n') - if err != nil { - break - } - c.Assert(str, Matches, zapLogPattern) - } - c.Assert(err, Equals, io.EOF) - -} - func (s *testLogSuite) TestZapLoggerWithKeys(c *C) { if runtime.GOOS == "windows" { // Skip this test on windows for two reason: @@ -211,7 +64,7 @@ func (s *testLogSuite) TestZapLoggerWithKeys(c *C) { c.Skip("skip on windows") } - fileCfg := FileLogConfig{zaplog.FileLogConfig{Filename: "zap_log", MaxSize: 4096}} + fileCfg := FileLogConfig{log.FileLogConfig{Filename: "zap_log", MaxSize: 4096}} conf := NewLogConfig("info", DefaultLogFormat, "", fileCfg, false) err := InitZapLogger(conf) c.Assert(err, IsNil) @@ -258,14 +111,14 @@ func (s *testLogSuite) TestSetLevel(c *C) { err := InitZapLogger(conf) c.Assert(err, IsNil) - c.Assert(zaplog.GetLevel(), Equals, zap.InfoLevel) + c.Assert(log.GetLevel(), Equals, zap.InfoLevel) err = SetLevel("warn") c.Assert(err, IsNil) - c.Assert(zaplog.GetLevel(), Equals, zap.WarnLevel) + c.Assert(log.GetLevel(), Equals, zap.WarnLevel) err = SetLevel("Error") c.Assert(err, IsNil) - c.Assert(zaplog.GetLevel(), Equals, zap.ErrorLevel) + c.Assert(log.GetLevel(), Equals, zap.ErrorLevel) err = SetLevel("DEBUG") c.Assert(err, IsNil) - c.Assert(zaplog.GetLevel(), Equals, zap.DebugLevel) + c.Assert(log.GetLevel(), Equals, zap.DebugLevel) } diff --git a/util/logutil/slow_query_logger.go b/util/logutil/slow_query_logger.go new file mode 100644 index 0000000000000..3910b6ecd1192 --- /dev/null +++ b/util/logutil/slow_query_logger.go @@ -0,0 +1,76 @@ +package logutil + +import ( + "fmt" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/log" + "go.uber.org/zap" + "go.uber.org/zap/buffer" + "go.uber.org/zap/zapcore" +) + +var _pool = buffer.NewPool() + +func newSlowQueryLogger(cfg *LogConfig) (*zap.Logger, error) { + + // reuse global config and override slow query log file + // if slow query log filename is empty, slow query log will behave the same as global log + sqConfig := &cfg.Config + if len(cfg.SlowQueryFile) != 0 { + sqConfig.File = log.FileLogConfig{ + MaxSize: cfg.File.MaxSize, + Filename: cfg.SlowQueryFile, + } + } + + // create the slow query logger + sqLogger, prop, err := log.InitLogger(sqConfig) + if err != nil { + return nil, errors.Trace(err) + } + + // replace 2018-12-19-unified-log-format text encoder with slow log encoder + sqLogger = sqLogger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { + return log.NewTextCore(&slowLogEncoder{}, prop.Syncer, prop.Level) + })) + + return sqLogger, nil +} + +type slowLogEncoder struct{} + +func (e *slowLogEncoder) EncodeEntry(entry zapcore.Entry, _ []zapcore.Field) (*buffer.Buffer, error) { + b := _pool.Get() + fmt.Fprintf(b, "# Time: %s\n", entry.Time.Format(SlowLogTimeFormat)) + fmt.Fprintf(b, "%s\n", entry.Message) + return b, nil +} + +func (e *slowLogEncoder) Clone() zapcore.Encoder { return e } +func (e *slowLogEncoder) AddArray(string, zapcore.ArrayMarshaler) error { return nil } +func (e *slowLogEncoder) AddObject(string, zapcore.ObjectMarshaler) error { return nil } +func (e *slowLogEncoder) AddBinary(string, []byte) {} +func (e *slowLogEncoder) AddByteString(string, []byte) {} +func (e *slowLogEncoder) AddBool(string, bool) {} +func (e *slowLogEncoder) AddComplex128(string, complex128) {} +func (e *slowLogEncoder) AddComplex64(string, complex64) {} +func (e *slowLogEncoder) AddDuration(string, time.Duration) {} +func (e *slowLogEncoder) AddFloat64(string, float64) {} +func (e *slowLogEncoder) AddFloat32(string, float32) {} +func (e *slowLogEncoder) AddInt(string, int) {} +func (e *slowLogEncoder) AddInt64(string, int64) {} +func (e *slowLogEncoder) AddInt32(string, int32) {} +func (e *slowLogEncoder) AddInt16(string, int16) {} +func (e *slowLogEncoder) AddInt8(string, int8) {} +func (e *slowLogEncoder) AddString(string, string) {} +func (e *slowLogEncoder) AddTime(string, time.Time) {} +func (e *slowLogEncoder) AddUint(string, uint) {} +func (e *slowLogEncoder) AddUint64(string, uint64) {} +func (e *slowLogEncoder) AddUint32(string, uint32) {} +func (e *slowLogEncoder) AddUint16(string, uint16) {} +func (e *slowLogEncoder) AddUint8(string, uint8) {} +func (e *slowLogEncoder) AddUintptr(string, uintptr) {} +func (e *slowLogEncoder) AddReflected(string, interface{}) error { return nil } +func (e *slowLogEncoder) OpenNamespace(string) {} diff --git a/util/memory/meminfo.go b/util/memory/meminfo.go index 6289a46c8b286..3ba93e826ad5e 100644 --- a/util/memory/meminfo.go +++ b/util/memory/meminfo.go @@ -14,7 +14,7 @@ package memory import ( - "io/ioutil" + "os" "strconv" "strings" "sync" @@ -138,7 +138,7 @@ func init() { } func inContainer() bool { - v, err := ioutil.ReadFile(selfCGroupPath) + v, err := os.ReadFile(selfCGroupPath) if err != nil { return false } @@ -171,7 +171,7 @@ func parseUint(s string, base, bitSize int) (uint64, error) { // refer to https://github.com/containerd/cgroups/blob/318312a373405e5e91134d8063d04d59768a1bff/utils.go#L243 func readUint(path string) (uint64, error) { - v, err := ioutil.ReadFile(path) + v, err := os.ReadFile(path) if err != nil { return 0, err } diff --git a/util/memory/tracker.go b/util/memory/tracker.go index 3c369724c229e..2525ee76c2e0a 100644 --- a/util/memory/tracker.go +++ b/util/memory/tracker.go @@ -491,4 +491,6 @@ const ( LabelForApplyCache int = -17 // LabelForSimpleTask represents the label of the simple task LabelForSimpleTask int = -18 + // LabelForCTEStorage represents the label of CTE storage + LabelForCTEStorage int = -19 ) diff --git a/util/misc.go b/util/misc.go index 13192f5811ccd..ccba554bb07d0 100644 --- a/util/misc.go +++ b/util/misc.go @@ -19,7 +19,6 @@ import ( "crypto/x509" "crypto/x509/pkix" "fmt" - "io/ioutil" "net" "net/http" "os" @@ -175,7 +174,7 @@ var ( // IsMemOrSysDB uses to check whether dbLowerName is memory database or system database. func IsMemOrSysDB(dbLowerName string) bool { - return IsMemDB(dbLowerName) || dbLowerName == mysql.SystemDB + return IsMemDB(dbLowerName) || IsSysDB(dbLowerName) } // IsMemDB checks whether dbLowerName is memory database. @@ -189,6 +188,11 @@ func IsMemDB(dbLowerName string) bool { return false } +// IsSysDB checks whether dbLowerName is system database. +func IsSysDB(dbLowerName string) bool { + return dbLowerName == mysql.SystemDB +} + // IsSystemView is similar to IsMemOrSyDB, but does not include the mysql schema func IsSystemView(dbLowerName string) bool { switch dbLowerName { @@ -451,7 +455,7 @@ func LoadTLSCertificates(ca, key, cert string) (tlsConfig *tls.Config, err error var certPool *x509.CertPool if len(ca) > 0 { var caCert []byte - caCert, err = ioutil.ReadFile(ca) + caCert, err = os.ReadFile(ca) if err != nil { logutil.BgLogger().Warn("read file failed", zap.Error(err)) err = errors.Trace(err) @@ -531,3 +535,12 @@ func GetLocalIP() string { } return "" } + +// QueryStrForLog trim the query if the query length more than 4096 +func QueryStrForLog(query string) string { + const size = 4096 + if len(query) > size { + return query[:size] + fmt.Sprintf("(len: %d)", len(query)) + } + return query +} diff --git a/util/mock/context.go b/util/mock/context.go index 4b329e0ff1f55..0dea6bac0fb92 100644 --- a/util/mock/context.go +++ b/util/mock/context.go @@ -26,7 +26,7 @@ import ( "github.com/pingcap/tidb/owner" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" - "github.com/pingcap/tidb/store/tikv/oracle" + "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/disk" "github.com/pingcap/tidb/util/kvcache" @@ -59,14 +59,6 @@ func (txn *wrapTxn) Valid() bool { return txn.Transaction != nil && txn.Transaction.Valid() } -// GetUnionStore implements GetUnionStore -func (txn *wrapTxn) GetUnionStore() kv.UnionStore { - if txn.Transaction == nil { - return nil - } - return txn.Transaction.GetUnionStore() -} - func (txn *wrapTxn) CacheTableInfo(id int64, info *model.TableInfo) { if txn.Transaction == nil { return @@ -152,6 +144,20 @@ func (c *Context) GetMPPClient() kv.MPPClient { return c.Store.GetMPPClient() } +// GetInfoSchema implements sessionctx.Context GetInfoSchema interface. +func (c *Context) GetInfoSchema() sessionctx.InfoschemaMetaVersion { + vars := c.GetSessionVars() + if snap, ok := vars.SnapshotInfoschema.(sessionctx.InfoschemaMetaVersion); ok { + return snap + } + if vars.TxnCtx != nil && vars.InTxn() { + if is, ok := vars.TxnCtx.InfoSchema.(sessionctx.InfoschemaMetaVersion); ok { + return is + } + } + return nil +} + // GetGlobalSysVar implements GlobalVarAccessor GetGlobalSysVar interface. func (c *Context) GetGlobalSysVar(ctx sessionctx.Context, name string) (string, error) { v := variable.GetSysVar(name) @@ -196,6 +202,11 @@ func (c *Context) NewTxn(context.Context) error { return nil } +// NewStaleTxnWithStartTS implements the sessionctx.Context interface. +func (c *Context) NewStaleTxnWithStartTS(ctx context.Context, startTS uint64) error { + return c.NewTxn(ctx) +} + // RefreshTxnCtx implements the sessionctx.Context interface. func (c *Context) RefreshTxnCtx(ctx context.Context) error { return errors.Trace(c.NewTxn(ctx)) @@ -212,7 +223,7 @@ func (c *Context) InitTxnWithStartTS(startTS uint64) error { return nil } if c.Store != nil { - txn, err := c.Store.BeginWithOption(kv.TransactionOption{}.SetTxnScope(oracle.GlobalTxnScope).SetStartTs(startTS)) + txn, err := c.Store.BeginWithOption(tikv.DefaultStartTSOption().SetTxnScope(kv.GlobalTxnScope).SetStartTS(startTS)) if err != nil { return errors.Trace(err) } @@ -221,11 +232,6 @@ func (c *Context) InitTxnWithStartTS(startTS uint64) error { return nil } -// NewTxnWithStalenessOption implements the sessionctx.Context interface. -func (c *Context) NewTxnWithStalenessOption(ctx context.Context, option sessionctx.StalenessTxnOption) error { - return c.NewTxn(ctx) -} - // GetStore gets the store of session. func (c *Context) GetStore() kv.Storage { return c.Store diff --git a/util/mock/store.go b/util/mock/store.go index 804f3d6a3f2d3..beefae3dc7171 100644 --- a/util/mock/store.go +++ b/util/mock/store.go @@ -16,7 +16,9 @@ package mock import ( "context" + deadlockpb "github.com/pingcap/kvproto/pkg/deadlock" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/oracle" ) @@ -38,7 +40,7 @@ func (s *Store) GetOracle() oracle.Oracle { return nil } func (s *Store) Begin() (kv.Transaction, error) { return nil, nil } // BeginWithOption implements kv.Storage interface. -func (s *Store) BeginWithOption(option kv.TransactionOption) (kv.Transaction, error) { +func (s *Store) BeginWithOption(option tikv.StartTSOption) (kv.Transaction, error) { return s.Begin() } @@ -72,3 +74,13 @@ func (s *Store) GetMemCache() kv.MemManager { // ShowStatus implements kv.Storage interface. func (s *Store) ShowStatus(ctx context.Context, key string) (interface{}, error) { return nil, nil } + +// GetMinSafeTS implements kv.Storage interface. +func (s *Store) GetMinSafeTS(txnScope string) uint64 { + return 0 +} + +// GetLockWaits implements kv.Storage interface. +func (s *Store) GetLockWaits() ([]*deadlockpb.WaitForEntry, error) { + return nil, nil +} diff --git a/util/plancodec/id.go b/util/plancodec/id.go index 0cd4a1b1476be..b0e8e16e2de62 100644 --- a/util/plancodec/id.go +++ b/util/plancodec/id.go @@ -120,6 +120,12 @@ const ( TypeIndexFullScan = "IndexFullScan" // TypeIndexRangeScan is the type of IndexRangeScan. TypeIndexRangeScan = "IndexRangeScan" + // TypeCTETable is the type of TypeCTETable. + TypeCTETable = "CTETable" + // TypeCTE is the type of CTEFullScan. + TypeCTE = "CTEFullScan" + // TypeCTEDefinition is the type of CTE definition + TypeCTEDefinition = "CTE" ) // plan id. @@ -174,6 +180,9 @@ const ( typeIndexRangeScan int = 47 typeExchangeReceiver int = 48 typeExchangeSender int = 49 + typeCTE int = 50 + typeCTEDefinition int = 51 + typeCTETable int = 52 ) // TypeStringToPhysicalID converts the plan type string to plan id. @@ -277,6 +286,12 @@ func TypeStringToPhysicalID(tp string) int { return typeExchangeReceiver case TypeExchangeSender: return typeExchangeSender + case TypeCTE: + return typeCTE + case TypeCTEDefinition: + return typeCTEDefinition + case TypeCTETable: + return typeCTETable } // Should never reach here. return 0 @@ -381,6 +396,12 @@ func PhysicalIDToTypeString(id int) string { return TypeExchangeReceiver case typeExchangeSender: return TypeExchangeSender + case typeCTE: + return TypeCTE + case typeCTEDefinition: + return TypeCTEDefinition + case typeCTETable: + return TypeCTETable } // Should never reach here. diff --git a/util/processinfo.go b/util/processinfo.go index 29716d914c3de..ebbf17094b80d 100644 --- a/util/processinfo.go +++ b/util/processinfo.go @@ -22,6 +22,7 @@ import ( "time" "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/session/txninfo" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/util/execdetails" @@ -161,6 +162,7 @@ func serverStatus2Str(state uint16) string { // kill statement rely on this interface. type SessionManager interface { ShowProcessList() map[uint64]*ProcessInfo + ShowTxnList() []*txninfo.TxnInfo GetProcessInfo(id uint64) (*ProcessInfo, bool) Kill(connectionID uint64, query bool) KillAllConnections() diff --git a/util/profile/profile.go b/util/profile/profile.go index 3e598a4328441..ff817032617fe 100644 --- a/util/profile/profile.go +++ b/util/profile/profile.go @@ -16,7 +16,6 @@ package profile import ( "bytes" "io" - "io/ioutil" "runtime/pprof" "strconv" "strings" @@ -103,7 +102,7 @@ func (c *Collector) ProfileGraph(name string) ([][]types.Datum, error) { // ParseGoroutines returns the groutine list for given string representation func (c *Collector) ParseGoroutines(reader io.Reader) ([][]types.Datum, error) { - content, err := ioutil.ReadAll(reader) + content, err := io.ReadAll(reader) if err != nil { return nil, err } diff --git a/util/profile/trackerRecorder.go b/util/profile/trackerRecorder.go index c5e5390a51bff..7c449ffa9113c 100644 --- a/util/profile/trackerRecorder.go +++ b/util/profile/trackerRecorder.go @@ -29,12 +29,10 @@ func HeapProfileForGlobalMemTracker(d time.Duration) { t := time.NewTicker(d) defer t.Stop() for { - select { - case <-t.C: - err := heapProfileForGlobalMemTracker() - if err != nil { - log.Warn("profile memory into tracker failed", zap.Error(err)) - } + <-t.C + err := heapProfileForGlobalMemTracker() + if err != nil { + log.Warn("profile memory into tracker failed", zap.Error(err)) } } } diff --git a/util/ranger/points.go b/util/ranger/points.go index d98b548dcbb7a..03b35e367cb00 100644 --- a/util/ranger/points.go +++ b/util/ranger/points.go @@ -246,7 +246,8 @@ func (r *builder) buildFormBinOp(expr *expression.ScalarFunction) []*point { if col.RetType.EvalType() == types.ETString && (value.Kind() == types.KindString || value.Kind() == types.KindBinaryLiteral) { value.SetString(value.GetString(), col.RetType.Collate) } - if col.GetType().Tp == mysql.TypeYear { + // If nulleq with null value, values.ToInt64 will return err + if col.GetType().Tp == mysql.TypeYear && !value.IsNull() { // If the original value is adjusted, we need to change the condition. // For example, col < 2156. Since the max year is 2155, 2156 is changed to 2155. // col < 2155 is wrong. It should be col <= 2155. @@ -456,10 +457,15 @@ func handleEnumFromBinOp(sc *stmtctx.StatementContext, ft *types.FieldType, val } tmpEnum := types.Enum{} - for i := range ft.Elems { - tmpEnum.Name = ft.Elems[i] - tmpEnum.Value = uint64(i) - d := types.NewMysqlEnumDatum(tmpEnum) + for i := 0; i <= len(ft.Elems); i++ { + if i == 0 { + tmpEnum = types.Enum{} + } else { + tmpEnum.Name = ft.Elems[i-1] + tmpEnum.Value = uint64(i) + } + + d := types.NewCollateMysqlEnumDatum(tmpEnum, ft.Collate) if v, err := d.CompareDatum(sc, &val); err == nil { switch op { case ast.LT: diff --git a/util/ranger/ranger.go b/util/ranger/ranger.go index 4f1efef7a7d90..d69c3dbc02392 100644 --- a/util/ranger/ranger.go +++ b/util/ranger/ranger.go @@ -474,7 +474,7 @@ func fixPrefixColRange(ranges []*Range, lengths []int, tp []*types.FieldType) bo for _, ran := range ranges { lowTail := len(ran.LowVal) - 1 for i := 0; i < lowTail; i++ { - CutDatumByPrefixLen(&ran.LowVal[i], lengths[i], tp[i]) + hasCut = CutDatumByPrefixLen(&ran.LowVal[i], lengths[i], tp[i]) || hasCut } lowCut := CutDatumByPrefixLen(&ran.LowVal[lowTail], lengths[lowTail], tp[lowTail]) // If the length of the last column of LowVal is equal to the prefix length, LowExclude should be set false. @@ -485,13 +485,13 @@ func fixPrefixColRange(ranges []*Range, lengths []int, tp []*types.FieldType) bo } highTail := len(ran.HighVal) - 1 for i := 0; i < highTail; i++ { - CutDatumByPrefixLen(&ran.HighVal[i], lengths[i], tp[i]) + hasCut = CutDatumByPrefixLen(&ran.HighVal[i], lengths[i], tp[i]) || hasCut } highCut := CutDatumByPrefixLen(&ran.HighVal[highTail], lengths[highTail], tp[highTail]) if highCut { ran.HighExclude = false } - hasCut = lowCut || highCut + hasCut = hasCut || lowCut || highCut } return hasCut } diff --git a/util/ranger/ranger_test.go b/util/ranger/ranger_test.go index 194ba1e779dc3..3f4e59848b044 100644 --- a/util/ranger/ranger_test.go +++ b/util/ranger/ranger_test.go @@ -304,10 +304,10 @@ func (s *testRangerSuite) TestTableRange(c *C) { stmts, err := session.Parse(sctx, sql) c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, tt.exprStr)) c.Assert(stmts, HasLen, 1) - is := domain.GetDomain(sctx).InfoSchema() - err = plannercore.Preprocess(sctx, stmts[0], is) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) c.Assert(err, IsNil, Commentf("error %v, for resolve name, expr %s", err, tt.exprStr)) - p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is) + p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], ret.InfoSchema) c.Assert(err, IsNil, Commentf("error %v, for build plan, expr %s", err, tt.exprStr)) selection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) conds := make([]expression.Expression, len(selection.Conditions)) @@ -347,11 +347,14 @@ create table t( d varchar(10), e binary(10), f varchar(10) collate utf8mb4_general_ci, + g enum('A','B','C') collate utf8mb4_general_ci, index idx_ab(a(50), b), index idx_cb(c, a), index idx_d(d(2)), index idx_e(e(2)), - index idx_f(f) + index idx_f(f), + index idx_de(d(2), e), + index idx_g(g) )`) tests := []struct { @@ -620,6 +623,20 @@ create table t( filterConds: "[like(test.t.f, @%, 92)]", resultStr: "[[NULL,+inf]]", }, + { + indexPos: 5, + exprStr: "d in ('aab', 'aac') and e = 'a'", + accessConds: "[in(test.t.d, aab, aac) eq(test.t.e, a)]", + filterConds: "[in(test.t.d, aab, aac)]", + resultStr: "[[\"aa\" 0x61,\"aa\" 0x61]]", + }, + { + indexPos: 6, + exprStr: "g = 'a'", + accessConds: "[eq(test.t.g, a)]", + filterConds: "[]", + resultStr: "[[\"A\",\"A\"]]", + }, } collate.SetNewCollationEnabledForTest(true) @@ -631,10 +648,10 @@ create table t( stmts, err := session.Parse(sctx, sql) c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, tt.exprStr)) c.Assert(stmts, HasLen, 1) - is := domain.GetDomain(sctx).InfoSchema() - err = plannercore.Preprocess(sctx, stmts[0], is) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) c.Assert(err, IsNil, Commentf("error %v, for resolve name, expr %s", err, tt.exprStr)) - p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is) + p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], ret.InfoSchema) c.Assert(err, IsNil, Commentf("error %v, for build plan, expr %s", err, tt.exprStr)) selection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) tbl := selection.Children()[0].(*plannercore.DataSource).TableInfo() @@ -822,10 +839,10 @@ create table t( stmts, err := session.Parse(sctx, sql) c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, tt.exprStr)) c.Assert(stmts, HasLen, 1) - is := domain.GetDomain(sctx).InfoSchema() - err = plannercore.Preprocess(sctx, stmts[0], is) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) c.Assert(err, IsNil, Commentf("error %v, for resolve name, expr %s", err, tt.exprStr)) - p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is) + p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], ret.InfoSchema) c.Assert(err, IsNil, Commentf("error %v, for build plan, expr %s", err, tt.exprStr)) selection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) tbl := selection.Children()[0].(*plannercore.DataSource).TableInfo() @@ -1186,10 +1203,10 @@ func (s *testRangerSuite) TestColumnRange(c *C) { stmts, err := session.Parse(sctx, sql) c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, tt.exprStr)) c.Assert(stmts, HasLen, 1) - is := domain.GetDomain(sctx).InfoSchema() - err = plannercore.Preprocess(sctx, stmts[0], is) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) c.Assert(err, IsNil, Commentf("error %v, for resolve name, expr %s", err, tt.exprStr)) - p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is) + p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], ret.InfoSchema) c.Assert(err, IsNil, Commentf("error %v, for build plan, expr %s", err, tt.exprStr)) sel := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) ds, ok := sel.Children()[0].(*plannercore.DataSource) @@ -1504,12 +1521,13 @@ func (s *testRangerSuite) TestIndexRangeForYear(c *C) { // test index range testKit.MustExec("DROP TABLE IF EXISTS t") testKit.MustExec("CREATE TABLE t (a year(4), key(a))") - testKit.MustExec("INSERT INTO t VALUES (1), (70), (99), (0), ('0')") + testKit.MustExec("INSERT INTO t VALUES (1), (70), (99), (0), ('0'), (NULL)") testKit.MustQuery("SELECT * FROM t WHERE a < 15698").Check(testkit.Rows("0", "1970", "1999", "2000", "2001")) testKit.MustQuery("SELECT * FROM t WHERE a <= 0").Check(testkit.Rows("0")) testKit.MustQuery("SELECT * FROM t WHERE a <= 1").Check(testkit.Rows("0", "1970", "1999", "2000", "2001")) testKit.MustQuery("SELECT * FROM t WHERE a < 2000").Check(testkit.Rows("0", "1970", "1999")) testKit.MustQuery("SELECT * FROM t WHERE a > -1").Check(testkit.Rows("0", "1970", "1999", "2000", "2001")) + testKit.MustQuery("SELECT * FROM t WHERE a <=> NULL").Check(testkit.Rows("")) tests := []struct { indexPos int @@ -1609,10 +1627,10 @@ func (s *testRangerSuite) TestIndexRangeForYear(c *C) { stmts, err := session.Parse(sctx, sql) c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, tt.exprStr)) c.Assert(stmts, HasLen, 1) - is := domain.GetDomain(sctx).InfoSchema() - err = plannercore.Preprocess(sctx, stmts[0], is) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) c.Assert(err, IsNil, Commentf("error %v, for resolve name, expr %s", err, tt.exprStr)) - p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is) + p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], ret.InfoSchema) c.Assert(err, IsNil, Commentf("error %v, for build plan, expr %s", err, tt.exprStr)) selection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) tbl := selection.Children()[0].(*plannercore.DataSource).TableInfo() @@ -1680,10 +1698,10 @@ func (s *testRangerSuite) TestPrefixIndexRangeScan(c *C) { stmts, err := session.Parse(sctx, sql) c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, tt.exprStr)) c.Assert(stmts, HasLen, 1) - is := domain.GetDomain(sctx).InfoSchema() - err = plannercore.Preprocess(sctx, stmts[0], is) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) c.Assert(err, IsNil, Commentf("error %v, for resolve name, expr %s", err, tt.exprStr)) - p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is) + p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], ret.InfoSchema) c.Assert(err, IsNil, Commentf("error %v, for build plan, expr %s", err, tt.exprStr)) selection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) tbl := selection.Children()[0].(*plannercore.DataSource).TableInfo() diff --git a/util/ranger/testdata/ranger_suite_out.json b/util/ranger/testdata/ranger_suite_out.json index f56a1567e6043..9def6a398c661 100644 --- a/util/ranger/testdata/ranger_suite_out.json +++ b/util/ranger/testdata/ranger_suite_out.json @@ -265,8 +265,8 @@ { "SQL": "select * from t where a = 1 and ((b = 1) or (b = 2 and c = 3));", "Plan": [ - "TableReader_6 2.00 root data:TableRangeScan_5", - "└─TableRangeScan_5 2.00 cop[tikv] table:t range:[1 1,1 1], [1 2 3,1 2 3], keep order:false" + "TableReader_6 1.71 root data:TableRangeScan_5", + "└─TableRangeScan_5 1.71 cop[tikv] table:t range:[1 1,1 1], [1 2 3,1 2 3], keep order:false" ], "Result": [ "1 1 1" @@ -293,8 +293,8 @@ { "SQL": "select * from t use index(primary) where ((a = 1) or (a = 2 and b = 2)) and c = 3;", "Plan": [ - "TableReader_7 0.75 root data:Selection_6", - "└─Selection_6 0.75 cop[tikv] eq(test.t.c, 3), or(eq(test.t.a, 1), and(eq(test.t.a, 2), eq(test.t.b, 2)))", + "TableReader_7 0.68 root data:Selection_6", + "└─Selection_6 0.68 cop[tikv] eq(test.t.c, 3), or(eq(test.t.a, 1), and(eq(test.t.a, 2), eq(test.t.b, 2)))", " └─TableRangeScan_5 2.00 cop[tikv] table:t range:[1,1], [2,2], keep order:false" ], "Result": [ @@ -304,7 +304,7 @@ { "SQL": "select * from t where (a,b) in ((1,1),(2,2)) and c > 2 and (a,b,c) in ((1,1,1),(2,2,3));", "Plan": [ - "Selection_6 0.56 root gt(test.t.c, 2), or(and(eq(test.t.a, 1), eq(test.t.b, 1)), and(eq(test.t.a, 2), eq(test.t.b, 2)))", + "Selection_6 0.44 root gt(test.t.c, 2), or(and(eq(test.t.a, 1), eq(test.t.b, 1)), and(eq(test.t.a, 2), eq(test.t.b, 2)))", "└─Batch_Point_Get_5 2.00 root table:t, clustered index:PRIMARY(a, b, c) keep order:false, desc:false" ], "Result": [ @@ -314,8 +314,8 @@ { "SQL": "select * from t where (a,b) in ((1,1),(2,2)) and c > 2;", "Plan": [ - "TableReader_6 1.00 root data:TableRangeScan_5", - "└─TableRangeScan_5 1.00 cop[tikv] table:t range:(1 1 2,1 1 +inf], (2 2 2,2 2 +inf], keep order:false" + "TableReader_6 1.19 root data:TableRangeScan_5", + "└─TableRangeScan_5 1.19 cop[tikv] table:t range:(1 1 2,1 1 +inf], (2 2 2,2 2 +inf], keep order:false" ], "Result": [ "2 2 3" @@ -324,8 +324,8 @@ { "SQL": "select * from t where ((a = 1 and b = 1) or (a = 2 and b = 2)) and c > 2;", "Plan": [ - "TableReader_6 1.00 root data:TableRangeScan_5", - "└─TableRangeScan_5 1.00 cop[tikv] table:t range:(1 1 2,1 1 +inf], (2 2 2,2 2 +inf], keep order:false" + "TableReader_6 1.19 root data:TableRangeScan_5", + "└─TableRangeScan_5 1.19 cop[tikv] table:t range:(1 1 2,1 1 +inf], (2 2 2,2 2 +inf], keep order:false" ], "Result": [ "2 2 3" @@ -364,8 +364,8 @@ { "SQL": "select * from t2 where t='aaaa';", "Plan": [ - "TableReader_7 0.00 root data:Selection_6", - "└─Selection_6 0.00 cop[tikv] eq(test.t2.t, \"aaaa\")", + "TableReader_7 1.00 root data:Selection_6", + "└─Selection_6 1.00 cop[tikv] eq(test.t2.t, \"aaaa\")", " └─TableRangeScan_5 2.00 cop[tikv] table:t2 range:[0,+inf], keep order:false" ], "Result": [ @@ -375,8 +375,8 @@ { "SQL": "select * from t2 where t='aaaa' or t = 'a';", "Plan": [ - "TableReader_7 0.80 root data:Selection_6", - "└─Selection_6 0.80 cop[tikv] or(eq(test.t2.t, \"aaaa\"), eq(test.t2.t, \"a\"))", + "TableReader_7 1.60 root data:Selection_6", + "└─Selection_6 1.60 cop[tikv] or(eq(test.t2.t, \"aaaa\"), eq(test.t2.t, \"a\"))", " └─TableRangeScan_5 2.00 cop[tikv] table:t2 range:[0,+inf], keep order:false" ], "Result": [ @@ -392,11 +392,11 @@ { "SQL": "select * from t;", "Plan": [ - "PartitionUnion_8 4.00 root ", - "├─TableReader_10 1.00 root data:TableFullScan_9", - "│ └─TableFullScan_9 1.00 cop[tikv] table:t, partition:p0 keep order:false", - "└─TableReader_12 3.00 root data:TableFullScan_11", - " └─TableFullScan_11 3.00 cop[tikv] table:t, partition:p1 keep order:false" + "PartitionUnion_8 4.00 root ", + "├─TableReader_10 1.00 root data:TableFullScan_9", + "│ └─TableFullScan_9 1.00 cop[tikv] table:t, partition:p0 keep order:false", + "└─TableReader_12 3.00 root data:TableFullScan_11", + " └─TableFullScan_11 3.00 cop[tikv] table:t, partition:p1 keep order:false" ], "Result": [ "\u0000 0", @@ -463,7 +463,7 @@ "└─PartitionUnion_10 4.00 root ", " ├─TableReader_12 1.00 root data:TableFullScan_11", " │ └─TableFullScan_11 1.00 cop[tikv] table:t, partition:p0 keep order:false", - " └─TableReader_14 3.00 root data:TableFullScan_13", + " └─TableReader_14 3.00 root data:TableFullScan_13", " └─TableFullScan_13 3.00 cop[tikv] table:t, partition:p1 keep order:false" ], "Result": [ @@ -478,7 +478,7 @@ " ├─TableReader_12 1.00 root data:TableFullScan_11", " │ └─TableFullScan_11 1.00 cop[tikv] table:t, partition:p0 keep order:false", " └─TableReader_14 3.00 root data:TableFullScan_13", - " └─TableFullScan_13 3.00 cop[tikv] table:t, partition:p1 keep order:false" + " └─TableFullScan_13 3.00 cop[tikv] table:t, partition:p1 keep order:false" ], "Result": [ "\u0000 0", diff --git a/util/resourcegrouptag/resource_group_tag.go b/util/resourcegrouptag/resource_group_tag.go new file mode 100644 index 0000000000000..03150a0393ea4 --- /dev/null +++ b/util/resourcegrouptag/resource_group_tag.go @@ -0,0 +1,40 @@ +package resourcegrouptag + +import ( + "github.com/pingcap/errors" + "github.com/pingcap/parser" + "github.com/pingcap/tipb/go-tipb" +) + +// EncodeResourceGroupTag encodes sql digest and plan digest into resource group tag. +func EncodeResourceGroupTag(sqlDigest, planDigest *parser.Digest) []byte { + if sqlDigest == nil && planDigest == nil { + return nil + } + + tag := &tipb.ResourceGroupTag{} + if sqlDigest != nil { + tag.SqlDigest = sqlDigest.Bytes() + } + if planDigest != nil { + tag.PlanDigest = planDigest.Bytes() + } + b, err := tag.Marshal() + if err != nil { + return nil + } + return b +} + +// DecodeResourceGroupTag decodes a resource group tag and return the sql digest. +func DecodeResourceGroupTag(data []byte) (sqlDigest []byte, err error) { + if len(data) == 0 { + return nil, nil + } + tag := &tipb.ResourceGroupTag{} + err = tag.Unmarshal(data) + if err != nil { + return nil, errors.Errorf("invalid resource group tag data %x", data) + } + return tag.SqlDigest, nil +} diff --git a/util/resourcegrouptag/resource_group_tag_test.go b/util/resourcegrouptag/resource_group_tag_test.go new file mode 100644 index 0000000000000..f5334aacbd17f --- /dev/null +++ b/util/resourcegrouptag/resource_group_tag_test.go @@ -0,0 +1,108 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package resourcegrouptag + +import ( + "crypto/sha256" + "math/rand" + "testing" + + . "github.com/pingcap/check" + "github.com/pingcap/parser" + "github.com/pingcap/tidb/util/hack" + "github.com/pingcap/tipb/go-tipb" +) + +type testUtilsSuite struct{} + +var _ = Suite(&testUtilsSuite{}) + +func TestT(t *testing.T) { + TestingT(t) +} + +func (s *testUtilsSuite) TestResourceGroupTagEncoding(c *C) { + sqlDigest := parser.NewDigest(nil) + tag := EncodeResourceGroupTag(sqlDigest, nil) + c.Assert(len(tag), Equals, 0) + decodedSQLDigest, err := DecodeResourceGroupTag(tag) + c.Assert(err, IsNil) + c.Assert(len(decodedSQLDigest), Equals, 0) + + sqlDigest = parser.NewDigest([]byte{'a', 'a'}) + tag = EncodeResourceGroupTag(sqlDigest, nil) + // version(1) + prefix(1) + length(1) + content(2hex -> 1byte) + c.Assert(len(tag), Equals, 4) + decodedSQLDigest, err = DecodeResourceGroupTag(tag) + c.Assert(err, IsNil) + c.Assert(decodedSQLDigest, DeepEquals, sqlDigest.Bytes()) + + sqlDigest = parser.NewDigest(genRandHex(64)) + tag = EncodeResourceGroupTag(sqlDigest, nil) + decodedSQLDigest, err = DecodeResourceGroupTag(tag) + c.Assert(err, IsNil) + c.Assert(decodedSQLDigest, DeepEquals, sqlDigest.Bytes()) + + sqlDigest = parser.NewDigest(genRandHex(510)) + tag = EncodeResourceGroupTag(sqlDigest, nil) + decodedSQLDigest, err = DecodeResourceGroupTag(tag) + c.Assert(err, IsNil) + c.Assert(decodedSQLDigest, DeepEquals, sqlDigest.Bytes()) +} + +func genRandHex(length int) []byte { + const chars = "0123456789abcdef" + res := make([]byte, length) + for i := 0; i < length; i++ { + res[i] = chars[rand.Intn(len(chars))] + } + return res +} + +func genDigest(str string) []byte { + hasher := sha256.New() + hasher.Write(hack.Slice(str)) + return hasher.Sum(nil) +} + +func (s *testUtilsSuite) TestResourceGroupTagEncodingPB(c *C) { + digest1 := genDigest("abc") + digest2 := genDigest("abcdefg") + // Test for protobuf + resourceTag := &tipb.ResourceGroupTag{ + SqlDigest: digest1, + PlanDigest: digest2, + } + buf, err := resourceTag.Marshal() + c.Assert(err, IsNil) + c.Assert(len(buf), Equals, 68) + tag := &tipb.ResourceGroupTag{} + err = tag.Unmarshal(buf) + c.Assert(err, IsNil) + c.Assert(tag.SqlDigest, DeepEquals, digest1) + c.Assert(tag.PlanDigest, DeepEquals, digest2) + + // Test for protobuf sql_digest only + resourceTag = &tipb.ResourceGroupTag{ + SqlDigest: digest1, + } + buf, err = resourceTag.Marshal() + c.Assert(err, IsNil) + c.Assert(len(buf), Equals, 34) + tag = &tipb.ResourceGroupTag{} + err = tag.Unmarshal(buf) + c.Assert(err, IsNil) + c.Assert(tag.SqlDigest, DeepEquals, digest1) + c.Assert(tag.PlanDigest, IsNil) +} diff --git a/util/rowDecoder/decoder.go b/util/rowDecoder/decoder.go index 3fd63c05fae96..35413904fe3d8 100644 --- a/util/rowDecoder/decoder.go +++ b/util/rowDecoder/decoder.go @@ -164,6 +164,8 @@ func (rd *RowDecoder) DecodeTheExistedColumnMap(ctx sessionctx.Context, handle k if err != nil { return nil, err } + // Fill the default value into map. + row[colInfo.ID] = val rd.mutRow.SetValue(colInfo.Offset, val.GetValue()) } // return the existed column map here. diff --git a/util/rowDecoder/decoder_test.go b/util/rowDecoder/decoder_test.go index 04d71dda0d19a..4439ecd90bdd7 100644 --- a/util/rowDecoder/decoder_test.go +++ b/util/rowDecoder/decoder_test.go @@ -28,7 +28,7 @@ import ( "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/mock" - "github.com/pingcap/tidb/util/rowDecoder" + decoder "github.com/pingcap/tidb/util/rowDecoder" "github.com/pingcap/tidb/util/rowcodec" "github.com/pingcap/tidb/util/testleak" "github.com/pingcap/tidb/util/testutil" diff --git a/util/rowcodec/decoder.go b/util/rowcodec/decoder.go index 69a78d1de7d43..0efd50ecaf27c 100644 --- a/util/rowcodec/decoder.go +++ b/util/rowcodec/decoder.go @@ -260,10 +260,7 @@ func (decoder *ChunkDecoder) tryAppendHandleColumn(colIdx int, col *ColInfo, han } coder := codec.NewDecoder(chk, decoder.loc) _, err := coder.DecodeOne(handle.EncodedCol(i), colIdx, col.Ft) - if err != nil { - return false - } - return true + return err == nil } } return false diff --git a/util/sem/sem.go b/util/sem/sem.go index 8c3d2b456d991..1aac6d0a9a999 100644 --- a/util/sem/sem.go +++ b/util/sem/sem.go @@ -14,6 +14,7 @@ package sem import ( + "os" "strings" "sync/atomic" @@ -70,6 +71,7 @@ var ( func Enable() { atomic.StoreInt32(&semEnabled, 1) variable.SetSysVar(variable.TiDBEnableEnhancedSecurity, variable.On) + variable.SetSysVar(variable.Hostname, variable.DefHostname) // write to log so users understand why some operations are weird. logutil.BgLogger().Info("tidb-server is operating with security enhanced mode (SEM) enabled") } @@ -79,6 +81,9 @@ func Enable() { func Disable() { atomic.StoreInt32(&semEnabled, 0) variable.SetSysVar(variable.TiDBEnableEnhancedSecurity, variable.Off) + if hostname, err := os.Hostname(); err != nil { + variable.SetSysVar(variable.Hostname, hostname) + } } // IsEnabled checks if Security Enhanced Mode (SEM) is enabled @@ -125,6 +130,35 @@ func IsInvisibleStatusVar(varName string) bool { return varName == tidbGCLeaderDesc } +// IsInvisibleSysVar returns true if the sysvar needs to be hidden +func IsInvisibleSysVar(varNameInLower string) bool { + switch varNameInLower { + case variable.TiDBDDLSlowOprThreshold, // ddl_slow_threshold + variable.TiDBAllowRemoveAutoInc, + variable.TiDBCheckMb4ValueInUTF8, + variable.TiDBConfig, + variable.TiDBEnableSlowLog, + variable.TiDBEnableTelemetry, + variable.TiDBExpensiveQueryTimeThreshold, + variable.TiDBForcePriority, + variable.TiDBGeneralLog, + variable.TiDBMetricSchemaRangeDuration, + variable.TiDBMetricSchemaStep, + variable.TiDBOptWriteRowID, + variable.TiDBPProfSQLCPU, + variable.TiDBRecordPlanInSlowLog, + variable.TiDBRowFormatVersion, + variable.TiDBSlowQueryFile, + variable.TiDBSlowLogThreshold, + variable.TiDBEnableCollectExecutionInfo, + variable.TiDBMemoryUsageAlarmRatio, + variable.TiDBRedactLog, + variable.TiDBSlowLogMasking: + return true + } + return false +} + // IsRestrictedPrivilege returns true if the privilege shuld not be satisfied by SUPER // As most dynamic privileges are. func IsRestrictedPrivilege(privNameInUpper string) bool { diff --git a/util/sem/sem_test.go b/util/sem/sem_test.go index c303d2195c7f4..c2a54170dcf99 100644 --- a/util/sem/sem_test.go +++ b/util/sem/sem_test.go @@ -17,6 +17,7 @@ import ( "testing" "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/sessionctx/variable" . "github.com/pingcap/check" ) @@ -74,3 +75,29 @@ func (s *testSecurity) TestIsInvisibleStatusVar(c *C) { c.Assert(IsInvisibleStatusVar("ddl_schema_version"), IsFalse) c.Assert(IsInvisibleStatusVar("Ssl_version"), IsFalse) } + +func (s *testSecurity) TestIsInvisibleSysVar(c *C) { + c.Assert(IsInvisibleSysVar(variable.Hostname), IsFalse) // changes the value to default, but is not invisible + c.Assert(IsInvisibleSysVar(variable.TiDBEnableEnhancedSecurity), IsFalse) // should be able to see the mode is on. + + c.Assert(IsInvisibleSysVar(variable.TiDBAllowRemoveAutoInc), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBCheckMb4ValueInUTF8), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBConfig), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBEnableSlowLog), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBExpensiveQueryTimeThreshold), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBForcePriority), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBGeneralLog), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBMetricSchemaRangeDuration), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBMetricSchemaStep), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBOptWriteRowID), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBPProfSQLCPU), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBRecordPlanInSlowLog), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBSlowQueryFile), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBSlowLogThreshold), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBEnableCollectExecutionInfo), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBMemoryUsageAlarmRatio), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBEnableTelemetry), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBRowFormatVersion), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBRedactLog), IsTrue) + c.Assert(IsInvisibleSysVar(variable.TiDBSlowLogMasking), IsTrue) +} diff --git a/util/sli/sli.go b/util/sli/sli.go index 1c48f9a03a510..80d8cdc6f7868 100644 --- a/util/sli/sli.go +++ b/util/sli/sli.go @@ -94,12 +94,12 @@ func (t *TxnWriteThroughputSLI) IsInvalid() bool { const ( smallTxnAffectRow = 20 - smallTxnWriteSize = 1 * 1024 * 1024 // 1MB + smallTxnSize = 1 * 1024 * 1024 // 1MB ) // IsSmallTxn exports for testing. func (t *TxnWriteThroughputSLI) IsSmallTxn() bool { - return t.affectRow <= smallTxnAffectRow && t.writeSize <= smallTxnWriteSize + return t.affectRow <= smallTxnAffectRow && t.writeSize <= smallTxnSize } // Reset exports for testing. @@ -117,3 +117,10 @@ func (t *TxnWriteThroughputSLI) String() string { return fmt.Sprintf("invalid: %v, affectRow: %v, writeSize: %v, readKeys: %v, writeKeys: %v, writeTime: %v", t.invalid, t.affectRow, t.writeSize, t.readKeys, t.writeKeys, t.writeTime.String()) } + +// ObserveReadSLI observes the read SLI metric. +func ObserveReadSLI(readKeys uint64, readTime float64) { + if readKeys <= smallTxnAffectRow && readKeys != 0 && readTime != 0 { + metrics.TiKVSmallReadDuration.Observe(readTime) + } +} diff --git a/util/sqlexec/restricted_sql_executor.go b/util/sqlexec/restricted_sql_executor.go index e350118a77715..4d714a474806e 100644 --- a/util/sqlexec/restricted_sql_executor.go +++ b/util/sqlexec/restricted_sql_executor.go @@ -96,7 +96,7 @@ type SQLExecutor interface { // But a session already has a parser bind in it, so we define this interface and use session as its implementation, // thus avoid allocating new parser. See session.SQLParser for more information. type SQLParser interface { - ParseSQL(sql, charset, collation string) ([]ast.StmtNode, error) + ParseSQL(ctx context.Context, sql, charset, collation string) ([]ast.StmtNode, []error, error) } // Statement is an interface for SQL execution. diff --git a/util/stmtsummary/evicted.go b/util/stmtsummary/evicted.go new file mode 100644 index 0000000000000..d3cf1ff0abc29 --- /dev/null +++ b/util/stmtsummary/evicted.go @@ -0,0 +1,192 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package stmtsummary + +import ( + "container/list" + "time" + + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/types" +) + +// stmtSummaryByDigestEvicted contents digests evicted from stmtSummaryByDigestMap +type stmtSummaryByDigestEvicted struct { + // record evicted data in intervals + // latest history data is Back() + history *list.List +} + +// element being stored in stmtSummaryByDigestEvicted +type stmtSummaryByDigestEvictedElement struct { + // beginTime is the begin time of current interval + beginTime int64 + // endTime is the end time of current interval + endTime int64 + // *Kinds* of digest being evicted + digestKeyMap map[string]struct{} +} + +// spawn a new pointer to stmtSummaryByDigestEvicted +func newStmtSummaryByDigestEvicted() *stmtSummaryByDigestEvicted { + return &stmtSummaryByDigestEvicted{ + history: list.New(), + } +} + +// spawn a new pointer to stmtSummaryByDigestEvictedElement +func newStmtSummaryByDigestEvictedElement(beginTime int64, endTime int64) *stmtSummaryByDigestEvictedElement { + return &stmtSummaryByDigestEvictedElement{ + beginTime: beginTime, + endTime: endTime, + digestKeyMap: make(map[string]struct{}), + } +} + +// AddEvicted is used add an evicted record to stmtSummaryByDigestEvicted +func (ssbde *stmtSummaryByDigestEvicted) AddEvicted(evictedKey *stmtSummaryByDigestKey, evictedValue *stmtSummaryByDigest, historySize int) { + if evictedValue == nil { + return + } + + evictedValue.Lock() + defer evictedValue.Unlock() + + if evictedValue.history == nil { + return + } + for e, h := evictedValue.history.Back(), ssbde.history.Back(); e != nil; e = e.Prev() { + evictedElement := e.Value.(*stmtSummaryByDigestElement) + + // use closure to minimize time holding lock + func() { + evictedElement.Lock() + defer evictedElement.Unlock() + // no record in ssbde.history, direct insert + if ssbde.history.Len() == 0 && historySize != 0 { + + eBeginTime := evictedElement.beginTime + eEndTime := evictedElement.endTime + record := newStmtSummaryByDigestEvictedElement(eBeginTime, eEndTime) + record.addEvicted(evictedKey, evictedElement) + ssbde.history.PushFront(record) + + h = ssbde.history.Back() + return + } + + // look for matching history interval + MATCHING: + for ; h != nil; h = h.Prev() { + historyElement := h.Value.(*stmtSummaryByDigestEvictedElement) + + switch historyElement.matchAndAdd(evictedKey, evictedElement) { + case isMatch: + // automatically added + break MATCHING + // not matching, create a new record and insert + case isTooYoung: + { + eBeginTime := evictedElement.beginTime + eEndTime := evictedElement.endTime + record := newStmtSummaryByDigestEvictedElement(eBeginTime, eEndTime) + record.addEvicted(evictedKey, evictedElement) + ssbde.history.InsertAfter(record, h) + break MATCHING + } + default: // isTooOld + { + if h == ssbde.history.Front() { + // if digest older than all records in ssbde.history. + eBeginTime := evictedElement.beginTime + eEndTime := evictedElement.endTime + record := newStmtSummaryByDigestEvictedElement(eBeginTime, eEndTime) + record.addEvicted(evictedKey, evictedElement) + ssbde.history.PushFront(record) + break MATCHING + } + } + } + } + }() + + // prevent exceeding history size + for ssbde.history.Len() > historySize && ssbde.history.Len() > 0 { + ssbde.history.Remove(ssbde.history.Front()) + } + } +} + +// Clear up all records in stmtSummaryByDigestEvicted +func (ssbde *stmtSummaryByDigestEvicted) Clear() { + ssbde.history.Init() +} + +// add an evicted record to stmtSummaryByDigestEvictedElement +func (seElement *stmtSummaryByDigestEvictedElement) addEvicted(digestKey *stmtSummaryByDigestKey, digestValue *stmtSummaryByDigestElement) { + if digestKey != nil { + seElement.digestKeyMap[string(digestKey.Hash())] = struct{}{} + } +} + +const ( + isMatch = 0 + isTooOld = 1 + isTooYoung = 2 +) + +// matchAndAdd check time interval of seElement and digestValue. +// if matches, it will add the digest and return enum match +// if digest too old, it will return enum tooOld and do nothing +// if digest too young, it will return enum tooYoung and do nothing +func (seElement *stmtSummaryByDigestEvictedElement) matchAndAdd(digestKey *stmtSummaryByDigestKey, digestValue *stmtSummaryByDigestElement) (statement int) { + if seElement == nil || digestValue == nil { + return isTooYoung + } + sBeginTime, sEndTime := seElement.beginTime, seElement.endTime + eBeginTime, eEndTime := digestValue.beginTime, digestValue.endTime + if sBeginTime <= eBeginTime && eEndTime <= sEndTime { + seElement.addEvicted(digestKey, digestValue) + return isMatch + } else if eEndTime <= sBeginTime { + return isTooOld + } else { + return isTooYoung + } +} + +// ToEvictedCountDatum converts history evicted record to `evicted count` record's datum +func (ssbde *stmtSummaryByDigestEvicted) ToEvictedCountDatum() [][]types.Datum { + records := make([][]types.Datum, 0, ssbde.history.Len()) + for e := ssbde.history.Back(); e != nil; e = e.Prev() { + if record := e.Value.(*stmtSummaryByDigestEvictedElement).toEvictedCountDatum(); record != nil { + records = append(records, record) + } + } + return records +} + +// toEvictedCountDatum converts evicted record to `EvictedCount` record's datum +func (seElement *stmtSummaryByDigestEvictedElement) toEvictedCountDatum() []types.Datum { + datum := types.MakeDatums( + types.NewTime(types.FromGoTime(time.Unix(seElement.beginTime, 0)), mysql.TypeTimestamp, 0), + types.NewTime(types.FromGoTime(time.Unix(seElement.endTime, 0)), mysql.TypeTimestamp, 0), + int64(len(seElement.digestKeyMap)), + ) + return datum +} + +func (ssMap *stmtSummaryByDigestMap) ToEvictedCountDatum() [][]types.Datum { + return ssMap.other.ToEvictedCountDatum() +} diff --git a/util/stmtsummary/evicted_test.go b/util/stmtsummary/evicted_test.go new file mode 100644 index 0000000000000..36861eb4cfd1e --- /dev/null +++ b/util/stmtsummary/evicted_test.go @@ -0,0 +1,337 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package stmtsummary + +import ( + "bytes" + "container/list" + "fmt" + "time" + + . "github.com/pingcap/check" + "github.com/pingcap/log" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/types" +) + +func newInduceSsbd(beginTime int64, endTime int64) *stmtSummaryByDigest { + newSsbd := &stmtSummaryByDigest{ + history: list.New(), + } + newSsbd.history.PushBack(newInduceSsbde(beginTime, endTime)) + return newSsbd +} +func newInduceSsbde(beginTime int64, endTime int64) *stmtSummaryByDigestElement { + newSsbde := &stmtSummaryByDigestElement{ + beginTime: beginTime, + endTime: endTime, + minLatency: time.Duration.Round(1<<63-1, time.Nanosecond), + } + return newSsbde +} + +// generate new stmtSummaryByDigestKey and stmtSummaryByDigest +func (s *testStmtSummarySuite) generateStmtSummaryByDigestKeyValue(schema string, beginTime int64, endTime int64) (*stmtSummaryByDigestKey, *stmtSummaryByDigest) { + key := &stmtSummaryByDigestKey{ + schemaName: schema, + } + value := newInduceSsbd(beginTime, endTime) + return key, value +} + +// Test stmtSummaryByDigestMap.ToEvictedCountDatum +func (s *testStmtSummarySuite) TestMapToEvictedCountDatum(c *C) { + ssMap := newStmtSummaryByDigestMap() + ssMap.Clear() + now := time.Now().Unix() + interval := ssMap.refreshInterval() + ssMap.beginTimeForCurInterval = now + interval + + // set summaryMap capacity to 1. + err := ssMap.summaryMap.SetCapacity(1) + if err != nil { + log.Fatal(err.Error()) + } + ssMap.Clear() + + sei0 := generateAnyExecInfo() + sei1 := generateAnyExecInfo() + + sei0.SchemaName = "I'll occupy this cache! :(" + ssMap.AddStatement(sei0) + n := ssMap.beginTimeForCurInterval + sei1.SchemaName = "sorry, it's mine now. =)" + ssMap.AddStatement(sei1) + + expectedEvictedCount := []interface{}{ + types.NewTime(types.FromGoTime(time.Unix(n, 0)), mysql.TypeTimestamp, types.DefaultFsp), + types.NewTime(types.FromGoTime(time.Unix(n+interval, 0)), mysql.TypeTimestamp, types.DefaultFsp), + int64(1), + } + + // test stmtSummaryByDigestMap.toEvictedCountDatum + match(c, ssMap.ToEvictedCountDatum()[0], expectedEvictedCount...) + + // test multiple intervals + ssMap.Clear() + err = ssMap.SetRefreshInterval("60", false) + interval = ssMap.refreshInterval() + c.Assert(err, IsNil) + err = ssMap.SetMaxStmtCount("1", false) + c.Assert(err, IsNil) + err = ssMap.SetHistorySize("100", false) + c.Assert(err, IsNil) + + ssMap.beginTimeForCurInterval = now + interval + // insert one statement every other interval. + for i := 0; i < 50; i++ { + ssMap.AddStatement(generateAnyExecInfo()) + ssMap.beginTimeForCurInterval += interval * 2 + } + c.Assert(ssMap.summaryMap.Size(), Equals, 1) + val := ssMap.summaryMap.Values()[0] + c.Assert(val, NotNil) + digest := val.(*stmtSummaryByDigest) + c.Assert(digest.history.Len(), Equals, 50) + + err = ssMap.SetHistorySize("25", false) + c.Assert(err, IsNil) + // update begin time + ssMap.beginTimeForCurInterval += interval * 2 + banditSei := generateAnyExecInfo() + banditSei.SchemaName = "Kick you out >:(" + ssMap.AddStatement(banditSei) + + evictedCountDatums := ssMap.ToEvictedCountDatum() + c.Assert(len(evictedCountDatums), Equals, 25) + + // update begin time + banditSei.SchemaName = "Yet another kicker" + ssMap.AddStatement(banditSei) + + evictedCountDatums = ssMap.ToEvictedCountDatum() + // test young digest + c.Assert(len(evictedCountDatums), Equals, 25) + n = ssMap.beginTimeForCurInterval + newlyEvicted := evictedCountDatums[0] + expectedEvictedCount = []interface{}{ + types.NewTime(types.FromGoTime(time.Unix(n, 0)), mysql.TypeTimestamp, types.DefaultFsp), + types.NewTime(types.FromGoTime(time.Unix(n+interval, 0)), mysql.TypeTimestamp, types.DefaultFsp), + int64(1), + } + match(c, newlyEvicted, expectedEvictedCount...) +} + +// Test stmtSummaryByDigestEvicted +func (s *testStmtSummarySuite) TestSimpleStmtSummaryByDigestEvicted(c *C) { + ssbde := newStmtSummaryByDigestEvicted() + evictedKey, evictedValue := s.generateStmtSummaryByDigestKeyValue("a", 1, 2) + + // test NULL + ssbde.AddEvicted(nil, nil, 10) + c.Assert(ssbde.history.Len(), Equals, 0) + ssbde.Clear() + // passing NULL key is used as *refresh*. + ssbde.AddEvicted(nil, evictedValue, 10) + c.Assert(ssbde.history.Len(), Equals, 1) + ssbde.Clear() + ssbde.AddEvicted(evictedKey, nil, 10) + c.Assert(ssbde.history.Len(), Equals, 0) + ssbde.Clear() + + // test zero historySize + ssbde.AddEvicted(evictedKey, evictedValue, 0) + c.Assert(ssbde.history.Len(), Equals, 0) + + ssbde = newStmtSummaryByDigestEvicted() + ssbde.AddEvicted(evictedKey, evictedValue, 1) + c.Assert(getAllEvicted(ssbde), Equals, "{begin: 1, end: 2, count: 1}") + // test insert same *kind* of digest + ssbde.AddEvicted(evictedKey, evictedValue, 1) + c.Assert(getAllEvicted(ssbde), Equals, "{begin: 1, end: 2, count: 1}") + + evictedKey, evictedValue = s.generateStmtSummaryByDigestKeyValue("b", 1, 2) + ssbde.AddEvicted(evictedKey, evictedValue, 1) + c.Assert(getAllEvicted(ssbde), Equals, "{begin: 1, end: 2, count: 2}") + + evictedKey, evictedValue = s.generateStmtSummaryByDigestKeyValue("b", 5, 6) + ssbde.AddEvicted(evictedKey, evictedValue, 2) + c.Assert(getAllEvicted(ssbde), Equals, "{begin: 5, end: 6, count: 1}, {begin: 1, end: 2, count: 2}") + + evictedKey, evictedValue = s.generateStmtSummaryByDigestKeyValue("b", 3, 4) + ssbde.AddEvicted(evictedKey, evictedValue, 3) + c.Assert(getAllEvicted(ssbde), Equals, "{begin: 5, end: 6, count: 1}, {begin: 3, end: 4, count: 1}, {begin: 1, end: 2, count: 2}") + + // test evicted element with multi-time range value. + ssbde = newStmtSummaryByDigestEvicted() + evictedKey, evictedValue = s.generateStmtSummaryByDigestKeyValue("a", 1, 2) + evictedValue.history.PushBack(newInduceSsbde(2, 3)) + evictedValue.history.PushBack(newInduceSsbde(5, 6)) + evictedValue.history.PushBack(newInduceSsbde(8, 9)) + ssbde.AddEvicted(evictedKey, evictedValue, 3) + c.Assert(getAllEvicted(ssbde), Equals, "{begin: 8, end: 9, count: 1}, {begin: 5, end: 6, count: 1}, {begin: 2, end: 3, count: 1}") + + evictedKey = &stmtSummaryByDigestKey{schemaName: "b"} + ssbde.AddEvicted(evictedKey, evictedValue, 4) + c.Assert(getAllEvicted(ssbde), Equals, "{begin: 8, end: 9, count: 2}, {begin: 5, end: 6, count: 2}, {begin: 2, end: 3, count: 2}, {begin: 1, end: 2, count: 1}") + + evictedKey, evictedValue = s.generateStmtSummaryByDigestKeyValue("c", 4, 5) + evictedValue.history.PushBack(newInduceSsbde(5, 6)) + evictedValue.history.PushBack(newInduceSsbde(7, 8)) + ssbde.AddEvicted(evictedKey, evictedValue, 4) + c.Assert(getAllEvicted(ssbde), Equals, "{begin: 8, end: 9, count: 2}, {begin: 7, end: 8, count: 1}, {begin: 5, end: 6, count: 3}, {begin: 4, end: 5, count: 1}") + + evictedKey, evictedValue = s.generateStmtSummaryByDigestKeyValue("d", 7, 8) + ssbde.AddEvicted(evictedKey, evictedValue, 4) + c.Assert(getAllEvicted(ssbde), Equals, "{begin: 8, end: 9, count: 2}, {begin: 7, end: 8, count: 2}, {begin: 5, end: 6, count: 3}, {begin: 4, end: 5, count: 1}") + + // test for too old + evictedKey, evictedValue = s.generateStmtSummaryByDigestKeyValue("d", 0, 1) + evictedValue.history.PushBack(newInduceSsbde(1, 2)) + evictedValue.history.PushBack(newInduceSsbde(2, 3)) + evictedValue.history.PushBack(newInduceSsbde(4, 5)) + ssbde.AddEvicted(evictedKey, evictedValue, 4) + c.Assert(getAllEvicted(ssbde), Equals, "{begin: 8, end: 9, count: 2}, {begin: 7, end: 8, count: 2}, {begin: 5, end: 6, count: 3}, {begin: 4, end: 5, count: 2}") + + // test for too young + evictedKey, evictedValue = s.generateStmtSummaryByDigestKeyValue("d", 1, 2) + evictedValue.history.PushBack(newInduceSsbde(9, 10)) + ssbde.AddEvicted(evictedKey, evictedValue, 4) + c.Assert(getAllEvicted(ssbde), Equals, "{begin: 9, end: 10, count: 1}, {begin: 8, end: 9, count: 2}, {begin: 7, end: 8, count: 2}, {begin: 5, end: 6, count: 3}") +} + +// Test stmtSummaryByDigestEvictedElement.ToEvictedCountDatum +func (s *testStmtSummarySuite) TestStmtSummaryByDigestEvictedElement(c *C) { + record := newStmtSummaryByDigestEvictedElement(0, 1) + evictedKey, evictedValue := s.generateStmtSummaryByDigestKeyValue("alpha", 0, 1) + digestValue := evictedValue.history.Back().Value.(*stmtSummaryByDigestElement) + + // test poisoning will NULL key. + record.addEvicted(nil, nil) + c.Assert(getEvicted(record), Equals, "{begin: 0, end: 1, count: 0}") + record.addEvicted(nil, digestValue) + c.Assert(getEvicted(record), Equals, "{begin: 0, end: 1, count: 0}") + + // test add evictedKey and evicted stmtSummaryByDigestElement + record.addEvicted(evictedKey, digestValue) + c.Assert(getEvicted(record), Equals, "{begin: 0, end: 1, count: 1}") + + // test add same *kind* of values. + record.addEvicted(evictedKey, digestValue) + c.Assert(getEvicted(record), Equals, "{begin: 0, end: 1, count: 1}") + + // test add different *kind* of values. + evictedKey, evictedValue = s.generateStmtSummaryByDigestKeyValue("bravo", 0, 1) + digestValue = evictedValue.history.Back().Value.(*stmtSummaryByDigestElement) + record.addEvicted(evictedKey, digestValue) + c.Assert(getEvicted(record), Equals, "{begin: 0, end: 1, count: 2}") +} + +// test stmtSummaryByDigestEvicted.addEvicted +// test evicted count's detail +func (s *testStmtSummarySuite) TestEvictedCountDetailed(c *C) { + ssMap := newStmtSummaryByDigestMap() + ssMap.Clear() + err := ssMap.SetRefreshInterval("60", false) + c.Assert(err, IsNil) + err = ssMap.SetHistorySize("100", false) + c.Assert(err, IsNil) + now := time.Now().Unix() + interval := int64(60) + ssMap.beginTimeForCurInterval = now + interval + // set capacity to 1 + err = ssMap.summaryMap.SetCapacity(1) + c.Assert(err, IsNil) + + // test stmtSummaryByDigest's history length + for i := 0; i < 100; i++ { + if i == 0 { + c.Assert(ssMap.summaryMap.Size(), Equals, 0) + } else { + c.Assert(ssMap.summaryMap.Size(), Equals, 1) + val := ssMap.summaryMap.Values()[0] + c.Assert(val, NotNil) + digest := val.(*stmtSummaryByDigest) + c.Assert(digest.history.Len(), Equals, i) + } + ssMap.AddStatement(generateAnyExecInfo()) + ssMap.beginTimeForCurInterval += interval + } + ssMap.beginTimeForCurInterval -= interval + + banditSei := generateAnyExecInfo() + banditSei.SchemaName = "kick you out >:(" + ssMap.AddStatement(banditSei) + evictedCountDatums := ssMap.ToEvictedCountDatum() + n := ssMap.beginTimeForCurInterval + for _, evictedCountDatum := range evictedCountDatums { + expectedDatum := []interface{}{ + types.NewTime(types.FromGoTime(time.Unix(n, 0)), mysql.TypeTimestamp, types.DefaultFsp), + types.NewTime(types.FromGoTime(time.Unix(n+60, 0)), mysql.TypeTimestamp, types.DefaultFsp), + int64(1), + } + match(c, evictedCountDatum, expectedDatum...) + n -= 60 + } + + // test more than one eviction in single interval + banditSei.SchemaName = "Yet another kicker" + n = ssMap.beginTimeForCurInterval + expectedDatum := []interface{}{ + types.NewTime(types.FromGoTime(time.Unix(n, 0)), mysql.TypeTimestamp, types.DefaultFsp), + types.NewTime(types.FromGoTime(time.Unix(n+60, 0)), mysql.TypeTimestamp, types.DefaultFsp), + int64(2), + } + ssMap.AddStatement(banditSei) + evictedCountDatums = ssMap.ToEvictedCountDatum() + match(c, evictedCountDatums[0], expectedDatum...) + + ssMap.Clear() + other := ssMap.other + // test poisoning with empty-history digestValue + other.AddEvicted(new(stmtSummaryByDigestKey), new(stmtSummaryByDigest), 100) + c.Assert(other.history.Len(), Equals, 0) +} + +func (s *testStmtSummarySuite) TestNewStmtSummaryByDigestEvictedElement(c *C) { + now := time.Now().Unix() + end := now + 60 + stmtEvictedElement := newStmtSummaryByDigestEvictedElement(now, end) + c.Assert(stmtEvictedElement.beginTime, Equals, now) + c.Assert(stmtEvictedElement.endTime, Equals, end) + c.Assert(len(stmtEvictedElement.digestKeyMap), Equals, 0) +} + +func (s *testStmtSummarySuite) TestStmtSummaryByDigestEvicted(c *C) { + stmtEvicted := newStmtSummaryByDigestEvicted() + c.Assert(stmtEvicted.history.Len(), Equals, 0) +} + +func getAllEvicted(ssdbe *stmtSummaryByDigestEvicted) string { + buf := bytes.NewBuffer(nil) + for e := ssdbe.history.Back(); e != nil; e = e.Prev() { + if buf.Len() != 0 { + buf.WriteString(", ") + } + val := e.Value.(*stmtSummaryByDigestEvictedElement) + buf.WriteString(fmt.Sprintf("{begin: %v, end: %v, count: %v}", val.beginTime, val.endTime, len(val.digestKeyMap))) + } + return buf.String() +} + +func getEvicted(ssbdee *stmtSummaryByDigestEvictedElement) string { + buf := bytes.NewBuffer(nil) + buf.WriteString(fmt.Sprintf("{begin: %v, end: %v, count: %v}", ssbdee.beginTime, ssbdee.endTime, len(ssbdee.digestKeyMap))) + return buf.String() +} diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go index f42089c9e7a01..149196e0a4c31 100644 --- a/util/stmtsummary/statement_summary.go +++ b/util/stmtsummary/statement_summary.go @@ -73,6 +73,9 @@ type stmtSummaryByDigestMap struct { // sysVars encapsulates system variables needed to control statement summary. sysVars *systemVars + + // other stores summary of evicted data. + other *stmtSummaryByDigestEvicted } // StmtSummaryByDigestMap is a global map containing all statement summaries. @@ -172,7 +175,7 @@ type stmtSummaryByDigestElement struct { sumTxnRetry int64 maxTxnRetry int sumBackoffTimes int64 - backoffTypes map[fmt.Stringer]int + backoffTypes map[string]int authUsers map[string]struct{} // other sumMem int64 @@ -235,11 +238,20 @@ type StmtExecInfo struct { // newStmtSummaryByDigestMap creates an empty stmtSummaryByDigestMap. func newStmtSummaryByDigestMap() *stmtSummaryByDigestMap { sysVars := newSysVars() + + ssbde := newStmtSummaryByDigestEvicted() + maxStmtCount := uint(sysVars.getVariable(typeMaxStmtCount)) - return &stmtSummaryByDigestMap{ + newSsMap := &stmtSummaryByDigestMap{ summaryMap: kvcache.NewSimpleLRUCache(maxStmtCount, 0, 0), sysVars: sysVars, + other: ssbde, } + newSsMap.summaryMap.SetOnEvict(func(k kvcache.Key, v kvcache.Value) { + historySize := newSsMap.historySize() + newSsMap.other.AddEvicted(k.(*stmtSummaryByDigestKey), v.(*stmtSummaryByDigest), historySize) + }) + return newSsMap } // AddStatement adds a statement to StmtSummaryByDigestMap. @@ -291,7 +303,6 @@ func (ssMap *stmtSummaryByDigestMap) AddStatement(sei *StmtExecInfo) { summary.isInternal = summary.isInternal && sei.IsInternal return summary, beginTime }() - // Lock a single entry, not the whole cache. if summary != nil { summary.add(sei, beginTime, intervalSeconds, historySize) @@ -304,6 +315,7 @@ func (ssMap *stmtSummaryByDigestMap) Clear() { defer ssMap.Unlock() ssMap.summaryMap.DeleteAll() + ssMap.other.Clear() ssMap.beginTimeForCurInterval = 0 } @@ -635,7 +647,7 @@ func newStmtSummaryByDigestElement(sei *StmtExecInfo, beginTime int64, intervalS minLatency: sei.TotalLatency, firstSeen: sei.StartTime, lastSeen: sei.StartTime, - backoffTypes: make(map[fmt.Stringer]int), + backoffTypes: make(map[string]int), authUsers: make(map[string]struct{}), planInCache: false, planCacheHits: 0, @@ -771,11 +783,6 @@ func (ssElement *stmtSummaryByDigestElement) add(sei *StmtExecInfo, intervalSeco if commitDetails.GetCommitTsTime > ssElement.maxGetCommitTsTime { ssElement.maxGetCommitTsTime = commitDetails.GetCommitTsTime } - commitBackoffTime := atomic.LoadInt64(&commitDetails.CommitBackoffTime) - ssElement.sumCommitBackoffTime += commitBackoffTime - if commitBackoffTime > ssElement.maxCommitBackoffTime { - ssElement.maxCommitBackoffTime = commitBackoffTime - } resolveLockTime := atomic.LoadInt64(&commitDetails.ResolveLockTime) ssElement.sumResolveLockTime += resolveLockTime if resolveLockTime > ssElement.maxResolveLockTime { @@ -803,6 +810,11 @@ func (ssElement *stmtSummaryByDigestElement) add(sei *StmtExecInfo, intervalSeco ssElement.maxTxnRetry = commitDetails.TxnRetry } commitDetails.Mu.Lock() + commitBackoffTime := commitDetails.Mu.CommitBackoffTime + ssElement.sumCommitBackoffTime += commitBackoffTime + if commitBackoffTime > ssElement.maxCommitBackoffTime { + ssElement.maxCommitBackoffTime = commitBackoffTime + } ssElement.sumBackoffTimes += int64(len(commitDetails.Mu.BackoffTypes)) for _, backoffType := range commitDetails.Mu.BackoffTypes { ssElement.backoffTypes[backoffType] += 1 @@ -971,9 +983,9 @@ func formatSQL(sql string) string { } // Format the backoffType map to a string or nil. -func formatBackoffTypes(backoffMap map[fmt.Stringer]int) interface{} { +func formatBackoffTypes(backoffMap map[string]int) interface{} { type backoffStat struct { - backoffType fmt.Stringer + backoffType string count int } diff --git a/util/stmtsummary/statement_summary_test.go b/util/stmtsummary/statement_summary_test.go index 5971e83e7980f..f09398df68423 100644 --- a/util/stmtsummary/statement_summary_test.go +++ b/util/stmtsummary/statement_summary_test.go @@ -27,7 +27,6 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/sessionctx/stmtctx" - "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/util" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/execdetails" @@ -62,6 +61,10 @@ func TestT(t *testing.T) { TestingT(t) } +const ( + boTxnLockName = "txnlock" +) + // Test stmtSummaryByDigest.AddStatement. func (s *testStmtSummarySuite) TestAddStatement(c *C) { s.ssMap.Clear() @@ -73,13 +76,14 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { // first statement stmtExecInfo1 := generateAnyExecInfo() - stmtExecInfo1.ExecDetail.CommitDetail.Mu.BackoffTypes = make([]fmt.Stringer, 0) + stmtExecInfo1.ExecDetail.CommitDetail.Mu.BackoffTypes = make([]string, 0) key := &stmtSummaryByDigestKey{ schemaName: stmtExecInfo1.SchemaName, digest: stmtExecInfo1.Digest, planDigest: stmtExecInfo1.PlanDigest, } samplePlan, _ := stmtExecInfo1.PlanGenerator() + stmtExecInfo1.ExecDetail.CommitDetail.Mu.Lock() expectedSummaryElement := stmtSummaryByDigestElement{ beginTime: now + 60, endTime: now + 1860, @@ -117,8 +121,8 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { maxCommitTime: stmtExecInfo1.ExecDetail.CommitDetail.CommitTime, sumLocalLatchTime: stmtExecInfo1.ExecDetail.CommitDetail.LocalLatchTime, maxLocalLatchTime: stmtExecInfo1.ExecDetail.CommitDetail.LocalLatchTime, - sumCommitBackoffTime: stmtExecInfo1.ExecDetail.CommitDetail.CommitBackoffTime, - maxCommitBackoffTime: stmtExecInfo1.ExecDetail.CommitDetail.CommitBackoffTime, + sumCommitBackoffTime: stmtExecInfo1.ExecDetail.CommitDetail.Mu.CommitBackoffTime, + maxCommitBackoffTime: stmtExecInfo1.ExecDetail.CommitDetail.Mu.CommitBackoffTime, sumResolveLockTime: stmtExecInfo1.ExecDetail.CommitDetail.ResolveLockTime, maxResolveLockTime: stmtExecInfo1.ExecDetail.CommitDetail.ResolveLockTime, sumWriteKeys: int64(stmtExecInfo1.ExecDetail.CommitDetail.WriteKeys), @@ -129,7 +133,7 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { maxPrewriteRegionNum: stmtExecInfo1.ExecDetail.CommitDetail.PrewriteRegionNum, sumTxnRetry: int64(stmtExecInfo1.ExecDetail.CommitDetail.TxnRetry), maxTxnRetry: stmtExecInfo1.ExecDetail.CommitDetail.TxnRetry, - backoffTypes: make(map[fmt.Stringer]int), + backoffTypes: make(map[string]int), sumMem: stmtExecInfo1.MemMax, maxMem: stmtExecInfo1.MemMax, sumDisk: stmtExecInfo1.DiskMax, @@ -138,6 +142,7 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { firstSeen: stmtExecInfo1.StartTime, lastSeen: stmtExecInfo1.StartTime, } + stmtExecInfo1.ExecDetail.CommitDetail.Mu.Unlock() history := list.New() history.PushBack(&expectedSummaryElement) expectedSummary := stmtSummaryByDigest{ @@ -183,16 +188,17 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { BackoffTime: 180, RequestCount: 20, CommitDetail: &util.CommitDetails{ - GetCommitTsTime: 500, - PrewriteTime: 50000, - CommitTime: 5000, - LocalLatchTime: 50, - CommitBackoffTime: 1000, + GetCommitTsTime: 500, + PrewriteTime: 50000, + CommitTime: 5000, + LocalLatchTime: 50, Mu: struct { sync.Mutex - BackoffTypes []fmt.Stringer + CommitBackoffTime int64 + BackoffTypes []string }{ - BackoffTypes: []fmt.Stringer{tikv.BoTxnLock}, + CommitBackoffTime: 1000, + BackoffTypes: []string{boTxnLockName}, }, ResolveLockTime: 10000, WriteKeys: 100000, @@ -255,8 +261,10 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { expectedSummaryElement.maxCommitTime = stmtExecInfo2.ExecDetail.CommitDetail.CommitTime expectedSummaryElement.sumLocalLatchTime += stmtExecInfo2.ExecDetail.CommitDetail.LocalLatchTime expectedSummaryElement.maxLocalLatchTime = stmtExecInfo2.ExecDetail.CommitDetail.LocalLatchTime - expectedSummaryElement.sumCommitBackoffTime += stmtExecInfo2.ExecDetail.CommitDetail.CommitBackoffTime - expectedSummaryElement.maxCommitBackoffTime = stmtExecInfo2.ExecDetail.CommitDetail.CommitBackoffTime + stmtExecInfo2.ExecDetail.CommitDetail.Mu.Lock() + expectedSummaryElement.sumCommitBackoffTime += stmtExecInfo2.ExecDetail.CommitDetail.Mu.CommitBackoffTime + expectedSummaryElement.maxCommitBackoffTime = stmtExecInfo2.ExecDetail.CommitDetail.Mu.CommitBackoffTime + stmtExecInfo2.ExecDetail.CommitDetail.Mu.Unlock() expectedSummaryElement.sumResolveLockTime += stmtExecInfo2.ExecDetail.CommitDetail.ResolveLockTime expectedSummaryElement.maxResolveLockTime = stmtExecInfo2.ExecDetail.CommitDetail.ResolveLockTime expectedSummaryElement.sumWriteKeys += int64(stmtExecInfo2.ExecDetail.CommitDetail.WriteKeys) @@ -268,7 +276,7 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { expectedSummaryElement.sumTxnRetry += int64(stmtExecInfo2.ExecDetail.CommitDetail.TxnRetry) expectedSummaryElement.maxTxnRetry = stmtExecInfo2.ExecDetail.CommitDetail.TxnRetry expectedSummaryElement.sumBackoffTimes += 1 - expectedSummaryElement.backoffTypes[tikv.BoTxnLock] = 1 + expectedSummaryElement.backoffTypes[boTxnLockName] = 1 expectedSummaryElement.sumMem += stmtExecInfo2.MemMax expectedSummaryElement.maxMem = stmtExecInfo2.MemMax expectedSummaryElement.sumDisk += stmtExecInfo2.DiskMax @@ -310,16 +318,17 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { BackoffTime: 18, RequestCount: 2, CommitDetail: &util.CommitDetails{ - GetCommitTsTime: 50, - PrewriteTime: 5000, - CommitTime: 500, - LocalLatchTime: 5, - CommitBackoffTime: 100, + GetCommitTsTime: 50, + PrewriteTime: 5000, + CommitTime: 500, + LocalLatchTime: 5, Mu: struct { sync.Mutex - BackoffTypes []fmt.Stringer + CommitBackoffTime int64 + BackoffTypes []string }{ - BackoffTypes: []fmt.Stringer{tikv.BoTxnLock}, + CommitBackoffTime: 100, + BackoffTypes: []string{boTxnLockName}, }, ResolveLockTime: 1000, WriteKeys: 10000, @@ -367,14 +376,16 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { expectedSummaryElement.sumPrewriteTime += stmtExecInfo3.ExecDetail.CommitDetail.PrewriteTime expectedSummaryElement.sumCommitTime += stmtExecInfo3.ExecDetail.CommitDetail.CommitTime expectedSummaryElement.sumLocalLatchTime += stmtExecInfo3.ExecDetail.CommitDetail.LocalLatchTime - expectedSummaryElement.sumCommitBackoffTime += stmtExecInfo3.ExecDetail.CommitDetail.CommitBackoffTime + stmtExecInfo3.ExecDetail.CommitDetail.Mu.Lock() + expectedSummaryElement.sumCommitBackoffTime += stmtExecInfo3.ExecDetail.CommitDetail.Mu.CommitBackoffTime + stmtExecInfo3.ExecDetail.CommitDetail.Mu.Unlock() expectedSummaryElement.sumResolveLockTime += stmtExecInfo3.ExecDetail.CommitDetail.ResolveLockTime expectedSummaryElement.sumWriteKeys += int64(stmtExecInfo3.ExecDetail.CommitDetail.WriteKeys) expectedSummaryElement.sumWriteSize += int64(stmtExecInfo3.ExecDetail.CommitDetail.WriteSize) expectedSummaryElement.sumPrewriteRegionNum += int64(stmtExecInfo3.ExecDetail.CommitDetail.PrewriteRegionNum) expectedSummaryElement.sumTxnRetry += int64(stmtExecInfo3.ExecDetail.CommitDetail.TxnRetry) expectedSummaryElement.sumBackoffTimes += 1 - expectedSummaryElement.backoffTypes[tikv.BoTxnLock] = 2 + expectedSummaryElement.backoffTypes[boTxnLockName] = 2 expectedSummaryElement.sumMem += stmtExecInfo3.MemMax expectedSummaryElement.sumDisk += stmtExecInfo3.DiskMax expectedSummaryElement.sumAffectedRows += stmtExecInfo3.StmtCtx.AffectedRows() @@ -566,16 +577,17 @@ func generateAnyExecInfo() *StmtExecInfo { BackoffTime: 80, RequestCount: 10, CommitDetail: &util.CommitDetails{ - GetCommitTsTime: 100, - PrewriteTime: 10000, - CommitTime: 1000, - LocalLatchTime: 10, - CommitBackoffTime: 200, + GetCommitTsTime: 100, + PrewriteTime: 10000, + CommitTime: 1000, + LocalLatchTime: 10, Mu: struct { sync.Mutex - BackoffTypes []fmt.Stringer + CommitBackoffTime int64 + BackoffTypes []string }{ - BackoffTypes: []fmt.Stringer{tikv.BoTxnLock}, + CommitBackoffTime: 200, + BackoffTypes: []string{boTxnLockName}, }, ResolveLockTime: 2000, WriteKeys: 20000, @@ -625,6 +637,7 @@ func (s *testStmtSummarySuite) TestToDatum(c *C) { n := types.NewTime(types.FromGoTime(time.Unix(s.ssMap.beginTimeForCurInterval, 0)), mysql.TypeTimestamp, types.DefaultFsp) e := types.NewTime(types.FromGoTime(time.Unix(s.ssMap.beginTimeForCurInterval+1800, 0)), mysql.TypeTimestamp, types.DefaultFsp) t := types.NewTime(types.FromGoTime(stmtExecInfo1.StartTime), mysql.TypeTimestamp, types.DefaultFsp) + stmtExecInfo1.ExecDetail.CommitDetail.Mu.Lock() expectedDatum := []interface{}{n, e, "Select", stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, stmtExecInfo1.NormalizedSQL, "db1.tb1,db2.tb2", "a", stmtExecInfo1.User, 1, 0, 0, int64(stmtExecInfo1.TotalLatency), int64(stmtExecInfo1.TotalLatency), int64(stmtExecInfo1.TotalLatency), int64(stmtExecInfo1.TotalLatency), @@ -643,16 +656,17 @@ func (s *testStmtSummarySuite) TestToDatum(c *C) { int64(stmtExecInfo1.ExecDetail.CommitDetail.PrewriteTime), int64(stmtExecInfo1.ExecDetail.CommitDetail.PrewriteTime), int64(stmtExecInfo1.ExecDetail.CommitDetail.CommitTime), int64(stmtExecInfo1.ExecDetail.CommitDetail.CommitTime), int64(stmtExecInfo1.ExecDetail.CommitDetail.GetCommitTsTime), int64(stmtExecInfo1.ExecDetail.CommitDetail.GetCommitTsTime), - stmtExecInfo1.ExecDetail.CommitDetail.CommitBackoffTime, stmtExecInfo1.ExecDetail.CommitDetail.CommitBackoffTime, + stmtExecInfo1.ExecDetail.CommitDetail.Mu.CommitBackoffTime, stmtExecInfo1.ExecDetail.CommitDetail.Mu.CommitBackoffTime, stmtExecInfo1.ExecDetail.CommitDetail.ResolveLockTime, stmtExecInfo1.ExecDetail.CommitDetail.ResolveLockTime, int64(stmtExecInfo1.ExecDetail.CommitDetail.LocalLatchTime), int64(stmtExecInfo1.ExecDetail.CommitDetail.LocalLatchTime), stmtExecInfo1.ExecDetail.CommitDetail.WriteKeys, stmtExecInfo1.ExecDetail.CommitDetail.WriteKeys, stmtExecInfo1.ExecDetail.CommitDetail.WriteSize, stmtExecInfo1.ExecDetail.CommitDetail.WriteSize, stmtExecInfo1.ExecDetail.CommitDetail.PrewriteRegionNum, stmtExecInfo1.ExecDetail.CommitDetail.PrewriteRegionNum, stmtExecInfo1.ExecDetail.CommitDetail.TxnRetry, stmtExecInfo1.ExecDetail.CommitDetail.TxnRetry, 0, 0, 1, - "txnLock:1", stmtExecInfo1.MemMax, stmtExecInfo1.MemMax, stmtExecInfo1.DiskMax, stmtExecInfo1.DiskMax, + fmt.Sprintf("%s:1", boTxnLockName), stmtExecInfo1.MemMax, stmtExecInfo1.MemMax, stmtExecInfo1.DiskMax, stmtExecInfo1.DiskMax, 0, 0, 0, 0, 0, stmtExecInfo1.StmtCtx.AffectedRows(), t, t, 0, 0, 0, stmtExecInfo1.OriginalSQL, stmtExecInfo1.PrevSQL, "plan_digest", ""} + stmtExecInfo1.ExecDetail.CommitDetail.Mu.Unlock() match(c, datums[0], expectedDatum...) datums = s.ssMap.ToHistoryDatum(nil, true) c.Assert(len(datums), Equals, 1) @@ -957,14 +971,15 @@ func (s *testStmtSummarySuite) TestGetMoreThanOnceBindableStmt(c *C) { // Test `formatBackoffTypes`. func (s *testStmtSummarySuite) TestFormatBackoffTypes(c *C) { - backoffMap := make(map[fmt.Stringer]int) + backoffMap := make(map[string]int) c.Assert(formatBackoffTypes(backoffMap), IsNil) + bo1 := "pdrpc" + backoffMap[bo1] = 1 + c.Assert(formatBackoffTypes(backoffMap), Equals, "pdrpc:1") + bo2 := "txnlock" + backoffMap[bo2] = 2 - backoffMap[tikv.BoPDRPC] = 1 - c.Assert(formatBackoffTypes(backoffMap), Equals, "pdRPC:1") - - backoffMap[tikv.BoTxnLock] = 2 - c.Assert(formatBackoffTypes(backoffMap), Equals, "txnLock:2,pdRPC:1") + c.Assert(formatBackoffTypes(backoffMap), Equals, "txnlock:2,pdrpc:1") } // Test refreshing current statement summary periodically. diff --git a/util/stmtsummary/variables.go b/util/stmtsummary/variables.go index b179929002fd5..96f6b0fac7076 100644 --- a/util/stmtsummary/variables.go +++ b/util/stmtsummary/variables.go @@ -16,12 +16,12 @@ package stmtsummary import ( "fmt" "strconv" + "strings" "sync" "sync/atomic" "github.com/pingcap/errors" "github.com/pingcap/tidb/config" - svariable "github.com/pingcap/tidb/sessionctx/variable" ) const ( @@ -104,7 +104,7 @@ func getBoolFinalVariable(varType int, sessionValue, globalValue string) int64 { // normalizeEnableValue converts 'ON' or '1' to 1 and 'OFF' or '0' to 0. func normalizeEnableValue(value string) int64 { - if svariable.TiDBOptOn(value) { + if strings.EqualFold(value, "ON") || value == "1" { return 1 } return 0 diff --git a/util/tableutil/tableutil.go b/util/tableutil/tableutil.go new file mode 100644 index 0000000000000..bf5d7caac2732 --- /dev/null +++ b/util/tableutil/tableutil.go @@ -0,0 +1,43 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package tableutil + +import ( + "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/meta/autoid" +) + +// TempTable is used to store transaction-specific or session-specific information for global / local temporary tables. +// For example, stats and autoID should have their own copies of data, instead of being shared by all sessions. +type TempTable interface { + // GetAutoIDAllocator gets the autoID allocator of this table. + GetAutoIDAllocator() autoid.Allocator + + // SetModified sets that the table is modified. + SetModified(bool) + + // GetModified queries whether the table is modified. + GetModified() bool + + // The stats of this table (*statistics.Table). + // Define the return type as interface{} here to avoid cycle imports. + GetStats() interface{} + + GetSize() int64 + SetSize(int64) +} + +// TempTableFromMeta builds a TempTable from *model.TableInfo. +// Currently, it is assigned to tables.TempTableFromMeta in tidb package's init function. +var TempTableFromMeta func(tblInfo *model.TableInfo) TempTable diff --git a/util/testkit/testkit.go b/util/testkit/testkit.go index 4992e28663b1a..ef0a0858f76cf 100644 --- a/util/testkit/testkit.go +++ b/util/testkit/testkit.go @@ -255,6 +255,37 @@ func (tk *TestKit) MustNoGlobalStats(table string) bool { return true } +// MustPartition checks if the result execution plan must read specific partitions. +func (tk *TestKit) MustPartition(sql string, partitions string, args ...interface{}) *Result { + rs := tk.MustQuery("explain "+sql, args...) + ok := len(partitions) == 0 + for i := range rs.rows { + if len(partitions) == 0 && strings.Contains(rs.rows[i][3], "partition:") { + ok = false + } + if len(partitions) != 0 && strings.Compare(rs.rows[i][3], "partition:"+partitions) == 0 { + ok = true + } + } + tk.c.Assert(ok, check.IsTrue) + return tk.MustQuery(sql, args...) +} + +// UsedPartitions returns the partition names that will be used or all/dual. +func (tk *TestKit) UsedPartitions(sql string, args ...interface{}) *Result { + rs := tk.MustQuery("explain "+sql, args...) + var usedPartitions [][]string + for i := range rs.rows { + index := strings.Index(rs.rows[i][3], "partition:") + if index != -1 { + p := rs.rows[i][3][index+len("partition:"):] + partitions := strings.Split(strings.SplitN(p, " ", 2)[0], ",") + usedPartitions = append(usedPartitions, partitions) + } + } + return &Result{rows: usedPartitions, c: tk.c, comment: check.Commentf("sql:%s, args:%v", sql, args)} +} + // MustUseIndex checks if the result execution plan contains specific index(es). func (tk *TestKit) MustUseIndex(sql string, index string, args ...interface{}) bool { rs := tk.MustQuery("explain "+sql, args...) @@ -296,6 +327,19 @@ func (tk *TestKit) MustQuery(sql string, args ...interface{}) *Result { return tk.ResultSetToResult(rs, comment) } +// MayQuery query the statements and returns result rows if result set is returned. +// If expected result is set it asserts the query result equals expected result. +func (tk *TestKit) MayQuery(sql string, args ...interface{}) *Result { + comment := check.Commentf("sql:%s, args:%v", sql, args) + rs, err := tk.Exec(sql, args...) + tk.c.Assert(errors.ErrorStack(err), check.Equals, "", comment) + if rs == nil { + var emptyStringAoA [][]string + return &Result{rows: emptyStringAoA, c: tk.c, comment: comment} + } + return tk.ResultSetToResult(rs, comment) +} + // QueryToErr executes a sql statement and discard results. func (tk *TestKit) QueryToErr(sql string, args ...interface{}) error { comment := check.Commentf("sql:%s, args:%v", sql, args) diff --git a/util/testutil/testutil.go b/util/testutil/testutil.go index d1ecd061a5624..5bb70ac21cf87 100644 --- a/util/testutil/testutil.go +++ b/util/testutil/testutil.go @@ -20,7 +20,7 @@ import ( "encoding/json" "flag" "fmt" - "io/ioutil" + "io" "os" "path/filepath" "reflect" @@ -287,7 +287,7 @@ func loadTestSuiteCases(filePath string) (res []testCases, err error) { err = err1 } }() - byteValue, err := ioutil.ReadAll(jsonFile) + byteValue, err := io.ReadAll(jsonFile) if err != nil { return res, err } diff --git a/util/topsql/reporter/client.go b/util/topsql/reporter/client.go new file mode 100644 index 0000000000000..dc5854611bb51 --- /dev/null +++ b/util/topsql/reporter/client.go @@ -0,0 +1,190 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package reporter + +import ( + "context" + "math" + "sync" + "time" + + "github.com/pingcap/tidb/util/logutil" + "github.com/pingcap/tipb/go-tipb" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" +) + +// ReportClient send data to the target server. +type ReportClient interface { + Send(ctx context.Context, addr string, sqlMetas []*tipb.SQLMeta, planMetas []*tipb.PlanMeta, records []*tipb.CPUTimeRecord) error + Close() +} + +// GRPCReportClient reports data to grpc servers. +type GRPCReportClient struct { + curRPCAddr string + conn *grpc.ClientConn +} + +// NewGRPCReportClient returns a new GRPCReportClient +func NewGRPCReportClient() *GRPCReportClient { + return &GRPCReportClient{} +} + +// Send implements the ReportClient interface. +func (r *GRPCReportClient) Send( + ctx context.Context, targetRPCAddr string, + sqlMetas []*tipb.SQLMeta, planMetas []*tipb.PlanMeta, records []*tipb.CPUTimeRecord) error { + if targetRPCAddr == "" { + return nil + } + err := r.tryEstablishConnection(ctx, targetRPCAddr) + if err != nil { + return err + } + + var wg sync.WaitGroup + errCh := make(chan error, 3) + wg.Add(3) + + go func() { + defer wg.Done() + errCh <- r.sendBatchSQLMeta(ctx, sqlMetas) + }() + go func() { + defer wg.Done() + errCh <- r.sendBatchPlanMeta(ctx, planMetas) + }() + go func() { + defer wg.Done() + errCh <- r.sendBatchCPUTimeRecord(ctx, records) + }() + wg.Wait() + close(errCh) + for err := range errCh { + if err != nil { + return err + } + } + return nil +} + +// Close uses to close grpc connection. +func (r *GRPCReportClient) Close() { + if r.conn == nil { + return + } + err := r.conn.Close() + if err != nil { + logutil.BgLogger().Warn("[top-sql] grpc client close connection failed", zap.Error(err)) + } + r.conn = nil +} + +// sendBatchCPUTimeRecord sends a batch of TopSQL records by stream. +func (r *GRPCReportClient) sendBatchCPUTimeRecord(ctx context.Context, records []*tipb.CPUTimeRecord) error { + if len(records) == 0 { + return nil + } + client := tipb.NewTopSQLAgentClient(r.conn) + stream, err := client.ReportCPUTimeRecords(ctx) + if err != nil { + return err + } + for _, record := range records { + if err := stream.Send(record); err != nil { + break + } + } + // See https://pkg.go.dev/google.golang.org/grpc#ClientConn.NewStream for how to avoid leaking the stream + _, err = stream.CloseAndRecv() + return err +} + +// sendBatchSQLMeta sends a batch of SQL metas by stream. +func (r *GRPCReportClient) sendBatchSQLMeta(ctx context.Context, metas []*tipb.SQLMeta) error { + if len(metas) == 0 { + return nil + } + client := tipb.NewTopSQLAgentClient(r.conn) + stream, err := client.ReportSQLMeta(ctx) + if err != nil { + return err + } + for _, meta := range metas { + if err := stream.Send(meta); err != nil { + break + } + } + _, err = stream.CloseAndRecv() + return err +} + +// sendBatchSQLMeta sends a batch of SQL metas by stream. +func (r *GRPCReportClient) sendBatchPlanMeta(ctx context.Context, metas []*tipb.PlanMeta) error { + if len(metas) == 0 { + return nil + } + client := tipb.NewTopSQLAgentClient(r.conn) + stream, err := client.ReportPlanMeta(ctx) + if err != nil { + return err + } + for _, meta := range metas { + if err := stream.Send(meta); err != nil { + break + } + } + _, err = stream.CloseAndRecv() + return err +} + +// tryEstablishConnection establishes the gRPC connection if connection is not established. +func (r *GRPCReportClient) tryEstablishConnection(ctx context.Context, targetRPCAddr string) (err error) { + if r.curRPCAddr == targetRPCAddr && r.conn != nil { + // Address is not changed, skip. + return nil + } + r.conn, err = r.dial(ctx, targetRPCAddr) + if err != nil { + return err + } + r.curRPCAddr = targetRPCAddr + return nil +} + +func (r *GRPCReportClient) dial(ctx context.Context, targetRPCAddr string) (*grpc.ClientConn, error) { + dialCtx, cancel := context.WithTimeout(ctx, dialTimeout) + defer cancel() + return grpc.DialContext( + dialCtx, + targetRPCAddr, + grpc.WithBlock(), + grpc.WithInsecure(), + grpc.WithInitialWindowSize(grpcInitialWindowSize), + grpc.WithInitialConnWindowSize(grpcInitialConnWindowSize), + grpc.WithDefaultCallOptions( + grpc.MaxCallRecvMsgSize(math.MaxInt64), + ), + grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: backoff.Config{ + BaseDelay: 100 * time.Millisecond, // Default was 1s. + Multiplier: 1.6, // Default + Jitter: 0.2, // Default + MaxDelay: 3 * time.Second, // Default was 120s. + }, + }), + ) +} diff --git a/util/topsql/reporter/mock/server.go b/util/topsql/reporter/mock/server.go new file mode 100644 index 0000000000000..42479bc7a6201 --- /dev/null +++ b/util/topsql/reporter/mock/server.go @@ -0,0 +1,144 @@ +package mock + +import ( + "fmt" + "io" + "net" + "sync" + "time" + + "github.com/pingcap/tidb/util/logutil" + "github.com/pingcap/tipb/go-tipb" + "go.uber.org/zap" + "google.golang.org/grpc" +) + +type mockAgentServer struct { + sync.Mutex + addr string + grpcServer *grpc.Server + sqlMetas map[string]string + planMetas map[string]string + records []*tipb.CPUTimeRecord +} + +// StartMockAgentServer starts the mock agent server. +func StartMockAgentServer() (*mockAgentServer, error) { + addr := "127.0.0.1:0" + lis, err := net.Listen("tcp", addr) + if err != nil { + return nil, err + } + server := grpc.NewServer() + agentServer := &mockAgentServer{ + addr: fmt.Sprintf("127.0.0.1:%d", lis.Addr().(*net.TCPAddr).Port), + grpcServer: server, + sqlMetas: make(map[string]string, 5000), + planMetas: make(map[string]string, 5000), + } + tipb.RegisterTopSQLAgentServer(server, agentServer) + + go func() { + err := server.Serve(lis) + if err != nil { + logutil.BgLogger().Warn("[top-sql] mock agent server serve failed", zap.Error(err)) + } + }() + + return agentServer, nil +} + +func (svr *mockAgentServer) ReportCPUTimeRecords(stream tipb.TopSQLAgent_ReportCPUTimeRecordsServer) error { + for { + req, err := stream.Recv() + if err == io.EOF { + break + } else if err != nil { + return err + } + svr.Lock() + svr.records = append(svr.records, req) + svr.Unlock() + } + return stream.SendAndClose(&tipb.EmptyResponse{}) +} + +func (svr *mockAgentServer) ReportSQLMeta(stream tipb.TopSQLAgent_ReportSQLMetaServer) error { + for { + req, err := stream.Recv() + if err == io.EOF { + break + } else if err != nil { + return err + } + svr.Lock() + svr.sqlMetas[string(req.SqlDigest)] = req.NormalizedSql + svr.Unlock() + } + return stream.SendAndClose(&tipb.EmptyResponse{}) +} + +func (svr *mockAgentServer) ReportPlanMeta(stream tipb.TopSQLAgent_ReportPlanMetaServer) error { + for { + req, err := stream.Recv() + if err == io.EOF { + break + } else if err != nil { + return err + } + svr.Lock() + svr.planMetas[string(req.PlanDigest)] = req.NormalizedPlan + svr.Unlock() + } + return stream.SendAndClose(&tipb.EmptyResponse{}) +} + +func (svr *mockAgentServer) WaitServerCollect(recordCount int, timeout time.Duration) { + start := time.Now() + for { + svr.Lock() + if len(svr.records) >= recordCount { + svr.Unlock() + return + } + svr.Unlock() + if time.Since(start) > timeout { + return + } + time.Sleep(time.Millisecond) + } +} + +func (svr *mockAgentServer) GetSQLMetas() map[string]string { + svr.Lock() + m := svr.sqlMetas + svr.sqlMetas = make(map[string]string) + svr.Unlock() + return m +} + +func (svr *mockAgentServer) GetPlanMetas() map[string]string { + svr.Lock() + m := svr.planMetas + svr.planMetas = make(map[string]string) + svr.Unlock() + return m +} + +func (svr *mockAgentServer) GetRecords() []*tipb.CPUTimeRecord { + svr.Lock() + records := svr.records + svr.records = []*tipb.CPUTimeRecord{} + svr.Unlock() + return records +} + +func (svr *mockAgentServer) Address() string { + return svr.addr +} + +func (svr *mockAgentServer) Stop() { + if svr.grpcServer != nil { + svr.grpcServer.Stop() + } +} diff --git a/util/topsql/reporter/reporter.go b/util/topsql/reporter/reporter.go new file mode 100644 index 0000000000000..7723a62d6aa2a --- /dev/null +++ b/util/topsql/reporter/reporter.go @@ -0,0 +1,368 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package reporter + +import ( + "bytes" + "context" + "sync" + "sync/atomic" + "time" + + "github.com/pingcap/tidb/sessionctx/variable" + "github.com/pingcap/tidb/util" + "github.com/pingcap/tidb/util/logutil" + "github.com/pingcap/tidb/util/topsql/tracecpu" + "github.com/pingcap/tipb/go-tipb" + "github.com/wangjohn/quickselect" + "go.uber.org/zap" +) + +const ( + dialTimeout = 5 * time.Second + reportTimeout = 40 * time.Second + grpcInitialWindowSize = 1 << 30 + grpcInitialConnWindowSize = 1 << 30 +) + +var _ TopSQLReporter = &RemoteTopSQLReporter{} + +// TopSQLReporter collects Top SQL metrics. +type TopSQLReporter interface { + tracecpu.Collector + RegisterSQL(sqlDigest []byte, normalizedSQL string) + RegisterPlan(planDigest []byte, normalizedPlan string) + Close() +} + +type cpuData struct { + timestamp uint64 + records []tracecpu.SQLCPUTimeRecord +} + +// dataPoints represents the cumulative SQL plan CPU time in current minute window +type dataPoints struct { + SQLDigest []byte + PlanDigest []byte + TimestampList []uint64 + CPUTimeMsList []uint32 + CPUTimeMsTotal uint64 +} + +// cpuTimeSort is used to sort TopSQL records by total CPU time +type cpuTimeSort struct { + Key string + SQLDigest []byte + PlanDigest []byte + CPUTimeMsTotal uint64 // The sorting field +} + +type cpuTimeSortSlice []cpuTimeSort + +func (t cpuTimeSortSlice) Len() int { + return len(t) +} + +func (t cpuTimeSortSlice) Less(i, j int) bool { + // We need find the kth largest value, so here should use > + return t[i].CPUTimeMsTotal > t[j].CPUTimeMsTotal +} +func (t cpuTimeSortSlice) Swap(i, j int) { + t[i], t[j] = t[j], t[i] +} + +type planBinaryDecodeFunc func(string) (string, error) + +// RemoteTopSQLReporter implements a TopSQL reporter that sends data to a remote agent +// This should be called periodically to collect TopSQL resource usage metrics +type RemoteTopSQLReporter struct { + ctx context.Context + cancel context.CancelFunc + client ReportClient + + // normalizedSQLMap is an map, whose keys are SQL digest strings and values are normalized SQL strings + normalizedSQLMap atomic.Value // sync.Map + + // normalizedPlanMap is an map, whose keys are plan digest strings and values are normalized plans **in binary**. + // The normalized plans in binary can be decoded to string using the `planBinaryDecoder`. + normalizedPlanMap atomic.Value // sync.Map + + // calling this can take a while, so should not block critical paths + planBinaryDecoder planBinaryDecodeFunc + + collectCPUDataChan chan cpuData + reportDataChan chan reportData +} + +// NewRemoteTopSQLReporter creates a new TopSQL reporter +// +// planBinaryDecoder is a decoding function which will be called asynchronously to decode the plan binary to string +// MaxStatementsNum is the maximum SQL and plan number, which will restrict the memory usage of the internal LFU cache +func NewRemoteTopSQLReporter(client ReportClient, planDecodeFn planBinaryDecodeFunc) *RemoteTopSQLReporter { + + ctx, cancel := context.WithCancel(context.Background()) + tsr := &RemoteTopSQLReporter{ + ctx: ctx, + cancel: cancel, + client: client, + planBinaryDecoder: planDecodeFn, + collectCPUDataChan: make(chan cpuData, 1), + reportDataChan: make(chan reportData, 1), + } + tsr.normalizedSQLMap.Store(&sync.Map{}) + tsr.normalizedPlanMap.Store(&sync.Map{}) + + go tsr.collectWorker() + go tsr.reportWorker() + + return tsr +} + +// RegisterSQL registers a normalized SQL string to a SQL digest. +// This function is thread-safe and efficient. +// +// Note that the normalized SQL string can be of >1M long. +// This function should be thread-safe, which means parallelly calling it in several goroutines should be fine. +// It should also return immediately, and do any CPU-intensive job asynchronously. +func (tsr *RemoteTopSQLReporter) RegisterSQL(sqlDigest []byte, normalizedSQL string) { + m := tsr.normalizedSQLMap.Load().(*sync.Map) + key := string(sqlDigest) + m.LoadOrStore(key, normalizedSQL) +} + +// RegisterPlan is like RegisterSQL, but for normalized plan strings. +// This function is thread-safe and efficient. +func (tsr *RemoteTopSQLReporter) RegisterPlan(planDigest []byte, normalizedBinaryPlan string) { + m := tsr.normalizedPlanMap.Load().(*sync.Map) + key := string(planDigest) + m.LoadOrStore(key, normalizedBinaryPlan) +} + +// Collect receives CPU time records for processing. WARN: It will drop the records if the processing is not in time. +// This function is thread-safe and efficient. +func (tsr *RemoteTopSQLReporter) Collect(timestamp uint64, records []tracecpu.SQLCPUTimeRecord) { + if len(records) == 0 { + return + } + select { + case tsr.collectCPUDataChan <- cpuData{ + timestamp: timestamp, + records: records, + }: + default: + // ignore if chan blocked + } +} + +// Close uses to close and release the reporter resource. +func (tsr *RemoteTopSQLReporter) Close() { + tsr.cancel() + tsr.client.Close() +} + +func (tsr *RemoteTopSQLReporter) collectWorker() { + defer util.Recover("top-sql", "collectWorker", nil, false) + + collectedData := make(map[string]*dataPoints) + + currentReportInterval := variable.TopSQLVariable.ReportIntervalSeconds.Load() + reportTicker := time.NewTicker(time.Second * time.Duration(currentReportInterval)) + + for { + select { + case data := <-tsr.collectCPUDataChan: + // On receiving data to collect: Write to local data array, and retain records with most CPU time. + tsr.doCollect(collectedData, data.timestamp, data.records) + case <-reportTicker.C: + tsr.takeDataAndSendToReportChan(&collectedData) + + // Update `reportTicker` if report interval changed. + if newInterval := variable.TopSQLVariable.ReportIntervalSeconds.Load(); newInterval != currentReportInterval { + currentReportInterval = newInterval + reportTicker.Reset(time.Second * time.Duration(currentReportInterval)) + } + case <-tsr.ctx.Done(): + return + } + } +} + +func encodeKey(buf *bytes.Buffer, sqlDigest, planDigest []byte) string { + buf.Reset() + buf.Write(sqlDigest) + buf.Write(planDigest) + return buf.String() +} + +// doCollect uses a hashmap to store records in every second, and evict when necessary. +func (tsr *RemoteTopSQLReporter) doCollect( + collectTarget map[string]*dataPoints, timestamp uint64, records []tracecpu.SQLCPUTimeRecord) { + defer util.Recover("top-sql", "doCollect", nil, false) + + keyBuf := bytes.NewBuffer(make([]byte, 0, 64)) + listCapacity := int(variable.TopSQLVariable.ReportIntervalSeconds.Load()/variable.TopSQLVariable.PrecisionSeconds.Load() + 1) + if listCapacity < 1 { + listCapacity = 1 + } + for _, record := range records { + key := encodeKey(keyBuf, record.SQLDigest, record.PlanDigest) + entry, exist := collectTarget[key] + if !exist { + entry = &dataPoints{ + SQLDigest: record.SQLDigest, + PlanDigest: record.PlanDigest, + CPUTimeMsList: make([]uint32, 1, listCapacity), + TimestampList: make([]uint64, 1, listCapacity), + } + entry.CPUTimeMsList[0] = record.CPUTimeMs + entry.TimestampList[0] = timestamp + collectTarget[key] = entry + } else { + entry.CPUTimeMsList = append(entry.CPUTimeMsList, record.CPUTimeMs) + entry.TimestampList = append(entry.TimestampList, timestamp) + } + entry.CPUTimeMsTotal += uint64(record.CPUTimeMs) + } + + // evict records according to `MaxStatementCount` variable. + // TODO: Better to pass in the variable in the constructor, instead of referencing directly. + maxStmt := int(variable.TopSQLVariable.MaxStatementCount.Load()) + if len(collectTarget) <= maxStmt { + return + } + + // find the max CPUTimeMsTotal that should be evicted + digestCPUTimeList := make([]cpuTimeSort, len(collectTarget)) + idx := 0 + for key, value := range collectTarget { + digestCPUTimeList[idx] = cpuTimeSort{ + Key: key, + SQLDigest: value.SQLDigest, + PlanDigest: value.PlanDigest, + CPUTimeMsTotal: value.CPUTimeMsTotal, + } + idx++ + } + + // QuickSelect will only return error when the second parameter is out of range + if err := quickselect.QuickSelect(cpuTimeSortSlice(digestCPUTimeList), maxStmt); err != nil { + // skip eviction + return + } + + itemsToEvict := digestCPUTimeList[maxStmt:] + normalizedSQLMap := tsr.normalizedSQLMap.Load().(*sync.Map) + normalizedPlanMap := tsr.normalizedPlanMap.Load().(*sync.Map) + for _, evict := range itemsToEvict { + delete(collectTarget, evict.Key) + normalizedSQLMap.Delete(string(evict.SQLDigest)) + normalizedPlanMap.Delete(string(evict.PlanDigest)) + } +} + +// takeDataAndSendToReportChan takes out (resets) collected data. These data will be send to a report channel +// for reporting later. +func (tsr *RemoteTopSQLReporter) takeDataAndSendToReportChan(collectedDataPtr *map[string]*dataPoints) { + data := reportData{ + collectedData: *collectedDataPtr, + normalizedSQLMap: tsr.normalizedSQLMap.Load().(*sync.Map), + normalizedPlanMap: tsr.normalizedPlanMap.Load().(*sync.Map), + } + + // Reset data for next report. + *collectedDataPtr = make(map[string]*dataPoints) + tsr.normalizedSQLMap.Store(&sync.Map{}) + tsr.normalizedPlanMap.Store(&sync.Map{}) + + // Send to report channel. When channel is full, data will be dropped. + select { + case tsr.reportDataChan <- data: + default: + } +} + +type reportData struct { + collectedData map[string]*dataPoints + normalizedSQLMap *sync.Map + normalizedPlanMap *sync.Map +} + +// prepareReportDataForSending prepares the data that need to reported. +func (tsr *RemoteTopSQLReporter) prepareReportDataForSending(data reportData) (sqlMetas []*tipb.SQLMeta, planMetas []*tipb.PlanMeta, records []*tipb.CPUTimeRecord) { + sqlMetas = make([]*tipb.SQLMeta, 0, len(data.collectedData)) + data.normalizedSQLMap.Range(func(key, value interface{}) bool { + sqlMetas = append(sqlMetas, &tipb.SQLMeta{ + SqlDigest: []byte(key.(string)), + NormalizedSql: value.(string), + }) + return true + }) + + planMetas = make([]*tipb.PlanMeta, 0, len(data.collectedData)) + data.normalizedPlanMap.Range(func(key, value interface{}) bool { + planDecoded, err := tsr.planBinaryDecoder(value.(string)) + if err != nil { + logutil.BgLogger().Warn("[top-sql] decode plan failed", zap.Error(err)) + return true + } + planMetas = append(planMetas, &tipb.PlanMeta{ + PlanDigest: []byte(key.(string)), + NormalizedPlan: planDecoded, + }) + return true + }) + + records = make([]*tipb.CPUTimeRecord, 0, len(data.collectedData)) + for _, value := range data.collectedData { + records = append(records, &tipb.CPUTimeRecord{ + TimestampList: value.TimestampList, + CpuTimeMsList: value.CPUTimeMsList, + SqlDigest: value.SQLDigest, + PlanDigest: value.PlanDigest, + }) + } + return sqlMetas, planMetas, records +} + +// reportWorker sends data to the gRPC endpoint from the `reportDataChan` one by one. +func (tsr *RemoteTopSQLReporter) reportWorker() { + defer util.Recover("top-sql", "reportWorker", nil, false) + + for { + select { + case data := <-tsr.reportDataChan: + // When `reportDataChan` receives something, there could be ongoing `RegisterSQL` and `RegisterPlan` running, + // who writes to the data structure that `data` contains. So we wait for a little while to ensure that + // these writes are finished. + time.Sleep(time.Millisecond * 100) + tsr.doReport(data) + case <-tsr.ctx.Done(): + return + } + } +} + +func (tsr *RemoteTopSQLReporter) doReport(data reportData) { + defer util.Recover("top-sql", "doReport", nil, false) + + sqlMetas, planMetas, records := tsr.prepareReportDataForSending(data) + agentAddr := variable.TopSQLVariable.AgentAddress.Load() + + ctx, cancel := context.WithTimeout(tsr.ctx, reportTimeout) + err := tsr.client.Send(ctx, agentAddr, sqlMetas, planMetas, records) + if err != nil { + logutil.BgLogger().Warn("[top-sql] client failed to send data", zap.Error(err)) + } + cancel() +} diff --git a/util/topsql/reporter/reporter_test.go b/util/topsql/reporter/reporter_test.go new file mode 100644 index 0000000000000..7962d970b97af --- /dev/null +++ b/util/topsql/reporter/reporter_test.go @@ -0,0 +1,192 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package reporter + +import ( + "strconv" + "strings" + "testing" + "time" + + . "github.com/pingcap/check" + "github.com/pingcap/tidb/sessionctx/variable" + "github.com/pingcap/tidb/util/topsql/reporter/mock" + "github.com/pingcap/tidb/util/topsql/tracecpu" +) + +const ( + maxSQLNum = 5000 +) + +func TestT(t *testing.T) { + TestingT(t) +} + +var _ = SerialSuites(&testTopSQLReporter{}) + +type testTopSQLReporter struct{} + +func (s *testTopSQLReporter) SetUpSuite(c *C) {} + +func (s *testTopSQLReporter) SetUpTest(c *C) {} + +func testPlanBinaryDecoderFunc(plan string) (string, error) { + return plan, nil +} + +func populateCache(tsr *RemoteTopSQLReporter, begin, end int, timestamp uint64) { + // register normalized sql + for i := begin; i < end; i++ { + key := []byte("sqlDigest" + strconv.Itoa(i+1)) + value := "sqlNormalized" + strconv.Itoa(i+1) + tsr.RegisterSQL(key, value) + } + // register normalized plan + for i := begin; i < end; i++ { + key := []byte("planDigest" + strconv.Itoa(i+1)) + value := "planNormalized" + strconv.Itoa(i+1) + tsr.RegisterPlan(key, value) + } + // collect + var records []tracecpu.SQLCPUTimeRecord + for i := begin; i < end; i++ { + records = append(records, tracecpu.SQLCPUTimeRecord{ + SQLDigest: []byte("sqlDigest" + strconv.Itoa(i+1)), + PlanDigest: []byte("planDigest" + strconv.Itoa(i+1)), + CPUTimeMs: uint32(i + 1), + }) + } + tsr.Collect(timestamp, records) + // sleep a while for the asynchronous collect + time.Sleep(100 * time.Millisecond) +} + +func setupRemoteTopSQLReporter(maxStatementsNum, interval int, addr string) *RemoteTopSQLReporter { + variable.TopSQLVariable.MaxStatementCount.Store(int64(maxStatementsNum)) + variable.TopSQLVariable.ReportIntervalSeconds.Store(int64(interval)) + variable.TopSQLVariable.AgentAddress.Store(addr) + + rc := NewGRPCReportClient() + ts := NewRemoteTopSQLReporter(rc, testPlanBinaryDecoderFunc) + return ts +} + +func initializeCache(maxStatementsNum, interval int, addr string) *RemoteTopSQLReporter { + ts := setupRemoteTopSQLReporter(maxStatementsNum, interval, addr) + populateCache(ts, 0, maxStatementsNum, 1) + return ts +} + +func (s *testTopSQLReporter) TestCollectAndSendBatch(c *C) { + agentServer, err := mock.StartMockAgentServer() + c.Assert(err, IsNil) + defer agentServer.Stop() + + tsr := setupRemoteTopSQLReporter(maxSQLNum, 1, agentServer.Address()) + defer tsr.Close() + populateCache(tsr, 0, maxSQLNum, 1) + + agentServer.WaitServerCollect(maxSQLNum, time.Second*5) + + c.Assert(agentServer.GetRecords(), HasLen, maxSQLNum) + + // check for equality of server received batch and the original data + records := agentServer.GetRecords() + sqlMetas := agentServer.GetSQLMetas() + planMetas := agentServer.GetPlanMetas() + for _, req := range records { + id := 0 + prefix := "sqlDigest" + if strings.HasPrefix(string(req.SqlDigest), prefix) { + n, err := strconv.Atoi(string(req.SqlDigest)[len(prefix):]) + c.Assert(err, IsNil) + id = n + } + c.Assert(req.CpuTimeMsList, HasLen, 1) + for i := range req.CpuTimeMsList { + c.Assert(req.CpuTimeMsList[i], Equals, uint32(id)) + } + c.Assert(req.TimestampList, HasLen, 1) + for i := range req.TimestampList { + c.Assert(req.TimestampList[i], Equals, uint64(1)) + } + normalizedSQL, exist := sqlMetas[string(req.SqlDigest)] + c.Assert(exist, IsTrue) + c.Assert(normalizedSQL, Equals, "sqlNormalized"+strconv.Itoa(id)) + normalizedPlan, exist := planMetas[string(req.PlanDigest)] + c.Assert(exist, IsTrue) + c.Assert(normalizedPlan, Equals, "planNormalized"+strconv.Itoa(id)) + } +} + +func (s *testTopSQLReporter) TestCollectAndEvicted(c *C) { + agentServer, err := mock.StartMockAgentServer() + c.Assert(err, IsNil) + defer agentServer.Stop() + + tsr := setupRemoteTopSQLReporter(maxSQLNum, 1, agentServer.Address()) + defer tsr.Close() + populateCache(tsr, 0, maxSQLNum*2, 2) + + agentServer.WaitServerCollect(maxSQLNum, time.Second*10) + + c.Assert(agentServer.GetRecords(), HasLen, maxSQLNum) + + // check for equality of server received batch and the original data + records := agentServer.GetRecords() + sqlMetas := agentServer.GetSQLMetas() + planMetas := agentServer.GetPlanMetas() + for _, req := range records { + id := 0 + prefix := "sqlDigest" + if strings.HasPrefix(string(req.SqlDigest), prefix) { + n, err := strconv.Atoi(string(req.SqlDigest)[len(prefix):]) + c.Assert(err, IsNil) + id = n + } + c.Assert(id >= maxSQLNum, IsTrue) + c.Assert(req.CpuTimeMsList, HasLen, 1) + for i := range req.CpuTimeMsList { + c.Assert(req.CpuTimeMsList[i], Equals, uint32(id)) + } + c.Assert(req.TimestampList, HasLen, 1) + for i := range req.TimestampList { + c.Assert(req.TimestampList[i], Equals, uint64(2)) + } + normalizedSQL, exist := sqlMetas[string(req.SqlDigest)] + c.Assert(exist, IsTrue) + c.Assert(normalizedSQL, Equals, "sqlNormalized"+strconv.Itoa(id)) + normalizedPlan, exist := planMetas[string(req.PlanDigest)] + c.Assert(exist, IsTrue) + c.Assert(normalizedPlan, Equals, "planNormalized"+strconv.Itoa(id)) + } +} + +func BenchmarkTopSQL_CollectAndIncrementFrequency(b *testing.B) { + tsr := initializeCache(maxSQLNum, 120, ":23333") + for i := 0; i < b.N; i++ { + populateCache(tsr, 0, maxSQLNum, uint64(i)) + } +} + +func BenchmarkTopSQL_CollectAndEvict(b *testing.B) { + tsr := initializeCache(maxSQLNum, 120, ":23333") + begin := 0 + end := maxSQLNum + for i := 0; i < b.N; i++ { + begin += maxSQLNum + end += maxSQLNum + populateCache(tsr, begin, end, uint64(i)) + } +} diff --git a/util/topsql/topsql.go b/util/topsql/topsql.go new file mode 100644 index 0000000000000..39e26b085a6fa --- /dev/null +++ b/util/topsql/topsql.go @@ -0,0 +1,85 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package topsql + +import ( + "context" + "runtime/pprof" + + "github.com/pingcap/parser" + "github.com/pingcap/tidb/util/plancodec" + "github.com/pingcap/tidb/util/topsql/reporter" + "github.com/pingcap/tidb/util/topsql/tracecpu" +) + +var globalTopSQLReport reporter.TopSQLReporter + +// SetupTopSQL sets up the top-sql worker. +func SetupTopSQL() { + rc := reporter.NewGRPCReportClient() + globalTopSQLReport = reporter.NewRemoteTopSQLReporter(rc, plancodec.DecodeNormalizedPlan) + tracecpu.GlobalSQLCPUProfiler.SetCollector(globalTopSQLReport) + tracecpu.GlobalSQLCPUProfiler.Run() +} + +// Close uses to close and release the top sql resource. +func Close() { + if globalTopSQLReport != nil { + globalTopSQLReport.Close() + } +} + +// AttachSQLInfo attach the sql information info top sql. +func AttachSQLInfo(ctx context.Context, normalizedSQL string, sqlDigest *parser.Digest, normalizedPlan string, planDigest *parser.Digest) context.Context { + if len(normalizedSQL) == 0 || sqlDigest == nil || len(sqlDigest.Bytes()) == 0 { + return ctx + } + var sqlDigestBytes, planDigestBytes []byte + sqlDigestBytes = sqlDigest.Bytes() + if planDigest != nil { + planDigestBytes = planDigest.Bytes() + } + ctx = tracecpu.CtxWithDigest(ctx, sqlDigestBytes, planDigestBytes) + pprof.SetGoroutineLabels(ctx) + + if len(normalizedPlan) == 0 || len(planDigestBytes) == 0 { + // If plan digest is '', indicate it is the first time to attach the SQL info, since it only know the sql digest. + linkSQLTextWithDigest(sqlDigestBytes, normalizedSQL) + } else { + linkPlanTextWithDigest(planDigestBytes, normalizedPlan) + } + return ctx +} + +func linkSQLTextWithDigest(sqlDigest []byte, normalizedSQL string) { + c := tracecpu.GlobalSQLCPUProfiler.GetCollector() + if c == nil { + return + } + topc, ok := c.(reporter.TopSQLReporter) + if ok { + topc.RegisterSQL(sqlDigest, normalizedSQL) + } +} + +func linkPlanTextWithDigest(planDigest []byte, normalizedPlan string) { + c := tracecpu.GlobalSQLCPUProfiler.GetCollector() + if c == nil { + return + } + topc, ok := c.(reporter.TopSQLReporter) + if ok { + topc.RegisterPlan(planDigest, normalizedPlan) + } +} diff --git a/util/topsql/topsql_test.go b/util/topsql/topsql_test.go new file mode 100644 index 0000000000000..8d7500900cb42 --- /dev/null +++ b/util/topsql/topsql_test.go @@ -0,0 +1,224 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package topsql_test + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/google/pprof/profile" + . "github.com/pingcap/check" + "github.com/pingcap/parser" + "github.com/pingcap/tidb/sessionctx/variable" + "github.com/pingcap/tidb/util/topsql" + "github.com/pingcap/tidb/util/topsql/reporter" + mockServer "github.com/pingcap/tidb/util/topsql/reporter/mock" + "github.com/pingcap/tidb/util/topsql/tracecpu" + "github.com/pingcap/tidb/util/topsql/tracecpu/mock" +) + +func TestT(t *testing.T) { + CustomVerboseFlag = true + TestingT(t) +} + +var _ = SerialSuites(&testSuite{}) + +type testSuite struct{} + +type collectorWrapper struct { + reporter.TopSQLReporter +} + +func (s *testSuite) SetUpSuite(c *C) { + variable.TopSQLVariable.Enable.Store(true) + variable.TopSQLVariable.AgentAddress.Store("mock") + variable.TopSQLVariable.PrecisionSeconds.Store(1) + tracecpu.GlobalSQLCPUProfiler.Run() +} + +func (s *testSuite) TestTopSQLCPUProfile(c *C) { + collector := mock.NewTopSQLCollector() + tracecpu.GlobalSQLCPUProfiler.SetCollector(&collectorWrapper{collector}) + reqs := []struct { + sql string + plan string + }{ + {"select * from t where a=?", "point-get"}, + {"select * from t where a>?", "table-scan"}, + {"insert into t values (?)", ""}, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for _, req := range reqs { + go func(sql, plan string) { + for { + select { + case <-ctx.Done(): + return + default: + s.mockExecuteSQL(sql, plan) + } + } + }(req.sql, req.plan) + } + + // test for StartCPUProfile. + buf := bytes.NewBuffer(nil) + err := tracecpu.StartCPUProfile(buf) + c.Assert(err, IsNil) + collector.WaitCollectCnt(2) + err = tracecpu.StopCPUProfile() + c.Assert(err, IsNil) + _, err = profile.Parse(buf) + c.Assert(err, IsNil) + + for _, req := range reqs { + stats := collector.GetSQLStatsBySQLWithRetry(req.sql, len(req.plan) > 0) + c.Assert(len(stats), Equals, 1) + sql := collector.GetSQL(stats[0].SQLDigest) + plan := collector.GetPlan(stats[0].PlanDigest) + c.Assert(sql, Equals, req.sql) + c.Assert(plan, Equals, req.plan) + } +} + +func (s *testSuite) TestIsEnabled(c *C) { + s.setTopSQLEnable(false) + c.Assert(tracecpu.GlobalSQLCPUProfiler.IsEnabled(), IsFalse) + + s.setTopSQLEnable(true) + err := tracecpu.StartCPUProfile(bytes.NewBuffer(nil)) + c.Assert(err, IsNil) + c.Assert(tracecpu.GlobalSQLCPUProfiler.IsEnabled(), IsTrue) + s.setTopSQLEnable(false) + c.Assert(tracecpu.GlobalSQLCPUProfiler.IsEnabled(), IsTrue) + err = tracecpu.StopCPUProfile() + c.Assert(err, IsNil) + + s.setTopSQLEnable(false) + c.Assert(tracecpu.GlobalSQLCPUProfiler.IsEnabled(), IsFalse) + s.setTopSQLEnable(true) + c.Assert(tracecpu.GlobalSQLCPUProfiler.IsEnabled(), IsTrue) +} + +func (s *testSuite) TestTopSQLReporter(c *C) { + server, err := mockServer.StartMockAgentServer() + c.Assert(err, IsNil) + variable.TopSQLVariable.MaxStatementCount.Store(200) + variable.TopSQLVariable.ReportIntervalSeconds.Store(1) + variable.TopSQLVariable.AgentAddress.Store(server.Address()) + + client := reporter.NewGRPCReportClient() + report := reporter.NewRemoteTopSQLReporter(client, func(s string) (string, error) { + return s, nil + }) + defer report.Close() + + tracecpu.GlobalSQLCPUProfiler.SetCollector(&collectorWrapper{report}) + reqs := []struct { + sql string + plan string + }{ + {"select * from t where a=?", "point-get"}, + {"select * from t where a>?", "table-scan"}, + {"insert into t values (?)", ""}, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sqlMap := make(map[string]string) + sql2plan := make(map[string]string) + for _, req := range reqs { + sql2plan[req.sql] = req.plan + sqlDigest := mock.GenSQLDigest(req.sql) + sqlMap[string(sqlDigest.Bytes())] = req.sql + + go func(sql, plan string) { + for { + select { + case <-ctx.Done(): + return + default: + s.mockExecuteSQL(sql, plan) + } + } + }(req.sql, req.plan) + } + + server.WaitServerCollect(6, time.Second*5) + + records := server.GetRecords() + sqlMetas := server.GetSQLMetas() + planMetas := server.GetPlanMetas() + + checkSQLPlanMap := map[string]struct{}{} + for _, req := range records { + c.Assert(len(req.CpuTimeMsList) > 0, IsTrue) + c.Assert(req.CpuTimeMsList[0] > 0, IsTrue) + c.Assert(req.TimestampList[0] > 0, IsTrue) + normalizedSQL, exist := sqlMetas[string(req.SqlDigest)] + c.Assert(exist, IsTrue) + expectedNormalizedSQL, exist := sqlMap[string(req.SqlDigest)] + c.Assert(exist, IsTrue) + c.Assert(normalizedSQL, Equals, expectedNormalizedSQL) + + expectedNormalizedPlan := sql2plan[expectedNormalizedSQL] + if expectedNormalizedPlan == "" || len(req.PlanDigest) == 0 { + c.Assert(len(req.PlanDigest), Equals, 0) + continue + } + normalizedPlan, exist := planMetas[string(req.PlanDigest)] + c.Assert(exist, IsTrue) + c.Assert(normalizedPlan, Equals, expectedNormalizedPlan) + checkSQLPlanMap[expectedNormalizedSQL] = struct{}{} + } + c.Assert(len(checkSQLPlanMap) == 2, IsTrue) +} + +func (s *testSuite) setTopSQLEnable(enabled bool) { + variable.TopSQLVariable.Enable.Store(enabled) +} + +func (s *testSuite) mockExecuteSQL(sql, plan string) { + ctx := context.Background() + sqlDigest := mock.GenSQLDigest(sql) + topsql.AttachSQLInfo(ctx, sql, sqlDigest, "", nil) + s.mockExecute(time.Millisecond * 100) + planDigest := genDigest(plan) + topsql.AttachSQLInfo(ctx, sql, sqlDigest, plan, planDigest) + s.mockExecute(time.Millisecond * 300) +} + +func genDigest(str string) *parser.Digest { + if str == "" { + return parser.NewDigest(nil) + } + return parser.DigestNormalized(str) +} + +func (s *testSuite) mockExecute(d time.Duration) { + start := time.Now() + for { + for i := 0; i < 10e5; i++ { + } + if time.Since(start) > d { + return + } + } +} diff --git a/util/topsql/tracecpu/mock/mock.go b/util/topsql/tracecpu/mock/mock.go new file mode 100644 index 0000000000000..a48100eb2eebb --- /dev/null +++ b/util/topsql/tracecpu/mock/mock.go @@ -0,0 +1,176 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package mock + +import ( + "bytes" + "sync" + "time" + + "github.com/pingcap/parser" + "github.com/pingcap/tidb/util/hack" + "github.com/pingcap/tidb/util/topsql/tracecpu" + "github.com/uber-go/atomic" +) + +// TopSQLCollector uses for testing. +type TopSQLCollector struct { + sync.Mutex + // sql_digest -> normalized SQL + sqlMap map[string]string + // plan_digest -> normalized plan + planMap map[string]string + // (sql + plan_digest) -> sql stats + sqlStatsMap map[string]*tracecpu.SQLCPUTimeRecord + collectCnt atomic.Int64 +} + +// NewTopSQLCollector uses for testing. +func NewTopSQLCollector() *TopSQLCollector { + return &TopSQLCollector{ + sqlMap: make(map[string]string), + planMap: make(map[string]string), + sqlStatsMap: make(map[string]*tracecpu.SQLCPUTimeRecord), + } +} + +// Collect uses for testing. +func (c *TopSQLCollector) Collect(ts uint64, stats []tracecpu.SQLCPUTimeRecord) { + defer c.collectCnt.Inc() + if len(stats) == 0 { + return + } + c.Lock() + defer c.Unlock() + for _, stmt := range stats { + hash := c.hash(stmt) + stats, ok := c.sqlStatsMap[hash] + if !ok { + stats = &tracecpu.SQLCPUTimeRecord{ + SQLDigest: stmt.SQLDigest, + PlanDigest: stmt.PlanDigest, + } + c.sqlStatsMap[hash] = stats + } + stats.CPUTimeMs += stmt.CPUTimeMs + } +} + +// GetSQLStatsBySQLWithRetry uses for testing. +func (c *TopSQLCollector) GetSQLStatsBySQLWithRetry(sql string, planIsNotNull bool) []*tracecpu.SQLCPUTimeRecord { + after := time.After(time.Second * 10) + for { + select { + case <-after: + return nil + default: + } + stats := c.GetSQLStatsBySQL(sql, planIsNotNull) + if len(stats) > 0 { + return stats + } + c.WaitCollectCnt(1) + } +} + +// GetSQLStatsBySQL uses for testing. +func (c *TopSQLCollector) GetSQLStatsBySQL(sql string, planIsNotNull bool) []*tracecpu.SQLCPUTimeRecord { + stats := make([]*tracecpu.SQLCPUTimeRecord, 0, 2) + sqlDigest := GenSQLDigest(sql) + c.Lock() + for _, stmt := range c.sqlStatsMap { + if bytes.Equal(stmt.SQLDigest, sqlDigest.Bytes()) { + if planIsNotNull { + plan := c.planMap[string(stmt.PlanDigest)] + if len(plan) > 0 { + stats = append(stats, stmt) + } + } else { + stats = append(stats, stmt) + } + } + } + c.Unlock() + return stats +} + +// GetSQL uses for testing. +func (c *TopSQLCollector) GetSQL(sqlDigest []byte) string { + c.Lock() + sql := c.sqlMap[string(sqlDigest)] + c.Unlock() + return sql +} + +// GetPlan uses for testing. +func (c *TopSQLCollector) GetPlan(planDigest []byte) string { + c.Lock() + plan := c.planMap[string(planDigest)] + c.Unlock() + return plan +} + +// RegisterSQL uses for testing. +func (c *TopSQLCollector) RegisterSQL(sqlDigest []byte, normalizedSQL string) { + digestStr := string(hack.String(sqlDigest)) + c.Lock() + _, ok := c.sqlMap[digestStr] + if !ok { + c.sqlMap[digestStr] = normalizedSQL + } + c.Unlock() + +} + +// RegisterPlan uses for testing. +func (c *TopSQLCollector) RegisterPlan(planDigest []byte, normalizedPlan string) { + digestStr := string(hack.String(planDigest)) + c.Lock() + _, ok := c.planMap[digestStr] + if !ok { + c.planMap[digestStr] = normalizedPlan + } + c.Unlock() +} + +// WaitCollectCnt uses for testing. +func (c *TopSQLCollector) WaitCollectCnt(count int64) { + timeout := time.After(time.Second * 10) + end := c.collectCnt.Load() + count + for { + // Wait for reporter to collect sql stats count >= expected count + if c.collectCnt.Load() >= end { + break + } + select { + case <-timeout: + break + default: + time.Sleep(time.Millisecond * 10) + } + } +} + +// Close implements the interface. +func (c *TopSQLCollector) Close() {} + +func (c *TopSQLCollector) hash(stat tracecpu.SQLCPUTimeRecord) string { + return string(stat.SQLDigest) + string(stat.PlanDigest) +} + +// GenSQLDigest uses for testing. +func GenSQLDigest(sql string) *parser.Digest { + _, digest := parser.NormalizeDigest(sql) + return digest +} diff --git a/util/topsql/tracecpu/profile.go b/util/topsql/tracecpu/profile.go new file mode 100644 index 0000000000000..c068278b26169 --- /dev/null +++ b/util/topsql/tracecpu/profile.go @@ -0,0 +1,431 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package tracecpu + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "runtime/pprof" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/google/pprof/profile" + "github.com/pingcap/tidb/sessionctx/variable" + "github.com/pingcap/tidb/util" + "github.com/pingcap/tidb/util/hack" + "github.com/pingcap/tidb/util/logutil" + "go.uber.org/zap" +) + +const ( + labelSQL = "sql" + labelSQLDigest = "sql_digest" + labelPlanDigest = "plan_digest" +) + +// GlobalSQLCPUProfiler is the global SQL stats profiler. +var GlobalSQLCPUProfiler = newSQLCPUProfiler() + +// Collector uses to collect SQL execution cpu time. +type Collector interface { + // Collect uses to collect the SQL execution cpu time. + // ts is a Unix time, unit is second. + Collect(ts uint64, stats []SQLCPUTimeRecord) +} + +// SQLCPUTimeRecord represents a single record of how much cpu time a sql plan consumes in one second. +// +// PlanDigest can be empty, because: +// 1. some sql statements has no plan, like `COMMIT` +// 2. when a sql statement is being compiled, there's no plan yet +type SQLCPUTimeRecord struct { + SQLDigest []byte + PlanDigest []byte + CPUTimeMs uint32 +} + +type sqlCPUProfiler struct { + taskCh chan *profileData + + mu struct { + sync.Mutex + ept *exportProfileTask + } + collector atomic.Value +} + +var ( + defaultProfileBufSize = 100 * 1024 + profileBufPool = sync.Pool{ + New: func() interface{} { + return bytes.NewBuffer(make([]byte, 0, defaultProfileBufSize)) + }, + } +) + +// newSQLCPUProfiler create a sqlCPUProfiler. +func newSQLCPUProfiler() *sqlCPUProfiler { + return &sqlCPUProfiler{ + taskCh: make(chan *profileData, 128), + } +} + +func (sp *sqlCPUProfiler) Run() { + logutil.BgLogger().Info("cpu profiler started") + go sp.startCPUProfileWorker() + go sp.startAnalyzeProfileWorker() +} + +func (sp *sqlCPUProfiler) SetCollector(c Collector) { + sp.collector.Store(c) +} + +func (sp *sqlCPUProfiler) GetCollector() Collector { + c, ok := sp.collector.Load().(Collector) + if !ok || c == nil { + return nil + } + return c +} + +func (sp *sqlCPUProfiler) startCPUProfileWorker() { + defer util.Recover("top-sql", "profileWorker", nil, false) + for { + if sp.IsEnabled() { + sp.doCPUProfile() + } else { + time.Sleep(time.Second) + } + } +} + +func (sp *sqlCPUProfiler) doCPUProfile() { + intervalSecond := variable.TopSQLVariable.PrecisionSeconds.Load() + task := sp.newProfileTask() + if err := pprof.StartCPUProfile(task.buf); err != nil { + // Sleep a while before retry. + time.Sleep(time.Second) + sp.putTaskToBuffer(task) + return + } + ns := int64(time.Second)*intervalSecond - int64(time.Now().Nanosecond()) + time.Sleep(time.Nanosecond * time.Duration(ns)) + pprof.StopCPUProfile() + task.end = time.Now().Unix() + if task.end < 0 { + task.end = 0 + } + sp.taskCh <- task +} + +func (sp *sqlCPUProfiler) startAnalyzeProfileWorker() { + defer util.Recover("top-sql", "analyzeProfileWorker", nil, false) + for { + task := <-sp.taskCh + p, err := profile.ParseData(task.buf.Bytes()) + if err != nil { + logutil.BgLogger().Error("parse profile error", zap.Error(err)) + sp.putTaskToBuffer(task) + continue + } + stats := sp.parseCPUProfileBySQLLabels(p) + sp.handleExportProfileTask(p) + if c := sp.GetCollector(); c != nil { + c.Collect(uint64(task.end), stats) + } + sp.putTaskToBuffer(task) + } +} + +type profileData struct { + buf *bytes.Buffer + end int64 +} + +func (sp *sqlCPUProfiler) newProfileTask() *profileData { + buf := profileBufPool.Get().(*bytes.Buffer) + return &profileData{ + buf: buf, + } +} + +func (sp *sqlCPUProfiler) putTaskToBuffer(task *profileData) { + task.buf.Reset() + profileBufPool.Put(task.buf) +} + +// parseCPUProfileBySQLLabels uses to aggregate the cpu-profile sample data by sql_digest and plan_digest labels, +// output the TopSQLCPUTimeRecord slice. Want to know more information about profile labels, see https://rakyll.org/profiler-labels/ +// The sql_digest label is been set by `SetSQLLabels` function after parse the SQL. +// The plan_digest label is been set by `SetSQLAndPlanLabels` function after build the SQL plan. +// Since `sqlCPUProfiler` only care about the cpu time that consume by (sql_digest,plan_digest), the other sample data +// without those label will be ignore. +func (sp *sqlCPUProfiler) parseCPUProfileBySQLLabels(p *profile.Profile) []SQLCPUTimeRecord { + sqlMap := make(map[string]*sqlStats) + idx := len(p.SampleType) - 1 + for _, s := range p.Sample { + digests, ok := s.Label[labelSQLDigest] + if !ok || len(digests) == 0 { + continue + } + for _, digest := range digests { + stmt, ok := sqlMap[digest] + if !ok { + stmt = &sqlStats{ + plans: make(map[string]int64), + total: 0, + } + sqlMap[digest] = stmt + } + stmt.total += s.Value[idx] + + plans := s.Label[labelPlanDigest] + for _, plan := range plans { + stmt.plans[plan] += s.Value[idx] + } + } + } + return sp.createSQLStats(sqlMap) +} + +func (sp *sqlCPUProfiler) createSQLStats(sqlMap map[string]*sqlStats) []SQLCPUTimeRecord { + stats := make([]SQLCPUTimeRecord, 0, len(sqlMap)) + for sqlDigest, stmt := range sqlMap { + stmt.tune() + for planDigest, val := range stmt.plans { + stats = append(stats, SQLCPUTimeRecord{ + SQLDigest: []byte(sqlDigest), + PlanDigest: []byte(planDigest), + CPUTimeMs: uint32(time.Duration(val).Milliseconds()), + }) + } + } + return stats +} + +type sqlStats struct { + plans map[string]int64 + total int64 +} + +// tune use to adjust sql stats. Consider following situation: +// The `sqlStats` maybe: +// plans: { +// "table_scan": 200ms, // The cpu time of the sql that plan with `table_scan` is 200ms. +// "index_scan": 300ms, // The cpu time of the sql that plan with `table_scan` is 300ms. +// }, +// total: 600ms, // The total cpu time of the sql is 600ms. +// total_time - table_scan_time - index_scan_time = 100ms, and this 100ms means those sample data only contain the +// sql_digest label, doesn't contain the plan_digest label. This is cause by the `pprof profile` is base on sample, +// and the plan digest can only be set after optimizer generated execution plan. So the remain 100ms means the plan +// optimizer takes time to generated plan. +// After this tune function, the `sqlStats` become to: +// plans: { +// "" : 100ms, // 600 - 200 - 300 = 100ms, indicate the optimizer generated plan time cost. +// "table_scan": 200ms, +// "index_scan": 300ms, +// }, +// total: 600ms, +func (s *sqlStats) tune() { + if len(s.plans) == 0 { + s.plans[""] = s.total + return + } + planTotal := int64(0) + for _, v := range s.plans { + planTotal += v + } + optimize := s.total - planTotal + if optimize <= 0 { + return + } + s.plans[""] += optimize +} + +func (sp *sqlCPUProfiler) handleExportProfileTask(p *profile.Profile) { + sp.mu.Lock() + defer sp.mu.Unlock() + if sp.mu.ept == nil { + return + } + sp.mu.ept.mergeProfile(p) +} + +func (sp *sqlCPUProfiler) hasExportProfileTask() bool { + sp.mu.Lock() + has := sp.mu.ept != nil + sp.mu.Unlock() + return has +} + +// IsEnabled return true if it is(should be) enabled. It exports for tests. +func (sp *sqlCPUProfiler) IsEnabled() bool { + return variable.TopSQLEnabled() || sp.hasExportProfileTask() +} + +// StartCPUProfile same like pprof.StartCPUProfile. +// Because the GlobalSQLCPUProfiler keep calling pprof.StartCPUProfile to fetch SQL cpu stats, other place (such pprof profile HTTP API handler) call pprof.StartCPUProfile will be failed, +// other place should call tracecpu.StartCPUProfile instead of pprof.StartCPUProfile. +func StartCPUProfile(w io.Writer) error { + if GlobalSQLCPUProfiler.IsEnabled() { + return GlobalSQLCPUProfiler.startExportCPUProfile(w) + } + return pprof.StartCPUProfile(w) +} + +// StopCPUProfile same like pprof.StopCPUProfile. +// other place should call tracecpu.StopCPUProfile instead of pprof.StopCPUProfile. +func StopCPUProfile() error { + if GlobalSQLCPUProfiler.IsEnabled() { + return GlobalSQLCPUProfiler.stopExportCPUProfile() + } + pprof.StopCPUProfile() + return nil +} + +// CtxWithDigest wrap the ctx with sql digest, if plan digest is not null, wrap with plan digest too. +func CtxWithDigest(ctx context.Context, sqlDigest, planDigest []byte) context.Context { + if len(planDigest) == 0 { + return pprof.WithLabels(ctx, pprof.Labels(labelSQLDigest, string(hack.String(sqlDigest)))) + } + return pprof.WithLabels(ctx, pprof.Labels(labelSQLDigest, string(hack.String(sqlDigest)), + labelPlanDigest, string(hack.String(planDigest)))) +} + +func (sp *sqlCPUProfiler) startExportCPUProfile(w io.Writer) error { + sp.mu.Lock() + defer sp.mu.Unlock() + if sp.mu.ept != nil { + return errors.New("cpu profiling already in use") + } + sp.mu.ept = &exportProfileTask{w: w} + return nil +} + +func (sp *sqlCPUProfiler) stopExportCPUProfile() error { + sp.mu.Lock() + ept := sp.mu.ept + sp.mu.ept = nil + sp.mu.Unlock() + if ept.err != nil { + return ept.err + } + if w := ept.w; w != nil && ept.cpuProfile != nil { + sp.removeLabel(ept.cpuProfile) + return ept.cpuProfile.Write(w) + } + return nil +} + +// removeLabel uses to remove labels for export cpu profile data. +// Since the sql_digest and plan_digest label is strange for other users. +// If `variable.EnablePProfSQLCPU` is true means wanto keep the `sql` label, otherwise, remove the `sql` label too. +func (sp *sqlCPUProfiler) removeLabel(p *profile.Profile) { + if p == nil { + return + } + keepLabelSQL := variable.EnablePProfSQLCPU.Load() + for _, s := range p.Sample { + for k := range s.Label { + switch k { + case labelSQL: + if !keepLabelSQL { + delete(s.Label, k) + } + case labelSQLDigest, labelPlanDigest: + delete(s.Label, k) + } + } + } +} + +type exportProfileTask struct { + cpuProfile *profile.Profile + err error + w io.Writer +} + +func (t *exportProfileTask) mergeProfile(p *profile.Profile) { + if t.err != nil || p == nil { + return + } + ps := make([]*profile.Profile, 0, 2) + if t.cpuProfile != nil { + ps = append(ps, t.cpuProfile) + } + ps = append(ps, p) + t.cpuProfile, t.err = profile.Merge(ps) +} + +// ProfileHTTPHandler is same as pprof.Profile. +// The difference is ProfileHTTPHandler uses tracecpu.StartCPUProfile/StopCPUProfile to fetch profile data. +func ProfileHTTPHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Content-Type-Options", "nosniff") + sec, err := strconv.ParseInt(r.FormValue("seconds"), 10, 64) + if sec <= 0 || err != nil { + sec = 30 + } + + if durationExceedsWriteTimeout(r, float64(sec)) { + serveError(w, http.StatusBadRequest, "profile duration exceeds server's WriteTimeout") + return + } + + // Set Content Type assuming StartCPUProfile will work, + // because if it does it starts writing. + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", `attachment; filename="profile"`) + + err = StartCPUProfile(w) + if err != nil { + serveError(w, http.StatusInternalServerError, "Could not enable CPU profiling: "+err.Error()) + return + } + // TODO: fix me. + // This can be fixed by always starts a 1 second profiling one by one, + // but to aggregate (merge) multiple profiles into one according to the precision. + // |<-- 1s -->| + // -|----------|----------|----------|----------|----------|-----------|-----> Background profile task timeline. + // |________________________________| + // (start cpu profile) v v (stop cpu profile) // expected profile timeline + // |________________________________| // actual profile timeline + time.Sleep(time.Second * time.Duration(sec)) + err = StopCPUProfile() + if err != nil { + serveError(w, http.StatusInternalServerError, "Could not enable CPU profiling: "+err.Error()) + return + } +} + +func durationExceedsWriteTimeout(r *http.Request, seconds float64) bool { + srv, ok := r.Context().Value(http.ServerContextKey).(*http.Server) + return ok && srv.WriteTimeout != 0 && seconds >= srv.WriteTimeout.Seconds() +} + +func serveError(w http.ResponseWriter, status int, txt string) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("X-Go-Pprof", "1") + w.Header().Del("Content-Disposition") + w.WriteHeader(status) + _, err := fmt.Fprintln(w, txt) + if err != nil { + logutil.BgLogger().Info("write http response error", zap.Error(err)) + } +}