From 42b4ad4fe384cc7a75da19e7d6d957f10544857a Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Fri, 7 Aug 2020 17:20:08 +0800 Subject: [PATCH 1/3] planner,executor: enable plan cache for partition table --- executor/prepared.go | 10 +++++++++- planner/core/cacheable_checker.go | 8 +++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/executor/prepared.go b/executor/prepared.go index c5c3b835d003d..98aae1f3fbd6e 100644 --- a/executor/prepared.go +++ b/executor/prepared.go @@ -177,7 +177,15 @@ func (e *PrepareExec) Next(ctx context.Context, req *chunk.Chunk) error { SchemaVersion: e.is.SchemaMetaVersion(), } - prepared.UseCache = plannercore.PreparedPlanCacheEnabled() && plannercore.Cacheable(stmt, e.is) + if !plannercore.PreparedPlanCacheEnabled() { + prepared.UseCache = false + } else { + if tryOldPartitionImplementation(e.ctx) { + prepared.UseCache = plannercore.Cacheable(stmt, e.is) + } else { + prepared.UseCache = plannercore.Cacheable(stmt, nil) + } + } // We try to build the real statement of preparedStmt. for i := range prepared.Params { diff --git a/planner/core/cacheable_checker.go b/planner/core/cacheable_checker.go index 64a6769185856..a2a4be7063385 100644 --- a/planner/core/cacheable_checker.go +++ b/planner/core/cacheable_checker.go @@ -117,9 +117,11 @@ func (checker *cacheableChecker) Enter(in ast.Node) (out ast.Node, skipChildren return in, true } case *ast.TableName: - if checker.isPartitionTable(node) { - checker.cacheable = false - return in, true + if checker.schema != nil { + if checker.isPartitionTable(node) { + checker.cacheable = false + return in, true + } } } return in, false From 72bc7d8341dc174fce89b54730fa9542798f7d2d Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Thu, 3 Sep 2020 21:21:21 +0800 Subject: [PATCH 2/3] address comment --- planner/core/common_plans.go | 69 ------------------- planner/core/prepare_test.go | 129 ++++++++++++++++++----------------- 2 files changed, 67 insertions(+), 131 deletions(-) diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 8a8a9c4cab7fe..a0f2f14e53e9b 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -30,7 +30,6 @@ import ( "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/privilege" "github.com/pingcap/tidb/sessionctx" - "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" @@ -441,15 +440,6 @@ func (e *Execute) rebuildRange(p Plan) error { if err != nil { return err } - if ts.Table.Partition != nil && ts.Table.Partition.Type == model.PartitionTypeHash { - pID, err := rebuildNewTableIDFromTable(e.ctx, ts, sc, pkCol) - if err != nil { - return err - } - if pID != -1 { - ts.physicalTableID = pID - } - } } else { ts.Ranges = ranger.FullIntRange(false) } @@ -460,32 +450,12 @@ func (e *Execute) rebuildRange(p Plan) error { if err != nil { return err } - if is.Table.Partition != nil && is.Table.Partition.Type == model.PartitionTypeHash { - pID, err := rebuildNewTableIDFromIndex(e.ctx, is, sc) - if err != nil { - return err - } - if pID != -1 { - is.physicalTableID = pID - } - } case *PhysicalIndexLookUpReader: is := x.IndexPlans[0].(*PhysicalIndexScan) is.Ranges, err = e.buildRangeForIndexScan(sctx, is) if err != nil { return err } - if is.Table.Partition != nil && is.Table.Partition.Type == model.PartitionTypeHash { - pID, err := rebuildNewTableIDFromIndex(e.ctx, is, sc) - if err != nil { - return err - } - if pID != -1 { - is.physicalTableID = pID - tblScan := x.TablePlans[0].(*PhysicalTableScan) - tblScan.physicalTableID = pID - } - } case *PointGetPlan: // if access condition is not nil, which means it's a point get generated by cbo. if x.AccessConditions != nil { @@ -1292,42 +1262,3 @@ func locateHashPartition(ctx sessionctx.Context, expr expression.Expression, pi } return int(ret % int64(pi.Num)), nil } - -func getPhysicalTableIDForPartition(ctx sessionctx.Context, pi *model.PartitionInfo, schema *expression.Schema, names types.NameSlice, val []types.Datum) (int64, error) { - expr, err := expression.ParseSimpleExprsWithNames(ctx, pi.Expr, schema, names) - if err != nil { - return 0, err - } - pos, err := locateHashPartition(ctx, expr[0], pi, val) - if err != nil { - return 0, err - } - pID := pi.Definitions[pos].ID - return pID, nil -} - -func rebuildNewTableIDFromIndex(ctx sessionctx.Context, is *PhysicalIndexScan, sc *stmtctx.StatementContext) (int64, error) { - pi := is.Table.Partition - if pi.Type == model.PartitionTypeHash && len(is.Ranges) == 1 && is.Ranges[0].IsPoint(sc) { - schema, names := buildSchemaAndNameFromIndex(is.IdxCols, is.DBName, is.Table, is.Index) - pID, err := getPhysicalTableIDForPartition(ctx, pi, schema, names, is.Ranges[0].LowVal) - if err != nil { - return -1, err - } - return pID, nil - } - return -1, nil -} - -func rebuildNewTableIDFromTable(ctx sessionctx.Context, ts *PhysicalTableScan, sc *stmtctx.StatementContext, pkCol *expression.Column) (int64, error) { - pi := ts.Table.Partition - if pi.Type == model.PartitionTypeHash && len(ts.Ranges) == 1 && ts.Ranges[0].IsPoint(sc) { - schema, names := buildSchemaAndNameFromPKCol(pkCol, ts.DBName, ts.Table) - pID, err := getPhysicalTableIDForPartition(ctx, pi, schema, names, ts.Ranges[0].LowVal) - if err != nil { - return -1, err - } - return pID, nil - } - return -1, nil -} diff --git a/planner/core/prepare_test.go b/planner/core/prepare_test.go index 296cdb00a8dee..57cbc4ec4dd8c 100644 --- a/planner/core/prepare_test.go +++ b/planner/core/prepare_test.go @@ -411,68 +411,73 @@ func (s *testPrepareSerialSuite) TestPrepareCacheForPartition(c *C) { c.Assert(err, IsNil) tk.MustExec("use test") - // Test for PointGet and IndexRead. - tk.MustExec("drop table if exists t_index_read") - tk.MustExec("create table t_index_read (id int, k int, c varchar(10), primary key (id, k)) partition by hash(id+k) partitions 10") - tk.MustExec("insert into t_index_read values (1, 2, 'abc'), (3, 4, 'def'), (5, 6, 'xyz')") - tk.MustExec("prepare stmt1 from 'select c from t_index_read where id = ? and k = ?;'") - tk.MustExec("set @id=1, @k=2") - // When executing one statement at the first time, we don't use cache, so we need to execute it at least twice to test the cache. - tk.MustQuery("execute stmt1 using @id, @k").Check(testkit.Rows("abc")) - tk.MustQuery("execute stmt1 using @id, @k").Check(testkit.Rows("abc")) - tk.MustExec("set @id=5, @k=6") - tk.MustQuery("execute stmt1 using @id, @k").Check(testkit.Rows("xyz")) - tk.MustExec("prepare stmt2 from 'select c from t_index_read where id = ? and k = ? and 1 = 1;'") - tk.MustExec("set @id=1, @k=2") - tk.MustQuery("execute stmt2 using @id, @k").Check(testkit.Rows("abc")) - tk.MustQuery("execute stmt2 using @id, @k").Check(testkit.Rows("abc")) - tk.MustExec("set @id=5, @k=6") - tk.MustQuery("execute stmt2 using @id, @k").Check(testkit.Rows("xyz")) - // Test for TableScan. - tk.MustExec("drop table if exists t_table_read") - tk.MustExec("create table t_table_read (id int, k int, c varchar(10), primary key(id)) partition by hash(id) partitions 10") - tk.MustExec("insert into t_table_read values (1, 2, 'abc'), (3, 4, 'def'), (5, 6, 'xyz')") - tk.MustExec("prepare stmt3 from 'select c from t_index_read where id = ?;'") - tk.MustExec("set @id=1") - // When executing one statement at the first time, we don't use cache, so we need to execute it at least twice to test the cache. - tk.MustQuery("execute stmt3 using @id").Check(testkit.Rows("abc")) - tk.MustQuery("execute stmt3 using @id").Check(testkit.Rows("abc")) - tk.MustExec("set @id=5") - tk.MustQuery("execute stmt3 using @id").Check(testkit.Rows("xyz")) - tk.MustExec("prepare stmt4 from 'select c from t_index_read where id = ? and k = ?'") - tk.MustExec("set @id=1, @k=2") - tk.MustQuery("execute stmt4 using @id, @k").Check(testkit.Rows("abc")) - tk.MustQuery("execute stmt4 using @id, @k").Check(testkit.Rows("abc")) - tk.MustExec("set @id=5, @k=6") - tk.MustQuery("execute stmt4 using @id, @k").Check(testkit.Rows("xyz")) - // Query on range partition tables should not raise error. - tk.MustExec("create table t_range_index (id int, k int, c varchar(10), primary key(id)) partition by range(id) ( PARTITION p0 VALUES LESS THAN (4), PARTITION p1 VALUES LESS THAN (14),PARTITION p2 VALUES LESS THAN (20) )") - tk.MustExec("insert into t_range_index values (1, 2, 'abc'), (5, 4, 'def'), (13, 6, 'xyz'), (17, 6, 'hij')") - tk.MustExec("prepare stmt5 from 'select c from t_range_index where id = ?'") - tk.MustExec("set @id=1") - tk.MustQuery("execute stmt5 using @id").Check(testkit.Rows("abc")) - tk.MustQuery("execute stmt5 using @id").Check(testkit.Rows("abc")) - tk.MustExec("set @id=5") - tk.MustQuery("execute stmt5 using @id").Check(testkit.Rows("def")) - tk.MustQuery("execute stmt5 using @id").Check(testkit.Rows("def")) - tk.MustExec("set @id=13") - tk.MustQuery("execute stmt5 using @id").Check(testkit.Rows("xyz")) - tk.MustExec("set @id=17") - tk.MustQuery("execute stmt5 using @id").Check(testkit.Rows("hij")) - - tk.MustExec("create table t_range_table (id int, k int, c varchar(10)) partition by range(id) ( PARTITION p0 VALUES LESS THAN (4), PARTITION p1 VALUES LESS THAN (14),PARTITION p2 VALUES LESS THAN (20) )") - tk.MustExec("insert into t_range_table values (1, 2, 'abc'), (5, 4, 'def'), (13, 6, 'xyz'), (17, 6, 'hij')") - tk.MustExec("prepare stmt6 from 'select c from t_range_table where id = ?'") - tk.MustExec("set @id=1") - tk.MustQuery("execute stmt6 using @id").Check(testkit.Rows("abc")) - tk.MustQuery("execute stmt6 using @id").Check(testkit.Rows("abc")) - tk.MustExec("set @id=5") - tk.MustQuery("execute stmt6 using @id").Check(testkit.Rows("def")) - tk.MustQuery("execute stmt6 using @id").Check(testkit.Rows("def")) - tk.MustExec("set @id=13") - tk.MustQuery("execute stmt6 using @id").Check(testkit.Rows("xyz")) - tk.MustExec("set @id=17") - tk.MustQuery("execute stmt6 using @id").Check(testkit.Rows("hij")) + for _, val := range []string{"1", "null"} { + tk.MustExec("set @try_old_partition_implementation = " + val) + // Test for PointGet and IndexRead. + tk.MustExec("drop table if exists t_index_read") + tk.MustExec("create table t_index_read (id int, k int, c varchar(10), primary key (id, k)) partition by hash(id+k) partitions 10") + tk.MustExec("insert into t_index_read values (1, 2, 'abc'), (3, 4, 'def'), (5, 6, 'xyz')") + tk.MustExec("prepare stmt1 from 'select c from t_index_read where id = ? and k = ?;'") + tk.MustExec("set @id=1, @k=2") + // When executing one statement at the first time, we don't use cache, so we need to execute it at least twice to test the cache. + tk.MustQuery("execute stmt1 using @id, @k").Check(testkit.Rows("abc")) + tk.MustQuery("execute stmt1 using @id, @k").Check(testkit.Rows("abc")) + tk.MustExec("set @id=5, @k=6") + tk.MustQuery("execute stmt1 using @id, @k").Check(testkit.Rows("xyz")) + tk.MustExec("prepare stmt2 from 'select c from t_index_read where id = ? and k = ? and 1 = 1;'") + tk.MustExec("set @id=1, @k=2") + tk.MustQuery("execute stmt2 using @id, @k").Check(testkit.Rows("abc")) + tk.MustQuery("execute stmt2 using @id, @k").Check(testkit.Rows("abc")) + tk.MustExec("set @id=5, @k=6") + tk.MustQuery("execute stmt2 using @id, @k").Check(testkit.Rows("xyz")) + // Test for TableScan. + tk.MustExec("drop table if exists t_table_read") + tk.MustExec("create table t_table_read (id int, k int, c varchar(10), primary key(id)) partition by hash(id) partitions 10") + tk.MustExec("insert into t_table_read values (1, 2, 'abc'), (3, 4, 'def'), (5, 6, 'xyz')") + tk.MustExec("prepare stmt3 from 'select c from t_index_read where id = ?;'") + tk.MustExec("set @id=1") + // When executing one statement at the first time, we don't use cache, so we need to execute it at least twice to test the cache. + tk.MustQuery("execute stmt3 using @id").Check(testkit.Rows("abc")) + tk.MustQuery("execute stmt3 using @id").Check(testkit.Rows("abc")) + tk.MustExec("set @id=5") + tk.MustQuery("execute stmt3 using @id").Check(testkit.Rows("xyz")) + tk.MustExec("prepare stmt4 from 'select c from t_index_read where id = ? and k = ?'") + tk.MustExec("set @id=1, @k=2") + tk.MustQuery("execute stmt4 using @id, @k").Check(testkit.Rows("abc")) + tk.MustQuery("execute stmt4 using @id, @k").Check(testkit.Rows("abc")) + tk.MustExec("set @id=5, @k=6") + tk.MustQuery("execute stmt4 using @id, @k").Check(testkit.Rows("xyz")) + // Query on range partition tables should not raise error. + tk.MustExec("drop table if exists t_range_index") + tk.MustExec("create table t_range_index (id int, k int, c varchar(10), primary key(id)) partition by range(id) ( PARTITION p0 VALUES LESS THAN (4), PARTITION p1 VALUES LESS THAN (14),PARTITION p2 VALUES LESS THAN (20) )") + tk.MustExec("insert into t_range_index values (1, 2, 'abc'), (5, 4, 'def'), (13, 6, 'xyz'), (17, 6, 'hij')") + tk.MustExec("prepare stmt5 from 'select c from t_range_index where id = ?'") + tk.MustExec("set @id=1") + tk.MustQuery("execute stmt5 using @id").Check(testkit.Rows("abc")) + tk.MustQuery("execute stmt5 using @id").Check(testkit.Rows("abc")) + tk.MustExec("set @id=5") + tk.MustQuery("execute stmt5 using @id").Check(testkit.Rows("def")) + tk.MustQuery("execute stmt5 using @id").Check(testkit.Rows("def")) + tk.MustExec("set @id=13") + tk.MustQuery("execute stmt5 using @id").Check(testkit.Rows("xyz")) + tk.MustExec("set @id=17") + tk.MustQuery("execute stmt5 using @id").Check(testkit.Rows("hij")) + + tk.MustExec("drop table if exists t_range_table") + tk.MustExec("create table t_range_table (id int, k int, c varchar(10)) partition by range(id) ( PARTITION p0 VALUES LESS THAN (4), PARTITION p1 VALUES LESS THAN (14),PARTITION p2 VALUES LESS THAN (20) )") + tk.MustExec("insert into t_range_table values (1, 2, 'abc'), (5, 4, 'def'), (13, 6, 'xyz'), (17, 6, 'hij')") + tk.MustExec("prepare stmt6 from 'select c from t_range_table where id = ?'") + tk.MustExec("set @id=1") + tk.MustQuery("execute stmt6 using @id").Check(testkit.Rows("abc")) + tk.MustQuery("execute stmt6 using @id").Check(testkit.Rows("abc")) + tk.MustExec("set @id=5") + tk.MustQuery("execute stmt6 using @id").Check(testkit.Rows("def")) + tk.MustQuery("execute stmt6 using @id").Check(testkit.Rows("def")) + tk.MustExec("set @id=13") + tk.MustQuery("execute stmt6 using @id").Check(testkit.Rows("xyz")) + tk.MustExec("set @id=17") + tk.MustQuery("execute stmt6 using @id").Check(testkit.Rows("hij")) + } } func newSession(c *C, store kv.Storage, dbName string) session.Session { From eef2100a7e3ae09abb1341430edd71bdca32a3d0 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Thu, 3 Sep 2020 21:34:40 +0800 Subject: [PATCH 3/3] tiny clean up --- planner/core/common_plans.go | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index a0f2f14e53e9b..927ce6d09ecc1 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -39,7 +39,6 @@ import ( "github.com/pingcap/tidb/util/hint" "github.com/pingcap/tidb/util/kvcache" "github.com/pingcap/tidb/util/logutil" - "github.com/pingcap/tidb/util/math" "github.com/pingcap/tidb/util/ranger" "github.com/pingcap/tidb/util/texttree" "go.uber.org/zap" @@ -483,6 +482,11 @@ func (e *Execute) rebuildRange(p Plan) error { } } } + // The code should never run here as long as we're not using point get for partition table. + // And if we change the logic one day, here work as defensive programming to cache the error. + if x.PartitionInfo != nil { + return errors.New("point get for partition table can not use plan cache") + } if x.HandleParam != nil { var iv int64 iv, err = x.HandleParam.Datum.ToInt64(sc) @@ -490,14 +494,6 @@ func (e *Execute) rebuildRange(p Plan) error { return err } x.Handle = kv.IntHandle(iv) - if x.PartitionInfo != nil { - if x.TblInfo.Partition.Type != model.PartitionTypeHash { - return errors.New("range partition table can not use plan cache") - } - num := x.TblInfo.Partition.Num - pos := math.Abs(iv) % int64(num) - x.PartitionInfo = &x.TblInfo.Partition.Definitions[pos] - } return nil } for i, param := range x.IndexValueParams { @@ -505,14 +501,6 @@ func (e *Execute) rebuildRange(p Plan) error { x.IndexValues[i] = param.Datum } } - if x.PartitionInfo != nil { - if x.TblInfo.Partition.Type != model.PartitionTypeHash { - return errors.New("range partition table can not use plan cache") - } - val := x.IndexValues[x.partitionColumnPos].GetInt64() - partitionID := val % int64(x.TblInfo.Partition.Num) - x.PartitionInfo = &x.TblInfo.Partition.Definitions[partitionID] - } return nil case *BatchPointGetPlan: // if access condition is not nil, which means it's a point get generated by cbo.