From ec9ff181c31e9ccac2eec9884ce9cbbb06196d69 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Tue, 30 Mar 2021 17:25:24 +0800 Subject: [PATCH 1/3] cherry pick #23652 to release-4.0 Signed-off-by: ti-srebot --- executor/batch_point_get.go | 5 ++ executor/builder.go | 7 ++ executor/partition_table_test.go | 108 ++++++++++++++++++++++++++++++ planner/core/cacheable_checker.go | 2 +- 4 files changed, 121 insertions(+), 1 deletion(-) diff --git a/executor/batch_point_get.go b/executor/batch_point_get.go index d6c1257651697..50a1aca5fc9a6 100644 --- a/executor/batch_point_get.go +++ b/executor/batch_point_get.go @@ -388,8 +388,13 @@ func (getter *PessimisticLockCacheGetter) Get(_ context.Context, key kv.Key) ([] return nil, kv.ErrNotExist } +<<<<<<< HEAD func getPhysID(tblInfo *model.TableInfo, val int64) int64 { pi := tblInfo.Partition +======= +func getPhysID(tblInfo *model.TableInfo, intVal int64) int64 { + pi := tblInfo.GetPartitionInfo() +>>>>>>> 67874c579... executor: fix a panic when batch point get is used for partition table (#23652) if pi == nil { return tblInfo.ID } diff --git a/executor/builder.go b/executor/builder.go index e8f4bca522a8f..a567d7047109e 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -2517,6 +2517,13 @@ func buildNoRangeIndexLookUpReader(b *executorBuilder, v *plannercore.PhysicalIn collectTable := false e.tableRequest.CollectRangeCounts = &collectTable collectIndex := statistics.CollectFeedback(b.ctx.GetSessionVars().StmtCtx, e.feedback, len(is.Ranges)) +<<<<<<< HEAD +======= + // Do not collect the feedback when the table is the partition table. + if collectIndex && tbl.Meta().GetPartitionInfo() != nil { + collectIndex = false + } +>>>>>>> 67874c579... executor: fix a panic when batch point get is used for partition table (#23652) if !collectIndex { e.feedback.Invalidate() } diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go index 4c731a96b5b1f..d3343bae570e9 100644 --- a/executor/partition_table_test.go +++ b/executor/partition_table_test.go @@ -15,6 +15,12 @@ package executor_test import ( . "github.com/pingcap/check" +<<<<<<< HEAD +======= + "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/sessionctx/variable" +>>>>>>> 67874c579... executor: fix a panic when batch point get is used for partition table (#23652) "github.com/pingcap/tidb/util/testkit" ) @@ -64,4 +70,106 @@ func (s *testSuite9) TestPartitionReaderUnderApply(c *C) { "3 vibrant shamir 6.300000", "4 hungry wilson 4.900000", "5 naughty swartz 9.524000")) +<<<<<<< HEAD +======= + + // For issue 19450 release-4.0 + tk.MustExec(`set @@tidb_partition_prune_mode='` + string(variable.Static) + `'`) + tk.MustQuery("select * from t1 where c_decimal in (select c_decimal from t2 where t1.c_int = t2.c_int or t1.c_int = t2.c_int and t1.c_str > t2.c_str)").Check(testkit.Rows( + "1 romantic robinson 4.436000", + "2 stoic chaplygin 9.826000", + "3 vibrant shamir 6.300000", + "4 hungry wilson 4.900000", + "5 naughty swartz 9.524000")) +} + +func (s *partitionTableSuite) TestImproveCoverage(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("use test") + tk.MustExec(`create table coverage_rr ( +pk1 varchar(35) NOT NULL, +pk2 int NOT NULL, +c int, +PRIMARY KEY (pk1,pk2)) partition by hash(pk2) partitions 4;`) + tk.MustExec("create table coverage_dt (pk1 varchar(35), pk2 int)") + tk.MustExec("insert into coverage_rr values ('ios', 3, 2),('android', 4, 7),('linux',5,1)") + tk.MustExec("insert into coverage_dt values ('apple',3),('ios',3),('linux',5)") + tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + tk.MustQuery("select /*+ INL_JOIN(dt, rr) */ * from coverage_dt dt join coverage_rr rr on (dt.pk1 = rr.pk1 and dt.pk2 = rr.pk2);").Sort().Check(testkit.Rows("ios 3 ios 3 2", "linux 5 linux 5 1")) + tk.MustQuery("select /*+ INL_MERGE_JOIN(dt, rr) */ * from coverage_dt dt join coverage_rr rr on (dt.pk1 = rr.pk1 and dt.pk2 = rr.pk2);").Sort().Check(testkit.Rows("ios 3 ios 3 2", "linux 5 linux 5 1")) +} + +func (s *partitionTableSuite) TestPartitionInfoDisable(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t_info_null") + tk.MustExec(`CREATE TABLE t_info_null ( + id bigint(20) unsigned NOT NULL AUTO_INCREMENT, + date date NOT NULL, + media varchar(32) NOT NULL DEFAULT '0', + app varchar(32) NOT NULL DEFAULT '', + xxx bigint(20) NOT NULL DEFAULT '0', + PRIMARY KEY (id, date), + UNIQUE KEY idx_media_id (media, date, app) +) PARTITION BY RANGE COLUMNS(date) ( + PARTITION p201912 VALUES LESS THAN ("2020-01-01"), + PARTITION p202001 VALUES LESS THAN ("2020-02-01"), + PARTITION p202002 VALUES LESS THAN ("2020-03-01"), + PARTITION p202003 VALUES LESS THAN ("2020-04-01"), + PARTITION p202004 VALUES LESS THAN ("2020-05-01"), + PARTITION p202005 VALUES LESS THAN ("2020-06-01"), + PARTITION p202006 VALUES LESS THAN ("2020-07-01"), + PARTITION p202007 VALUES LESS THAN ("2020-08-01"), + PARTITION p202008 VALUES LESS THAN ("2020-09-01"), + PARTITION p202009 VALUES LESS THAN ("2020-10-01"), + PARTITION p202010 VALUES LESS THAN ("2020-11-01"), + PARTITION p202011 VALUES LESS THAN ("2020-12-01") +)`) + is := infoschema.GetInfoSchema(tk.Se) + tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t_info_null")) + c.Assert(err, IsNil) + + tbInfo := tbl.Meta() + // Mock for a case that the tableInfo.Partition is not nil, but tableInfo.Partition.Enable is false. + // That may happen when upgrading from a old version TiDB. + tbInfo.Partition.Enable = false + tbInfo.Partition.Num = 0 + + tk.MustExec("set @@tidb_partition_prune_mode = 'static'") + tk.MustQuery("explain select * from t_info_null where (date = '2020-10-02' or date = '2020-10-06') and app = 'xxx' and media = '19003006'").Check(testkit.Rows("Batch_Point_Get_5 2.00 root table:t_info_null, index:idx_media_id(media, date, app) keep order:false, desc:false")) + tk.MustQuery("explain select * from t_info_null").Check(testkit.Rows("TableReader_5 10000.00 root data:TableFullScan_4", + "└─TableFullScan_4 10000.00 cop[tikv] table:t_info_null keep order:false, stats:pseudo")) + // No panic. + tk.MustQuery("select * from t_info_null where (date = '2020-10-02' or date = '2020-10-06') and app = 'xxx' and media = '19003006'").Check(testkit.Rows()) +} + +func (s *globalIndexSuite) TestGlobalIndexScan(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists p") + tk.MustExec(`create table p (id int, c int) partition by range (c) ( +partition p0 values less than (4), +partition p1 values less than (7), +partition p2 values less than (10))`) + tk.MustExec("alter table p add unique idx(id)") + tk.MustExec("insert into p values (1,3), (3,4), (5,6), (7,9)") + tk.MustQuery("select id from p use index (idx)").Check(testkit.Rows("1", "3", "5", "7")) +} + +func (s *globalIndexSuite) TestGlobalIndexDoubleRead(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists p") + tk.MustExec(`create table p (id int, c int) partition by range (c) ( +partition p0 values less than (4), +partition p1 values less than (7), +partition p2 values less than (10))`) + tk.MustExec("alter table p add unique idx(id)") + tk.MustExec("insert into p values (1,3), (3,4), (5,6), (7,9)") + tk.MustQuery("select * from p use index (idx)").Check(testkit.Rows("1 3", "3 4", "5 6", "7 9")) +} + +func (s *globalIndexSuite) TestIssue21731(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists p") + tk.MustExec("create table t (a int, b int, unique index idx(a)) partition by list columns(b) (partition p0 values in (1), partition p1 values in (2));") +>>>>>>> 67874c579... executor: fix a panic when batch point get is used for partition table (#23652) } diff --git a/planner/core/cacheable_checker.go b/planner/core/cacheable_checker.go index feacf56bef7b9..f17b6ad96b14e 100644 --- a/planner/core/cacheable_checker.go +++ b/planner/core/cacheable_checker.go @@ -131,7 +131,7 @@ func (checker *cacheableChecker) isPartitionTable(tn *ast.TableName) bool { logutil.BgLogger().Error("Error occur in checking cacheable", zap.Error(err)) return false } - if tb.Meta().Partition != nil { + if tb.Meta().GetPartitionInfo() != nil { return true } return false From b3639a2a62cc188b43bb86c04496ef87b51936b2 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Tue, 11 May 2021 12:20:07 +0800 Subject: [PATCH 2/3] resolve conflicts --- executor/batch_point_get.go | 5 --- executor/builder.go | 7 ---- executor/partition_table_test.go | 64 -------------------------------- 3 files changed, 76 deletions(-) diff --git a/executor/batch_point_get.go b/executor/batch_point_get.go index 50a1aca5fc9a6..a86543ef61bbd 100644 --- a/executor/batch_point_get.go +++ b/executor/batch_point_get.go @@ -388,13 +388,8 @@ func (getter *PessimisticLockCacheGetter) Get(_ context.Context, key kv.Key) ([] return nil, kv.ErrNotExist } -<<<<<<< HEAD func getPhysID(tblInfo *model.TableInfo, val int64) int64 { - pi := tblInfo.Partition -======= -func getPhysID(tblInfo *model.TableInfo, intVal int64) int64 { pi := tblInfo.GetPartitionInfo() ->>>>>>> 67874c579... executor: fix a panic when batch point get is used for partition table (#23652) if pi == nil { return tblInfo.ID } diff --git a/executor/builder.go b/executor/builder.go index a567d7047109e..e8f4bca522a8f 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -2517,13 +2517,6 @@ func buildNoRangeIndexLookUpReader(b *executorBuilder, v *plannercore.PhysicalIn collectTable := false e.tableRequest.CollectRangeCounts = &collectTable collectIndex := statistics.CollectFeedback(b.ctx.GetSessionVars().StmtCtx, e.feedback, len(is.Ranges)) -<<<<<<< HEAD -======= - // Do not collect the feedback when the table is the partition table. - if collectIndex && tbl.Meta().GetPartitionInfo() != nil { - collectIndex = false - } ->>>>>>> 67874c579... executor: fix a panic when batch point get is used for partition table (#23652) if !collectIndex { e.feedback.Invalidate() } diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go index d3343bae570e9..8443f2bba44cf 100644 --- a/executor/partition_table_test.go +++ b/executor/partition_table_test.go @@ -15,12 +15,6 @@ package executor_test import ( . "github.com/pingcap/check" -<<<<<<< HEAD -======= - "github.com/pingcap/parser/model" - "github.com/pingcap/tidb/infoschema" - "github.com/pingcap/tidb/sessionctx/variable" ->>>>>>> 67874c579... executor: fix a panic when batch point get is used for partition table (#23652) "github.com/pingcap/tidb/util/testkit" ) @@ -70,33 +64,6 @@ func (s *testSuite9) TestPartitionReaderUnderApply(c *C) { "3 vibrant shamir 6.300000", "4 hungry wilson 4.900000", "5 naughty swartz 9.524000")) -<<<<<<< HEAD -======= - - // For issue 19450 release-4.0 - tk.MustExec(`set @@tidb_partition_prune_mode='` + string(variable.Static) + `'`) - tk.MustQuery("select * from t1 where c_decimal in (select c_decimal from t2 where t1.c_int = t2.c_int or t1.c_int = t2.c_int and t1.c_str > t2.c_str)").Check(testkit.Rows( - "1 romantic robinson 4.436000", - "2 stoic chaplygin 9.826000", - "3 vibrant shamir 6.300000", - "4 hungry wilson 4.900000", - "5 naughty swartz 9.524000")) -} - -func (s *partitionTableSuite) TestImproveCoverage(c *C) { - tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustExec("use test") - tk.MustExec(`create table coverage_rr ( -pk1 varchar(35) NOT NULL, -pk2 int NOT NULL, -c int, -PRIMARY KEY (pk1,pk2)) partition by hash(pk2) partitions 4;`) - tk.MustExec("create table coverage_dt (pk1 varchar(35), pk2 int)") - tk.MustExec("insert into coverage_rr values ('ios', 3, 2),('android', 4, 7),('linux',5,1)") - tk.MustExec("insert into coverage_dt values ('apple',3),('ios',3),('linux',5)") - tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") - tk.MustQuery("select /*+ INL_JOIN(dt, rr) */ * from coverage_dt dt join coverage_rr rr on (dt.pk1 = rr.pk1 and dt.pk2 = rr.pk2);").Sort().Check(testkit.Rows("ios 3 ios 3 2", "linux 5 linux 5 1")) - tk.MustQuery("select /*+ INL_MERGE_JOIN(dt, rr) */ * from coverage_dt dt join coverage_rr rr on (dt.pk1 = rr.pk1 and dt.pk2 = rr.pk2);").Sort().Check(testkit.Rows("ios 3 ios 3 2", "linux 5 linux 5 1")) } func (s *partitionTableSuite) TestPartitionInfoDisable(c *C) { @@ -142,34 +109,3 @@ func (s *partitionTableSuite) TestPartitionInfoDisable(c *C) { // No panic. tk.MustQuery("select * from t_info_null where (date = '2020-10-02' or date = '2020-10-06') and app = 'xxx' and media = '19003006'").Check(testkit.Rows()) } - -func (s *globalIndexSuite) TestGlobalIndexScan(c *C) { - tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustExec("drop table if exists p") - tk.MustExec(`create table p (id int, c int) partition by range (c) ( -partition p0 values less than (4), -partition p1 values less than (7), -partition p2 values less than (10))`) - tk.MustExec("alter table p add unique idx(id)") - tk.MustExec("insert into p values (1,3), (3,4), (5,6), (7,9)") - tk.MustQuery("select id from p use index (idx)").Check(testkit.Rows("1", "3", "5", "7")) -} - -func (s *globalIndexSuite) TestGlobalIndexDoubleRead(c *C) { - tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustExec("drop table if exists p") - tk.MustExec(`create table p (id int, c int) partition by range (c) ( -partition p0 values less than (4), -partition p1 values less than (7), -partition p2 values less than (10))`) - tk.MustExec("alter table p add unique idx(id)") - tk.MustExec("insert into p values (1,3), (3,4), (5,6), (7,9)") - tk.MustQuery("select * from p use index (idx)").Check(testkit.Rows("1 3", "3 4", "5 6", "7 9")) -} - -func (s *globalIndexSuite) TestIssue21731(c *C) { - tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustExec("drop table if exists p") - tk.MustExec("create table t (a int, b int, unique index idx(a)) partition by list columns(b) (partition p0 values in (1), partition p1 values in (2));") ->>>>>>> 67874c579... executor: fix a panic when batch point get is used for partition table (#23652) -} From 85f917d1bce5252a73fb1217990f3759243844e0 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Tue, 11 May 2021 12:47:27 +0800 Subject: [PATCH 3/3] fix CI --- executor/partition_table_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go index 8443f2bba44cf..e834776216a49 100644 --- a/executor/partition_table_test.go +++ b/executor/partition_table_test.go @@ -15,6 +15,8 @@ package executor_test import ( . "github.com/pingcap/check" + "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/util/testkit" ) @@ -66,7 +68,7 @@ func (s *testSuite9) TestPartitionReaderUnderApply(c *C) { "5 naughty swartz 9.524000")) } -func (s *partitionTableSuite) TestPartitionInfoDisable(c *C) { +func (s *testSuite9) TestPartitionInfoDisable(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t_info_null") @@ -102,10 +104,6 @@ func (s *partitionTableSuite) TestPartitionInfoDisable(c *C) { tbInfo.Partition.Enable = false tbInfo.Partition.Num = 0 - tk.MustExec("set @@tidb_partition_prune_mode = 'static'") - tk.MustQuery("explain select * from t_info_null where (date = '2020-10-02' or date = '2020-10-06') and app = 'xxx' and media = '19003006'").Check(testkit.Rows("Batch_Point_Get_5 2.00 root table:t_info_null, index:idx_media_id(media, date, app) keep order:false, desc:false")) - tk.MustQuery("explain select * from t_info_null").Check(testkit.Rows("TableReader_5 10000.00 root data:TableFullScan_4", - "└─TableFullScan_4 10000.00 cop[tikv] table:t_info_null keep order:false, stats:pseudo")) // No panic. tk.MustQuery("select * from t_info_null where (date = '2020-10-02' or date = '2020-10-06') and app = 'xxx' and media = '19003006'").Check(testkit.Rows()) }