From 6ed5e77ef1f0603fbcba59a3771e999a28954aa2 Mon Sep 17 00:00:00 2001
From: fengou1 <feng.ou@pingcap.com>
Date: Mon, 12 Dec 2022 13:17:03 +0530
Subject: [PATCH 1/3] fix: restore failure on batch create table when size of
tables schema large than 6 mb
---
br/pkg/gluetidb/glue.go | 29 ++++++++++++++++++++++++-----
1 file changed, 24 insertions(+), 5 deletions(-)
diff --git a/br/pkg/gluetidb/glue.go b/br/pkg/gluetidb/glue.go
index 3b1675921afd3..2c8726310153b 100644
--- a/br/pkg/gluetidb/glue.go
+++ b/br/pkg/gluetidb/glue.go
@@ -203,13 +203,33 @@ func (gs *tidbSession) CreatePlacementPolicy(ctx context.Context, policy *model.
return d.CreatePlacementPolicyWithInfo(gs.se, policy, ddl.OnExistIgnore)
}
+// SplitBatchCreateTable provide a way to split batch into small batch when batch size is large than 6 MB.
+// The raft entry has limit size of 6 MB, a batch of CreateTables may hit this limitation
+// TODO: shall query string be set for each split batch create, it looks does not matter if we set once for all.
+func (gs *tidbSession) SplitBatchCreateTable(schema model.CIStr, info []*model.TableInfo, l int, r int, cs ...ddl.CreateTableWithInfoConfigurier) error {
+ var err error
+ d := domain.GetDomain(gs.se).DDL()
+ if err = d.BatchCreateTableWithInfo(gs.se, schema, info[l:r], append(cs, ddl.OnExistIgnore)...); kv.ErrEntryTooLarge.Equal(err) {
+ if r-l == 1 {
+ return err
+ }
+ err = gs.SplitBatchCreateTable(schema, info, l, (l+r)/2)
+ if err != nil {
+ return err
+ }
+ err = gs.SplitBatchCreateTable(schema, info, (l+r)/2, r)
+ if err != nil {
+ return err
+ }
+ return nil
+ }
+ return err
+}
+
// CreateTables implements glue.BatchCreateTableSession.
func (gs *tidbSession) CreateTables(ctx context.Context, tables map[string][]*model.TableInfo, cs ...ddl.CreateTableWithInfoConfigurier) error {
- d := domain.GetDomain(gs.se).DDL()
var dbName model.CIStr
- // Disable foreign key check when batch create tables.
- gs.se.GetSessionVars().ForeignKeyChecks = false
for db, tablesInDB := range tables {
dbName = model.NewCIStr(db)
queryBuilder := strings.Builder{}
@@ -233,8 +253,7 @@ func (gs *tidbSession) CreateTables(ctx context.Context, tables map[string][]*mo
cloneTables = append(cloneTables, table)
}
gs.se.SetValue(sessionctx.QueryString, queryBuilder.String())
- err := d.BatchCreateTableWithInfo(gs.se, dbName, cloneTables, append(cs, ddl.OnExistIgnore)...)
- if err != nil {
+ if err := gs.SplitBatchCreateTable(dbName, cloneTables, 0, len(cloneTables)); err != nil {
//It is possible to failure when TiDB does not support model.ActionCreateTables.
//In this circumstance, BatchCreateTableWithInfo returns errno.ErrInvalidDDLJob,
//we fall back to old way that creating table one by one
From 2e0df43482cc4bd06cabe36cbcec643dd5840709 Mon Sep 17 00:00:00 2001
From: fengou1 <feng.ou@pingcap.com>
Date: Tue, 13 Dec 2022 04:40:41 +0530
Subject: [PATCH 2/3] fix: refactor the func with less parameter
---
br/pkg/gluetidb/glue.go | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/br/pkg/gluetidb/glue.go b/br/pkg/gluetidb/glue.go
index 2c8726310153b..3b3d71ef8cff6 100644
--- a/br/pkg/gluetidb/glue.go
+++ b/br/pkg/gluetidb/glue.go
@@ -206,18 +206,19 @@ func (gs *tidbSession) CreatePlacementPolicy(ctx context.Context, policy *model.
// SplitBatchCreateTable provide a way to split batch into small batch when batch size is large than 6 MB.
// The raft entry has limit size of 6 MB, a batch of CreateTables may hit this limitation
// TODO: shall query string be set for each split batch create, it looks does not matter if we set once for all.
-func (gs *tidbSession) SplitBatchCreateTable(schema model.CIStr, info []*model.TableInfo, l int, r int, cs ...ddl.CreateTableWithInfoConfigurier) error {
+func (gs *tidbSession) SplitBatchCreateTable(schema model.CIStr, info []*model.TableInfo, cs ...ddl.CreateTableWithInfoConfigurier) error {
var err error
d := domain.GetDomain(gs.se).DDL()
- if err = d.BatchCreateTableWithInfo(gs.se, schema, info[l:r], append(cs, ddl.OnExistIgnore)...); kv.ErrEntryTooLarge.Equal(err) {
- if r-l == 1 {
+ if err = d.BatchCreateTableWithInfo(gs.se, schema, info, append(cs, ddl.OnExistIgnore)...); kv.ErrEntryTooLarge.Equal(err) {
+ if len(info) == 1 {
return err
}
- err = gs.SplitBatchCreateTable(schema, info, l, (l+r)/2)
+ mid := len(info) / 2
+ err = gs.SplitBatchCreateTable(schema, info[:mid])
if err != nil {
return err
}
- err = gs.SplitBatchCreateTable(schema, info, (l+r)/2, r)
+ err = gs.SplitBatchCreateTable(schema, info[mid:])
if err != nil {
return err
}
@@ -253,7 +254,7 @@ func (gs *tidbSession) CreateTables(ctx context.Context, tables map[string][]*mo
cloneTables = append(cloneTables, table)
}
gs.se.SetValue(sessionctx.QueryString, queryBuilder.String())
- if err := gs.SplitBatchCreateTable(dbName, cloneTables, 0, len(cloneTables)); err != nil {
+ if err := gs.SplitBatchCreateTable(dbName, cloneTables); err != nil {
//It is possible to failure when TiDB does not support model.ActionCreateTables.
//In this circumstance, BatchCreateTableWithInfo returns errno.ErrInvalidDDLJob,
//we fall back to old way that creating table one by one
From 0e606c57ca0c96e16248c67dff3143b73f666a24 Mon Sep 17 00:00:00 2001
From: fengou1 <feng.ou@pingcap.com>
Date: Tue, 13 Dec 2022 04:47:44 +0530
Subject: [PATCH 3/3] fix: revert unexpected change
---
br/pkg/gluetidb/glue.go | 2 ++
1 file changed, 2 insertions(+)
diff --git a/br/pkg/gluetidb/glue.go b/br/pkg/gluetidb/glue.go
index 3b3d71ef8cff6..19c3ef1c4c460 100644
--- a/br/pkg/gluetidb/glue.go
+++ b/br/pkg/gluetidb/glue.go
@@ -231,6 +231,8 @@ func (gs *tidbSession) SplitBatchCreateTable(schema model.CIStr, info []*model.T
func (gs *tidbSession) CreateTables(ctx context.Context, tables map[string][]*model.TableInfo, cs ...ddl.CreateTableWithInfoConfigurier) error {
var dbName model.CIStr
+ // Disable foreign key check when batch create tables.
+ gs.se.GetSessionVars().ForeignKeyChecks = false
for db, tablesInDB := range tables {
dbName = model.NewCIStr(db)
queryBuilder := strings.Builder{}