From 68918cfd16217a653add164c04aa6c4c014727bc Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 14 Dec 2020 19:08:45 +0800 Subject: [PATCH 1/3] skip split regions if engine total size is smaller than region size --- lightning/backend/local.go | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/lightning/backend/local.go b/lightning/backend/local.go index 1ecf6dd46..f9d490a0a 100644 --- a/lightning/backend/local.go +++ b/lightning/backend/local.go @@ -1122,19 +1122,27 @@ func (local *local) ImportEngine(ctx context.Context, engineUUID uuid.UUID) erro log.L().Info("start import engine", zap.Stringer("uuid", engineUUID), zap.Int("ranges", len(ranges))) - // split region by given ranges - for i := 0; i < maxRetryTimes; i++ { - err = local.SplitAndScatterRegionByRanges(ctx, ranges) - if err == nil { - break - } + // if all the kv can fit in one region, skip split regions. TiDB will split one region for + // the table when table is created. + if lf.TotalSize > local.regionSplitSize || lf.Length > regionMaxKeyCount { + // split region by given ranges + for i := 0; i < maxRetryTimes; i++ { + err = local.SplitAndScatterRegionByRanges(ctx, ranges) + if err == nil { + break + } - log.L().Warn("split and scatter failed in retry", zap.Stringer("uuid", engineUUID), - log.ShortError(err), zap.Int("retry", i)) - } - if err != nil { - log.L().Error("split & scatter ranges failed", zap.Stringer("uuid", engineUUID), log.ShortError(err)) - return err + log.L().Warn("split and scatter failed in retry", zap.Stringer("uuid", engineUUID), + log.ShortError(err), zap.Int("retry", i)) + } + if err != nil { + log.L().Error("split & scatter ranges failed", zap.Stringer("uuid", engineUUID), log.ShortError(err)) + return err + } + } else { + log.L().Info("engine total size is smaller than region split size, skip split regions", + zap.Stringer("uuid", engineUUID), zap.Int64("size", lf.TotalSize), + zap.Int64("count", lf.Length)) } // start to write to kv and ingest From d71e1a55d891a793c1b323199e18a1785a06bd54 Mon Sep 17 00:00:00 2001 From: glorv Date: Fri, 8 Jan 2021 10:14:12 +0800 Subject: [PATCH 2/3] fix unit test --- lightning/backend/localhelper_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lightning/backend/localhelper_test.go b/lightning/backend/localhelper_test.go index 50d1b9657..c8bddbbd3 100644 --- a/lightning/backend/localhelper_test.go +++ b/lightning/backend/localhelper_test.go @@ -363,7 +363,7 @@ func (s *localSuite) doTestBatchSplitRegionByRanges(c *C, hook clientHook, errPa start = end } - err = local.SplitAndScatterRegionByRanges(ctx, ranges) + err = local.SplitAndScatterRegionByRanges(ctx, ranges, true) if len(errPat) == 0 { c.Assert(err, IsNil) } else { @@ -504,7 +504,7 @@ func (s *localSuite) doTestBatchSplitByRangesWithClusteredIndex(c *C, hook clien start = e } - err := local.SplitAndScatterRegionByRanges(ctx, ranges) + err := local.SplitAndScatterRegionByRanges(ctx, ranges, true) c.Assert(err, IsNil) startKey := codec.EncodeBytes([]byte{}, rangeKeys[0]) From 2f2f81a2d7dde946e1fb6c7fb5faebff70f42572 Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 11 Jan 2021 14:28:18 +0800 Subject: [PATCH 3/3] apply suggest --- lightning/backend/local.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lightning/backend/local.go b/lightning/backend/local.go index 12bd22e9f..1607a6b2d 100644 --- a/lightning/backend/local.go +++ b/lightning/backend/local.go @@ -1192,7 +1192,7 @@ func (local *local) ImportEngine(ctx context.Context, engineUUID uuid.UUID) erro // if all the kv can fit in one region, skip split regions. TiDB will split one region for // the table when table is created. - needSplit := len(ranges) > 0 || lf.TotalSize > local.regionSplitSize || lf.Length > regionMaxKeyCount + needSplit := len(ranges) > 1 || lf.TotalSize > local.regionSplitSize || lf.Length > regionMaxKeyCount // split region by given ranges for i := 0; i < maxRetryTimes; i++ {