From 478cadce2eba62d4dc44af4cd6a9ae6fa79dc7b5 Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Tue, 12 Dec 2023 16:10:42 +0800 Subject: [PATCH 1/3] statistics: fix LoadStatsFromJSONNoUpdate when runtime.GOMAXPROCS(0) return 1 --- pkg/statistics/handle/storage/stats_read_writer.go | 12 +++++------- pkg/statistics/handle/types/interfaces.go | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pkg/statistics/handle/storage/stats_read_writer.go b/pkg/statistics/handle/storage/stats_read_writer.go index a8dd3175f976a..63cbef4f08ec8 100644 --- a/pkg/statistics/handle/storage/stats_read_writer.go +++ b/pkg/statistics/handle/storage/stats_read_writer.go @@ -558,7 +558,7 @@ type TestLoadStatsErr struct{} // LoadStatsFromJSON will load statistic from JSONTable, and save it to the storage. // In final, it will also udpate the stats cache. func (s *statsReadWriter) LoadStatsFromJSON(ctx context.Context, is infoschema.InfoSchema, - jsonTbl *util.JSONTable, concurrencyForPartition uint8) error { + jsonTbl *util.JSONTable, concurrencyForPartition int) error { if err := s.LoadStatsFromJSONNoUpdate(ctx, is, jsonTbl, concurrencyForPartition); err != nil { return errors.Trace(err) } @@ -567,14 +567,12 @@ func (s *statsReadWriter) LoadStatsFromJSON(ctx context.Context, is infoschema.I // LoadStatsFromJSONNoUpdate will load statistic from JSONTable, and save it to the storage. func (s *statsReadWriter) LoadStatsFromJSONNoUpdate(ctx context.Context, is infoschema.InfoSchema, - jsonTbl *util.JSONTable, concurrencyForPartition uint8) error { - nCPU := uint8(runtime.GOMAXPROCS(0)) + jsonTbl *util.JSONTable, concurrencyForPartition int) error { + nCPU := runtime.GOMAXPROCS(0) if concurrencyForPartition == 0 { - concurrencyForPartition = nCPU / 2 // default - } - if concurrencyForPartition > nCPU { - concurrencyForPartition = nCPU // for safety + concurrencyForPartition = (nCPU + 1) / 2 // default } + concurrencyForPartition = min(concurrencyForPartition, nCPU) // for safety table, err := is.TableByName(model.NewCIStr(jsonTbl.DatabaseName), model.NewCIStr(jsonTbl.TableName)) if err != nil { diff --git a/pkg/statistics/handle/types/interfaces.go b/pkg/statistics/handle/types/interfaces.go index d2c03f3e2dfe0..e3a80f7d4db34 100644 --- a/pkg/statistics/handle/types/interfaces.go +++ b/pkg/statistics/handle/types/interfaces.go @@ -299,10 +299,10 @@ type StatsReadWriter interface { // LoadStatsFromJSON will load statistic from JSONTable, and save it to the storage. // In final, it will also udpate the stats cache. - LoadStatsFromJSON(ctx context.Context, is infoschema.InfoSchema, jsonTbl *statsutil.JSONTable, concurrencyForPartition uint8) error + LoadStatsFromJSON(ctx context.Context, is infoschema.InfoSchema, jsonTbl *statsutil.JSONTable, concurrencyForPartition int) error // LoadStatsFromJSONNoUpdate will load statistic from JSONTable, and save it to the storage. - LoadStatsFromJSONNoUpdate(ctx context.Context, is infoschema.InfoSchema, jsonTbl *statsutil.JSONTable, concurrencyForPartition uint8) error + LoadStatsFromJSONNoUpdate(ctx context.Context, is infoschema.InfoSchema, jsonTbl *statsutil.JSONTable, concurrencyForPartition int) error // Methods for extended stast. From 69b7458d481529c68afa3450f0538a27ad3218fd Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Tue, 12 Dec 2023 16:12:53 +0800 Subject: [PATCH 2/3] add tests --- pkg/statistics/handle/storage/dump_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/statistics/handle/storage/dump_test.go b/pkg/statistics/handle/storage/dump_test.go index 454fd69fd6130..f582f45e24d26 100644 --- a/pkg/statistics/handle/storage/dump_test.go +++ b/pkg/statistics/handle/storage/dump_test.go @@ -19,6 +19,7 @@ import ( "encoding/json" "errors" "fmt" + "runtime" "strings" "testing" @@ -226,6 +227,10 @@ func TestLoadPartitionStats(t *testing.T) { func TestLoadPartitionStatsErrPanic(t *testing.T) { store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) + val := runtime.GOMAXPROCS(1) + defer func() { + runtime.GOMAXPROCS(val) + }() tk.MustExec("use test") tk.MustExec("set @@tidb_analyze_version = 2") tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") From 4ba93d0b29178d103b6e2309ab4d59eb3a338389 Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Tue, 12 Dec 2023 16:28:13 +0800 Subject: [PATCH 3/3] update --- pkg/statistics/handle/storage/stats_read_writer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/statistics/handle/storage/stats_read_writer.go b/pkg/statistics/handle/storage/stats_read_writer.go index 63cbef4f08ec8..8f7be58ea0c3c 100644 --- a/pkg/statistics/handle/storage/stats_read_writer.go +++ b/pkg/statistics/handle/storage/stats_read_writer.go @@ -594,7 +594,7 @@ func (s *statsReadWriter) LoadStatsFromJSONNoUpdate(ctx context.Context, is info close(taskCh) var wg sync.WaitGroup e := new(atomic.Pointer[error]) - for i := 0; i < int(concurrencyForPartition); i++ { + for i := 0; i < concurrencyForPartition; i++ { wg.Add(1) s.statsHandler.GPool().Go(func() { defer func() {