From b9240a06587779840e6a361e6fdd7ac03a53ab0d Mon Sep 17 00:00:00 2001 From: disksing Date: Wed, 10 Apr 2024 15:45:22 +0800 Subject: [PATCH 01/50] resource_group: add retry configurations (#8041) close tikv/pd#8040 resource_group: add retry configurations Signed-off-by: disksing --- client/resource_group/controller/config.go | 16 ++++++++++++ .../resource_group/controller/controller.go | 26 ++++++++++++++----- .../resourcemanager/resource_manager_test.go | 15 ++++++++--- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/client/resource_group/controller/config.go b/client/resource_group/controller/config.go index ffc360c385c3..a4176c073cc6 100644 --- a/client/resource_group/controller/config.go +++ b/client/resource_group/controller/config.go @@ -52,6 +52,10 @@ const ( defaultTargetPeriod = 5 * time.Second // defaultMaxWaitDuration is the max duration to wait for the token before throwing error. defaultMaxWaitDuration = 30 * time.Second + // defaultWaitRetryTimes is the times to retry when waiting for the token. + defaultWaitRetryTimes = 10 + // defaultWaitRetryInterval is the interval to retry when waiting for the token. + defaultWaitRetryInterval = 50 * time.Millisecond ) const ( @@ -85,6 +89,12 @@ type Config struct { // LTBMaxWaitDuration is the max wait time duration for local token bucket. LTBMaxWaitDuration Duration `toml:"ltb-max-wait-duration" json:"ltb-max-wait-duration"` + // WaitRetryInterval is the interval to retry when waiting for the token. + WaitRetryInterval Duration `toml:"wait-retry-interval" json:"wait-retry-interval"` + + // WaitRetryTimes is the times to retry when waiting for the token. + WaitRetryTimes int `toml:"wait-retry-times" json:"wait-retry-times"` + // RequestUnit is the configuration determines the coefficients of the RRU and WRU cost. // This configuration should be modified carefully. RequestUnit RequestUnitConfig `toml:"request-unit" json:"request-unit"` @@ -98,6 +108,8 @@ func DefaultConfig() *Config { return &Config{ DegradedModeWaitDuration: NewDuration(defaultDegradedModeWaitDuration), LTBMaxWaitDuration: NewDuration(defaultMaxWaitDuration), + WaitRetryInterval: NewDuration(defaultWaitRetryInterval), + WaitRetryTimes: defaultWaitRetryTimes, RequestUnit: DefaultRequestUnitConfig(), EnableControllerTraceLog: false, } @@ -155,6 +167,8 @@ type RUConfig struct { // some config for client LTBMaxWaitDuration time.Duration + WaitRetryInterval time.Duration + WaitRetryTimes int DegradedModeWaitDuration time.Duration } @@ -176,6 +190,8 @@ func GenerateRUConfig(config *Config) *RUConfig { WriteBytesCost: RequestUnit(config.RequestUnit.WriteCostPerByte), CPUMsCost: RequestUnit(config.RequestUnit.CPUMsCost), LTBMaxWaitDuration: config.LTBMaxWaitDuration.Duration, + WaitRetryInterval: config.WaitRetryInterval.Duration, + WaitRetryTimes: config.WaitRetryTimes, DegradedModeWaitDuration: config.DegradedModeWaitDuration.Duration, } } diff --git a/client/resource_group/controller/controller.go b/client/resource_group/controller/controller.go index a695aaf82bcb..750a3c6e48f8 100755 --- a/client/resource_group/controller/controller.go +++ b/client/resource_group/controller/controller.go @@ -39,8 +39,6 @@ import ( const ( controllerConfigPath = "resource_group/controller" - maxRetry = 10 - retryInterval = 50 * time.Millisecond maxNotificationChanLen = 200 needTokensAmplification = 1.1 trickleReserveDuration = 1250 * time.Millisecond @@ -105,6 +103,20 @@ func WithMaxWaitDuration(d time.Duration) ResourceControlCreateOption { } } +// WithWaitRetryInterval is the option to set the retry interval when waiting for the token. +func WithWaitRetryInterval(d time.Duration) ResourceControlCreateOption { + return func(controller *ResourceGroupsController) { + controller.ruConfig.WaitRetryInterval = d + } +} + +// WithWaitRetryTimes is the option to set the times to retry when waiting for the token. +func WithWaitRetryTimes(times int) ResourceControlCreateOption { + return func(controller *ResourceGroupsController) { + controller.ruConfig.WaitRetryTimes = times + } +} + var _ ResourceGroupKVInterceptor = (*ResourceGroupsController)(nil) // ResourceGroupsController implements ResourceGroupKVInterceptor. @@ -186,7 +198,7 @@ func loadServerConfig(ctx context.Context, provider ResourceGroupProvider) (*Con log.Warn("[resource group controller] server does not save config, load config failed") return DefaultConfig(), nil } - config := &Config{} + config := DefaultConfig() err = json.Unmarshal(kvs[0].GetValue(), config) if err != nil { return nil, err @@ -367,7 +379,7 @@ func (c *ResourceGroupsController) Start(ctx context.Context) { } for _, item := range resp { cfgRevision = item.Kv.ModRevision - config := &Config{} + config := DefaultConfig() if err := json.Unmarshal(item.Kv.Value, config); err != nil { continue } @@ -1206,7 +1218,7 @@ func (gc *groupCostController) onRequestWait( var i int var d time.Duration retryLoop: - for i = 0; i < maxRetry; i++ { + for i = 0; i < gc.mainCfg.WaitRetryTimes; i++ { switch gc.mode { case rmpb.GroupMode_RawMode: res := make([]*Reservation, 0, len(requestResourceLimitTypeList)) @@ -1230,8 +1242,8 @@ func (gc *groupCostController) onRequestWait( } } gc.requestRetryCounter.Inc() - time.Sleep(retryInterval) - waitDuration += retryInterval + time.Sleep(gc.mainCfg.WaitRetryInterval) + waitDuration += gc.mainCfg.WaitRetryInterval } if err != nil { gc.failedRequestCounter.Inc() diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go index aea0441c7d79..f1ea8736fda2 100644 --- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go +++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go @@ -34,6 +34,7 @@ import ( "github.com/tikv/pd/client/resource_group/controller" "github.com/tikv/pd/pkg/mcs/resourcemanager/server" "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/tests" "go.uber.org/goleak" @@ -1436,14 +1437,20 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupControllerConfigCh waitDuration := 10 * time.Second readBaseCost := 1.5 defaultCfg := controller.DefaultConfig() - // failpoint enableDegradedMode will setup and set it be 1s. - defaultCfg.DegradedModeWaitDuration.Duration = time.Second + expectCfg := server.ControllerConfig{ + // failpoint enableDegradedMode will setup and set it be 1s. + DegradedModeWaitDuration: typeutil.NewDuration(time.Second), + LTBMaxWaitDuration: typeutil.Duration(defaultCfg.LTBMaxWaitDuration), + RequestUnit: server.RequestUnitConfig(defaultCfg.RequestUnit), + EnableControllerTraceLog: defaultCfg.EnableControllerTraceLog, + } expectRUCfg := controller.GenerateRUConfig(defaultCfg) + expectRUCfg.DegradedModeWaitDuration = time.Second // initial config verification respString := sendRequest("GET", getAddr()+configURL, nil) - defaultString, err := json.Marshal(defaultCfg) + expectStr, err := json.Marshal(expectCfg) re.NoError(err) - re.JSONEq(string(respString), string(defaultString)) + re.JSONEq(string(respString), string(expectStr)) re.EqualValues(expectRUCfg, c1.GetConfig()) testCases := []struct { From f0eb74b15c4c66a20bc9e942c28073a6746e7d67 Mon Sep 17 00:00:00 2001 From: Hu# Date: Wed, 10 Apr 2024 18:25:52 +0800 Subject: [PATCH 02/50] *: make unit test great (#7952) ref tikv/pd#7969 Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- Makefile | 18 +- client/resource_group/controller/util_test.go | 2 - pkg/audit/audit_test.go | 3 - pkg/autoscaling/calculation_test.go | 5 - pkg/autoscaling/prometheus_test.go | 6 - pkg/balancer/balancer_test.go | 3 - pkg/cache/cache_test.go | 6 - pkg/codec/codec_test.go | 2 - pkg/core/rangetree/range_tree_test.go | 2 - pkg/core/storelimit/limit_test.go | 2 - pkg/encryption/config_test.go | 4 - pkg/encryption/crypter_test.go | 5 - pkg/encryption/master_key_test.go | 8 - pkg/encryption/region_crypter_test.go | 8 - pkg/errs/errs_test.go | 2 - pkg/mock/mockhbstream/mockhbstream_test.go | 1 - pkg/movingaverage/avg_over_time_test.go | 5 - pkg/movingaverage/max_filter_test.go | 1 - pkg/movingaverage/moving_average_test.go | 2 - pkg/movingaverage/weight_allocator_test.go | 1 - pkg/ratelimit/concurrency_limiter_test.go | 1 - pkg/ratelimit/controller_test.go | 4 - pkg/ratelimit/limiter_test.go | 3 - pkg/ratelimit/ratelimiter_test.go | 1 - pkg/slice/slice_test.go | 4 - pkg/utils/apiutil/apiutil_test.go | 3 - pkg/utils/assertutil/assertutil_test.go | 1 - pkg/utils/grpcutil/grpcutil_test.go | 1 - pkg/utils/jsonutil/jsonutil_test.go | 1 - pkg/utils/keyutil/util_test.go | 1 - pkg/utils/logutil/log_test.go | 2 - pkg/utils/metricutil/metricutil_test.go | 1 - pkg/utils/netutil/address_test.go | 2 - pkg/utils/reflectutil/tag_test.go | 3 - pkg/utils/requestutil/context_test.go | 2 - pkg/utils/typeutil/comparison_test.go | 5 - pkg/utils/typeutil/conversion_test.go | 3 - pkg/utils/typeutil/duration_test.go | 2 - pkg/utils/typeutil/size_test.go | 2 - pkg/utils/typeutil/string_slice_test.go | 2 - pkg/utils/typeutil/time_test.go | 3 - server/api/health_test.go | 2 +- server/api/member_test.go | 21 +- tools/go.mod | 1 + tools/go.sum | 4 + tools/pd-ut/README.md | 66 ++ tools/pd-ut/ut.go | 803 ++++++++++++++++++ tools/pd-ut/xprog.go | 119 +++ 48 files changed, 1010 insertions(+), 139 deletions(-) create mode 100644 tools/pd-ut/README.md create mode 100644 tools/pd-ut/ut.go create mode 100644 tools/pd-ut/xprog.go diff --git a/Makefile b/Makefile index 0d02189f508b..d78ddcdd65ed 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,7 @@ BUILD_BIN_PATH := $(ROOT_PATH)/bin build: pd-server pd-ctl pd-recover -tools: pd-tso-bench pd-heartbeat-bench regions-dump stores-dump pd-api-bench +tools: pd-tso-bench pd-heartbeat-bench regions-dump stores-dump pd-api-bench pd-ut PD_SERVER_DEP := ifeq ($(SWAGGER), 1) @@ -108,7 +108,6 @@ pd-server-basic: .PHONY: pre-build build tools pd-server pd-server-basic # Tools - pd-ctl: cd tools && GOEXPERIMENT=$(BUILD_GOEXPERIMENT) CGO_ENABLED=$(BUILD_TOOL_CGO_ENABLED) go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-ctl pd-ctl/main.go pd-tso-bench: @@ -127,8 +126,12 @@ regions-dump: cd tools && CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/regions-dump regions-dump/main.go stores-dump: cd tools && CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/stores-dump stores-dump/main.go +pd-ut: pd-xprog + cd tools && GOEXPERIMENT=$(BUILD_GOEXPERIMENT) CGO_ENABLED=$(BUILD_TOOL_CGO_ENABLED) go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-ut pd-ut/ut.go +pd-xprog: + cd tools && GOEXPERIMENT=$(BUILD_GOEXPERIMENT) CGO_ENABLED=$(BUILD_TOOL_CGO_ENABLED) go build -tags xprog -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/xprog pd-ut/xprog.go -.PHONY: pd-ctl pd-tso-bench pd-recover pd-analysis pd-heartbeat-bench simulator regions-dump stores-dump pd-api-bench +.PHONY: pd-ctl pd-tso-bench pd-recover pd-analysis pd-heartbeat-bench simulator regions-dump stores-dump pd-api-bench pd-ut #### Docker image #### @@ -225,6 +228,12 @@ failpoint-disable: install-tools #### Test #### +ut: pd-ut + @$(FAILPOINT_ENABLE) + ./bin/pd-ut run --race + @$(CLEAN_UT_BINARY) + @$(FAILPOINT_DISABLE) + PACKAGE_DIRECTORIES := $(subst $(PD_PKG)/,,$(PACKAGES)) TEST_PKGS := $(filter $(shell find . -iname "*_test.go" -exec dirname {} \; | \ sort -u | sed -e "s/^\./github.com\/tikv\/pd/"),$(PACKAGES)) @@ -303,6 +312,8 @@ split: clean: failpoint-disable clean-test clean-build +CLEAN_UT_BINARY := find . -name '*.test.bin'| xargs rm -f + clean-test: # Cleaning test tmp... rm -rf /tmp/test_pd* @@ -310,6 +321,7 @@ clean-test: rm -rf /tmp/test_etcd* rm -f $(REAL_CLUSTER_TEST_PATH)/playground.log go clean -testcache + @$(CLEAN_UT_BINARY) clean-build: # Cleaning building files... diff --git a/client/resource_group/controller/util_test.go b/client/resource_group/controller/util_test.go index a89ea08b955c..10fa7c345a50 100644 --- a/client/resource_group/controller/util_test.go +++ b/client/resource_group/controller/util_test.go @@ -27,7 +27,6 @@ type example struct { } func TestDurationJSON(t *testing.T) { - t.Parallel() re := require.New(t) example := &example{} @@ -41,7 +40,6 @@ func TestDurationJSON(t *testing.T) { } func TestDurationTOML(t *testing.T) { - t.Parallel() re := require.New(t) example := &example{} diff --git a/pkg/audit/audit_test.go b/pkg/audit/audit_test.go index 8098b36975ee..9066d81ebe36 100644 --- a/pkg/audit/audit_test.go +++ b/pkg/audit/audit_test.go @@ -32,7 +32,6 @@ import ( ) func TestLabelMatcher(t *testing.T) { - t.Parallel() re := require.New(t) matcher := &LabelMatcher{"testSuccess"} labels1 := &BackendLabels{Labels: []string{"testFail", "testSuccess"}} @@ -42,7 +41,6 @@ func TestLabelMatcher(t *testing.T) { } func TestPrometheusHistogramBackend(t *testing.T) { - t.Parallel() re := require.New(t) serviceAuditHistogramTest := prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -90,7 +88,6 @@ func TestPrometheusHistogramBackend(t *testing.T) { } func TestLocalLogBackendUsingFile(t *testing.T) { - t.Parallel() re := require.New(t) backend := NewLocalLogBackend(true) fname := testutil.InitTempFileLogger("info") diff --git a/pkg/autoscaling/calculation_test.go b/pkg/autoscaling/calculation_test.go index 05a348af59de..5d0c2ba11267 100644 --- a/pkg/autoscaling/calculation_test.go +++ b/pkg/autoscaling/calculation_test.go @@ -29,7 +29,6 @@ import ( ) func TestGetScaledTiKVGroups(t *testing.T) { - t.Parallel() re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -214,7 +213,6 @@ func (q *mockQuerier) Query(options *QueryOptions) (QueryResult, error) { } func TestGetTotalCPUUseTime(t *testing.T) { - t.Parallel() re := require.New(t) querier := &mockQuerier{} instances := []instance{ @@ -237,7 +235,6 @@ func TestGetTotalCPUUseTime(t *testing.T) { } func TestGetTotalCPUQuota(t *testing.T) { - t.Parallel() re := require.New(t) querier := &mockQuerier{} instances := []instance{ @@ -260,7 +257,6 @@ func TestGetTotalCPUQuota(t *testing.T) { } func TestScaleOutGroupLabel(t *testing.T) { - t.Parallel() re := require.New(t) var jsonStr = []byte(` { @@ -303,7 +299,6 @@ func TestScaleOutGroupLabel(t *testing.T) { } func TestStrategyChangeCount(t *testing.T) { - t.Parallel() re := require.New(t) var count uint64 = 2 strategy := &Strategy{ diff --git a/pkg/autoscaling/prometheus_test.go b/pkg/autoscaling/prometheus_test.go index b4cf9aefd918..3ee6cb94e37a 100644 --- a/pkg/autoscaling/prometheus_test.go +++ b/pkg/autoscaling/prometheus_test.go @@ -180,7 +180,6 @@ func (c *normalClient) Do(_ context.Context, req *http.Request) (response *http. } func TestRetrieveCPUMetrics(t *testing.T) { - t.Parallel() re := require.New(t) client := &normalClient{ mockData: make(map[string]*response), @@ -225,7 +224,6 @@ func (c *emptyResponseClient) Do(_ context.Context, req *http.Request) (r *http. } func TestEmptyResponse(t *testing.T) { - t.Parallel() re := require.New(t) client := &emptyResponseClient{} querier := NewPrometheusQuerier(client) @@ -253,7 +251,6 @@ func (c *errorHTTPStatusClient) Do(_ context.Context, req *http.Request) (r *htt } func TestErrorHTTPStatus(t *testing.T) { - t.Parallel() re := require.New(t) client := &errorHTTPStatusClient{} querier := NewPrometheusQuerier(client) @@ -279,7 +276,6 @@ func (c *errorPrometheusStatusClient) Do(_ context.Context, req *http.Request) ( } func TestErrorPrometheusStatus(t *testing.T) { - t.Parallel() re := require.New(t) client := &errorPrometheusStatusClient{} querier := NewPrometheusQuerier(client) @@ -290,7 +286,6 @@ func TestErrorPrometheusStatus(t *testing.T) { } func TestGetInstanceNameFromAddress(t *testing.T) { - t.Parallel() re := require.New(t) testCases := []struct { address string @@ -328,7 +323,6 @@ func TestGetInstanceNameFromAddress(t *testing.T) { } func TestGetDurationExpression(t *testing.T) { - t.Parallel() re := require.New(t) testCases := []struct { duration time.Duration diff --git a/pkg/balancer/balancer_test.go b/pkg/balancer/balancer_test.go index 996b4f1da357..2c760c6220c3 100644 --- a/pkg/balancer/balancer_test.go +++ b/pkg/balancer/balancer_test.go @@ -22,7 +22,6 @@ import ( ) func TestBalancerPutAndDelete(t *testing.T) { - t.Parallel() re := require.New(t) balancers := []Balancer[uint32]{ NewRoundRobin[uint32](), @@ -56,7 +55,6 @@ func TestBalancerPutAndDelete(t *testing.T) { } func TestBalancerDuplicate(t *testing.T) { - t.Parallel() re := require.New(t) balancers := []Balancer[uint32]{ NewRoundRobin[uint32](), @@ -77,7 +75,6 @@ func TestBalancerDuplicate(t *testing.T) { } func TestRoundRobin(t *testing.T) { - t.Parallel() re := require.New(t) balancer := NewRoundRobin[uint32]() for i := 0; i < 100; i++ { diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go index dbef41d27548..43e97dfa2b0e 100644 --- a/pkg/cache/cache_test.go +++ b/pkg/cache/cache_test.go @@ -25,7 +25,6 @@ import ( ) func TestExpireRegionCache(t *testing.T) { - t.Parallel() re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -121,7 +120,6 @@ func sortIDs(ids []uint64) []uint64 { } func TestLRUCache(t *testing.T) { - t.Parallel() re := require.New(t) cache := newLRU(3) @@ -199,7 +197,6 @@ func TestLRUCache(t *testing.T) { } func TestFifoCache(t *testing.T) { - t.Parallel() re := require.New(t) cache := NewFIFO(3) cache.Put(1, "1") @@ -227,7 +224,6 @@ func TestFifoCache(t *testing.T) { } func TestFifoFromLastSameElems(t *testing.T) { - t.Parallel() re := require.New(t) type testStruct struct { value string @@ -260,7 +256,6 @@ func TestFifoFromLastSameElems(t *testing.T) { } func TestTwoQueueCache(t *testing.T) { - t.Parallel() re := require.New(t) cache := newTwoQueue(3) cache.Put(1, "1") @@ -345,7 +340,6 @@ func (pq PriorityQueueItemTest) ID() uint64 { } func TestPriorityQueue(t *testing.T) { - t.Parallel() re := require.New(t) testData := []PriorityQueueItemTest{0, 1, 2, 3, 4, 5} pq := NewPriorityQueue(0) diff --git a/pkg/codec/codec_test.go b/pkg/codec/codec_test.go index f734d2e528e2..50bf552a60dd 100644 --- a/pkg/codec/codec_test.go +++ b/pkg/codec/codec_test.go @@ -21,7 +21,6 @@ import ( ) func TestDecodeBytes(t *testing.T) { - t.Parallel() re := require.New(t) key := "abcdefghijklmnopqrstuvwxyz" for i := 0; i < len(key); i++ { @@ -32,7 +31,6 @@ func TestDecodeBytes(t *testing.T) { } func TestTableID(t *testing.T) { - t.Parallel() re := require.New(t) key := EncodeBytes([]byte("t\x80\x00\x00\x00\x00\x00\x00\xff")) re.Equal(int64(0xff), key.TableID()) diff --git a/pkg/core/rangetree/range_tree_test.go b/pkg/core/rangetree/range_tree_test.go index 0664a7bdbefe..6955947cb1bc 100644 --- a/pkg/core/rangetree/range_tree_test.go +++ b/pkg/core/rangetree/range_tree_test.go @@ -85,7 +85,6 @@ func bucketDebrisFactory(startKey, endKey []byte, item RangeItem) []RangeItem { } func TestRingPutItem(t *testing.T) { - t.Parallel() re := require.New(t) bucketTree := NewRangeTree(2, bucketDebrisFactory) bucketTree.Update(newSimpleBucketItem([]byte("002"), []byte("100"))) @@ -120,7 +119,6 @@ func TestRingPutItem(t *testing.T) { } func TestDebris(t *testing.T) { - t.Parallel() re := require.New(t) ringItem := newSimpleBucketItem([]byte("010"), []byte("090")) var overlaps []RangeItem diff --git a/pkg/core/storelimit/limit_test.go b/pkg/core/storelimit/limit_test.go index 758653303114..e11618767a12 100644 --- a/pkg/core/storelimit/limit_test.go +++ b/pkg/core/storelimit/limit_test.go @@ -45,7 +45,6 @@ func TestStoreLimit(t *testing.T) { } func TestSlidingWindow(t *testing.T) { - t.Parallel() re := require.New(t) capacity := int64(defaultWindowSize) s := NewSlidingWindows() @@ -92,7 +91,6 @@ func TestSlidingWindow(t *testing.T) { } func TestWindow(t *testing.T) { - t.Parallel() re := require.New(t) capacity := int64(100 * 10) s := newWindow(capacity) diff --git a/pkg/encryption/config_test.go b/pkg/encryption/config_test.go index 6f7e4a41b034..4134d46c2f3b 100644 --- a/pkg/encryption/config_test.go +++ b/pkg/encryption/config_test.go @@ -23,7 +23,6 @@ import ( ) func TestAdjustDefaultValue(t *testing.T) { - t.Parallel() re := require.New(t) config := &Config{} err := config.Adjust() @@ -35,21 +34,18 @@ func TestAdjustDefaultValue(t *testing.T) { } func TestAdjustInvalidDataEncryptionMethod(t *testing.T) { - t.Parallel() re := require.New(t) config := &Config{DataEncryptionMethod: "unknown"} re.Error(config.Adjust()) } func TestAdjustNegativeRotationDuration(t *testing.T) { - t.Parallel() re := require.New(t) config := &Config{DataKeyRotationPeriod: typeutil.NewDuration(time.Duration(int64(-1)))} re.Error(config.Adjust()) } func TestAdjustInvalidMasterKeyType(t *testing.T) { - t.Parallel() re := require.New(t) config := &Config{MasterKey: MasterKeyConfig{Type: "unknown"}} re.Error(config.Adjust()) diff --git a/pkg/encryption/crypter_test.go b/pkg/encryption/crypter_test.go index 12a851d15639..9ac72bd78138 100644 --- a/pkg/encryption/crypter_test.go +++ b/pkg/encryption/crypter_test.go @@ -24,7 +24,6 @@ import ( ) func TestEncryptionMethodSupported(t *testing.T) { - t.Parallel() re := require.New(t) re.Error(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_PLAINTEXT)) re.Error(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_UNKNOWN)) @@ -34,7 +33,6 @@ func TestEncryptionMethodSupported(t *testing.T) { } func TestKeyLength(t *testing.T) { - t.Parallel() re := require.New(t) _, err := KeyLength(encryptionpb.EncryptionMethod_PLAINTEXT) re.Error(err) @@ -52,7 +50,6 @@ func TestKeyLength(t *testing.T) { } func TestNewIv(t *testing.T) { - t.Parallel() re := require.New(t) ivCtr, err := NewIvCTR() re.NoError(err) @@ -63,7 +60,6 @@ func TestNewIv(t *testing.T) { } func TestNewDataKey(t *testing.T) { - t.Parallel() re := require.New(t) for _, method := range []encryptionpb.EncryptionMethod{ encryptionpb.EncryptionMethod_AES128_CTR, @@ -82,7 +78,6 @@ func TestNewDataKey(t *testing.T) { } func TestAesGcmCrypter(t *testing.T) { - t.Parallel() re := require.New(t) key, err := hex.DecodeString("ed568fbd8c8018ed2d042a4e5d38d6341486922d401d2022fb81e47c900d3f07") re.NoError(err) diff --git a/pkg/encryption/master_key_test.go b/pkg/encryption/master_key_test.go index 4bc08dab7a50..31962e9e99d7 100644 --- a/pkg/encryption/master_key_test.go +++ b/pkg/encryption/master_key_test.go @@ -24,7 +24,6 @@ import ( ) func TestPlaintextMasterKey(t *testing.T) { - t.Parallel() re := require.New(t) config := &encryptionpb.MasterKey{ Backend: &encryptionpb.MasterKey_Plaintext{ @@ -50,7 +49,6 @@ func TestPlaintextMasterKey(t *testing.T) { } func TestEncrypt(t *testing.T) { - t.Parallel() re := require.New(t) keyHex := "2f07ec61e5a50284f47f2b402a962ec672e500b26cb3aa568bb1531300c74806" // #nosec G101 key, err := hex.DecodeString(keyHex) @@ -66,7 +64,6 @@ func TestEncrypt(t *testing.T) { } func TestDecrypt(t *testing.T) { - t.Parallel() re := require.New(t) keyHex := "2f07ec61e5a50284f47f2b402a962ec672e500b26cb3aa568bb1531300c74806" // #nosec G101 key, err := hex.DecodeString(keyHex) @@ -83,7 +80,6 @@ func TestDecrypt(t *testing.T) { } func TestNewFileMasterKeyMissingPath(t *testing.T) { - t.Parallel() re := require.New(t) config := &encryptionpb.MasterKey{ Backend: &encryptionpb.MasterKey_File{ @@ -97,7 +93,6 @@ func TestNewFileMasterKeyMissingPath(t *testing.T) { } func TestNewFileMasterKeyMissingFile(t *testing.T) { - t.Parallel() re := require.New(t) dir := t.TempDir() path := dir + "/key" @@ -113,7 +108,6 @@ func TestNewFileMasterKeyMissingFile(t *testing.T) { } func TestNewFileMasterKeyNotHexString(t *testing.T) { - t.Parallel() re := require.New(t) dir := t.TempDir() path := dir + "/key" @@ -130,7 +124,6 @@ func TestNewFileMasterKeyNotHexString(t *testing.T) { } func TestNewFileMasterKeyLengthMismatch(t *testing.T) { - t.Parallel() re := require.New(t) dir := t.TempDir() path := dir + "/key" @@ -147,7 +140,6 @@ func TestNewFileMasterKeyLengthMismatch(t *testing.T) { } func TestNewFileMasterKey(t *testing.T) { - t.Parallel() re := require.New(t) key := "2f07ec61e5a50284f47f2b402a962ec672e500b26cb3aa568bb1531300c74806" // #nosec G101 dir := t.TempDir() diff --git a/pkg/encryption/region_crypter_test.go b/pkg/encryption/region_crypter_test.go index 5fd9778a8c0e..b1ca558063c1 100644 --- a/pkg/encryption/region_crypter_test.go +++ b/pkg/encryption/region_crypter_test.go @@ -70,7 +70,6 @@ func (m *testKeyManager) GetKey(keyID uint64) (*encryptionpb.DataKey, error) { } func TestNilRegion(t *testing.T) { - t.Parallel() re := require.New(t) m := newTestKeyManager() region, err := EncryptRegion(nil, m) @@ -81,7 +80,6 @@ func TestNilRegion(t *testing.T) { } func TestEncryptRegionWithoutKeyManager(t *testing.T) { - t.Parallel() re := require.New(t) region := &metapb.Region{ Id: 10, @@ -98,7 +96,6 @@ func TestEncryptRegionWithoutKeyManager(t *testing.T) { } func TestEncryptRegionWhileEncryptionDisabled(t *testing.T) { - t.Parallel() re := require.New(t) region := &metapb.Region{ Id: 10, @@ -117,7 +114,6 @@ func TestEncryptRegionWhileEncryptionDisabled(t *testing.T) { } func TestEncryptRegion(t *testing.T) { - t.Parallel() re := require.New(t) startKey := []byte("abc") endKey := []byte("xyz") @@ -152,7 +148,6 @@ func TestEncryptRegion(t *testing.T) { } func TestDecryptRegionNotEncrypted(t *testing.T) { - t.Parallel() re := require.New(t) region := &metapb.Region{ Id: 10, @@ -170,7 +165,6 @@ func TestDecryptRegionNotEncrypted(t *testing.T) { } func TestDecryptRegionWithoutKeyManager(t *testing.T) { - t.Parallel() re := require.New(t) region := &metapb.Region{ Id: 10, @@ -186,7 +180,6 @@ func TestDecryptRegionWithoutKeyManager(t *testing.T) { } func TestDecryptRegionWhileKeyMissing(t *testing.T) { - t.Parallel() re := require.New(t) keyID := uint64(3) m := newTestKeyManager() @@ -207,7 +200,6 @@ func TestDecryptRegionWhileKeyMissing(t *testing.T) { } func TestDecryptRegion(t *testing.T) { - t.Parallel() re := require.New(t) keyID := uint64(1) startKey := []byte("abc") diff --git a/pkg/errs/errs_test.go b/pkg/errs/errs_test.go index d76c02dc1109..1dcabc32d9a0 100644 --- a/pkg/errs/errs_test.go +++ b/pkg/errs/errs_test.go @@ -97,7 +97,6 @@ func TestError(t *testing.T) { } func TestErrorEqual(t *testing.T) { - t.Parallel() re := require.New(t) err1 := ErrSchedulerNotFound.FastGenByArgs() err2 := ErrSchedulerNotFound.FastGenByArgs() @@ -134,7 +133,6 @@ func TestZapError(t *testing.T) { } func TestErrorWithStack(t *testing.T) { - t.Parallel() re := require.New(t) conf := &log.Config{Level: "debug", File: log.FileLogConfig{}, DisableTimestamp: true} lg := newZapTestLogger(conf) diff --git a/pkg/mock/mockhbstream/mockhbstream_test.go b/pkg/mock/mockhbstream/mockhbstream_test.go index a8e88f61aee6..aa1ca85279b4 100644 --- a/pkg/mock/mockhbstream/mockhbstream_test.go +++ b/pkg/mock/mockhbstream/mockhbstream_test.go @@ -29,7 +29,6 @@ import ( ) func TestActivity(t *testing.T) { - t.Parallel() re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/pkg/movingaverage/avg_over_time_test.go b/pkg/movingaverage/avg_over_time_test.go index 43553d9d6088..4a54e33d449b 100644 --- a/pkg/movingaverage/avg_over_time_test.go +++ b/pkg/movingaverage/avg_over_time_test.go @@ -23,7 +23,6 @@ import ( ) func TestPulse(t *testing.T) { - t.Parallel() re := require.New(t) aot := NewAvgOverTime(5 * time.Second) // warm up @@ -43,7 +42,6 @@ func TestPulse(t *testing.T) { } func TestPulse2(t *testing.T) { - t.Parallel() re := require.New(t) dur := 5 * time.Second aot := NewAvgOverTime(dur) @@ -57,7 +55,6 @@ func TestPulse2(t *testing.T) { } func TestChange(t *testing.T) { - t.Parallel() re := require.New(t) aot := NewAvgOverTime(5 * time.Second) @@ -91,7 +88,6 @@ func TestChange(t *testing.T) { } func TestMinFilled(t *testing.T) { - t.Parallel() re := require.New(t) interval := 10 * time.Second rate := 1.0 @@ -108,7 +104,6 @@ func TestMinFilled(t *testing.T) { } func TestUnstableInterval(t *testing.T) { - t.Parallel() re := require.New(t) aot := NewAvgOverTime(5 * time.Second) re.Equal(0., aot.Get()) diff --git a/pkg/movingaverage/max_filter_test.go b/pkg/movingaverage/max_filter_test.go index bba770cecc23..7d3906ec93c5 100644 --- a/pkg/movingaverage/max_filter_test.go +++ b/pkg/movingaverage/max_filter_test.go @@ -21,7 +21,6 @@ import ( ) func TestMaxFilter(t *testing.T) { - t.Parallel() re := require.New(t) var empty float64 = 0 data := []float64{2, 1, 3, 4, 1, 1, 3, 3, 2, 0, 5} diff --git a/pkg/movingaverage/moving_average_test.go b/pkg/movingaverage/moving_average_test.go index 49c20637c209..fd0a1a9fcf36 100644 --- a/pkg/movingaverage/moving_average_test.go +++ b/pkg/movingaverage/moving_average_test.go @@ -72,7 +72,6 @@ func checkInstantaneous(re *require.Assertions, ma MovingAvg) { } func TestMedianFilter(t *testing.T) { - t.Parallel() re := require.New(t) var empty float64 = 0 data := []float64{2, 4, 2, 800, 600, 6, 3} @@ -92,7 +91,6 @@ type testCase struct { } func TestMovingAvg(t *testing.T) { - t.Parallel() re := require.New(t) var empty float64 = 0 data := []float64{1, 1, 1, 1, 5, 1, 1, 1} diff --git a/pkg/movingaverage/weight_allocator_test.go b/pkg/movingaverage/weight_allocator_test.go index 631a71f10c95..405d8f728762 100644 --- a/pkg/movingaverage/weight_allocator_test.go +++ b/pkg/movingaverage/weight_allocator_test.go @@ -21,7 +21,6 @@ import ( ) func TestWeightAllocator(t *testing.T) { - t.Parallel() re := require.New(t) checkSumFunc := func(wa *WeightAllocator, length int) { diff --git a/pkg/ratelimit/concurrency_limiter_test.go b/pkg/ratelimit/concurrency_limiter_test.go index e77c79c8ebc3..a397b6ac50fe 100644 --- a/pkg/ratelimit/concurrency_limiter_test.go +++ b/pkg/ratelimit/concurrency_limiter_test.go @@ -26,7 +26,6 @@ import ( ) func TestConcurrencyLimiter(t *testing.T) { - t.Parallel() re := require.New(t) cl := NewConcurrencyLimiter(10) for i := 0; i < 10; i++ { diff --git a/pkg/ratelimit/controller_test.go b/pkg/ratelimit/controller_test.go index 48a5ee2054bc..50be24540e4a 100644 --- a/pkg/ratelimit/controller_test.go +++ b/pkg/ratelimit/controller_test.go @@ -78,7 +78,6 @@ func runMulitLabelLimiter(t *testing.T, limiter *Controller, testCase []labelCas } func TestControllerWithConcurrencyLimiter(t *testing.T) { - t.Parallel() re := require.New(t) limiter := NewController(context.Background(), "grpc", nil) defer limiter.Close() @@ -191,7 +190,6 @@ func TestControllerWithConcurrencyLimiter(t *testing.T) { } func TestBlockList(t *testing.T) { - t.Parallel() re := require.New(t) opts := []Option{AddLabelAllowList()} limiter := NewController(context.Background(), "grpc", nil) @@ -213,7 +211,6 @@ func TestBlockList(t *testing.T) { } func TestControllerWithQPSLimiter(t *testing.T) { - t.Parallel() re := require.New(t) limiter := NewController(context.Background(), "grpc", nil) defer limiter.Close() @@ -323,7 +320,6 @@ func TestControllerWithQPSLimiter(t *testing.T) { } func TestControllerWithTwoLimiters(t *testing.T) { - t.Parallel() re := require.New(t) limiter := NewController(context.Background(), "grpc", nil) defer limiter.Close() diff --git a/pkg/ratelimit/limiter_test.go b/pkg/ratelimit/limiter_test.go index fabb9d989172..36f339b47ac0 100644 --- a/pkg/ratelimit/limiter_test.go +++ b/pkg/ratelimit/limiter_test.go @@ -40,7 +40,6 @@ func (r *releaseUtil) append(d DoneFunc) { } func TestWithConcurrencyLimiter(t *testing.T) { - t.Parallel() re := require.New(t) limiter := newLimiter() @@ -103,7 +102,6 @@ func TestWithConcurrencyLimiter(t *testing.T) { } func TestWithQPSLimiter(t *testing.T) { - t.Parallel() re := require.New(t) limiter := newLimiter() status := limiter.updateQPSConfig(float64(rate.Every(time.Second)), 1) @@ -177,7 +175,6 @@ func TestWithQPSLimiter(t *testing.T) { } func TestWithTwoLimiters(t *testing.T) { - t.Parallel() re := require.New(t) cfg := &DimensionConfig{ QPS: 100, diff --git a/pkg/ratelimit/ratelimiter_test.go b/pkg/ratelimit/ratelimiter_test.go index 35b355e7b210..f16bb6a83d28 100644 --- a/pkg/ratelimit/ratelimiter_test.go +++ b/pkg/ratelimit/ratelimiter_test.go @@ -22,7 +22,6 @@ import ( ) func TestRateLimiter(t *testing.T) { - t.Parallel() re := require.New(t) limiter := NewRateLimiter(100, 100) diff --git a/pkg/slice/slice_test.go b/pkg/slice/slice_test.go index 1fe3fe79dcf7..019cd49c46a8 100644 --- a/pkg/slice/slice_test.go +++ b/pkg/slice/slice_test.go @@ -22,7 +22,6 @@ import ( ) func TestSlice(t *testing.T) { - t.Parallel() re := require.New(t) testCases := []struct { a []int @@ -45,7 +44,6 @@ func TestSlice(t *testing.T) { } func TestSliceContains(t *testing.T) { - t.Parallel() re := require.New(t) ss := []string{"a", "b", "c"} re.True(slice.Contains(ss, "a")) @@ -61,7 +59,6 @@ func TestSliceContains(t *testing.T) { } func TestSliceRemoveGenericTypes(t *testing.T) { - t.Parallel() re := require.New(t) ss := []string{"a", "b", "c"} ss = slice.Remove(ss, "a") @@ -77,7 +74,6 @@ func TestSliceRemoveGenericTypes(t *testing.T) { } func TestSliceRemove(t *testing.T) { - t.Parallel() re := require.New(t) is := []int64{} diff --git a/pkg/utils/apiutil/apiutil_test.go b/pkg/utils/apiutil/apiutil_test.go index 106d3fb21cbb..aee21621dd23 100644 --- a/pkg/utils/apiutil/apiutil_test.go +++ b/pkg/utils/apiutil/apiutil_test.go @@ -26,7 +26,6 @@ import ( ) func TestJsonRespondErrorOk(t *testing.T) { - t.Parallel() re := require.New(t) rd := render.New(render.Options{ IndentJSON: true, @@ -45,7 +44,6 @@ func TestJsonRespondErrorOk(t *testing.T) { } func TestJsonRespondErrorBadInput(t *testing.T) { - t.Parallel() re := require.New(t) rd := render.New(render.Options{ IndentJSON: true, @@ -71,7 +69,6 @@ func TestJsonRespondErrorBadInput(t *testing.T) { } func TestGetIPPortFromHTTPRequest(t *testing.T) { - t.Parallel() re := require.New(t) testCases := []struct { diff --git a/pkg/utils/assertutil/assertutil_test.go b/pkg/utils/assertutil/assertutil_test.go index 84bd21cef059..076cdd2ac934 100644 --- a/pkg/utils/assertutil/assertutil_test.go +++ b/pkg/utils/assertutil/assertutil_test.go @@ -22,7 +22,6 @@ import ( ) func TestNilFail(t *testing.T) { - t.Parallel() re := require.New(t) var failErr error checker := NewChecker() diff --git a/pkg/utils/grpcutil/grpcutil_test.go b/pkg/utils/grpcutil/grpcutil_test.go index 21b7e1a4acbf..2cbff4f3ebc1 100644 --- a/pkg/utils/grpcutil/grpcutil_test.go +++ b/pkg/utils/grpcutil/grpcutil_test.go @@ -37,7 +37,6 @@ func TestToTLSConfig(t *testing.T) { } }() - t.Parallel() re := require.New(t) tlsConfig := TLSConfig{ KeyPath: path.Join(certPath, "pd-server-key.pem"), diff --git a/pkg/utils/jsonutil/jsonutil_test.go b/pkg/utils/jsonutil/jsonutil_test.go index a046fbaf70a4..1e8c21917bab 100644 --- a/pkg/utils/jsonutil/jsonutil_test.go +++ b/pkg/utils/jsonutil/jsonutil_test.go @@ -31,7 +31,6 @@ type testJSONStructLevel2 struct { } func TestJSONUtil(t *testing.T) { - t.Parallel() re := require.New(t) father := &testJSONStructLevel1{ Name: "father", diff --git a/pkg/utils/keyutil/util_test.go b/pkg/utils/keyutil/util_test.go index 374faa1f7973..7bcb0a49c6ff 100644 --- a/pkg/utils/keyutil/util_test.go +++ b/pkg/utils/keyutil/util_test.go @@ -21,7 +21,6 @@ import ( ) func TestKeyUtil(t *testing.T) { - t.Parallel() re := require.New(t) startKey := []byte("a") endKey := []byte("b") diff --git a/pkg/utils/logutil/log_test.go b/pkg/utils/logutil/log_test.go index 7d4be7a88bd9..650ba62fe9d9 100644 --- a/pkg/utils/logutil/log_test.go +++ b/pkg/utils/logutil/log_test.go @@ -23,7 +23,6 @@ import ( ) func TestStringToZapLogLevel(t *testing.T) { - t.Parallel() re := require.New(t) re.Equal(zapcore.FatalLevel, StringToZapLogLevel("fatal")) re.Equal(zapcore.ErrorLevel, StringToZapLogLevel("ERROR")) @@ -35,7 +34,6 @@ func TestStringToZapLogLevel(t *testing.T) { } func TestRedactLog(t *testing.T) { - t.Parallel() re := require.New(t) testCases := []struct { name string diff --git a/pkg/utils/metricutil/metricutil_test.go b/pkg/utils/metricutil/metricutil_test.go index b817eb0112d4..acac9ce4d495 100644 --- a/pkg/utils/metricutil/metricutil_test.go +++ b/pkg/utils/metricutil/metricutil_test.go @@ -23,7 +23,6 @@ import ( ) func TestCamelCaseToSnakeCase(t *testing.T) { - t.Parallel() re := require.New(t) inputs := []struct { name string diff --git a/pkg/utils/netutil/address_test.go b/pkg/utils/netutil/address_test.go index faa3e2e1d04b..127c9a6d0f7d 100644 --- a/pkg/utils/netutil/address_test.go +++ b/pkg/utils/netutil/address_test.go @@ -22,7 +22,6 @@ import ( ) func TestResolveLoopBackAddr(t *testing.T) { - t.Parallel() re := require.New(t) nodes := []struct { address string @@ -40,7 +39,6 @@ func TestResolveLoopBackAddr(t *testing.T) { } func TestIsEnableHttps(t *testing.T) { - t.Parallel() re := require.New(t) re.False(IsEnableHTTPS(http.DefaultClient)) httpClient := &http.Client{ diff --git a/pkg/utils/reflectutil/tag_test.go b/pkg/utils/reflectutil/tag_test.go index f613f1f81b67..3e49e093912b 100644 --- a/pkg/utils/reflectutil/tag_test.go +++ b/pkg/utils/reflectutil/tag_test.go @@ -35,7 +35,6 @@ type testStruct3 struct { } func TestFindJSONFullTagByChildTag(t *testing.T) { - t.Parallel() re := require.New(t) key := "enable" result := FindJSONFullTagByChildTag(reflect.TypeOf(testStruct1{}), key) @@ -51,7 +50,6 @@ func TestFindJSONFullTagByChildTag(t *testing.T) { } func TestFindSameFieldByJSON(t *testing.T) { - t.Parallel() re := require.New(t) input := map[string]any{ "name": "test2", @@ -65,7 +63,6 @@ func TestFindSameFieldByJSON(t *testing.T) { } func TestFindFieldByJSONTag(t *testing.T) { - t.Parallel() re := require.New(t) t1 := testStruct1{} t2 := testStruct2{} diff --git a/pkg/utils/requestutil/context_test.go b/pkg/utils/requestutil/context_test.go index 298fc1ff8a34..e6bdcd7be46f 100644 --- a/pkg/utils/requestutil/context_test.go +++ b/pkg/utils/requestutil/context_test.go @@ -24,7 +24,6 @@ import ( ) func TestRequestInfo(t *testing.T) { - t.Parallel() re := require.New(t) ctx := context.Background() _, ok := RequestInfoFrom(ctx) @@ -53,7 +52,6 @@ func TestRequestInfo(t *testing.T) { } func TestEndTime(t *testing.T) { - t.Parallel() re := require.New(t) ctx := context.Background() _, ok := EndTimeFrom(ctx) diff --git a/pkg/utils/typeutil/comparison_test.go b/pkg/utils/typeutil/comparison_test.go index b296405b3d51..b53e961b4eea 100644 --- a/pkg/utils/typeutil/comparison_test.go +++ b/pkg/utils/typeutil/comparison_test.go @@ -23,7 +23,6 @@ import ( ) func TestMinUint64(t *testing.T) { - t.Parallel() re := require.New(t) re.Equal(uint64(1), MinUint64(1, 2)) re.Equal(uint64(1), MinUint64(2, 1)) @@ -31,7 +30,6 @@ func TestMinUint64(t *testing.T) { } func TestMaxUint64(t *testing.T) { - t.Parallel() re := require.New(t) re.Equal(uint64(2), MaxUint64(1, 2)) re.Equal(uint64(2), MaxUint64(2, 1)) @@ -39,7 +37,6 @@ func TestMaxUint64(t *testing.T) { } func TestMinDuration(t *testing.T) { - t.Parallel() re := require.New(t) re.Equal(time.Second, MinDuration(time.Minute, time.Second)) re.Equal(time.Second, MinDuration(time.Second, time.Minute)) @@ -47,7 +44,6 @@ func TestMinDuration(t *testing.T) { } func TestEqualFloat(t *testing.T) { - t.Parallel() re := require.New(t) f1 := rand.Float64() re.True(Float64Equal(f1, f1*1.000)) @@ -55,7 +51,6 @@ func TestEqualFloat(t *testing.T) { } func TestAreStringSlicesEquivalent(t *testing.T) { - t.Parallel() re := require.New(t) re.True(AreStringSlicesEquivalent(nil, nil)) re.True(AreStringSlicesEquivalent([]string{}, nil)) diff --git a/pkg/utils/typeutil/conversion_test.go b/pkg/utils/typeutil/conversion_test.go index 14b2f9dea522..7b17cfcbe2c1 100644 --- a/pkg/utils/typeutil/conversion_test.go +++ b/pkg/utils/typeutil/conversion_test.go @@ -23,7 +23,6 @@ import ( ) func TestBytesToUint64(t *testing.T) { - t.Parallel() re := require.New(t) str := "\x00\x00\x00\x00\x00\x00\x03\xe8" a, err := BytesToUint64([]byte(str)) @@ -32,7 +31,6 @@ func TestBytesToUint64(t *testing.T) { } func TestUint64ToBytes(t *testing.T) { - t.Parallel() re := require.New(t) var a uint64 = 1000 b := Uint64ToBytes(a) @@ -41,7 +39,6 @@ func TestUint64ToBytes(t *testing.T) { } func TestJSONToUint64Slice(t *testing.T) { - t.Parallel() re := require.New(t) type testArray struct { Array []uint64 `json:"array"` diff --git a/pkg/utils/typeutil/duration_test.go b/pkg/utils/typeutil/duration_test.go index cff7c3cd66cc..42db0be8cdc7 100644 --- a/pkg/utils/typeutil/duration_test.go +++ b/pkg/utils/typeutil/duration_test.go @@ -27,7 +27,6 @@ type example struct { } func TestDurationJSON(t *testing.T) { - t.Parallel() re := require.New(t) example := &example{} @@ -41,7 +40,6 @@ func TestDurationJSON(t *testing.T) { } func TestDurationTOML(t *testing.T) { - t.Parallel() re := require.New(t) example := &example{} diff --git a/pkg/utils/typeutil/size_test.go b/pkg/utils/typeutil/size_test.go index 57c246953e42..f2772ff8c6dc 100644 --- a/pkg/utils/typeutil/size_test.go +++ b/pkg/utils/typeutil/size_test.go @@ -23,7 +23,6 @@ import ( ) func TestSizeJSON(t *testing.T) { - t.Parallel() re := require.New(t) b := ByteSize(265421587) o, err := json.Marshal(b) @@ -40,7 +39,6 @@ func TestSizeJSON(t *testing.T) { } func TestParseMbFromText(t *testing.T) { - t.Parallel() re := require.New(t) testCases := []struct { body []string diff --git a/pkg/utils/typeutil/string_slice_test.go b/pkg/utils/typeutil/string_slice_test.go index 9a197eb68e44..9177cee0eb90 100644 --- a/pkg/utils/typeutil/string_slice_test.go +++ b/pkg/utils/typeutil/string_slice_test.go @@ -22,7 +22,6 @@ import ( ) func TestStringSliceJSON(t *testing.T) { - t.Parallel() re := require.New(t) b := StringSlice([]string{"zone", "rack"}) o, err := json.Marshal(b) @@ -36,7 +35,6 @@ func TestStringSliceJSON(t *testing.T) { } func TestEmpty(t *testing.T) { - t.Parallel() re := require.New(t) ss := StringSlice([]string{}) b, err := json.Marshal(ss) diff --git a/pkg/utils/typeutil/time_test.go b/pkg/utils/typeutil/time_test.go index 7a5baf55afad..b8078f63fa8c 100644 --- a/pkg/utils/typeutil/time_test.go +++ b/pkg/utils/typeutil/time_test.go @@ -23,7 +23,6 @@ import ( ) func TestParseTimestamp(t *testing.T) { - t.Parallel() re := require.New(t) for i := 0; i < 3; i++ { t := time.Now().Add(time.Second * time.Duration(rand.Int31n(1000))) @@ -39,7 +38,6 @@ func TestParseTimestamp(t *testing.T) { } func TestSubTimeByWallClock(t *testing.T) { - t.Parallel() re := require.New(t) for i := 0; i < 100; i++ { r := rand.Int63n(1000) @@ -63,7 +61,6 @@ func TestSubTimeByWallClock(t *testing.T) { } func TestSmallTimeDifference(t *testing.T) { - t.Parallel() re := require.New(t) t1, err := time.Parse("2006-01-02 15:04:05.999", "2021-04-26 00:44:25.682") re.NoError(err) diff --git a/server/api/health_test.go b/server/api/health_test.go index 6d2caec12cd7..89a9627bc377 100644 --- a/server/api/health_test.go +++ b/server/api/health_test.go @@ -26,7 +26,7 @@ import ( ) func checkSliceResponse(re *require.Assertions, body []byte, cfgs []*config.Config, unhealthy string) { - got := []Health{} + var got []Health re.NoError(json.Unmarshal(body, &got)) re.Len(cfgs, len(got)) diff --git a/server/api/member_test.go b/server/api/member_test.go index 65c0ff673608..5d692cda7d92 100644 --- a/server/api/member_test.go +++ b/server/api/member_test.go @@ -158,26 +158,7 @@ func (suite *memberTestSuite) changeLeaderPeerUrls(leader *pdpb.Member, id uint6 resp.Body.Close() } -type resignTestSuite struct { - suite.Suite - cfgs []*config.Config - servers []*server.Server - clean testutil.CleanupFunc -} - -func TestResignTestSuite(t *testing.T) { - suite.Run(t, new(resignTestSuite)) -} - -func (suite *resignTestSuite) SetupSuite() { - suite.cfgs, suite.servers, suite.clean = mustNewCluster(suite.Require(), 1) -} - -func (suite *resignTestSuite) TearDownSuite() { - suite.clean() -} - -func (suite *resignTestSuite) TestResignMyself() { +func (suite *memberTestSuite) TestResignMyself() { re := suite.Require() addr := suite.cfgs[0].ClientUrls + apiPrefix + "/api/v1/leader/resign" resp, err := testDialClient.Post(addr, "", nil) diff --git a/tools/go.mod b/tools/go.mod index 8287f8344718..9d8728f70347 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -31,6 +31,7 @@ require ( github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca + go.uber.org/automaxprocs v1.5.3 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 golang.org/x/text v0.14.0 diff --git a/tools/go.sum b/tools/go.sum index ac6cc75903ed..d7c7a4801b11 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -385,6 +385,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -514,6 +516,8 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= +go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/dig v1.9.0 h1:pJTDXKEhRqBI8W7rU7kwT5EgyRZuSMVSFcZolOvKK9U= go.uber.org/dig v1.9.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= go.uber.org/fx v1.12.0 h1:+1+3Cz9M0dFMPy9SW9XUIUHye8bnPUm7q7DroNGWYG4= diff --git a/tools/pd-ut/README.md b/tools/pd-ut/README.md new file mode 100644 index 000000000000..77b59bea4f77 --- /dev/null +++ b/tools/pd-ut/README.md @@ -0,0 +1,66 @@ +# pd-ut + +pd-ut is a tool to run unit tests for PD. + +## Build + +1. [Go](https://golang.org/) Version 1.21 or later +2. In the root directory of the [PD project](https://github.com/tikv/pd), use the `make pd-ut` command to compile and generate `bin/pd-ut` + +## Usage + +This section describes how to use the pd-ut tool. + +### brief run all tests +```shell +make ut +``` + + +### run by pd-ut + +- You should `make failpoint-enable` before running the tests. +- And after running the tests, you should `make failpoint-disable` and `make clean-test` to disable the failpoint and clean the environment. + +#### Flags description + +```shell +// run all tests +pd-ut + +// show usage +pd-ut -h + +// list all packages +pd-ut list + +// list test cases of a single package +pd-ut list $package + +// list test cases that match a pattern +pd-ut list $package 'r:$regex' + +// run all tests +pd-ut run + +// run test all cases of a single package +pd-ut run $package + +// run test cases of a single package +pd-ut run $package $test + +// run test cases that match a pattern +pd-ut run $package 'r:$regex' + +// build all test package +pd-ut build + +// build a test package +pd-ut build xxx + +// write the junitfile +pd-ut run --junitfile xxx + +// test with race flag +pd-ut run --race +``` diff --git a/tools/pd-ut/ut.go b/tools/pd-ut/ut.go new file mode 100644 index 000000000000..5d50bbd51a85 --- /dev/null +++ b/tools/pd-ut/ut.go @@ -0,0 +1,803 @@ +// Copyright 2024 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/xml" + "errors" + "fmt" + "io" + "log" + "math/rand" + "os" + "os/exec" + "path" + "regexp" + "runtime" + "strconv" + "strings" + "sync" + "time" + + // Set the correct value when it runs inside docker. + _ "go.uber.org/automaxprocs" +) + +func usage() bool { + msg := `// run all tests +pd-ut + +// show usage +pd-ut -h + +// list all packages +pd-ut list + +// list test cases of a single package +pd-ut list $package + +// list test cases that match a pattern +pd-ut list $package 'r:$regex' + +// run all tests +pd-ut run + +// run test all cases of a single package +pd-ut run $package + +// run test cases of a single package +pd-ut run $package $test + +// run test cases that match a pattern +pd-ut run $package 'r:$regex' + +// build all test package +pd-ut build + +// build a test package +pd-ut build xxx + +// write the junitfile +pd-ut run --junitfile xxx + +// test with race flag +pd-ut run --race` + + fmt.Println(msg) + return true +} + +const modulePath = "github.com/tikv/pd" + +var ( + // runtime + p int + buildParallel int + workDir string + // arguments + race bool + junitFile string +) + +func main() { + race = handleFlag("--race") + junitFile = stripFlag("--junitfile") + + // Get the correct count of CPU if it's in docker. + p = runtime.GOMAXPROCS(0) + // We use 2 * p for `go build` to make it faster. + buildParallel = p * 2 + var err error + workDir, err = os.Getwd() + if err != nil { + fmt.Println("os.Getwd() error", err) + } + + var isSucceed bool + // run all tests + if len(os.Args) == 1 { + isSucceed = cmdRun() + } + + if len(os.Args) >= 2 { + switch os.Args[1] { + case "list": + isSucceed = cmdList(os.Args[2:]...) + case "build": + isSucceed = cmdBuild(os.Args[2:]...) + case "run": + isSucceed = cmdRun(os.Args[2:]...) + default: + isSucceed = usage() + } + } + if !isSucceed { + os.Exit(1) + } +} + +func cmdList(args ...string) bool { + pkgs, err := listPackages() + if err != nil { + log.Println("list package error", err) + return false + } + + // list all packages + if len(args) == 0 { + for _, pkg := range pkgs { + fmt.Println(pkg) + } + return false + } + + // list test case of a single package + if len(args) == 1 || len(args) == 2 { + pkg := args[0] + pkgs = filter(pkgs, func(s string) bool { return s == pkg }) + if len(pkgs) != 1 { + fmt.Println("package not exist", pkg) + return false + } + + err := buildTestBinary(pkg) + if err != nil { + log.Println("build package error", pkg, err) + return false + } + exist, err := testBinaryExist(pkg) + if err != nil { + log.Println("check test binary existence error", err) + return false + } + if !exist { + fmt.Println("no test case in ", pkg) + return false + } + + res := listTestCases(pkg, nil) + + if len(args) == 2 { + res, err = filterTestCases(res, args[1]) + if err != nil { + log.Println("filter test cases error", err) + return false + } + } + + for _, x := range res { + fmt.Println(x.test) + } + } + return true +} + +func cmdBuild(args ...string) bool { + pkgs, err := listPackages() + if err != nil { + log.Println("list package error", err) + return false + } + + // build all packages + if len(args) == 0 { + err := buildTestBinaryMulti(pkgs) + if err != nil { + fmt.Println("build package error", pkgs, err) + return false + } + return true + } + + // build test binary of a single package + if len(args) >= 1 { + pkg := args[0] + err := buildTestBinary(pkg) + if err != nil { + log.Println("build package error", pkg, err) + return false + } + } + return true +} + +func cmdRun(args ...string) bool { + var err error + pkgs, err := listPackages() + if err != nil { + fmt.Println("list packages error", err) + return false + } + tasks := make([]task, 0, 5000) + start := time.Now() + // run all tests + if len(args) == 0 { + err := buildTestBinaryMulti(pkgs) + if err != nil { + fmt.Println("build package error", pkgs, err) + return false + } + + for _, pkg := range pkgs { + exist, err := testBinaryExist(pkg) + if err != nil { + fmt.Println("check test binary existence error", err) + return false + } + if !exist { + fmt.Println("no test case in ", pkg) + continue + } + + tasks = listTestCases(pkg, tasks) + } + } + + // run tests for a single package + if len(args) == 1 { + pkg := args[0] + err := buildTestBinary(pkg) + if err != nil { + log.Println("build package error", pkg, err) + return false + } + exist, err := testBinaryExist(pkg) + if err != nil { + log.Println("check test binary existence error", err) + return false + } + + if !exist { + fmt.Println("no test case in ", pkg) + return false + } + tasks = listTestCases(pkg, tasks) + } + + // run a single test + if len(args) == 2 { + pkg := args[0] + err := buildTestBinary(pkg) + if err != nil { + log.Println("build package error", pkg, err) + return false + } + exist, err := testBinaryExist(pkg) + if err != nil { + log.Println("check test binary existence error", err) + return false + } + if !exist { + fmt.Println("no test case in ", pkg) + return false + } + + tasks = listTestCases(pkg, tasks) + tasks, err = filterTestCases(tasks, args[1]) + if err != nil { + log.Println("filter test cases error", err) + return false + } + } + + fmt.Printf("building task finish, parallelism=%d, count=%d, takes=%v\n", buildParallel, len(tasks), time.Since(start)) + + taskCh := make(chan task, 100) + works := make([]numa, p) + var wg sync.WaitGroup + for i := 0; i < p; i++ { + wg.Add(1) + go works[i].worker(&wg, taskCh) + } + + shuffle(tasks) + + start = time.Now() + for _, task := range tasks { + taskCh <- task + } + close(taskCh) + wg.Wait() + fmt.Println("run all tasks takes", time.Since(start)) + + if junitFile != "" { + out := collectTestResults(works) + f, err := os.Create(junitFile) + if err != nil { + fmt.Println("create junit file fail:", err) + return false + } + if err := write(f, out); err != nil { + fmt.Println("write junit file error:", err) + return false + } + } + + for _, work := range works { + if work.Fail { + return false + } + } + return true +} + +// stripFlag strip the '--flag xxx' from the command line os.Args +// Example of the os.Args changes +// Before: ut run pkg TestXXX --junitfile yyy +// After: ut run pkg TestXXX +// The value of the flag is returned. +func stripFlag(flag string) string { + var res string + tmp := os.Args[:0] + // Iter to the flag + var i int + for ; i < len(os.Args); i++ { + if os.Args[i] == flag { + i++ + break + } + tmp = append(tmp, os.Args[i]) + } + // Handle the flag + if i < len(os.Args) { + res = os.Args[i] + i++ + } + // Iter the remain flags + for ; i < len(os.Args); i++ { + tmp = append(tmp, os.Args[i]) + } + + os.Args = tmp + return res +} + +func handleFlag(f string) (found bool) { + tmp := os.Args[:0] + for i := 0; i < len(os.Args); i++ { + if os.Args[i] == f { + found = true + continue + } + tmp = append(tmp, os.Args[i]) + } + os.Args = tmp + return +} + +type task struct { + pkg string + test string +} + +func (t *task) String() string { + return t.pkg + " " + t.test +} + +func listTestCases(pkg string, tasks []task) []task { + newCases := listNewTestCases(pkg) + for _, c := range newCases { + tasks = append(tasks, task{pkg, c}) + } + + return tasks +} + +func filterTestCases(tasks []task, arg1 string) ([]task, error) { + if strings.HasPrefix(arg1, "r:") { + r, err := regexp.Compile(arg1[2:]) + if err != nil { + return nil, err + } + tmp := tasks[:0] + for _, task := range tasks { + if r.MatchString(task.test) { + tmp = append(tmp, task) + } + } + return tmp, nil + } + tmp := tasks[:0] + for _, task := range tasks { + if strings.Contains(task.test, arg1) { + tmp = append(tmp, task) + } + } + return tmp, nil +} + +func listPackages() ([]string, error) { + cmd := exec.Command("go", "list", "./...") + ss, err := cmdToLines(cmd) + if err != nil { + return nil, withTrace(err) + } + + ret := ss[:0] + for _, s := range ss { + if !strings.HasPrefix(s, modulePath) { + continue + } + pkg := s[len(modulePath)+1:] + if skipDIR(pkg) { + continue + } + ret = append(ret, pkg) + } + return ret, nil +} + +type numa struct { + Fail bool + results []testResult +} + +func (n *numa) worker(wg *sync.WaitGroup, ch chan task) { + defer wg.Done() + for t := range ch { + res := n.runTestCase(t.pkg, t.test) + if res.Failure != nil { + fmt.Println("[FAIL] ", t.pkg, t.test) + fmt.Fprintf(os.Stderr, "err=%s\n%s", res.err.Error(), res.Failure.Contents) + n.Fail = true + } + n.results = append(n.results, res) + } +} + +type testResult struct { + JUnitTestCase + d time.Duration + err error +} + +func (n *numa) runTestCase(pkg string, fn string) testResult { + res := testResult{ + JUnitTestCase: JUnitTestCase{ + ClassName: path.Join(modulePath, pkg), + Name: fn, + }, + } + + var buf bytes.Buffer + var err error + var start time.Time + for i := 0; i < 3; i++ { + cmd := n.testCommand(pkg, fn) + cmd.Dir = path.Join(workDir, pkg) + // Combine the test case output, so the run result for failed cases can be displayed. + cmd.Stdout = &buf + cmd.Stderr = &buf + + start = time.Now() + err = cmd.Run() + if err != nil { + var exitError *exec.ExitError + if errors.As(err, &exitError) { + // Retry 3 times to get rid of the weird error: + switch err.Error() { + case "signal: segmentation fault (core dumped)": + buf.Reset() + continue + case "signal: trace/breakpoint trap (core dumped)": + buf.Reset() + continue + } + if strings.Contains(buf.String(), "panic during panic") { + buf.Reset() + continue + } + } + } + break + } + if err != nil { + res.Failure = &JUnitFailure{ + Message: "Failed", + Contents: buf.String(), + } + res.err = err + } + + res.d = time.Since(start) + res.Time = formatDurationAsSeconds(res.d) + return res +} + +func collectTestResults(workers []numa) JUnitTestSuites { + version := goVersion() + // pkg => test cases + pkgs := make(map[string][]JUnitTestCase) + durations := make(map[string]time.Duration) + + // The test result in workers are shuffled, so group by the packages here + for _, n := range workers { + for _, res := range n.results { + cases, ok := pkgs[res.ClassName] + if !ok { + cases = make([]JUnitTestCase, 0, 10) + } + cases = append(cases, res.JUnitTestCase) + pkgs[res.ClassName] = cases + durations[res.ClassName] += res.d + } + } + + suites := JUnitTestSuites{} + // Turn every package result to a suite. + for pkg, cases := range pkgs { + suite := JUnitTestSuite{ + Tests: len(cases), + Failures: failureCases(cases), + Time: formatDurationAsSeconds(durations[pkg]), + Name: pkg, + Properties: packageProperties(version), + TestCases: cases, + } + suites.Suites = append(suites.Suites, suite) + } + return suites +} + +func failureCases(input []JUnitTestCase) int { + sum := 0 + for _, v := range input { + if v.Failure != nil { + sum++ + } + } + return sum +} + +func (n *numa) testCommand(pkg string, fn string) *exec.Cmd { + args := make([]string, 0, 10) + exe := "./" + testFileName(pkg) + args = append(args, "-test.cpu", "1") + if !race { + args = append(args, []string{"-test.timeout", "2m"}...) + } else { + // it takes a longer when race is enabled. so it is set more timeout value. + args = append(args, []string{"-test.timeout", "5m"}...) + } + + // core.test -test.run TestClusteredPrefixColum + args = append(args, "-test.run", "^"+fn+"$") + + return exec.Command(exe, args...) +} + +func skipDIR(pkg string) bool { + skipDir := []string{"tests", "bin", "cmd", "tools"} + for _, ignore := range skipDir { + if strings.HasPrefix(pkg, ignore) { + return true + } + } + return false +} + +// buildTestBinaryMulti is much faster than build the test packages one by one. +func buildTestBinaryMulti(pkgs []string) error { + // go test --exec=xprog --tags=tso_function_test,deadlock -vet=off --count=0 $(pkgs) + xprogPath := path.Join(workDir, "bin/xprog") + packages := make([]string, 0, len(pkgs)) + for _, pkg := range pkgs { + packages = append(packages, path.Join(modulePath, pkg)) + } + + p := strconv.Itoa(buildParallel) + cmd := exec.Command("go", "test", "-p", p, "--exec", xprogPath, "-vet", "off", "--tags=tso_function_test,deadlock") + cmd.Args = append(cmd.Args, packages...) + cmd.Dir = workDir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return withTrace(err) + } + return nil +} + +func buildTestBinary(pkg string) error { + //nolint:gosec + cmd := exec.Command("go", "test", "-c", "-vet", "off", "--tags=tso_function_test,deadlock", "-o", testFileName(pkg), "-v") + if race { + cmd.Args = append(cmd.Args, "-race") + } + cmd.Dir = path.Join(workDir, pkg) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return withTrace(err) + } + return nil +} + +func testBinaryExist(pkg string) (bool, error) { + _, err := os.Stat(testFileFullPath(pkg)) + if err != nil { + var pathError *os.PathError + if errors.As(err, &pathError) { + return false, nil + } + } + return true, withTrace(err) +} + +func testFileName(pkg string) string { + _, file := path.Split(pkg) + return file + ".test.bin" +} + +func testFileFullPath(pkg string) string { + return path.Join(workDir, pkg, testFileName(pkg)) +} + +func listNewTestCases(pkg string) []string { + exe := "./" + testFileName(pkg) + + // core.test -test.list Test + cmd := exec.Command(exe, "-test.list", "Test") + cmd.Dir = path.Join(workDir, pkg) + var buf bytes.Buffer + cmd.Stdout = &buf + err := cmd.Run() + res := strings.Split(buf.String(), "\n") + if err != nil && len(res) == 0 { + fmt.Println("err ==", err) + } + return filter(res, func(s string) bool { + return strings.HasPrefix(s, "Test") && s != "TestT" && s != "TestBenchDaily" + }) +} + +func cmdToLines(cmd *exec.Cmd) ([]string, error) { + res, err := cmd.Output() + if err != nil { + return nil, withTrace(err) + } + ss := bytes.Split(res, []byte{'\n'}) + ret := make([]string, len(ss)) + for i, s := range ss { + ret[i] = string(s) + } + return ret, nil +} + +func filter(input []string, f func(string) bool) []string { + ret := input[:0] + for _, s := range input { + if f(s) { + ret = append(ret, s) + } + } + return ret +} + +func shuffle(tasks []task) { + for i := 0; i < len(tasks); i++ { + pos := rand.Intn(len(tasks)) + tasks[i], tasks[pos] = tasks[pos], tasks[i] + } +} + +type errWithStack struct { + err error + buf []byte +} + +func (e *errWithStack) Error() string { + return e.err.Error() + "\n" + string(e.buf) +} + +func withTrace(err error) error { + if err == nil { + return err + } + var errStack *errWithStack + if errors.As(err, &errStack) { + return err + } + var stack [4096]byte + sz := runtime.Stack(stack[:], false) + return &errWithStack{err, stack[:sz]} +} + +func formatDurationAsSeconds(d time.Duration) string { + return fmt.Sprintf("%f", d.Seconds()) +} + +func packageProperties(goVersion string) []JUnitProperty { + return []JUnitProperty{ + {Name: "go.version", Value: goVersion}, + } +} + +// goVersion returns the version as reported by the go binary in PATH. This +// version will not be the same as runtime.Version, which is always the version +// of go used to build the gotestsum binary. +// +// To skip the os/exec call set the GOVERSION environment variable to the +// desired value. +func goVersion() string { + if version, ok := os.LookupEnv("GOVERSION"); ok { + return version + } + cmd := exec.Command("go", "version") + out, err := cmd.Output() + if err != nil { + return "unknown" + } + return strings.TrimPrefix(strings.TrimSpace(string(out)), "go version ") +} + +func write(out io.Writer, suites JUnitTestSuites) error { + doc, err := xml.MarshalIndent(suites, "", "\t") + if err != nil { + return err + } + _, err = out.Write([]byte(xml.Header)) + if err != nil { + return err + } + _, err = out.Write(doc) + return err +} + +// JUnitTestSuites is a collection of JUnit test suites. +type JUnitTestSuites struct { + XMLName xml.Name `xml:"testsuites"` + Suites []JUnitTestSuite +} + +// JUnitTestSuite is a single JUnit test suite which may contain many +// testcases. +type JUnitTestSuite struct { + XMLName xml.Name `xml:"testsuite"` + Tests int `xml:"tests,attr"` + Failures int `xml:"failures,attr"` + Time string `xml:"time,attr"` + Name string `xml:"name,attr"` + Properties []JUnitProperty `xml:"properties>property,omitempty"` + TestCases []JUnitTestCase +} + +// JUnitTestCase is a single test case with its result. +type JUnitTestCase struct { + XMLName xml.Name `xml:"testcase"` + ClassName string `xml:"classname,attr"` + Name string `xml:"name,attr"` + Time string `xml:"time,attr"` + SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"` + Failure *JUnitFailure `xml:"failure,omitempty"` +} + +// JUnitSkipMessage contains the reason why a testcase was skipped. +type JUnitSkipMessage struct { + Message string `xml:"message,attr"` +} + +// JUnitProperty represents a key/value pair used to define properties. +type JUnitProperty struct { + Name string `xml:"name,attr"` + Value string `xml:"value,attr"` +} + +// JUnitFailure contains data related to a failed test. +type JUnitFailure struct { + Message string `xml:"message,attr"` + Type string `xml:"type,attr"` + Contents string `xml:",chardata"` +} diff --git a/tools/pd-ut/xprog.go b/tools/pd-ut/xprog.go new file mode 100644 index 000000000000..cf3e9b295e29 --- /dev/null +++ b/tools/pd-ut/xprog.go @@ -0,0 +1,119 @@ +// Copyright 2024 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//go:build xprog +// +build xprog + +package main + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +func main() { + // See https://github.com/golang/go/issues/15513#issuecomment-773994959 + // go test --exec=xprog ./... + // Command line args looks like: + // '$CWD/xprog /tmp/go-build2662369829/b1382/aggfuncs.test -test.paniconexit0 -test.timeout=10m0s' + // This program moves the test binary /tmp/go-build2662369829/b1382/aggfuncs.test to someplace else for later use. + + // Extract the current work directory + cwd := os.Args[0] + cwd = cwd[:len(cwd)-len("bin/xprog")] + + testBinaryPath := os.Args[1] + dir, _ := filepath.Split(testBinaryPath) + + // Extract the package info from /tmp/go-build2662369829/b1382/importcfg.link + pkg := getPackageInfo(dir) + + const prefix = "github.com/tikv/pd/" + if !strings.HasPrefix(pkg, prefix) { + os.Exit(-3) + } + + // github.com/tikv/pd/server/api/api.test => server/api/api + pkg = pkg[len(prefix) : len(pkg)-len(".test")] + + _, file := filepath.Split(pkg) + + // The path of the destination file looks like $CWD/server/api/api.test.bin + newName := filepath.Join(cwd, pkg, file+".test.bin") + + if err1 := os.Rename(testBinaryPath, newName); err1 != nil { + // Rename fail, handle error like "invalid cross-device linkcd tools/check" + err1 = MoveFile(testBinaryPath, newName) + if err1 != nil { + os.Exit(-4) + } + } +} + +func getPackageInfo(dir string) string { + // Read the /tmp/go-build2662369829/b1382/importcfg.link file to get the package information + f, err := os.Open(filepath.Join(dir, "importcfg.link")) + if err != nil { + os.Exit(-1) + } + defer f.Close() + + r := bufio.NewReader(f) + line, _, err := r.ReadLine() + if err != nil { + os.Exit(-2) + } + start := strings.IndexByte(string(line), ' ') + end := strings.IndexByte(string(line), '=') + pkg := string(line[start+1 : end]) + return pkg +} + +func MoveFile(srcPath, dstPath string) error { + inputFile, err := os.Open(srcPath) + if err != nil { + return fmt.Errorf("couldn't open source file: %s", err) + } + outputFile, err := os.Create(dstPath) + if err != nil { + inputFile.Close() + return fmt.Errorf("couldn't open dst file: %s", err) + } + defer outputFile.Close() + _, err = io.Copy(outputFile, inputFile) + inputFile.Close() + if err != nil { + return fmt.Errorf("writing to output file failed: %s", err) + } + + // Handle the permissions + si, err := os.Stat(srcPath) + if err != nil { + return fmt.Errorf("stat error: %s", err) + } + err = os.Chmod(dstPath, si.Mode()) + if err != nil { + return fmt.Errorf("chmod error: %s", err) + } + + // The copy was successful, so now delete the original file + err = os.Remove(srcPath) + if err != nil { + return fmt.Errorf("failed removing original file: %s", err) + } + return nil +} From 96a69fc623dbb680ca2eb69a0f5b432d9ce9827f Mon Sep 17 00:00:00 2001 From: ShuNing Date: Thu, 11 Apr 2024 14:51:22 +0800 Subject: [PATCH 03/50] pkg/schedule: put merge operators together (#8050) ref tikv/pd#7897, close tikv/pd#8049 pkg/schedule: put merge operators together to maintain atomicity Signed-off-by: nolouch Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/schedule/operator/operator_controller.go | 6 ++++-- pkg/schedule/operator/waiting_operator.go | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/pkg/schedule/operator/operator_controller.go b/pkg/schedule/operator/operator_controller.go index f05c232904f1..b9294ad970df 100644 --- a/pkg/schedule/operator/operator_controller.go +++ b/pkg/schedule/operator/operator_controller.go @@ -324,14 +324,16 @@ func (oc *Controller) AddWaitingOperator(ops ...*Operator) int { } continue } - oc.wop.PutOperator(op) + if isMerge { // count two merge operators as one, so wopStatus.ops[desc] should // not be updated here // TODO: call checkAddOperator ... + oc.wop.PutMergeOperators([]*Operator{op, ops[i+1]}) i++ added++ - oc.wop.PutOperator(ops[i]) + } else { + oc.wop.PutOperator(op) } operatorCounter.WithLabelValues(desc, "put").Inc() oc.wopStatus.incCount(desc) diff --git a/pkg/schedule/operator/waiting_operator.go b/pkg/schedule/operator/waiting_operator.go index b3b1b8856636..f75dcf25cd8a 100644 --- a/pkg/schedule/operator/waiting_operator.go +++ b/pkg/schedule/operator/waiting_operator.go @@ -26,6 +26,7 @@ var priorityWeight = []float64{1.0, 4.0, 9.0, 16.0} // WaitingOperator is an interface of waiting operators. type WaitingOperator interface { PutOperator(op *Operator) + PutMergeOperators(op []*Operator) GetOperator() []*Operator ListOperator() []*Operator } @@ -66,6 +67,21 @@ func (b *randBuckets) PutOperator(op *Operator) { bucket.ops = append(bucket.ops, op) } +// PutMergeOperators puts two operators into the random buckets. +func (b *randBuckets) PutMergeOperators(ops []*Operator) { + b.mu.Lock() + defer b.mu.Unlock() + if len(ops) != 2 && (ops[0].Kind()&OpMerge == 0 || ops[1].Kind()&OpMerge == 0) { + return + } + priority := ops[0].GetPriorityLevel() + bucket := b.buckets[priority] + if len(bucket.ops) == 0 { + b.totalWeight += bucket.weight + } + bucket.ops = append(bucket.ops, ops...) +} + // ListOperator lists all operator in the random buckets. func (b *randBuckets) ListOperator() []*Operator { b.mu.Lock() From 67461ddfa5092d375f51e33d59d7048eefb2c9b9 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Thu, 11 Apr 2024 15:00:23 +0800 Subject: [PATCH 04/50] client: introduce RPCClient (#8048) ref tikv/pd#8047 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/client.go | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/client/client.go b/client/client.go index b9535aa504ed..82d43181e7a9 100644 --- a/client/client.go +++ b/client/client.go @@ -69,16 +69,10 @@ type GlobalConfigItem struct { PayLoad []byte } -// Client is a PD (Placement Driver) RPC client. -// It should not be used after calling Close(). -type Client interface { - // GetClusterID gets the cluster ID from PD. - GetClusterID(ctx context.Context) uint64 +// RPCClient is a PD (Placement Driver) RPC and related mcs client which can only call RPC. +type RPCClient interface { // GetAllMembers gets the members Info from PD GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) - // GetLeaderURL returns current leader's URL. It returns "" before - // syncing leader from server. - GetLeaderURL() string // GetRegion gets a region and its leader Peer from PD by key. // The region may expire after split. Caller is responsible for caching and // taking care of region change. @@ -133,17 +127,12 @@ type Client interface { StoreGlobalConfig(ctx context.Context, configPath string, items []GlobalConfigItem) error // WatchGlobalConfig returns a stream with all global config and updates WatchGlobalConfig(ctx context.Context, configPath string, revision int64) (chan []GlobalConfigItem, error) - // UpdateOption updates the client option. - UpdateOption(option DynamicOption, value any) error // GetExternalTimestamp returns external timestamp GetExternalTimestamp(ctx context.Context) (uint64, error) // SetExternalTimestamp sets external timestamp SetExternalTimestamp(ctx context.Context, timestamp uint64) error - // GetServiceDiscovery returns ServiceDiscovery - GetServiceDiscovery() ServiceDiscovery - // TSOClient is the TSO client. TSOClient // MetaStorageClient is the meta storage client. @@ -154,6 +143,24 @@ type Client interface { GCClient // ResourceManagerClient manages resource group metadata and token assignment. ResourceManagerClient +} + +// Client is a PD (Placement Driver) RPC client. +// It should not be used after calling Close(). +type Client interface { + RPCClient + + // GetClusterID gets the cluster ID from PD. + GetClusterID(ctx context.Context) uint64 + // GetLeaderURL returns current leader's URL. It returns "" before + // syncing leader from server. + GetLeaderURL() string + // GetServiceDiscovery returns ServiceDiscovery + GetServiceDiscovery() ServiceDiscovery + + // UpdateOption updates the client option. + UpdateOption(option DynamicOption, value any) error + // Close closes the client. Close() } From 7e18a69734f2ae391516625ec0230c05b363f5c5 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 11 Apr 2024 17:00:22 +0800 Subject: [PATCH 05/50] *: improve the linter and fix some bugs (#8015) close tikv/pd#8019 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- .golangci.yml | 147 ++++++++++++++++++ Makefile | 3 - client/Makefile | 2 - client/client.go | 8 +- client/keyspace_client.go | 2 +- client/mock_pd_service_discovery.go | 28 ++-- client/pd_service_discovery.go | 8 +- .../resource_group/controller/controller.go | 13 +- client/resource_group/controller/limiter.go | 2 +- client/resource_group/controller/model.go | 7 +- client/resource_group/controller/testutil.go | 2 +- client/retry/backoff_test.go | 4 +- client/testutil/check_env_dummy.go | 2 +- client/tlsutil/tlsconfig.go | 6 +- client/tso_batch_controller.go | 2 +- client/tso_dispatcher.go | 3 +- client/tso_service_discovery.go | 6 +- client/tso_stream.go | 4 +- go.mod | 6 - go.sum | 21 --- pkg/audit/audit.go | 2 +- pkg/autoscaling/calculation.go | 2 +- pkg/autoscaling/calculation_test.go | 2 +- pkg/autoscaling/prometheus_test.go | 14 +- pkg/btree/btree_generic.go | 2 +- pkg/btree/btree_generic_test.go | 2 +- pkg/cgroup/cgroup.go | 5 - pkg/cgroup/cgroup_memory.go | 1 - pkg/core/metrics.go | 26 ++-- pkg/core/region.go | 4 +- pkg/core/store_test.go | 2 +- pkg/core/storelimit/sliding_window.go | 5 +- pkg/core/storelimit/store_limit.go | 6 +- pkg/dashboard/adapter/redirector_test.go | 4 +- pkg/dashboard/dashboard.go | 2 +- pkg/election/leadership_test.go | 10 +- pkg/encryption/key_manager_test.go | 6 +- pkg/encryption/kms.go | 2 +- pkg/errs/errs_test.go | 4 +- pkg/mcs/metastorage/server/grpc_service.go | 4 +- pkg/mcs/resourcemanager/server/config.go | 45 +++--- pkg/mcs/resourcemanager/server/config_test.go | 2 +- .../resourcemanager/server/grpc_service.go | 14 +- pkg/mcs/scheduling/server/apis/v1/api.go | 2 +- pkg/mcs/scheduling/server/cluster.go | 6 +- pkg/mcs/scheduling/server/config/config.go | 16 +- pkg/mcs/scheduling/server/config/watcher.go | 4 +- pkg/mcs/scheduling/server/grpc_service.go | 12 +- pkg/mcs/scheduling/server/rule/watcher.go | 18 +-- pkg/mcs/server/server.go | 2 +- pkg/mcs/tso/server/config.go | 4 +- pkg/mcs/tso/server/config_test.go | 2 +- pkg/mcs/tso/server/grpc_service.go | 8 +- pkg/mcs/tso/server/server.go | 8 +- pkg/member/participant.go | 4 +- pkg/mock/mockcluster/mockcluster.go | 2 +- pkg/mock/mockhbstream/mockhbstream.go | 4 +- pkg/mock/mockid/mockid.go | 2 +- pkg/ratelimit/controller_test.go | 4 +- pkg/ratelimit/runner.go | 2 +- pkg/ratelimit/runner_test.go | 4 +- pkg/replication/replication_mode_test.go | 2 +- pkg/schedule/checker/merge_checker.go | 2 +- pkg/schedule/checker/replica_checker.go | 2 +- pkg/schedule/checker/replica_strategy.go | 6 +- pkg/schedule/checker/rule_checker.go | 6 +- pkg/schedule/checker/split_checker.go | 2 +- pkg/schedule/config/config.go | 4 +- pkg/schedule/filter/candidates_test.go | 8 +- pkg/schedule/filter/filters.go | 44 +++--- pkg/schedule/filter/region_filters.go | 4 +- pkg/schedule/handler/handler.go | 8 +- pkg/schedule/labeler/labeler_test.go | 2 +- pkg/schedule/operator/create_operator.go | 4 +- pkg/schedule/operator/create_operator_test.go | 2 +- pkg/schedule/operator/operator_controller.go | 6 +- .../operator/operator_controller_test.go | 16 +- pkg/schedule/operator/operator_test.go | 76 +++++---- pkg/schedule/operator/status_tracker.go | 3 +- pkg/schedule/operator/step.go | 56 +++---- .../placement/region_rule_cache_test.go | 3 +- pkg/schedule/schedulers/balance_leader.go | 6 +- pkg/schedule/schedulers/balance_region.go | 2 +- pkg/schedule/schedulers/balance_test.go | 4 +- pkg/schedule/schedulers/balance_witness.go | 4 +- pkg/schedule/schedulers/base_scheduler.go | 14 +- pkg/schedule/schedulers/evict_leader.go | 12 +- pkg/schedule/schedulers/evict_slow_store.go | 8 +- pkg/schedule/schedulers/evict_slow_trend.go | 10 +- pkg/schedule/schedulers/grant_hot_region.go | 10 +- pkg/schedule/schedulers/grant_leader.go | 12 +- pkg/schedule/schedulers/hot_region.go | 12 +- pkg/schedule/schedulers/hot_region_config.go | 2 +- pkg/schedule/schedulers/hot_region_test.go | 6 +- pkg/schedule/schedulers/hot_region_v2.go | 8 +- pkg/schedule/schedulers/init.go | 50 +++--- pkg/schedule/schedulers/label.go | 6 +- pkg/schedule/schedulers/random_merge.go | 8 +- pkg/schedule/schedulers/scatter_range.go | 6 +- pkg/schedule/schedulers/scheduler.go | 2 +- pkg/schedule/schedulers/shuffle_hot_region.go | 6 +- pkg/schedule/schedulers/shuffle_leader.go | 6 +- pkg/schedule/schedulers/shuffle_region.go | 6 +- .../schedulers/shuffle_region_config.go | 2 +- pkg/schedule/schedulers/split_bucket.go | 6 +- .../schedulers/transfer_witness_leader.go | 14 +- pkg/schedule/splitter/region_splitter.go | 3 +- pkg/schedule/splitter/region_splitter_test.go | 4 +- pkg/statistics/buckets/hot_bucket_task.go | 4 +- pkg/statistics/collector.go | 10 +- pkg/statistics/hot_cache_task.go | 2 +- pkg/statistics/hot_peer_cache.go | 26 ++-- pkg/statistics/slow_stat.go | 4 +- pkg/statistics/store_collection.go | 6 +- pkg/statistics/store_collection_test.go | 2 +- pkg/storage/endpoint/keyspace.go | 10 +- pkg/storage/endpoint/meta.go | 4 +- pkg/storage/endpoint/rule.go | 12 +- pkg/storage/endpoint/tso_keyspace_group.go | 6 +- pkg/syncer/client_test.go | 6 +- pkg/tso/global_allocator.go | 4 +- pkg/tso/keyspace_group_manager.go | 12 +- pkg/tso/local_allocator.go | 4 +- pkg/tso/tso.go | 4 +- pkg/utils/apiutil/apiutil.go | 5 +- pkg/utils/etcdutil/etcdutil_test.go | 14 +- pkg/utils/etcdutil/health_checker.go | 4 +- pkg/utils/logutil/log.go | 2 +- pkg/utils/metricutil/metricutil_test.go | 2 +- pkg/utils/tempurl/check_env_dummy.go | 2 +- pkg/utils/testutil/api_check.go | 3 +- pkg/utils/tsoutil/tso_dispatcher.go | 8 +- pkg/utils/tsoutil/tso_proto_factory.go | 4 +- pkg/utils/tsoutil/tso_request.go | 6 +- pkg/window/counter_test.go | 2 +- pkg/window/policy_test.go | 2 +- plugin/scheduler_example/evict_leader.go | 16 +- revive.toml | 49 ------ server/api/admin.go | 2 +- server/api/cluster.go | 4 +- server/api/config.go | 10 +- server/api/health.go | 4 +- server/api/hot_status.go | 2 +- server/api/member.go | 6 +- server/api/member_test.go | 4 +- server/api/operator.go | 2 +- server/api/plugin_disable.go | 6 +- server/api/pprof.go | 18 +-- server/api/region.go | 2 +- server/api/region_test.go | 42 ++--- server/api/router.go | 4 +- server/api/service_gc_safepoint.go | 2 +- server/api/service_middleware.go | 2 +- server/api/stats_test.go | 2 +- server/api/status.go | 2 +- server/api/store_test.go | 12 +- server/api/trend.go | 6 +- server/api/version.go | 2 +- server/cluster/cluster.go | 4 +- server/cluster/cluster_test.go | 52 +++---- server/cluster/cluster_worker.go | 12 +- server/cluster/scheduling_controller.go | 8 +- server/config/persist_options.go | 4 +- server/forward.go | 10 +- server/grpc_service.go | 49 +++--- server/handler.go | 9 -- server/keyspace_service.go | 2 +- server/server.go | 7 +- server/testutil.go | 2 +- tests/cluster.go | 8 +- tests/dashboard/service_test.go | 4 +- tests/integrations/Makefile | 2 - tests/integrations/client/client_test.go | 55 ++++--- tests/integrations/client/client_tls_test.go | 6 +- tests/integrations/client/gc_client_test.go | 1 + .../integrations/client/global_config_test.go | 62 ++++---- tests/integrations/client/http_client_test.go | 22 +-- .../mcs/discovery/register_test.go | 8 +- .../mcs/keyspace/tso_keyspace_group_test.go | 32 +++- .../resourcemanager/resource_manager_test.go | 28 ++-- tests/integrations/mcs/tso/api_test.go | 2 +- tests/integrations/mcs/tso/proxy_test.go | 30 ++-- tests/integrations/realcluster/Makefile | 2 - tests/registry/registry_test.go | 6 +- tests/server/api/api_test.go | 10 +- tests/server/api/checker_test.go | 12 +- tests/server/api/operator_test.go | 30 +--- tests/server/api/region_test.go | 18 +-- tests/server/api/rule_test.go | 48 +++--- tests/server/api/scheduler_test.go | 44 +++--- tests/server/cluster/cluster_test.go | 6 +- tests/server/config/config_test.go | 22 +-- tests/server/join/join_test.go | 4 +- tests/server/keyspace/keyspace_test.go | 2 +- tests/server/member/member_test.go | 2 +- .../region_syncer/region_syncer_test.go | 8 +- tests/server/server_test.go | 2 +- .../server/storage/hot_region_storage_test.go | 6 +- tests/server/tso/allocator_test.go | 2 +- tests/server/tso/consistency_test.go | 2 +- tests/server/tso/global_tso_test.go | 2 +- tests/server/watch/leader_watch_test.go | 4 +- tests/testutil.go | 6 +- tests/tso_cluster.go | 2 +- tools.go | 1 - tools/Makefile | 2 - tools/pd-analysis/analysis/parse_log.go | 16 +- tools/pd-analysis/analysis/parse_log_test.go | 2 +- tools/pd-api-bench/cases/cases.go | 45 +++--- tools/pd-api-bench/cases/controller.go | 6 +- tools/pd-api-bench/config/config.go | 8 +- tools/pd-api-bench/main.go | 1 - tools/pd-backup/pdbackup/backup_test.go | 6 +- tools/pd-ctl/pdctl/command/config_command.go | 30 ++-- tools/pd-ctl/pdctl/command/exit_command.go | 2 +- .../pdctl/command/gc_safepoint_command.go | 2 +- tools/pd-ctl/pdctl/command/global.go | 2 +- tools/pd-ctl/pdctl/command/health_command.go | 2 +- tools/pd-ctl/pdctl/command/hot_command.go | 2 +- tools/pd-ctl/pdctl/command/label_command.go | 2 +- tools/pd-ctl/pdctl/command/member_command.go | 6 +- tools/pd-ctl/pdctl/command/min_resolved_ts.go | 2 +- tools/pd-ctl/pdctl/command/operator.go | 1 - tools/pd-ctl/pdctl/command/ping_command.go | 2 +- tools/pd-ctl/pdctl/command/region_command.go | 4 +- tools/pd-ctl/pdctl/command/scheduler.go | 2 +- tools/pd-ctl/pdctl/command/store_command.go | 4 +- tools/pd-ctl/pdctl/command/unsafe_command.go | 2 +- tools/pd-ctl/pdctl/ctl.go | 2 +- tools/pd-ctl/tests/config/config_test.go | 32 ++-- tools/pd-ctl/tests/global_test.go | 2 +- tools/pd-ctl/tests/hot/hot_test.go | 6 +- .../tests/keyspace/keyspace_group_test.go | 12 +- tools/pd-ctl/tests/keyspace/keyspace_test.go | 2 +- tools/pd-ctl/tests/label/label_test.go | 2 +- tools/pd-ctl/tests/operator/operator_test.go | 2 +- .../resource_manager_command_test.go | 10 +- tools/pd-ctl/tests/store/store_test.go | 2 +- tools/pd-heartbeat-bench/main.go | 4 +- tools/pd-simulator/main.go | 7 +- .../pd-simulator/simulator/cases/add_nodes.go | 2 +- .../simulator/cases/add_nodes_dynamic.go | 2 +- .../simulator/cases/balance_leader.go | 2 +- .../simulator/cases/balance_region.go | 2 +- .../simulator/cases/delete_nodes.go | 2 +- .../cases/diagnose_label_isolation.go | 6 +- .../simulator/cases/diagnose_rule.go | 4 +- .../simulator/cases/event_inner.go | 10 +- .../pd-simulator/simulator/cases/hot_read.go | 4 +- .../pd-simulator/simulator/cases/hot_write.go | 4 +- .../simulator/cases/import_data.go | 2 +- .../simulator/cases/makeup_down_replica.go | 2 +- .../simulator/cases/region_merge.go | 2 +- .../simulator/cases/region_split.go | 4 +- tools/pd-simulator/simulator/client.go | 2 +- tools/pd-simulator/simulator/task.go | 10 +- tools/pd-tso-bench/main.go | 6 +- tools/pd-ut/ut.go | 2 +- 258 files changed, 1202 insertions(+), 1160 deletions(-) delete mode 100644 revive.toml diff --git a/.golangci.yml b/.golangci.yml index 0e5028634aee..283de8e96b0a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -13,6 +13,7 @@ linters: - bodyclose - testifylint - gofmt + - revive disable: - errcheck linters-settings: @@ -52,3 +53,149 @@ linters-settings: rewrite-rules: - pattern: "interface{}" replacement: "any" + revive: + ignore-generated-header: false + severity: error + confidence: 0.8 + rules: + - name: atomic + severity: warning + exclude: [""] + disabled: false + - name: blank-imports + severity: warning + exclude: [""] + disabled: false + - name: confusing-naming + severity: warning + disabled: false + exclude: [""] + - name: confusing-results + severity: warning + disabled: false + exclude: [""] + - name: context-as-argument + severity: warning + disabled: false + exclude: [""] + arguments: + - allowTypesBefore: "*testing.T,*github.com/user/repo/testing.Harness" + - name: datarace + severity: warning + disabled: false + exclude: [""] + - name: defer + severity: warning + disabled: false + exclude: [""] + arguments: + - ["call-chain", "loop"] + - name: dot-imports + severity: warning + disabled: false + exclude: [""] + - name: duplicated-imports + severity: warning + disabled: false + exclude: [""] + - name: empty-block + severity: warning + disabled: false + exclude: [""] + - name: empty-lines + severity: warning + disabled: false + exclude: [""] + - name: error-return + severity: warning + disabled: false + exclude: [""] + - name: error-strings + severity: warning + disabled: false + exclude: [""] + - name: error-naming + severity: warning + disabled: false + exclude: [""] + - name: exported + severity: warning + disabled: false + exclude: [""] + arguments: + - "checkPrivateReceivers" + - "sayRepetitiveInsteadOfStutters" + - name: identical-branches + severity: warning + disabled: false + exclude: [""] + - name: if-return + severity: warning + disabled: false + exclude: [""] + - name: modifies-parameter + severity: warning + disabled: false + exclude: [""] + - name: optimize-operands-order + severity: warning + disabled: false + exclude: [""] + - name: package-comments + severity: warning + disabled: false + exclude: [""] + - name: range + severity: warning + disabled: false + exclude: [""] + - name: range-val-in-closure + severity: warning + disabled: false + exclude: [""] + - name: range-val-address + severity: warning + disabled: false + exclude: [""] + - name: receiver-naming + severity: warning + disabled: false + exclude: [""] + - name: indent-error-flow + severity: warning + disabled: false + exclude: [""] + - name: superfluous-else + severity: warning + disabled: false + exclude: [""] + - name: unnecessary-stmt + severity: warning + disabled: false + exclude: [""] + - name: unreachable-code + severity: warning + disabled: false + exclude: [""] + - name: unused-parameter + severity: warning + disabled: false + exclude: [""] + arguments: + - allowRegex: "^_" + - name: unused-receiver + severity: warning + disabled: false + exclude: [""] + - name: useless-break + severity: warning + disabled: false + exclude: [""] + - name: var-naming + severity: warning + disabled: false + exclude: [""] + - name: waitgroup-by-value + severity: warning + disabled: false + exclude: [""] diff --git a/Makefile b/Makefile index d78ddcdd65ed..205896c377af 100644 --- a/Makefile +++ b/Makefile @@ -184,9 +184,6 @@ static: install-tools pre-build @ gofmt -s -l -d $(PACKAGE_DIRECTORIES) 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }' @ echo "golangci-lint ..." @ golangci-lint run --verbose $(PACKAGE_DIRECTORIES) --allow-parallel-runners - @ echo "revive ..." - @ revive -formatter friendly -config revive.toml $(PACKAGES) - @ for mod in $(SUBMODULES); do cd $$mod && $(MAKE) static && cd $(ROOT_PATH) > /dev/null; done # Because CI downloads the dashboard code and runs gofmt, we can't add this check into static now. diff --git a/client/Makefile b/client/Makefile index 3328bfe8d116..3e8f6b0d3836 100644 --- a/client/Makefile +++ b/client/Makefile @@ -45,8 +45,6 @@ static: install-tools @ gofmt -s -l -d . 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }' @ echo "golangci-lint ..." @ golangci-lint run -c ../.golangci.yml --verbose ./... --allow-parallel-runners - @ echo "revive ..." - @ revive -formatter friendly -config ../revive.toml ./... tidy: @ go mod tidy diff --git a/client/client.go b/client/client.go index 82d43181e7a9..1852b77e4c6e 100644 --- a/client/client.go +++ b/client/client.go @@ -438,12 +438,12 @@ func NewAPIContextV1() APIContext { } // GetAPIVersion returns the API version. -func (apiCtx *apiContextV1) GetAPIVersion() (version APIVersion) { +func (*apiContextV1) GetAPIVersion() (version APIVersion) { return V1 } // GetKeyspaceName returns the keyspace name. -func (apiCtx *apiContextV1) GetKeyspaceName() (keyspaceName string) { +func (*apiContextV1) GetKeyspaceName() (keyspaceName string) { return "" } @@ -460,7 +460,7 @@ func NewAPIContextV2(keyspaceName string) APIContext { } // GetAPIVersion returns the API version. -func (apiCtx *apiContextV2) GetAPIVersion() (version APIVersion) { +func (*apiContextV2) GetAPIVersion() (version APIVersion) { return V2 } @@ -919,7 +919,7 @@ func handleRegionResponse(res *pdpb.GetRegionResponse) *Region { return r } -func (c *client) GetRegionFromMember(ctx context.Context, key []byte, memberURLs []string, opts ...GetRegionOption) (*Region, error) { +func (c *client) GetRegionFromMember(ctx context.Context, key []byte, memberURLs []string, _ ...GetRegionOption) (*Region, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.GetRegionFromMember", opentracing.ChildOf(span.Context())) defer span.Finish() diff --git a/client/keyspace_client.go b/client/keyspace_client.go index 340ecd0250ed..e52a4f85f05b 100644 --- a/client/keyspace_client.go +++ b/client/keyspace_client.go @@ -128,7 +128,7 @@ func (c *client) UpdateKeyspaceState(ctx context.Context, id uint32, state keysp // It returns a stream of slices of keyspace metadata. // The first message in stream contains all current keyspaceMeta, // all subsequent messages contains new put events for all keyspaces. -func (c *client) WatchKeyspaces(ctx context.Context) (chan []*keyspacepb.KeyspaceMeta, error) { +func (*client) WatchKeyspaces(context.Context) (chan []*keyspacepb.KeyspaceMeta, error) { return nil, errors.Errorf("WatchKeyspaces unimplemented") } diff --git a/client/mock_pd_service_discovery.go b/client/mock_pd_service_discovery.go index 17613a2f9e44..f1fabd0a1d23 100644 --- a/client/mock_pd_service_discovery.go +++ b/client/mock_pd_service_discovery.go @@ -56,19 +56,19 @@ func (m *mockPDServiceDiscovery) GetAllServiceClients() []ServiceClient { return m.clients } -func (m *mockPDServiceDiscovery) GetClusterID() uint64 { return 0 } -func (m *mockPDServiceDiscovery) GetKeyspaceID() uint32 { return 0 } -func (m *mockPDServiceDiscovery) GetKeyspaceGroupID() uint32 { return 0 } -func (m *mockPDServiceDiscovery) GetServiceURLs() []string { return nil } -func (m *mockPDServiceDiscovery) GetServingEndpointClientConn() *grpc.ClientConn { return nil } -func (m *mockPDServiceDiscovery) GetClientConns() *sync.Map { return nil } -func (m *mockPDServiceDiscovery) GetServingURL() string { return "" } -func (m *mockPDServiceDiscovery) GetBackupURLs() []string { return nil } -func (m *mockPDServiceDiscovery) GetServiceClient() ServiceClient { return nil } -func (m *mockPDServiceDiscovery) GetOrCreateGRPCConn(url string) (*grpc.ClientConn, error) { +func (*mockPDServiceDiscovery) GetClusterID() uint64 { return 0 } +func (*mockPDServiceDiscovery) GetKeyspaceID() uint32 { return 0 } +func (*mockPDServiceDiscovery) GetKeyspaceGroupID() uint32 { return 0 } +func (*mockPDServiceDiscovery) GetServiceURLs() []string { return nil } +func (*mockPDServiceDiscovery) GetServingEndpointClientConn() *grpc.ClientConn { return nil } +func (*mockPDServiceDiscovery) GetClientConns() *sync.Map { return nil } +func (*mockPDServiceDiscovery) GetServingURL() string { return "" } +func (*mockPDServiceDiscovery) GetBackupURLs() []string { return nil } +func (*mockPDServiceDiscovery) GetServiceClient() ServiceClient { return nil } +func (*mockPDServiceDiscovery) GetOrCreateGRPCConn(string) (*grpc.ClientConn, error) { return nil, nil } -func (m *mockPDServiceDiscovery) ScheduleCheckMemberChanged() {} -func (m *mockPDServiceDiscovery) CheckMemberChanged() error { return nil } -func (m *mockPDServiceDiscovery) AddServingURLSwitchedCallback(callbacks ...func()) {} -func (m *mockPDServiceDiscovery) AddServiceURLsSwitchedCallback(callbacks ...func()) {} +func (*mockPDServiceDiscovery) ScheduleCheckMemberChanged() {} +func (*mockPDServiceDiscovery) CheckMemberChanged() error { return nil } +func (*mockPDServiceDiscovery) AddServingURLSwitchedCallback(...func()) {} +func (*mockPDServiceDiscovery) AddServiceURLsSwitchedCallback(...func()) {} diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index defb797b7cac..97e82ec3321b 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -247,9 +247,9 @@ func (c *pdServiceClient) NeedRetry(pdErr *pdpb.Error, err error) bool { return !(err == nil && pdErr == nil) } -type errFn func(pdErr *pdpb.Error) bool +type errFn func(*pdpb.Error) bool -func emptyErrorFn(pdErr *pdpb.Error) bool { +func emptyErrorFn(*pdpb.Error) bool { return false } @@ -618,7 +618,7 @@ func (c *pdServiceDiscovery) checkLeaderHealth(ctx context.Context) { } func (c *pdServiceDiscovery) checkFollowerHealth(ctx context.Context) { - c.followers.Range(func(key, value any) bool { + c.followers.Range(func(_, value any) bool { // To ensure that the leader's healthy check is not delayed, shorten the duration. ctx, cancel := context.WithTimeout(ctx, MemberHealthCheckInterval/3) defer cancel() @@ -661,7 +661,7 @@ func (c *pdServiceDiscovery) SetKeyspaceID(keyspaceID uint32) { } // GetKeyspaceGroupID returns the ID of the keyspace group -func (c *pdServiceDiscovery) GetKeyspaceGroupID() uint32 { +func (*pdServiceDiscovery) GetKeyspaceGroupID() uint32 { // PD/API service only supports the default keyspace group return defaultKeySpaceGroupID } diff --git a/client/resource_group/controller/controller.go b/client/resource_group/controller/controller.go index 750a3c6e48f8..79bd6a9c3a6d 100755 --- a/client/resource_group/controller/controller.go +++ b/client/resource_group/controller/controller.go @@ -398,8 +398,7 @@ func (c *ResourceGroupsController) Start(ctx context.Context) { } case gc := <-c.tokenBucketUpdateChan: - now := gc.run.now - go gc.handleTokenBucketUpdateEvent(c.loopCtx, now) + go gc.handleTokenBucketUpdateEvent(c.loopCtx) } } }() @@ -473,7 +472,7 @@ func (c *ResourceGroupsController) cleanUpResourceGroup() { } func (c *ResourceGroupsController) executeOnAllGroups(f func(controller *groupCostController)) { - c.groupsController.Range(func(name, value any) bool { + c.groupsController.Range(func(_, value any) bool { f(value.(*groupCostController)) return true }) @@ -504,7 +503,7 @@ func (c *ResourceGroupsController) handleTokenBucketResponse(resp []*rmpb.TokenB func (c *ResourceGroupsController) collectTokenBucketRequests(ctx context.Context, source string, typ selectType) { c.run.currentRequests = make([]*rmpb.TokenBucketRequest, 0) - c.groupsController.Range(func(name, value any) bool { + c.groupsController.Range(func(_, value any) bool { gc := value.(*groupCostController) request := gc.collectRequestAndConsumption(typ) if request != nil { @@ -856,7 +855,7 @@ func (gc *groupCostController) resetEmergencyTokenAcquisition() { } } -func (gc *groupCostController) handleTokenBucketUpdateEvent(ctx context.Context, now time.Time) { +func (gc *groupCostController) handleTokenBucketUpdateEvent(ctx context.Context) { switch gc.mode { case rmpb.GroupMode_RawMode: for _, counter := range gc.run.resourceTokens { @@ -873,7 +872,7 @@ func (gc *groupCostController) handleTokenBucketUpdateEvent(ctx context.Context, counter.notify.setupNotificationCh = nil threshold := counter.notify.setupNotificationThreshold counter.notify.mu.Unlock() - counter.limiter.SetupNotificationThreshold(now, threshold) + counter.limiter.SetupNotificationThreshold(threshold) case <-ctx.Done(): return } @@ -894,7 +893,7 @@ func (gc *groupCostController) handleTokenBucketUpdateEvent(ctx context.Context, counter.notify.setupNotificationCh = nil threshold := counter.notify.setupNotificationThreshold counter.notify.mu.Unlock() - counter.limiter.SetupNotificationThreshold(now, threshold) + counter.limiter.SetupNotificationThreshold(threshold) case <-ctx.Done(): return } diff --git a/client/resource_group/controller/limiter.go b/client/resource_group/controller/limiter.go index 7e76934643f5..230ad46ecf1c 100644 --- a/client/resource_group/controller/limiter.go +++ b/client/resource_group/controller/limiter.go @@ -218,7 +218,7 @@ func (lim *Limiter) Reserve(ctx context.Context, waitDuration time.Duration, now } // SetupNotificationThreshold enables the notification at the given threshold. -func (lim *Limiter) SetupNotificationThreshold(now time.Time, threshold float64) { +func (lim *Limiter) SetupNotificationThreshold(threshold float64) { lim.mu.Lock() defer lim.mu.Unlock() lim.notifyThreshold = threshold diff --git a/client/resource_group/controller/model.go b/client/resource_group/controller/model.go index dedc2ed73592..9e86de69abbc 100644 --- a/client/resource_group/controller/model.go +++ b/client/resource_group/controller/model.go @@ -75,8 +75,7 @@ func newKVCalculator(cfg *RUConfig) *KVCalculator { } // Trickle ... -func (kc *KVCalculator) Trickle(*rmpb.Consumption) { -} +func (*KVCalculator) Trickle(*rmpb.Consumption) {} // BeforeKVRequest ... func (kc *KVCalculator) BeforeKVRequest(consumption *rmpb.Consumption, req RequestInfo) { @@ -166,11 +165,11 @@ func (dsc *SQLCalculator) Trickle(consumption *rmpb.Consumption) { } // BeforeKVRequest ... -func (dsc *SQLCalculator) BeforeKVRequest(consumption *rmpb.Consumption, req RequestInfo) { +func (*SQLCalculator) BeforeKVRequest(*rmpb.Consumption, RequestInfo) { } // AfterKVRequest ... -func (dsc *SQLCalculator) AfterKVRequest(consumption *rmpb.Consumption, req RequestInfo, res ResponseInfo) { +func (*SQLCalculator) AfterKVRequest(*rmpb.Consumption, RequestInfo, ResponseInfo) { } func getRUValueFromConsumption(custom *rmpb.Consumption, typ rmpb.RequestUnitType) float64 { diff --git a/client/resource_group/controller/testutil.go b/client/resource_group/controller/testutil.go index 4df8c9bba0d3..01a9c3af1fc8 100644 --- a/client/resource_group/controller/testutil.go +++ b/client/resource_group/controller/testutil.go @@ -52,7 +52,7 @@ func (tri *TestRequestInfo) StoreID() uint64 { } // ReplicaNumber implements the RequestInfo interface. -func (tri *TestRequestInfo) ReplicaNumber() int64 { +func (*TestRequestInfo) ReplicaNumber() int64 { return 1 } diff --git a/client/retry/backoff_test.go b/client/retry/backoff_test.go index c877860b5aeb..8df06b75f941 100644 --- a/client/retry/backoff_test.go +++ b/client/retry/backoff_test.go @@ -95,7 +95,7 @@ func TestBackoffer(t *testing.T) { // Test the retryable checker. execCount = 0 bo = InitialBackoffer(base, max, total) - bo.SetRetryableChecker(func(err error) bool { + bo.SetRetryableChecker(func(error) bool { return execCount < 2 }) err = bo.Exec(ctx, func() error { @@ -169,7 +169,7 @@ func (w *testingWriter) Write(p []byte) (n int, err error) { w.messages = append(w.messages, m) return n, nil } -func (w *testingWriter) Sync() error { +func (*testingWriter) Sync() error { return nil } diff --git a/client/testutil/check_env_dummy.go b/client/testutil/check_env_dummy.go index 2fbcbd1a9e70..c8f4d268c9de 100644 --- a/client/testutil/check_env_dummy.go +++ b/client/testutil/check_env_dummy.go @@ -16,6 +16,6 @@ package testutil -func environmentCheck(addr string) bool { +func environmentCheck(_ string) bool { return true } diff --git a/client/tlsutil/tlsconfig.go b/client/tlsutil/tlsconfig.go index c9cee5987bb1..a8bac17f676f 100644 --- a/client/tlsutil/tlsconfig.go +++ b/client/tlsutil/tlsconfig.go @@ -131,7 +131,7 @@ func (info TLSInfo) baseConfig() (*tls.Config, error) { } if info.AllowedCN != "" { - cfg.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + cfg.VerifyPeerCertificate = func(_ [][]byte, verifiedChains [][]*x509.Certificate) error { for _, chains := range verifiedChains { if len(chains) != 0 { if info.AllowedCN == chains[0].Subject.CommonName { @@ -145,10 +145,10 @@ func (info TLSInfo) baseConfig() (*tls.Config, error) { // this only reloads certs when there's a client request // TODO: support server-side refresh (e.g. inotify, SIGHUP), caching - cfg.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { + cfg.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return NewCert(info.CertFile, info.KeyFile, info.parseFunc) } - cfg.GetClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) { + cfg.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { return NewCert(info.CertFile, info.KeyFile, info.parseFunc) } return cfg, nil diff --git a/client/tso_batch_controller.go b/client/tso_batch_controller.go index 5f3b08c28951..d7ba5d7e74bb 100644 --- a/client/tso_batch_controller.go +++ b/client/tso_batch_controller.go @@ -140,7 +140,7 @@ func (tbc *tsoBatchController) finishCollectedRequests(physical, firstLogical in for i := 0; i < tbc.collectedRequestCount; i++ { tsoReq := tbc.collectedRequests[i] tsoReq.physical, tsoReq.logical = physical, tsoutil.AddLogical(firstLogical, int64(i), suffixBits) - defer trace.StartRegion(tsoReq.requestCtx, "pdclient.tsoReqDequeue").End() + defer trace.StartRegion(tsoReq.requestCtx, "pdclient.tsoReqDequeue").End() // nolint tsoReq.tryDone(err) } // Prevent the finished requests from being processed again. diff --git a/client/tso_dispatcher.go b/client/tso_dispatcher.go index 88f8ffd61b58..ad3aa1c5d74a 100644 --- a/client/tso_dispatcher.go +++ b/client/tso_dispatcher.go @@ -580,7 +580,7 @@ func (c *tsoClient) allowTSOFollowerProxy(dc string) bool { // chooseStream uses the reservoir sampling algorithm to randomly choose a connection. // connectionCtxs will only have only one stream to choose when the TSO Follower Proxy is off. -func (c *tsoClient) chooseStream(connectionCtxs *sync.Map) (connectionCtx *tsoConnectionContext) { +func (*tsoClient) chooseStream(connectionCtxs *sync.Map) (connectionCtx *tsoConnectionContext) { idx := 0 connectionCtxs.Range(func(_, cc any) bool { j := rand.Intn(idx + 1) @@ -797,6 +797,7 @@ func (c *tsoClient) processRequests( stream tsoStream, dcLocation string, tbc *tsoBatchController, ) error { requests := tbc.getCollectedRequests() + // nolint for _, req := range requests { defer trace.StartRegion(req.requestCtx, "pdclient.tsoReqSend").End() if span := opentracing.SpanFromContext(req.requestCtx); span != nil && span.Tracer() != nil { diff --git a/client/tso_service_discovery.go b/client/tso_service_discovery.go index f6c46346d5d3..34ef16f88b0b 100644 --- a/client/tso_service_discovery.go +++ b/client/tso_service_discovery.go @@ -349,13 +349,11 @@ func (c *tsoServiceDiscovery) CheckMemberChanged() error { // AddServingURLSwitchedCallback adds callbacks which will be called when the primary in // a primary/secondary configured cluster is switched. -func (c *tsoServiceDiscovery) AddServingURLSwitchedCallback(callbacks ...func()) { -} +func (*tsoServiceDiscovery) AddServingURLSwitchedCallback(...func()) {} // AddServiceURLsSwitchedCallback adds callbacks which will be called when any primary/secondary // in a primary/secondary configured cluster is changed. -func (c *tsoServiceDiscovery) AddServiceURLsSwitchedCallback(callbacks ...func()) { -} +func (*tsoServiceDiscovery) AddServiceURLsSwitchedCallback(...func()) {} // SetTSOLocalServURLsUpdatedCallback adds a callback which will be called when the local tso // allocator leader list is updated. diff --git a/client/tso_stream.go b/client/tso_stream.go index 83c0f08d4e00..14b72bc697b6 100644 --- a/client/tso_stream.go +++ b/client/tso_stream.go @@ -34,13 +34,13 @@ type tsoStreamBuilderFactory interface { type pdTSOStreamBuilderFactory struct{} -func (f *pdTSOStreamBuilderFactory) makeBuilder(cc *grpc.ClientConn) tsoStreamBuilder { +func (*pdTSOStreamBuilderFactory) makeBuilder(cc *grpc.ClientConn) tsoStreamBuilder { return &pdTSOStreamBuilder{client: pdpb.NewPDClient(cc), serverURL: cc.Target()} } type tsoTSOStreamBuilderFactory struct{} -func (f *tsoTSOStreamBuilderFactory) makeBuilder(cc *grpc.ClientConn) tsoStreamBuilder { +func (*tsoTSOStreamBuilderFactory) makeBuilder(cc *grpc.ClientConn) tsoStreamBuilder { return &tsoTSOStreamBuilder{client: tsopb.NewTSOClient(cc), serverURL: cc.Target()} } diff --git a/go.mod b/go.mod index 2620f5ad0a73..c76242f3753c 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,6 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/joho/godotenv v1.4.0 github.com/mailru/easyjson v0.7.6 - github.com/mgechev/revive v1.0.2 github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c @@ -95,7 +94,6 @@ require ( github.com/dnephin/pflag v1.0.7 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/color v1.10.0 // indirect - github.com/fatih/structtag v1.2.0 // indirect github.com/fogleman/gg v1.3.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect @@ -138,15 +136,11 @@ require ( github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.19 // indirect - github.com/mattn/go-runewidth v0.0.8 // indirect github.com/mattn/go-sqlite3 v1.14.15 // indirect - github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81 // indirect github.com/minio/sio v0.3.0 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/oleiade/reflections v1.0.1 // indirect - github.com/olekukonko/tablewriter v0.0.4 // indirect github.com/onsi/gomega v1.20.1 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 // indirect diff --git a/go.sum b/go.sum index d99804c887c2..d11fad07aa6c 100644 --- a/go.sum +++ b/go.sum @@ -111,11 +111,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= -github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -141,7 +138,6 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -316,31 +312,19 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0= -github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81 h1:QASJXOGm2RZ5Ardbc86qNFvby9AqkLDibfChMtAg5QM= -github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg= -github.com/mgechev/revive v1.0.2 h1:v0NxxQ7fSFz/u1NQydPo6EGdq7va0J1BtsZmae6kzUg= -github.com/mgechev/revive v1.0.2/go.mod h1:rb0dQy1LVAxW9SWy5R3LPUjevzUbUS316U5MFySA2lo= github.com/microsoft/go-mssqldb v0.17.0 h1:Fto83dMZPnYv1Zwx5vHHxpNraeEaUlQ/hhHLgZiaenE= github.com/microsoft/go-mssqldb v0.17.0/go.mod h1:OkoNGhGEs8EZqchVTtochlXruEhEOaO4S0d2sB5aeGQ= github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -355,8 +339,6 @@ github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oleiade/reflections v1.0.1 h1:D1XO3LVEYroYskEsoSiGItp9RUxG6jWnCVvrqH0HHQM= github.com/oleiade/reflections v1.0.1/go.mod h1:rdFxbxq4QXVZWj0F+e9jqjDkc7dbp97vkRixKo2JR60= -github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= -github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -644,7 +626,6 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -712,7 +693,6 @@ golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191114200427-caa0b0f7d508/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200225230052-807dcd883420/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201125231158-b5590deeca9b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -807,7 +787,6 @@ gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= moul.io/zapgorm2 v1.1.0 h1:qwAlMBYf+qJkJ7PAzJl4oCe6eS6QGiKAXUPeis0+RBE= moul.io/zapgorm2 v1.1.0/go.mod h1:emRfKjNqSzVj5lcgasBdovIXY1jSOwFz2GQZn1Rddks= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/pkg/audit/audit.go b/pkg/audit/audit.go index b971b09ed7ee..f84d035f8c91 100644 --- a/pkg/audit/audit.go +++ b/pkg/audit/audit.go @@ -118,7 +118,7 @@ func NewLocalLogBackend(before bool) Backend { } // ProcessHTTPRequest is used to implement audit.Backend -func (l *LocalLogBackend) ProcessHTTPRequest(r *http.Request) bool { +func (*LocalLogBackend) ProcessHTTPRequest(r *http.Request) bool { requestInfo, ok := requestutil.RequestInfoFrom(r.Context()) if !ok { return false diff --git a/pkg/autoscaling/calculation.go b/pkg/autoscaling/calculation.go index d85af498e47e..8c8783dd6180 100644 --- a/pkg/autoscaling/calculation.go +++ b/pkg/autoscaling/calculation.go @@ -409,7 +409,7 @@ func buildPlans(planMap map[string]map[string]struct{}, resourceTypeMap map[stri } // TODO: implement heterogeneous logic and take cluster information into consideration. -func findBestGroupToScaleIn(strategy *Strategy, scaleInQuota float64, groups []*Plan) Plan { +func findBestGroupToScaleIn(_ *Strategy, _ float64, groups []*Plan) Plan { return *groups[0] } diff --git a/pkg/autoscaling/calculation_test.go b/pkg/autoscaling/calculation_test.go index 5d0c2ba11267..9eb4ad648df6 100644 --- a/pkg/autoscaling/calculation_test.go +++ b/pkg/autoscaling/calculation_test.go @@ -203,7 +203,7 @@ func TestGetScaledTiKVGroups(t *testing.T) { type mockQuerier struct{} -func (q *mockQuerier) Query(options *QueryOptions) (QueryResult, error) { +func (*mockQuerier) Query(options *QueryOptions) (QueryResult, error) { result := make(QueryResult) for _, addr := range options.addresses { result[addr] = mockResultValue diff --git a/pkg/autoscaling/prometheus_test.go b/pkg/autoscaling/prometheus_test.go index 3ee6cb94e37a..9fe69e810d19 100644 --- a/pkg/autoscaling/prometheus_test.go +++ b/pkg/autoscaling/prometheus_test.go @@ -168,7 +168,7 @@ func makeJSONResponse(promResp *response) (*http.Response, []byte, error) { return response, body, nil } -func (c *normalClient) URL(ep string, args map[string]string) *url.URL { +func (*normalClient) URL(ep string, args map[string]string) *url.URL { return doURL(ep, args) } @@ -206,11 +206,11 @@ func TestRetrieveCPUMetrics(t *testing.T) { type emptyResponseClient struct{} -func (c *emptyResponseClient) URL(ep string, args map[string]string) *url.URL { +func (*emptyResponseClient) URL(ep string, args map[string]string) *url.URL { return doURL(ep, args) } -func (c *emptyResponseClient) Do(_ context.Context, req *http.Request) (r *http.Response, body []byte, err error) { +func (*emptyResponseClient) Do(context.Context, *http.Request) (r *http.Response, body []byte, err error) { promResp := &response{ Status: "success", Data: data{ @@ -235,11 +235,11 @@ func TestEmptyResponse(t *testing.T) { type errorHTTPStatusClient struct{} -func (c *errorHTTPStatusClient) URL(ep string, args map[string]string) *url.URL { +func (*errorHTTPStatusClient) URL(ep string, args map[string]string) *url.URL { return doURL(ep, args) } -func (c *errorHTTPStatusClient) Do(_ context.Context, req *http.Request) (r *http.Response, body []byte, err error) { +func (*errorHTTPStatusClient) Do(context.Context, *http.Request) (r *http.Response, body []byte, err error) { promResp := &response{} r, body, err = makeJSONResponse(promResp) @@ -262,11 +262,11 @@ func TestErrorHTTPStatus(t *testing.T) { type errorPrometheusStatusClient struct{} -func (c *errorPrometheusStatusClient) URL(ep string, args map[string]string) *url.URL { +func (*errorPrometheusStatusClient) URL(ep string, args map[string]string) *url.URL { return doURL(ep, args) } -func (c *errorPrometheusStatusClient) Do(_ context.Context, req *http.Request) (r *http.Response, body []byte, err error) { +func (*errorPrometheusStatusClient) Do(_ context.Context, _ *http.Request) (r *http.Response, body []byte, err error) { promResp := &response{ Status: "error", } diff --git a/pkg/btree/btree_generic.go b/pkg/btree/btree_generic.go index f918a8ac6862..599614678eb7 100644 --- a/pkg/btree/btree_generic.go +++ b/pkg/btree/btree_generic.go @@ -73,7 +73,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//revive:disable +// nolint package btree import ( diff --git a/pkg/btree/btree_generic_test.go b/pkg/btree/btree_generic_test.go index 9aa118fb8adf..fd0df3e5aafc 100644 --- a/pkg/btree/btree_generic_test.go +++ b/pkg/btree/btree_generic_test.go @@ -475,7 +475,7 @@ func BenchmarkSeek(b *testing.B) { b.StartTimer() for i := 0; i < b.N; i++ { - tr.AscendGreaterOrEqual(Int(i%size), func(i Int) bool { return false }) + tr.AscendGreaterOrEqual(Int(i%size), func(_ Int) bool { return false }) } } diff --git a/pkg/cgroup/cgroup.go b/pkg/cgroup/cgroup.go index e45dcbc09299..133bd3158c89 100644 --- a/pkg/cgroup/cgroup.go +++ b/pkg/cgroup/cgroup.go @@ -143,7 +143,6 @@ func combineErrors(err1, err2 error) error { func readFile(filepath string) (res []byte, err error) { var f *os.File - //nolint:gosec f, err = os.Open(filepath) if err != nil { return nil, err @@ -185,7 +184,6 @@ func controllerMatch(field string, controller string) bool { // The controller is defined via either type `memory` for cgroup v1 or via empty type for cgroup v2, // where the type is the second field in /proc/[pid]/cgroup file func detectControlPath(cgroupFilePath string, controller string) (string, error) { - //nolint:gosec cgroup, err := os.Open(cgroupFilePath) if err != nil { return "", errors.Wrapf(err, "failed to read %s cgroup from cgroups file: %s", controller, cgroupFilePath) @@ -229,7 +227,6 @@ func detectControlPath(cgroupFilePath string, controller string) (string, error) // See http://man7.org/linux/man-pages/man5/proc.5.html for `mountinfo` format. func getCgroupDetails(mountInfoPath string, cRoot string, controller string) (mount []string, version []int, err error) { - //nolint:gosec info, err := os.Open(mountInfoPath) if err != nil { return nil, nil, errors.Wrapf(err, "failed to read mounts info from file: %s", mountInfoPath) @@ -411,7 +408,6 @@ func detectCPUQuotaInV2(cRoot string) (period, quota int64, err error) { func detectCPUUsageInV2(cRoot string) (stime, utime uint64, err error) { statFilePath := filepath.Join(cRoot, cgroupV2CPUStat) var stat *os.File - //nolint:gosec stat, err = os.Open(statFilePath) if err != nil { return 0, 0, errors.Wrapf(err, "can't read cpu usage from cgroup v2 at %s", statFilePath) @@ -444,7 +440,6 @@ func detectCPUUsageInV2(cRoot string) (stime, utime uint64, err error) { func readInt64Value(root, filename string, cgVersion int) (value uint64, err error) { filePath := filepath.Join(root, filename) - //nolint:gosec file, err := os.Open(filePath) if err != nil { return 0, errors.Wrapf(err, "can't read %s from cgroup v%d", filename, cgVersion) diff --git a/pkg/cgroup/cgroup_memory.go b/pkg/cgroup/cgroup_memory.go index fb8e8f212dc6..2a6d581023ea 100644 --- a/pkg/cgroup/cgroup_memory.go +++ b/pkg/cgroup/cgroup_memory.go @@ -177,7 +177,6 @@ func detectMemInactiveFileUsageInV2(root string) (uint64, error) { func detectMemStatValue(cRoot, filename, key string, cgVersion int) (value uint64, err error) { statFilePath := filepath.Join(cRoot, filename) - //nolint:gosec stat, err := os.Open(statFilePath) if err != nil { return 0, errors.Wrapf(err, "can't read file %s from cgroup v%d", filename, cgVersion) diff --git a/pkg/core/metrics.go b/pkg/core/metrics.go index e6f3535b1d74..d23cf9dfcaaa 100644 --- a/pkg/core/metrics.go +++ b/pkg/core/metrics.go @@ -123,19 +123,19 @@ func NewNoopHeartbeatProcessTracer() RegionHeartbeatProcessTracer { return &noopHeartbeatProcessTracer{} } -func (n *noopHeartbeatProcessTracer) Begin() {} -func (n *noopHeartbeatProcessTracer) OnPreCheckFinished() {} -func (n *noopHeartbeatProcessTracer) OnAsyncHotStatsFinished() {} -func (n *noopHeartbeatProcessTracer) OnRegionGuideFinished() {} -func (n *noopHeartbeatProcessTracer) OnSaveCacheBegin() {} -func (n *noopHeartbeatProcessTracer) OnSaveCacheFinished() {} -func (n *noopHeartbeatProcessTracer) OnCheckOverlapsFinished() {} -func (n *noopHeartbeatProcessTracer) OnValidateRegionFinished() {} -func (n *noopHeartbeatProcessTracer) OnSetRegionFinished() {} -func (n *noopHeartbeatProcessTracer) OnUpdateSubTreeFinished() {} -func (n *noopHeartbeatProcessTracer) OnCollectRegionStatsFinished() {} -func (n *noopHeartbeatProcessTracer) OnAllStageFinished() {} -func (n *noopHeartbeatProcessTracer) LogFields() []zap.Field { +func (*noopHeartbeatProcessTracer) Begin() {} +func (*noopHeartbeatProcessTracer) OnPreCheckFinished() {} +func (*noopHeartbeatProcessTracer) OnAsyncHotStatsFinished() {} +func (*noopHeartbeatProcessTracer) OnRegionGuideFinished() {} +func (*noopHeartbeatProcessTracer) OnSaveCacheBegin() {} +func (*noopHeartbeatProcessTracer) OnSaveCacheFinished() {} +func (*noopHeartbeatProcessTracer) OnCheckOverlapsFinished() {} +func (*noopHeartbeatProcessTracer) OnValidateRegionFinished() {} +func (*noopHeartbeatProcessTracer) OnSetRegionFinished() {} +func (*noopHeartbeatProcessTracer) OnUpdateSubTreeFinished() {} +func (*noopHeartbeatProcessTracer) OnCollectRegionStatsFinished() {} +func (*noopHeartbeatProcessTracer) OnAllStageFinished() {} +func (*noopHeartbeatProcessTracer) LogFields() []zap.Field { return nil } diff --git a/pkg/core/region.go b/pkg/core/region.go index f7a4ef5f0fd5..baabafa1fa97 100644 --- a/pkg/core/region.go +++ b/pkg/core/region.go @@ -716,7 +716,7 @@ type RegionGuideFunc func(region, origin *RegionInfo) (saveKV, saveCache, needSy // GenerateRegionGuideFunc is used to generate a RegionGuideFunc. Control the log output by specifying the log function. // nil means do not print the log. func GenerateRegionGuideFunc(enableLog bool) RegionGuideFunc { - noLog := func(msg string, fields ...zap.Field) {} + noLog := func(string, ...zap.Field) {} debug, info := noLog, noLog if enableLog { debug = log.Debug @@ -964,7 +964,7 @@ func (r *RegionsInfo) AtomicCheckAndPutRegion(region *RegionInfo, trace RegionHe } // GetRelevantRegions returns the relevant regions for a given region. -func (r *RegionsInfo) GetRelevantRegions(region *RegionInfo, trace RegionHeartbeatProcessTracer) (origin *RegionInfo, overlaps []*regionItem) { +func (r *RegionsInfo) GetRelevantRegions(region *RegionInfo, _ RegionHeartbeatProcessTracer) (origin *RegionInfo, overlaps []*regionItem) { r.t.RLock() defer r.t.RUnlock() origin = r.getRegionLocked(region.GetID()) diff --git a/pkg/core/store_test.go b/pkg/core/store_test.go index 67618a63ea98..5cb324e56359 100644 --- a/pkg/core/store_test.go +++ b/pkg/core/store_test.go @@ -62,7 +62,7 @@ func TestDistinctScore(t *testing.T) { re.Equal(float64(0), DistinctScore(labels, stores, store)) } -func TestCloneStore(t *testing.T) { +func TestCloneStore(_ *testing.T) { meta := &metapb.Store{Id: 1, Address: "mock://tikv-1", Labels: []*metapb.StoreLabel{{Key: "zone", Value: "z1"}, {Key: "host", Value: "h1"}}} store := NewStoreInfo(meta) start := time.Now() diff --git a/pkg/core/storelimit/sliding_window.go b/pkg/core/storelimit/sliding_window.go index 0a70eb548d05..8feb0a2094d3 100644 --- a/pkg/core/storelimit/sliding_window.go +++ b/pkg/core/storelimit/sliding_window.go @@ -50,7 +50,7 @@ func NewSlidingWindows() *SlidingWindows { } // Version returns v2 -func (s *SlidingWindows) Version() string { +func (*SlidingWindows) Version() string { return VersionV2 } @@ -75,8 +75,7 @@ func (s *SlidingWindows) Feedback(e float64) { } // Reset does nothing because the capacity depends on the feedback. -func (s *SlidingWindows) Reset(_ float64, _ Type) { -} +func (*SlidingWindows) Reset(_ float64, _ Type) {} func (s *SlidingWindows) set(cap float64, typ Type) { if typ != SendSnapshot { diff --git a/pkg/core/storelimit/store_limit.go b/pkg/core/storelimit/store_limit.go index dc1de88e09f3..8d70b2918a13 100644 --- a/pkg/core/storelimit/store_limit.go +++ b/pkg/core/storelimit/store_limit.go @@ -82,15 +82,15 @@ func NewStoreRateLimit(ratePerSec float64) StoreLimit { } // Ack does nothing. -func (l *StoreRateLimit) Ack(_ int64, _ Type) {} +func (*StoreRateLimit) Ack(_ int64, _ Type) {} // Version returns v1 -func (l *StoreRateLimit) Version() string { +func (*StoreRateLimit) Version() string { return VersionV1 } // Feedback does nothing. -func (l *StoreRateLimit) Feedback(_ float64) {} +func (*StoreRateLimit) Feedback(_ float64) {} // Available returns the number of available tokens. // notice that the priority level is not used. diff --git a/pkg/dashboard/adapter/redirector_test.go b/pkg/dashboard/adapter/redirector_test.go index fff052f1d501..7767a6fda34c 100644 --- a/pkg/dashboard/adapter/redirector_test.go +++ b/pkg/dashboard/adapter/redirector_test.go @@ -42,14 +42,14 @@ func TestRedirectorTestSuite(t *testing.T) { func (suite *redirectorTestSuite) SetupSuite() { suite.tempText = "temp1" - suite.tempServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + suite.tempServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, suite.tempText) })) suite.testName = "test1" suite.redirector = NewRedirector(suite.testName, nil) suite.noRedirectHTTPClient = &http.Client{ - CheckRedirect: func(req *http.Request, via []*http.Request) error { + CheckRedirect: func(*http.Request, []*http.Request) error { // ErrUseLastResponse can be returned by Client.CheckRedirect hooks to // control how redirects are processed. If returned, the next request // is not sent and the most recent response is returned with its body diff --git a/pkg/dashboard/dashboard.go b/pkg/dashboard/dashboard.go index 9cd61a6f3323..998127d0f1b8 100644 --- a/pkg/dashboard/dashboard.go +++ b/pkg/dashboard/dashboard.go @@ -69,7 +69,7 @@ func GetServiceBuilders() []server.HandlerBuilder { // The order of execution must be sequential. return []server.HandlerBuilder{ // Dashboard API Service - func(ctx context.Context, srv *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { + func(_ context.Context, srv *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { distroutil.MustLoadAndReplaceStrings() if cfg, err = adapter.GenDashboardConfig(srv); err != nil { diff --git a/pkg/election/leadership_test.go b/pkg/election/leadership_test.go index de2e4b1129b4..1fde4ddeba7e 100644 --- a/pkg/election/leadership_test.go +++ b/pkg/election/leadership_test.go @@ -117,35 +117,35 @@ func TestExitWatch(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/election/fastTick", "return(true)")) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/fastTick", "return(true)")) // Case1: close the client before the watch loop starts - checkExitWatch(t, leaderKey, func(server *embed.Etcd, client *clientv3.Client) func() { + checkExitWatch(t, leaderKey, func(_ *embed.Etcd, client *clientv3.Client) func() { re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayWatcher", `pause`)) client.Close() re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayWatcher")) return func() {} }) // Case2: close the client when the watch loop is running - checkExitWatch(t, leaderKey, func(server *embed.Etcd, client *clientv3.Client) func() { + checkExitWatch(t, leaderKey, func(_ *embed.Etcd, client *clientv3.Client) func() { // Wait for the watch loop to start time.Sleep(500 * time.Millisecond) client.Close() return func() {} }) // Case3: delete the leader key - checkExitWatch(t, leaderKey, func(server *embed.Etcd, client *clientv3.Client) func() { + checkExitWatch(t, leaderKey, func(_ *embed.Etcd, client *clientv3.Client) func() { leaderKey := leaderKey _, err := client.Delete(context.Background(), leaderKey) re.NoError(err) return func() {} }) // Case4: close the server before the watch loop starts - checkExitWatch(t, leaderKey, func(server *embed.Etcd, client *clientv3.Client) func() { + checkExitWatch(t, leaderKey, func(server *embed.Etcd, _ *clientv3.Client) func() { re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayWatcher", `pause`)) server.Close() re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayWatcher")) return func() {} }) // Case5: close the server when the watch loop is running - checkExitWatch(t, leaderKey, func(server *embed.Etcd, client *clientv3.Client) func() { + checkExitWatch(t, leaderKey, func(server *embed.Etcd, _ *clientv3.Client) func() { // Wait for the watch loop to start time.Sleep(500 * time.Millisecond) server.Close() diff --git a/pkg/encryption/key_manager_test.go b/pkg/encryption/key_manager_test.go index 74f8b9a3b470..26453eeb5b3e 100644 --- a/pkg/encryption/key_manager_test.go +++ b/pkg/encryption/key_manager_test.go @@ -774,7 +774,7 @@ func TestSetLeadershipMasterKeyWithCiphertextKey(t *testing.T) { outputMasterKey, _ := hex.DecodeString(testMasterKey) outputCiphertextKey, _ := hex.DecodeString(testCiphertextKey) helper.newMasterKey = func( - meta *encryptionpb.MasterKey, + _ *encryptionpb.MasterKey, ciphertext []byte, ) (*MasterKey, error) { if newMasterKeyCalled < 2 { @@ -905,7 +905,7 @@ func TestKeyRotation(t *testing.T) { mockNow := int64(1601679533) helper.now = func() time.Time { return time.Unix(atomic.LoadInt64(&mockNow), 0) } mockTick := make(chan time.Time) - helper.tick = func(ticker *time.Ticker) <-chan time.Time { return mockTick } + helper.tick = func(_ *time.Ticker) <-chan time.Time { return mockTick } // Listen on watcher event reloadEvent := make(chan struct{}, 10) helper.eventAfterReloadByWatcher = func() { @@ -1001,7 +1001,7 @@ func TestKeyRotationConflict(t *testing.T) { mockNow := int64(1601679533) helper.now = func() time.Time { return time.Unix(atomic.LoadInt64(&mockNow), 0) } mockTick := make(chan time.Time, 10) - helper.tick = func(ticker *time.Ticker) <-chan time.Time { return mockTick } + helper.tick = func(_ *time.Ticker) <-chan time.Time { return mockTick } // Listen on ticker event tickerEvent := make(chan struct{}, 10) helper.eventAfterTicker = func() { diff --git a/pkg/encryption/kms.go b/pkg/encryption/kms.go index 7c52b4280c20..99dcf9619a37 100644 --- a/pkg/encryption/kms.go +++ b/pkg/encryption/kms.go @@ -60,7 +60,7 @@ func newMasterKeyFromKMS( roleArn := os.Getenv(envAwsRoleArn) tokenFile := os.Getenv(envAwsWebIdentityTokenFile) sessionName := os.Getenv(envAwsRoleSessionName) - optFn := func(options *kms.Options) {} + optFn := func(*kms.Options) {} // Session name is optional. if roleArn != "" && tokenFile != "" { client := sts.NewFromConfig(cfg) diff --git a/pkg/errs/errs_test.go b/pkg/errs/errs_test.go index 1dcabc32d9a0..01b7de461b8b 100644 --- a/pkg/errs/errs_test.go +++ b/pkg/errs/errs_test.go @@ -43,7 +43,7 @@ func (w *testingWriter) Write(p []byte) (n int, err error) { return n, nil } -func (w *testingWriter) Sync() error { +func (*testingWriter) Sync() error { return nil } @@ -124,7 +124,7 @@ func TestErrorEqual(t *testing.T) { re.False(errors.ErrorEqual(err1, err2)) } -func TestZapError(t *testing.T) { +func TestZapError(_ *testing.T) { err := errors.New("test") log.Info("test", ZapError(err)) err1 := ErrSchedulerNotFound diff --git a/pkg/mcs/metastorage/server/grpc_service.go b/pkg/mcs/metastorage/server/grpc_service.go index f5de50765e85..f018dc72f9f8 100644 --- a/pkg/mcs/metastorage/server/grpc_service.go +++ b/pkg/mcs/metastorage/server/grpc_service.go @@ -39,13 +39,13 @@ var ( var _ meta_storagepb.MetaStorageServer = (*Service)(nil) // SetUpRestHandler is a hook to sets up the REST service. -var SetUpRestHandler = func(srv *Service) (http.Handler, apiutil.APIServiceGroup) { +var SetUpRestHandler = func(*Service) (http.Handler, apiutil.APIServiceGroup) { return dummyRestService{}, apiutil.APIServiceGroup{} } type dummyRestService struct{} -func (d dummyRestService) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func (dummyRestService) ServeHTTP(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNotImplemented) w.Write([]byte("not implemented")) } diff --git a/pkg/mcs/resourcemanager/server/config.go b/pkg/mcs/resourcemanager/server/config.go index 508f37fe069f..dfb2babe676e 100644 --- a/pkg/mcs/resourcemanager/server/config.go +++ b/pkg/mcs/resourcemanager/server/config.go @@ -112,10 +112,13 @@ func (rmc *ControllerConfig) Adjust(meta *configutil.ConfigMetaData) { if rmc == nil { return } - rmc.RequestUnit.Adjust() - - configutil.AdjustDuration(&rmc.DegradedModeWaitDuration, defaultDegradedModeWaitDuration) - configutil.AdjustDuration(&rmc.LTBMaxWaitDuration, defaultMaxWaitDuration) + rmc.RequestUnit.Adjust(meta.Child("request-unit")) + if !meta.IsDefined("degraded-mode-wait-duration") { + configutil.AdjustDuration(&rmc.DegradedModeWaitDuration, defaultDegradedModeWaitDuration) + } + if !meta.IsDefined("ltb-max-wait-duration") { + configutil.AdjustDuration(&rmc.LTBMaxWaitDuration, defaultMaxWaitDuration) + } failpoint.Inject("enableDegradedMode", func() { configutil.AdjustDuration(&rmc.DegradedModeWaitDuration, time.Second) }) @@ -144,30 +147,30 @@ type RequestUnitConfig struct { } // Adjust adjusts the configuration and initializes it with the default value if necessary. -func (ruc *RequestUnitConfig) Adjust() { +func (ruc *RequestUnitConfig) Adjust(meta *configutil.ConfigMetaData) { if ruc == nil { return } - if ruc.ReadBaseCost == 0 { - ruc.ReadBaseCost = defaultReadBaseCost + if !meta.IsDefined("read-base-cost") { + configutil.AdjustFloat64(&ruc.ReadBaseCost, defaultReadBaseCost) } - if ruc.ReadPerBatchBaseCost == 0 { - ruc.ReadPerBatchBaseCost = defaultReadPerBatchBaseCost + if !meta.IsDefined("read-per-batch-base-cost") { + configutil.AdjustFloat64(&ruc.ReadPerBatchBaseCost, defaultReadPerBatchBaseCost) } - if ruc.ReadCostPerByte == 0 { - ruc.ReadCostPerByte = defaultReadCostPerByte + if !meta.IsDefined("read-cost-per-byte") { + configutil.AdjustFloat64(&ruc.ReadCostPerByte, defaultReadCostPerByte) } - if ruc.WriteBaseCost == 0 { - ruc.WriteBaseCost = defaultWriteBaseCost + if !meta.IsDefined("write-base-cost") { + configutil.AdjustFloat64(&ruc.WriteBaseCost, defaultWriteBaseCost) } - if ruc.WritePerBatchBaseCost == 0 { - ruc.WritePerBatchBaseCost = defaultWritePerBatchBaseCost + if !meta.IsDefined("write-per-batch-base-cost") { + configutil.AdjustFloat64(&ruc.WritePerBatchBaseCost, defaultWritePerBatchBaseCost) } - if ruc.WriteCostPerByte == 0 { - ruc.WriteCostPerByte = defaultWriteCostPerByte + if !meta.IsDefined("write-cost-per-byte") { + configutil.AdjustFloat64(&ruc.WriteCostPerByte, defaultWriteCostPerByte) } - if ruc.CPUMsCost == 0 { - ruc.CPUMsCost = defaultCPUMsCost + if !meta.IsDefined("read-cpu-ms-cost") { + configutil.AdjustFloat64(&ruc.CPUMsCost, defaultCPUMsCost) } } @@ -201,11 +204,11 @@ func (c *Config) Parse(flagSet *pflag.FlagSet) error { configutil.AdjustCommandLineString(flagSet, &c.ListenAddr, "listen-addr") configutil.AdjustCommandLineString(flagSet, &c.AdvertiseListenAddr, "advertise-listen-addr") - return c.Adjust(meta, false) + return c.Adjust(meta) } // Adjust is used to adjust the resource manager configurations. -func (c *Config) Adjust(meta *toml.MetaData, reloading bool) error { +func (c *Config) Adjust(meta *toml.MetaData) error { configMetaData := configutil.NewConfigMetadata(meta) if err := configMetaData.CheckUndecoded(); err != nil { c.WarningMsgs = append(c.WarningMsgs, err.Error()) diff --git a/pkg/mcs/resourcemanager/server/config_test.go b/pkg/mcs/resourcemanager/server/config_test.go index 64fd133ea733..2d57100468ec 100644 --- a/pkg/mcs/resourcemanager/server/config_test.go +++ b/pkg/mcs/resourcemanager/server/config_test.go @@ -39,7 +39,7 @@ read-cpu-ms-cost = 5.0 cfg := NewConfig() meta, err := toml.Decode(cfgData, &cfg) re.NoError(err) - err = cfg.Adjust(&meta, false) + err = cfg.Adjust(&meta) re.NoError(err) re.Equal(time.Second*2, cfg.Controller.DegradedModeWaitDuration.Duration) diff --git a/pkg/mcs/resourcemanager/server/grpc_service.go b/pkg/mcs/resourcemanager/server/grpc_service.go index cf985a147642..2f35042c48fd 100644 --- a/pkg/mcs/resourcemanager/server/grpc_service.go +++ b/pkg/mcs/resourcemanager/server/grpc_service.go @@ -41,13 +41,13 @@ var ( var _ rmpb.ResourceManagerServer = (*Service)(nil) // SetUpRestHandler is a hook to sets up the REST service. -var SetUpRestHandler = func(srv *Service) (http.Handler, apiutil.APIServiceGroup) { +var SetUpRestHandler = func(*Service) (http.Handler, apiutil.APIServiceGroup) { return dummyRestService{}, apiutil.APIServiceGroup{} } type dummyRestService struct{} -func (d dummyRestService) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func (dummyRestService) ServeHTTP(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNotImplemented) w.Write([]byte("not implemented")) } @@ -94,7 +94,7 @@ func (s *Service) checkServing() error { } // GetResourceGroup implements ResourceManagerServer.GetResourceGroup. -func (s *Service) GetResourceGroup(ctx context.Context, req *rmpb.GetResourceGroupRequest) (*rmpb.GetResourceGroupResponse, error) { +func (s *Service) GetResourceGroup(_ context.Context, req *rmpb.GetResourceGroupRequest) (*rmpb.GetResourceGroupResponse, error) { if err := s.checkServing(); err != nil { return nil, err } @@ -108,7 +108,7 @@ func (s *Service) GetResourceGroup(ctx context.Context, req *rmpb.GetResourceGro } // ListResourceGroups implements ResourceManagerServer.ListResourceGroups. -func (s *Service) ListResourceGroups(ctx context.Context, req *rmpb.ListResourceGroupsRequest) (*rmpb.ListResourceGroupsResponse, error) { +func (s *Service) ListResourceGroups(_ context.Context, req *rmpb.ListResourceGroupsRequest) (*rmpb.ListResourceGroupsResponse, error) { if err := s.checkServing(); err != nil { return nil, err } @@ -123,7 +123,7 @@ func (s *Service) ListResourceGroups(ctx context.Context, req *rmpb.ListResource } // AddResourceGroup implements ResourceManagerServer.AddResourceGroup. -func (s *Service) AddResourceGroup(ctx context.Context, req *rmpb.PutResourceGroupRequest) (*rmpb.PutResourceGroupResponse, error) { +func (s *Service) AddResourceGroup(_ context.Context, req *rmpb.PutResourceGroupRequest) (*rmpb.PutResourceGroupResponse, error) { if err := s.checkServing(); err != nil { return nil, err } @@ -135,7 +135,7 @@ func (s *Service) AddResourceGroup(ctx context.Context, req *rmpb.PutResourceGro } // DeleteResourceGroup implements ResourceManagerServer.DeleteResourceGroup. -func (s *Service) DeleteResourceGroup(ctx context.Context, req *rmpb.DeleteResourceGroupRequest) (*rmpb.DeleteResourceGroupResponse, error) { +func (s *Service) DeleteResourceGroup(_ context.Context, req *rmpb.DeleteResourceGroupRequest) (*rmpb.DeleteResourceGroupResponse, error) { if err := s.checkServing(); err != nil { return nil, err } @@ -147,7 +147,7 @@ func (s *Service) DeleteResourceGroup(ctx context.Context, req *rmpb.DeleteResou } // ModifyResourceGroup implements ResourceManagerServer.ModifyResourceGroup. -func (s *Service) ModifyResourceGroup(ctx context.Context, req *rmpb.PutResourceGroupRequest) (*rmpb.PutResourceGroupResponse, error) { +func (s *Service) ModifyResourceGroup(_ context.Context, req *rmpb.PutResourceGroupRequest) (*rmpb.PutResourceGroupResponse, error) { if err := s.checkServing(); err != nil { return nil, err } diff --git a/pkg/mcs/scheduling/server/apis/v1/api.go b/pkg/mcs/scheduling/server/apis/v1/api.go index 36451e5f0318..be3277f3fc63 100644 --- a/pkg/mcs/scheduling/server/apis/v1/api.go +++ b/pkg/mcs/scheduling/server/apis/v1/api.go @@ -1292,7 +1292,7 @@ func scatterRegions(c *gin.Context) { if !ok { return 0, nil, errors.New("regions_id is invalid") } - return handler.ScatterRegionsByID(ids, group, retryLimit, false) + return handler.ScatterRegionsByID(ids, group, retryLimit) }() if err != nil { c.String(http.StatusInternalServerError, err.Error()) diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index 1b915b6874d2..7ee7ae88cd1d 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -473,7 +473,7 @@ func (c *Cluster) runMetricsCollectionJob() { select { case <-c.ctx.Done(): log.Info("metrics are reset") - c.resetMetrics() + resetMetrics() log.Info("metrics collection job has been stopped") return case <-ticker.C: @@ -487,7 +487,7 @@ func (c *Cluster) collectMetrics() { stores := c.GetStores() for _, s := range stores { statsMap.Observe(s) - statsMap.ObserveHotStat(s, c.hotStat.StoresStats) + statistics.ObserveHotStat(s, c.hotStat.StoresStats) } statsMap.Collect() @@ -504,7 +504,7 @@ func (c *Cluster) collectMetrics() { c.RegionsInfo.CollectWaitLockMetrics() } -func (c *Cluster) resetMetrics() { +func resetMetrics() { statistics.Reset() schedulers.ResetSchedulerMetrics() schedule.ResetHotSpotMetrics() diff --git a/pkg/mcs/scheduling/server/config/config.go b/pkg/mcs/scheduling/server/config/config.go index 3e347afc12ef..148a7015d116 100644 --- a/pkg/mcs/scheduling/server/config/config.go +++ b/pkg/mcs/scheduling/server/config/config.go @@ -294,7 +294,7 @@ func (o *PersistConfig) SetScheduleConfig(cfg *sc.ScheduleConfig) { } // AdjustScheduleCfg adjusts the schedule config during the initialization. -func (o *PersistConfig) AdjustScheduleCfg(scheduleCfg *sc.ScheduleConfig) { +func AdjustScheduleCfg(scheduleCfg *sc.ScheduleConfig) { // In case we add new default schedulers. for _, ps := range sc.DefaultSchedulers { if slice.NoneOf(scheduleCfg.Schedulers, func(i int) bool { @@ -374,7 +374,7 @@ func (o *PersistConfig) IsUseJointConsensus() bool { } // GetKeyType returns the key type. -func (o *PersistConfig) GetKeyType() constant.KeyType { +func (*PersistConfig) GetKeyType() constant.KeyType { return constant.StringToKeyType("table") } @@ -685,7 +685,7 @@ func (o *PersistConfig) SetSplitMergeInterval(splitMergeInterval time.Duration) } // SetHaltScheduling set HaltScheduling. -func (o *PersistConfig) SetHaltScheduling(halt bool, source string) { +func (o *PersistConfig) SetHaltScheduling(halt bool, _ string) { v := o.GetScheduleConfig().Clone() v.HaltScheduling = halt o.SetScheduleConfig(v) @@ -735,25 +735,25 @@ func (o *PersistConfig) IsRaftKV2() bool { // AddSchedulerCfg adds the scheduler configurations. // This method is a no-op since we only use configurations derived from one-way synchronization from API server now. -func (o *PersistConfig) AddSchedulerCfg(string, []string) {} +func (*PersistConfig) AddSchedulerCfg(string, []string) {} // RemoveSchedulerCfg removes the scheduler configurations. // This method is a no-op since we only use configurations derived from one-way synchronization from API server now. -func (o *PersistConfig) RemoveSchedulerCfg(tp string) {} +func (*PersistConfig) RemoveSchedulerCfg(string) {} // CheckLabelProperty checks if the label property is satisfied. -func (o *PersistConfig) CheckLabelProperty(typ string, labels []*metapb.StoreLabel) bool { +func (*PersistConfig) CheckLabelProperty(string, []*metapb.StoreLabel) bool { return false } // IsTraceRegionFlow returns if the region flow is tracing. // If the accuracy cannot reach 0.1 MB, it is considered not. -func (o *PersistConfig) IsTraceRegionFlow() bool { +func (*PersistConfig) IsTraceRegionFlow() bool { return false } // Persist saves the configuration to the storage. -func (o *PersistConfig) Persist(storage endpoint.ConfigStorage) error { +func (*PersistConfig) Persist(endpoint.ConfigStorage) error { return nil } diff --git a/pkg/mcs/scheduling/server/config/watcher.go b/pkg/mcs/scheduling/server/config/watcher.go index 8db5e656279d..d1ca99bd36dc 100644 --- a/pkg/mcs/scheduling/server/config/watcher.go +++ b/pkg/mcs/scheduling/server/config/watcher.go @@ -129,14 +129,14 @@ func (cw *Watcher) initializeConfigWatcher() error { return err } log.Info("update scheduling config", zap.Reflect("new", cfg)) - cw.AdjustScheduleCfg(&cfg.Schedule) + AdjustScheduleCfg(&cfg.Schedule) cw.SetClusterVersion(&cfg.ClusterVersion) cw.SetScheduleConfig(&cfg.Schedule) cw.SetReplicationConfig(&cfg.Replication) cw.SetStoreConfig(&cfg.Store) return nil } - deleteFn := func(kv *mvccpb.KeyValue) error { + deleteFn := func(*mvccpb.KeyValue) error { return nil } cw.configWatcher = etcdutil.NewLoopWatcher( diff --git a/pkg/mcs/scheduling/server/grpc_service.go b/pkg/mcs/scheduling/server/grpc_service.go index ebce73e3303e..62ec1c1118fd 100644 --- a/pkg/mcs/scheduling/server/grpc_service.go +++ b/pkg/mcs/scheduling/server/grpc_service.go @@ -45,13 +45,13 @@ var ( ) // SetUpRestHandler is a hook to sets up the REST service. -var SetUpRestHandler = func(srv *Service) (http.Handler, apiutil.APIServiceGroup) { +var SetUpRestHandler = func(*Service) (http.Handler, apiutil.APIServiceGroup) { return dummyRestService{}, apiutil.APIServiceGroup{} } type dummyRestService struct{} -func (d dummyRestService) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func (dummyRestService) ServeHTTP(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNotImplemented) w.Write([]byte("not implemented")) } @@ -169,7 +169,7 @@ func (s *Service) RegionHeartbeat(stream schedulingpb.Scheduling_RegionHeartbeat } // StoreHeartbeat implements gRPC SchedulingServer. -func (s *Service) StoreHeartbeat(ctx context.Context, request *schedulingpb.StoreHeartbeatRequest) (*schedulingpb.StoreHeartbeatResponse, error) { +func (s *Service) StoreHeartbeat(_ context.Context, request *schedulingpb.StoreHeartbeatRequest) (*schedulingpb.StoreHeartbeatResponse, error) { c := s.GetCluster() if c == nil { // TODO: add metrics @@ -203,7 +203,7 @@ func (s *Service) SplitRegions(ctx context.Context, request *schedulingpb.SplitR } // ScatterRegions implements gRPC SchedulingServer. -func (s *Service) ScatterRegions(ctx context.Context, request *schedulingpb.ScatterRegionsRequest) (*schedulingpb.ScatterRegionsResponse, error) { +func (s *Service) ScatterRegions(_ context.Context, request *schedulingpb.ScatterRegionsRequest) (*schedulingpb.ScatterRegionsResponse, error) { c := s.GetCluster() if c == nil { return &schedulingpb.ScatterRegionsResponse{Header: s.notBootstrappedHeader()}, nil @@ -235,7 +235,7 @@ func (s *Service) ScatterRegions(ctx context.Context, request *schedulingpb.Scat } // GetOperator gets information about the operator belonging to the specify region. -func (s *Service) GetOperator(ctx context.Context, request *schedulingpb.GetOperatorRequest) (*schedulingpb.GetOperatorResponse, error) { +func (s *Service) GetOperator(_ context.Context, request *schedulingpb.GetOperatorRequest) (*schedulingpb.GetOperatorResponse, error) { c := s.GetCluster() if c == nil { return &schedulingpb.GetOperatorResponse{Header: s.notBootstrappedHeader()}, nil @@ -262,7 +262,7 @@ func (s *Service) GetOperator(ctx context.Context, request *schedulingpb.GetOper } // AskBatchSplit implements gRPC SchedulingServer. -func (s *Service) AskBatchSplit(ctx context.Context, request *schedulingpb.AskBatchSplitRequest) (*schedulingpb.AskBatchSplitResponse, error) { +func (s *Service) AskBatchSplit(_ context.Context, request *schedulingpb.AskBatchSplitRequest) (*schedulingpb.AskBatchSplitResponse, error) { c := s.GetCluster() if c == nil { return &schedulingpb.AskBatchSplitResponse{Header: s.notBootstrappedHeader()}, nil diff --git a/pkg/mcs/scheduling/server/rule/watcher.go b/pkg/mcs/scheduling/server/rule/watcher.go index d8a8dd3e6096..ea90b9d4e49e 100644 --- a/pkg/mcs/scheduling/server/rule/watcher.go +++ b/pkg/mcs/scheduling/server/rule/watcher.go @@ -109,7 +109,7 @@ func NewWatcher( func (rw *Watcher) initializeRuleWatcher() error { var suspectKeyRanges *core.KeyRanges - preEventsFn := func(events []*clientv3.Event) error { + preEventsFn := func([]*clientv3.Event) error { // It will be locked until the postEventsFn is finished. rw.ruleManager.Lock() rw.patch = rw.ruleManager.BeginPatch() @@ -149,10 +149,9 @@ func (rw *Watcher) initializeRuleWatcher() error { suspectKeyRanges.Append(rule.StartKey, rule.EndKey) } return nil - } else { - log.Warn("unknown key when updating placement rule", zap.String("key", key)) - return nil } + log.Warn("unknown key when updating placement rule", zap.String("key", key)) + return nil } deleteFn := func(kv *mvccpb.KeyValue) error { key := string(kv.Key) @@ -181,12 +180,11 @@ func (rw *Watcher) initializeRuleWatcher() error { suspectKeyRanges.Append(rule.StartKey, rule.EndKey) } return nil - } else { - log.Warn("unknown key when deleting placement rule", zap.String("key", key)) - return nil } + log.Warn("unknown key when deleting placement rule", zap.String("key", key)) + return nil } - postEventsFn := func(events []*clientv3.Event) error { + postEventsFn := func([]*clientv3.Event) error { defer rw.ruleManager.Unlock() if err := rw.ruleManager.TryCommitPatchLocked(rw.patch); err != nil { log.Error("failed to commit patch", zap.Error(err)) @@ -213,7 +211,7 @@ func (rw *Watcher) initializeRuleWatcher() error { func (rw *Watcher) initializeRegionLabelWatcher() error { prefixToTrim := rw.regionLabelPathPrefix + "/" // TODO: use txn in region labeler. - preEventsFn := func(events []*clientv3.Event) error { + preEventsFn := func([]*clientv3.Event) error { // It will be locked until the postEventsFn is finished. rw.regionLabeler.Lock() return nil @@ -231,7 +229,7 @@ func (rw *Watcher) initializeRegionLabelWatcher() error { log.Info("delete region label rule", zap.String("key", key)) return rw.regionLabeler.DeleteLabelRuleLocked(strings.TrimPrefix(key, prefixToTrim)) } - postEventsFn := func(events []*clientv3.Event) error { + postEventsFn := func([]*clientv3.Event) error { defer rw.regionLabeler.Unlock() rw.regionLabeler.BuildRangeListLocked() return nil diff --git a/pkg/mcs/server/server.go b/pkg/mcs/server/server.go index 2c008e8f5e85..6aec799278c6 100644 --- a/pkg/mcs/server/server.go +++ b/pkg/mcs/server/server.go @@ -171,7 +171,7 @@ func (bs *BaseServer) StartTimestamp() int64 { // CloseClientConns closes all client connections. func (bs *BaseServer) CloseClientConns() { - bs.clientConns.Range(func(key, value any) bool { + bs.clientConns.Range(func(_, value any) bool { conn := value.(*grpc.ClientConn) if err := conn.Close(); err != nil { log.Error("close client connection meet error") diff --git a/pkg/mcs/tso/server/config.go b/pkg/mcs/tso/server/config.go index eedf3a2f1b1a..c117dd72e384 100644 --- a/pkg/mcs/tso/server/config.go +++ b/pkg/mcs/tso/server/config.go @@ -177,11 +177,11 @@ func (c *Config) Parse(flagSet *pflag.FlagSet) error { configutil.AdjustCommandLineString(flagSet, &c.ListenAddr, "listen-addr") configutil.AdjustCommandLineString(flagSet, &c.AdvertiseListenAddr, "advertise-listen-addr") - return c.Adjust(meta, false) + return c.Adjust(meta) } // Adjust is used to adjust the TSO configurations. -func (c *Config) Adjust(meta *toml.MetaData, reloading bool) error { +func (c *Config) Adjust(meta *toml.MetaData) error { configMetaData := configutil.NewConfigMetadata(meta) if err := configMetaData.CheckUndecoded(); err != nil { c.WarningMsgs = append(c.WarningMsgs, err.Error()) diff --git a/pkg/mcs/tso/server/config_test.go b/pkg/mcs/tso/server/config_test.go index 9f5bc298964c..2cb9c8e019a6 100644 --- a/pkg/mcs/tso/server/config_test.go +++ b/pkg/mcs/tso/server/config_test.go @@ -83,7 +83,7 @@ max-gap-reset-ts = "1h" cfg := NewConfig() meta, err := toml.Decode(cfgData, &cfg) re.NoError(err) - err = cfg.Adjust(&meta, false) + err = cfg.Adjust(&meta) re.NoError(err) re.Equal("tso-test-name", cfg.GetName()) diff --git a/pkg/mcs/tso/server/grpc_service.go b/pkg/mcs/tso/server/grpc_service.go index 31a74f2a6886..03250d9ed37f 100644 --- a/pkg/mcs/tso/server/grpc_service.go +++ b/pkg/mcs/tso/server/grpc_service.go @@ -42,13 +42,13 @@ var ( var _ tsopb.TSOServer = (*Service)(nil) // SetUpRestHandler is a hook to sets up the REST service. -var SetUpRestHandler = func(srv *Service) (http.Handler, apiutil.APIServiceGroup) { +var SetUpRestHandler = func(*Service) (http.Handler, apiutil.APIServiceGroup) { return dummyRestService{}, apiutil.APIServiceGroup{} } type dummyRestService struct{} -func (d dummyRestService) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func (dummyRestService) ServeHTTP(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNotImplemented) w.Write([]byte("not implemented")) } @@ -135,7 +135,7 @@ func (s *Service) Tso(stream tsopb.TSO_TsoServer) error { // FindGroupByKeyspaceID returns the keyspace group that the keyspace belongs to. func (s *Service) FindGroupByKeyspaceID( - ctx context.Context, request *tsopb.FindGroupByKeyspaceIDRequest, + _ context.Context, request *tsopb.FindGroupByKeyspaceIDRequest, ) (*tsopb.FindGroupByKeyspaceIDResponse, error) { respKeyspaceGroup := request.GetHeader().GetKeyspaceGroupId() if errorType, err := s.validRequest(request.GetHeader()); err != nil { @@ -189,7 +189,7 @@ func (s *Service) FindGroupByKeyspaceID( // GetMinTS gets the minimum timestamp across all keyspace groups served by the TSO server // who receives and handles the request. func (s *Service) GetMinTS( - ctx context.Context, request *tsopb.GetMinTSRequest, + _ context.Context, request *tsopb.GetMinTSRequest, ) (*tsopb.GetMinTSResponse, error) { respKeyspaceGroup := request.GetHeader().GetKeyspaceGroupId() if errorType, err := s.validRequest(request.GetHeader()); err != nil { diff --git a/pkg/mcs/tso/server/server.go b/pkg/mcs/tso/server/server.go index f5f46a29504e..c38c7142730b 100644 --- a/pkg/mcs/tso/server/server.go +++ b/pkg/mcs/tso/server/server.go @@ -250,7 +250,7 @@ func (s *Server) ResignPrimary(keyspaceID, keyspaceGroupID uint32) error { // AddServiceReadyCallback implements basicserver. // It adds callbacks when it's ready for providing tso service. -func (s *Server) AddServiceReadyCallback(callbacks ...func(context.Context) error) { +func (*Server) AddServiceReadyCallback(...func(context.Context) error) { // Do nothing here. The primary of each keyspace group assigned to this host // will respond to the requests accordingly. } @@ -278,7 +278,7 @@ func (s *Server) GetTSOAllocatorManager(keyspaceGroupID uint32) (*tso.AllocatorM } // IsLocalRequest checks if the forwarded host is the current host -func (s *Server) IsLocalRequest(forwardedHost string) bool { +func (*Server) IsLocalRequest(forwardedHost string) bool { // TODO: Check if the forwarded host is the current host. // The logic is depending on etcd service mode -- if the TSO service // uses the embedded etcd, check against ClientUrls; otherwise check @@ -310,13 +310,13 @@ func (s *Server) ValidateRequest(header *tsopb.RequestHeader) error { // GetExternalTS returns external timestamp from the cache or the persistent storage. // TODO: Implement GetExternalTS -func (s *Server) GetExternalTS() uint64 { +func (*Server) GetExternalTS() uint64 { return 0 } // SetExternalTS saves external timestamp to cache and the persistent storage. // TODO: Implement SetExternalTS -func (s *Server) SetExternalTS(externalTS uint64) error { +func (*Server) SetExternalTS(uint64) error { return nil } diff --git a/pkg/member/participant.go b/pkg/member/participant.go index 0bf3bcc547e9..8a0ffadd31e2 100644 --- a/pkg/member/participant.go +++ b/pkg/member/participant.go @@ -200,7 +200,7 @@ func (m *Participant) KeepLeader(ctx context.Context) { // PreCheckLeader does some pre-check before checking whether or not it's the leader. // It returns true if it passes the pre-check, false otherwise. -func (m *Participant) PreCheckLeader() error { +func (*Participant) PreCheckLeader() error { // No specific thing to check. Returns no error. return nil } @@ -280,7 +280,7 @@ func (m *Participant) IsSameLeader(leader participant) bool { } // CheckPriority checks whether there is another participant has higher priority and resign it as the leader if so. -func (m *Participant) CheckPriority(ctx context.Context) { +func (*Participant) CheckPriority(_ context.Context) { // TODO: implement weighted-election when it's in need } diff --git a/pkg/mock/mockcluster/mockcluster.go b/pkg/mock/mockcluster/mockcluster.go index 6cf7ae143dfd..e5b3e39a5020 100644 --- a/pkg/mock/mockcluster/mockcluster.go +++ b/pkg/mock/mockcluster/mockcluster.go @@ -123,7 +123,7 @@ func (mc *Cluster) AllocID() (uint64, error) { } // UpdateRegionsLabelLevelStats updates the label level stats for the regions. -func (mc *Cluster) UpdateRegionsLabelLevelStats(regions []*core.RegionInfo) {} +func (*Cluster) UpdateRegionsLabelLevelStats(_ []*core.RegionInfo) {} // LoadRegion puts region info without leader func (mc *Cluster) LoadRegion(regionID uint64, peerStoreIDs ...uint64) { diff --git a/pkg/mock/mockhbstream/mockhbstream.go b/pkg/mock/mockhbstream/mockhbstream.go index 289f31d63dde..ac8f246f86af 100644 --- a/pkg/mock/mockhbstream/mockhbstream.go +++ b/pkg/mock/mockhbstream/mockhbstream.go @@ -46,10 +46,10 @@ func (s HeartbeatStream) Send(m core.RegionHeartbeatResponse) error { } // SendMsg is used to send the message. -func (s HeartbeatStream) SendMsg(region *core.RegionInfo, msg *pdpb.RegionHeartbeatResponse) {} +func (HeartbeatStream) SendMsg(*core.RegionInfo, *pdpb.RegionHeartbeatResponse) {} // BindStream mock method. -func (s HeartbeatStream) BindStream(storeID uint64, stream hbstream.HeartbeatStream) {} +func (HeartbeatStream) BindStream(uint64, hbstream.HeartbeatStream) {} // Recv mocks method. func (s HeartbeatStream) Recv() core.RegionHeartbeatResponse { diff --git a/pkg/mock/mockid/mockid.go b/pkg/mock/mockid/mockid.go index 4c0e7540653f..7b4902a6a042 100644 --- a/pkg/mock/mockid/mockid.go +++ b/pkg/mock/mockid/mockid.go @@ -38,6 +38,6 @@ func (alloc *IDAllocator) SetBase(newBase uint64) error { } // Rebase implements the IDAllocator interface. -func (alloc *IDAllocator) Rebase() error { +func (*IDAllocator) Rebase() error { return nil } diff --git a/pkg/ratelimit/controller_test.go b/pkg/ratelimit/controller_test.go index 50be24540e4a..d4093555ba79 100644 --- a/pkg/ratelimit/controller_test.go +++ b/pkg/ratelimit/controller_test.go @@ -108,7 +108,7 @@ func TestControllerWithConcurrencyLimiter(t *testing.T) { status := limiter.Update(label, o) re.NotZero(status & ConcurrencyNoChange) }, - checkStatusFunc: func(label string) {}, + checkStatusFunc: func(_ string) {}, }, { opt: UpdateConcurrencyLimiter(5), @@ -240,7 +240,7 @@ func TestControllerWithQPSLimiter(t *testing.T) { status := limiter.Update(label, o) re.NotZero(status & QPSNoChange) }, - checkStatusFunc: func(label string) {}, + checkStatusFunc: func(_ string) {}, }, { opt: UpdateQPSLimiter(5, 5), diff --git a/pkg/ratelimit/runner.go b/pkg/ratelimit/runner.go index 661668af3b91..dd92a10179dc 100644 --- a/pkg/ratelimit/runner.go +++ b/pkg/ratelimit/runner.go @@ -162,7 +162,7 @@ func NewSyncRunner() *SyncRunner { } // RunTask runs the task synchronously. -func (s *SyncRunner) RunTask(ctx context.Context, opt TaskOpts, f func(context.Context)) error { +func (*SyncRunner) RunTask(ctx context.Context, _ TaskOpts, f func(context.Context)) error { f(ctx) return nil } diff --git a/pkg/ratelimit/runner_test.go b/pkg/ratelimit/runner_test.go index 8a9eff773799..9b8dca231d11 100644 --- a/pkg/ratelimit/runner_test.go +++ b/pkg/ratelimit/runner_test.go @@ -36,7 +36,7 @@ func TestAsyncRunner(t *testing.T) { err := runner.RunTask(context.Background(), TaskOpts{ TaskName: "test1", Limit: limiter, - }, func(ctx context.Context) { + }, func(context.Context) { defer wg.Done() time.Sleep(100 * time.Millisecond) }) @@ -55,7 +55,7 @@ func TestAsyncRunner(t *testing.T) { err := runner.RunTask(context.Background(), TaskOpts{ TaskName: "test2", Limit: limiter, - }, func(ctx context.Context) { + }, func(context.Context) { defer wg.Done() time.Sleep(100 * time.Millisecond) }) diff --git a/pkg/replication/replication_mode_test.go b/pkg/replication/replication_mode_test.go index 038807d7d94e..d19a4f70d66a 100644 --- a/pkg/replication/replication_mode_test.go +++ b/pkg/replication/replication_mode_test.go @@ -144,7 +144,7 @@ func (rep *mockFileReplicator) GetMembers() ([]*pdpb.Member, error) { return members, nil } -func (rep *mockFileReplicator) ReplicateFileToMember(ctx context.Context, member *pdpb.Member, name string, data []byte) error { +func (rep *mockFileReplicator) ReplicateFileToMember(_ context.Context, member *pdpb.Member, _ string, data []byte) error { if err := rep.errors[member.GetMemberId()]; err != nil { return err } diff --git a/pkg/schedule/checker/merge_checker.go b/pkg/schedule/checker/merge_checker.go index 1ce7bddd1dc3..821c21cc119d 100644 --- a/pkg/schedule/checker/merge_checker.go +++ b/pkg/schedule/checker/merge_checker.go @@ -94,7 +94,7 @@ func NewMergeChecker(ctx context.Context, cluster sche.CheckerCluster, conf conf } // GetType return MergeChecker's type -func (m *MergeChecker) GetType() string { +func (*MergeChecker) GetType() string { return "merge-checker" } diff --git a/pkg/schedule/checker/replica_checker.go b/pkg/schedule/checker/replica_checker.go index 3e23f3bdcacb..6324fd2ca10d 100644 --- a/pkg/schedule/checker/replica_checker.go +++ b/pkg/schedule/checker/replica_checker.go @@ -76,7 +76,7 @@ func NewReplicaChecker(cluster sche.CheckerCluster, conf config.CheckerConfigPro } // GetType return ReplicaChecker's type -func (r *ReplicaChecker) GetType() string { +func (*ReplicaChecker) GetType() string { return replicaCheckerName } diff --git a/pkg/schedule/checker/replica_strategy.go b/pkg/schedule/checker/replica_strategy.go index ad85e307bbe9..e234189fe96f 100644 --- a/pkg/schedule/checker/replica_strategy.go +++ b/pkg/schedule/checker/replica_strategy.go @@ -97,7 +97,7 @@ func (s *ReplicaStrategy) SelectStoreToFix(coLocationStores []*core.StoreInfo, o return 0, false } // trick to avoid creating a slice with `old` removed. - s.swapStoreToFirst(coLocationStores, old) + swapStoreToFirst(coLocationStores, old) // If the coLocationStores only has one store, no need to remove. // Otherwise, the other stores will be filtered. if len(coLocationStores) > 1 { @@ -113,7 +113,7 @@ func (s *ReplicaStrategy) SelectStoreToImprove(coLocationStores []*core.StoreInf return 0, false } // trick to avoid creating a slice with `old` removed. - s.swapStoreToFirst(coLocationStores, old) + swapStoreToFirst(coLocationStores, old) oldStore := s.cluster.GetStore(old) if oldStore == nil { return 0, false @@ -127,7 +127,7 @@ func (s *ReplicaStrategy) SelectStoreToImprove(coLocationStores []*core.StoreInf return s.SelectStoreToAdd(coLocationStores[1:], filters...) } -func (s *ReplicaStrategy) swapStoreToFirst(stores []*core.StoreInfo, id uint64) { +func swapStoreToFirst(stores []*core.StoreInfo, id uint64) { for i, s := range stores { if s.GetID() == id { stores[0], stores[i] = stores[i], stores[0] diff --git a/pkg/schedule/checker/rule_checker.go b/pkg/schedule/checker/rule_checker.go index 464f5e97be82..66b958911b1c 100644 --- a/pkg/schedule/checker/rule_checker.go +++ b/pkg/schedule/checker/rule_checker.go @@ -107,7 +107,7 @@ func NewRuleChecker(ctx context.Context, cluster sche.CheckerCluster, ruleManage } // GetType returns RuleChecker's Type -func (c *RuleChecker) GetType() string { +func (*RuleChecker) GetType() string { return ruleCheckerName } @@ -347,7 +347,7 @@ func (c *RuleChecker) fixLooseMatchPeer(region *core.RegionInfo, fit *placement. if region.GetLeader().GetId() != peer.GetId() && rf.Rule.Role == placement.Leader { ruleCheckerFixLeaderRoleCounter.Inc() if c.allowLeader(fit, peer) { - return operator.CreateTransferLeaderOperator("fix-leader-role", c.cluster, region, region.GetLeader().GetStoreId(), peer.GetStoreId(), []uint64{}, 0) + return operator.CreateTransferLeaderOperator("fix-leader-role", c.cluster, region, peer.GetStoreId(), []uint64{}, 0) } ruleCheckerNotAllowLeaderCounter.Inc() return nil, errPeerCannotBeLeader @@ -356,7 +356,7 @@ func (c *RuleChecker) fixLooseMatchPeer(region *core.RegionInfo, fit *placement. ruleCheckerFixFollowerRoleCounter.Inc() for _, p := range region.GetPeers() { if c.allowLeader(fit, p) { - return operator.CreateTransferLeaderOperator("fix-follower-role", c.cluster, region, peer.GetStoreId(), p.GetStoreId(), []uint64{}, 0) + return operator.CreateTransferLeaderOperator("fix-follower-role", c.cluster, region, p.GetStoreId(), []uint64{}, 0) } } ruleCheckerNoNewLeaderCounter.Inc() diff --git a/pkg/schedule/checker/split_checker.go b/pkg/schedule/checker/split_checker.go index 072bdcf7a2e7..3a34eee8c905 100644 --- a/pkg/schedule/checker/split_checker.go +++ b/pkg/schedule/checker/split_checker.go @@ -51,7 +51,7 @@ func NewSplitChecker(cluster sche.CheckerCluster, ruleManager *placement.RuleMan } // GetType returns the checker type. -func (c *SplitChecker) GetType() string { +func (*SplitChecker) GetType() string { return "split-checker" } diff --git a/pkg/schedule/config/config.go b/pkg/schedule/config/config.go index 56038ddcb098..abf4c776f8a5 100644 --- a/pkg/schedule/config/config.go +++ b/pkg/schedule/config/config.go @@ -407,7 +407,7 @@ func (c *ScheduleConfig) Adjust(meta *configutil.ConfigMetaData, reloading bool) adjustSchedulers(&c.Schedulers, DefaultSchedulers) for k, b := range c.migrateConfigurationMap() { - v, err := c.parseDeprecatedFlag(meta, k, *b[0], *b[1]) + v, err := parseDeprecatedFlag(meta, k, *b[0], *b[1]) if err != nil { return err } @@ -456,7 +456,7 @@ func (c *ScheduleConfig) GetMaxMergeRegionKeys() uint64 { return c.MaxMergeRegionSize * 10000 } -func (c *ScheduleConfig) parseDeprecatedFlag(meta *configutil.ConfigMetaData, name string, old, new bool) (bool, error) { +func parseDeprecatedFlag(meta *configutil.ConfigMetaData, name string, old, new bool) (bool, error) { oldName, newName := "disable-"+name, "enable-"+name defineOld, defineNew := meta.IsDefined(oldName), meta.IsDefined(newName) switch { diff --git a/pkg/schedule/filter/candidates_test.go b/pkg/schedule/filter/candidates_test.go index 13e8ed661cc2..0d805312ba78 100644 --- a/pkg/schedule/filter/candidates_test.go +++ b/pkg/schedule/filter/candidates_test.go @@ -48,9 +48,9 @@ func idComparer2(a, b *core.StoreInfo) int { type idFilter func(uint64) bool -func (f idFilter) Scope() string { return "idFilter" } -func (f idFilter) Type() filterType { return filterType(0) } -func (f idFilter) Source(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { +func (idFilter) Scope() string { return "idFilter" } +func (idFilter) Type() filterType { return filterType(0) } +func (f idFilter) Source(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if f(store.GetID()) { return statusOK } @@ -58,7 +58,7 @@ func (f idFilter) Source(conf config.SharedConfigProvider, store *core.StoreInfo return statusStoreScoreDisallowed } -func (f idFilter) Target(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { +func (f idFilter) Target(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if f(store.GetID()) { return statusOK } diff --git a/pkg/schedule/filter/filters.go b/pkg/schedule/filter/filters.go index 0d188e69180a..1838f0104f45 100644 --- a/pkg/schedule/filter/filters.go +++ b/pkg/schedule/filter/filters.go @@ -185,18 +185,18 @@ func (f *excludedFilter) Scope() string { return f.scope } -func (f *excludedFilter) Type() filterType { +func (*excludedFilter) Type() filterType { return excluded } -func (f *excludedFilter) Source(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { +func (f *excludedFilter) Source(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if _, ok := f.sources[store.GetID()]; ok { return statusStoreAlreadyHasPeer } return statusOK } -func (f *excludedFilter) Target(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { +func (f *excludedFilter) Target(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if _, ok := f.targets[store.GetID()]; ok { return statusStoreAlreadyHasPeer } @@ -215,15 +215,15 @@ func (f *storageThresholdFilter) Scope() string { return f.scope } -func (f *storageThresholdFilter) Type() filterType { +func (*storageThresholdFilter) Type() filterType { return storageThreshold } -func (f *storageThresholdFilter) Source(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { +func (*storageThresholdFilter) Source(config.SharedConfigProvider, *core.StoreInfo) *plan.Status { return statusOK } -func (f *storageThresholdFilter) Target(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { +func (*storageThresholdFilter) Target(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if !store.IsLowSpace(conf.GetLowSpaceRatio()) { return statusOK } @@ -283,11 +283,11 @@ func (f *distinctScoreFilter) Scope() string { return f.scope } -func (f *distinctScoreFilter) Type() filterType { +func (*distinctScoreFilter) Type() filterType { return distinctScore } -func (f *distinctScoreFilter) Source(_ config.SharedConfigProvider, _ *core.StoreInfo) *plan.Status { +func (*distinctScoreFilter) Source(config.SharedConfigProvider, *core.StoreInfo) *plan.Status { return statusOK } @@ -387,7 +387,7 @@ func (f *StoreStateFilter) pauseLeaderTransfer(_ config.SharedConfigProvider, st return statusOK } -func (f *StoreStateFilter) slowStoreEvicted(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { +func (f *StoreStateFilter) slowStoreEvicted(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if store.EvictedAsSlowStore() { f.Reason = storeStateSlow return statusStoreRejectLeader @@ -583,12 +583,12 @@ func (f labelConstraintFilter) Scope() string { } // Type returns the name of the filter. -func (f labelConstraintFilter) Type() filterType { +func (labelConstraintFilter) Type() filterType { return labelConstraint } // Source filters stores when select them as schedule source. -func (f labelConstraintFilter) Source(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { +func (f labelConstraintFilter) Source(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if placement.MatchLabelConstraints(store, f.constraints) { return statusOK } @@ -634,11 +634,11 @@ func (f *ruleFitFilter) Scope() string { return f.scope } -func (f *ruleFitFilter) Type() filterType { +func (*ruleFitFilter) Type() filterType { return ruleFit } -func (f *ruleFitFilter) Source(_ config.SharedConfigProvider, _ *core.StoreInfo) *plan.Status { +func (*ruleFitFilter) Source(config.SharedConfigProvider, *core.StoreInfo) *plan.Status { return statusOK } @@ -687,11 +687,11 @@ func (f *ruleLeaderFitFilter) Scope() string { return f.scope } -func (f *ruleLeaderFitFilter) Type() filterType { +func (*ruleLeaderFitFilter) Type() filterType { return ruleLeader } -func (f *ruleLeaderFitFilter) Source(_ config.SharedConfigProvider, _ *core.StoreInfo) *plan.Status { +func (*ruleLeaderFitFilter) Source(config.SharedConfigProvider, *core.StoreInfo) *plan.Status { return statusOK } @@ -743,11 +743,11 @@ func (f *ruleWitnessFitFilter) Scope() string { return f.scope } -func (f *ruleWitnessFitFilter) Type() filterType { +func (*ruleWitnessFitFilter) Type() filterType { return ruleFit } -func (f *ruleWitnessFitFilter) Source(_ config.SharedConfigProvider, _ *core.StoreInfo) *plan.Status { +func (*ruleWitnessFitFilter) Source(config.SharedConfigProvider, *core.StoreInfo) *plan.Status { return statusOK } @@ -815,7 +815,7 @@ func (f *engineFilter) Scope() string { return f.scope } -func (f *engineFilter) Type() filterType { +func (*engineFilter) Type() filterType { return engine } @@ -858,7 +858,7 @@ func (f *specialUseFilter) Scope() string { return f.scope } -func (f *specialUseFilter) Type() filterType { +func (*specialUseFilter) Type() filterType { return specialUse } @@ -869,7 +869,7 @@ func (f *specialUseFilter) Source(conf config.SharedConfigProvider, store *core. return statusStoreNotMatchRule } -func (f *specialUseFilter) Target(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { +func (f *specialUseFilter) Target(_ config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { if !f.constraint.MatchStore(store) { return statusOK } @@ -932,11 +932,11 @@ func (f *isolationFilter) Scope() string { return f.scope } -func (f *isolationFilter) Type() filterType { +func (*isolationFilter) Type() filterType { return isolation } -func (f *isolationFilter) Source(conf config.SharedConfigProvider, store *core.StoreInfo) *plan.Status { +func (*isolationFilter) Source(config.SharedConfigProvider, *core.StoreInfo) *plan.Status { return statusOK } diff --git a/pkg/schedule/filter/region_filters.go b/pkg/schedule/filter/region_filters.go index 799cee7d90c8..7cd015412c27 100644 --- a/pkg/schedule/filter/region_filters.go +++ b/pkg/schedule/filter/region_filters.go @@ -76,7 +76,7 @@ func NewRegionPendingFilter() RegionFilter { return ®ionPendingFilter{} } -func (f *regionPendingFilter) Select(region *core.RegionInfo) *plan.Status { +func (*regionPendingFilter) Select(region *core.RegionInfo) *plan.Status { if hasPendingPeers(region) { return statusRegionPendingPeer } @@ -91,7 +91,7 @@ func NewRegionDownFilter() RegionFilter { return ®ionDownFilter{} } -func (f *regionDownFilter) Select(region *core.RegionInfo) *plan.Status { +func (*regionDownFilter) Select(region *core.RegionInfo) *plan.Status { if hasDownPeers(region) { return statusRegionDownPeer } diff --git a/pkg/schedule/handler/handler.go b/pkg/schedule/handler/handler.go index a9b89e4e3a43..0541a2d65672 100644 --- a/pkg/schedule/handler/handler.go +++ b/pkg/schedule/handler/handler.go @@ -417,7 +417,7 @@ func (h *Handler) AddTransferLeaderOperator(regionID uint64, storeID uint64) err return errors.Errorf("region has no voter in store %v", storeID) } - op, err := operator.CreateTransferLeaderOperator("admin-transfer-leader", c, region, region.GetLeader().GetStoreId(), newLeader.GetStoreId(), []uint64{}, operator.OpAdmin) + op, err := operator.CreateTransferLeaderOperator("admin-transfer-leader", c, region, newLeader.GetStoreId(), []uint64{}, operator.OpAdmin) if err != nil { log.Debug("fail to create transfer leader operator", errs.ZapError(err)) return err @@ -1157,7 +1157,7 @@ func (h *Handler) AccelerateRegionsScheduleInRanges(startKeys [][]byte, endKeys } // AdjustLimit adjusts the limit of regions to schedule. -func (h *Handler) AdjustLimit(limitStr string, defaultLimits ...int) (int, error) { +func (*Handler) AdjustLimit(limitStr string, defaultLimits ...int) (int, error) { limit := defaultRegionLimit if len(defaultLimits) > 0 { limit = defaultLimits[0] @@ -1181,7 +1181,7 @@ type ScatterRegionsResponse struct { } // BuildScatterRegionsResp builds ScatterRegionsResponse. -func (h *Handler) BuildScatterRegionsResp(opsCount int, failures map[uint64]error) *ScatterRegionsResponse { +func (*Handler) BuildScatterRegionsResp(opsCount int, failures map[uint64]error) *ScatterRegionsResponse { // If there existed any operator failed to be added into Operator Controller, add its regions into unProcessedRegions percentage := 100 if len(failures) > 0 { @@ -1217,7 +1217,7 @@ func (h *Handler) ScatterRegionsByRange(rawStartKey, rawEndKey string, group str } // ScatterRegionsByID scatters regions by id. -func (h *Handler) ScatterRegionsByID(ids []uint64, group string, retryLimit int, skipStoreLimit bool) (int, map[uint64]error, error) { +func (h *Handler) ScatterRegionsByID(ids []uint64, group string, retryLimit int) (int, map[uint64]error, error) { co := h.GetCoordinator() if co == nil { return 0, nil, errs.ErrNotBootstrapped.GenWithStackByArgs() diff --git a/pkg/schedule/labeler/labeler_test.go b/pkg/schedule/labeler/labeler_test.go index 364f79b7a146..bd51bab7d83b 100644 --- a/pkg/schedule/labeler/labeler_test.go +++ b/pkg/schedule/labeler/labeler_test.go @@ -404,7 +404,7 @@ func TestLabelerRuleTTL(t *testing.T) { func checkRuleInMemoryAndStorage(re *require.Assertions, labeler *RegionLabeler, ruleID string, exist bool) { re.Equal(exist, labeler.labelRules[ruleID] != nil) existInStorage := false - labeler.storage.LoadRegionRules(func(k, v string) { + labeler.storage.LoadRegionRules(func(k, _ string) { if k == ruleID { existInStorage = true } diff --git a/pkg/schedule/operator/create_operator.go b/pkg/schedule/operator/create_operator.go index 1c96128ab324..638230e30975 100644 --- a/pkg/schedule/operator/create_operator.go +++ b/pkg/schedule/operator/create_operator.go @@ -78,7 +78,7 @@ func CreateRemovePeerOperator(desc string, ci sche.SharedCluster, kind OpKind, r } // CreateTransferLeaderOperator creates an operator that transfers the leader from a source store to a target store. -func CreateTransferLeaderOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, sourceStoreID uint64, targetStoreID uint64, targetStoreIDs []uint64, kind OpKind) (*Operator, error) { +func CreateTransferLeaderOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, targetStoreID uint64, targetStoreIDs []uint64, kind OpKind) (*Operator, error) { return NewBuilder(desc, ci, region, SkipOriginJointStateCheck). SetLeader(targetStoreID). SetLeaders(targetStoreIDs). @@ -86,7 +86,7 @@ func CreateTransferLeaderOperator(desc string, ci sche.SharedCluster, region *co } // CreateForceTransferLeaderOperator creates an operator that transfers the leader from a source store to a target store forcible. -func CreateForceTransferLeaderOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, sourceStoreID uint64, targetStoreID uint64, kind OpKind) (*Operator, error) { +func CreateForceTransferLeaderOperator(desc string, ci sche.SharedCluster, region *core.RegionInfo, targetStoreID uint64, kind OpKind) (*Operator, error) { return NewBuilder(desc, ci, region, SkipOriginJointStateCheck, SkipPlacementRulesCheck). SetLeader(targetStoreID). EnableForceTargetLeader(). diff --git a/pkg/schedule/operator/create_operator_test.go b/pkg/schedule/operator/create_operator_test.go index 80c6cac4a04d..d481334bbcbb 100644 --- a/pkg/schedule/operator/create_operator_test.go +++ b/pkg/schedule/operator/create_operator_test.go @@ -423,7 +423,7 @@ func (suite *createOperatorTestSuite) TestCreateTransferLeaderOperator() { } for _, testCase := range testCases { region := core.NewRegionInfo(&metapb.Region{Id: 1, Peers: testCase.originPeers}, testCase.originPeers[0]) - op, err := CreateTransferLeaderOperator("test", suite.cluster, region, testCase.originPeers[0].StoreId, testCase.targetLeaderStoreID, []uint64{}, 0) + op, err := CreateTransferLeaderOperator("test", suite.cluster, region, testCase.targetLeaderStoreID, []uint64{}, 0) if testCase.isErr { re.Error(err) diff --git a/pkg/schedule/operator/operator_controller.go b/pkg/schedule/operator/operator_controller.go index b9294ad970df..86e51fe70d63 100644 --- a/pkg/schedule/operator/operator_controller.go +++ b/pkg/schedule/operator/operator_controller.go @@ -222,7 +222,7 @@ func (oc *Controller) checkStaleOperator(op *Operator, step OpStep, region *core return false } -func (oc *Controller) getNextPushOperatorTime(step OpStep, now time.Time) time.Time { +func getNextPushOperatorTime(step OpStep, now time.Time) time.Time { nextTime := slowNotifyInterval switch step.(type) { case TransferLeader, PromoteLearner, ChangePeerV2Enter, ChangePeerV2Leave: @@ -270,7 +270,7 @@ func (oc *Controller) pollNeedDispatchRegion() (r *core.RegionInfo, next bool) { } // pushes with new notify time. - item.time = oc.getNextPushOperatorTime(step, now) + item.time = getNextPushOperatorTime(step, now) oc.opNotifierQueue.Push(item) return r, true } @@ -561,7 +561,7 @@ func (oc *Controller) addOperatorInner(op *Operator) bool { } } - oc.opNotifierQueue.Push(&operatorWithTime{op: op, time: oc.getNextPushOperatorTime(step, time.Now())}) + oc.opNotifierQueue.Push(&operatorWithTime{op: op, time: getNextPushOperatorTime(step, time.Now())}) operatorCounter.WithLabelValues(op.Desc(), "create").Inc() for _, counter := range op.Counters { counter.Inc() diff --git a/pkg/schedule/operator/operator_controller_test.go b/pkg/schedule/operator/operator_controller_test.go index f2f2b7305cec..d3c50667fe07 100644 --- a/pkg/schedule/operator/operator_controller_test.go +++ b/pkg/schedule/operator/operator_controller_test.go @@ -108,7 +108,7 @@ func (suite *operatorControllerTestSuite) TestGetOpInfluence() { re.True(op2.Start()) oc.SetOperator(op2) go func(ctx context.Context) { - suite.checkRemoveOperatorSuccess(re, oc, op1) + checkRemoveOperatorSuccess(re, oc, op1) for { select { case <-ctx.Done(): @@ -550,7 +550,7 @@ func (suite *operatorControllerTestSuite) TestStoreLimit() { for i := uint64(1); i <= 5; i++ { op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, AddPeer{ToStore: 2, PeerID: i}) re.True(oc.AddOperator(op)) - suite.checkRemoveOperatorSuccess(re, oc, op) + checkRemoveOperatorSuccess(re, oc, op) } op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, AddPeer{ToStore: 2, PeerID: 1}) re.False(oc.AddOperator(op)) @@ -560,13 +560,13 @@ func (suite *operatorControllerTestSuite) TestStoreLimit() { for i := uint64(1); i <= 10; i++ { op = NewTestOperator(i, &metapb.RegionEpoch{}, OpRegion, AddPeer{ToStore: 2, PeerID: i}) re.True(oc.AddOperator(op)) - suite.checkRemoveOperatorSuccess(re, oc, op) + checkRemoveOperatorSuccess(re, oc, op) } tc.SetAllStoresLimit(storelimit.AddPeer, 60) for i := uint64(1); i <= 5; i++ { op = NewTestOperator(i, &metapb.RegionEpoch{}, OpRegion, AddPeer{ToStore: 2, PeerID: i}) re.True(oc.AddOperator(op)) - suite.checkRemoveOperatorSuccess(re, oc, op) + checkRemoveOperatorSuccess(re, oc, op) } op = NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, AddPeer{ToStore: 2, PeerID: 1}) re.False(oc.AddOperator(op)) @@ -576,7 +576,7 @@ func (suite *operatorControllerTestSuite) TestStoreLimit() { for i := uint64(1); i <= 5; i++ { op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, RemovePeer{FromStore: 2}) re.True(oc.AddOperator(op)) - suite.checkRemoveOperatorSuccess(re, oc, op) + checkRemoveOperatorSuccess(re, oc, op) } op = NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, RemovePeer{FromStore: 2}) re.False(oc.AddOperator(op)) @@ -586,13 +586,13 @@ func (suite *operatorControllerTestSuite) TestStoreLimit() { for i := uint64(1); i <= 10; i++ { op = NewTestOperator(i, &metapb.RegionEpoch{}, OpRegion, RemovePeer{FromStore: 2}) re.True(oc.AddOperator(op)) - suite.checkRemoveOperatorSuccess(re, oc, op) + checkRemoveOperatorSuccess(re, oc, op) } tc.SetAllStoresLimit(storelimit.RemovePeer, 60) for i := uint64(1); i <= 5; i++ { op = NewTestOperator(i, &metapb.RegionEpoch{}, OpRegion, RemovePeer{FromStore: 2}) re.True(oc.AddOperator(op)) - suite.checkRemoveOperatorSuccess(re, oc, op) + checkRemoveOperatorSuccess(re, oc, op) } op = NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, RemovePeer{FromStore: 2}) re.False(oc.AddOperator(op)) @@ -860,7 +860,7 @@ func newRegionInfo(id uint64, startKey, endKey string, size, keys int64, leader ) } -func (suite *operatorControllerTestSuite) checkRemoveOperatorSuccess(re *require.Assertions, oc *Controller, op *Operator) { +func checkRemoveOperatorSuccess(re *require.Assertions, oc *Controller, op *Operator) { re.True(oc.RemoveOperator(op)) re.True(op.IsEnd()) re.Equal(op, oc.GetOperatorStatus(op.RegionID()).Operator) diff --git a/pkg/schedule/operator/operator_test.go b/pkg/schedule/operator/operator_test.go index 4719df9408bc..693f5c174753 100644 --- a/pkg/schedule/operator/operator_test.go +++ b/pkg/schedule/operator/operator_test.go @@ -65,7 +65,7 @@ func (suite *operatorTestSuite) TearDownTest() { suite.cancel() } -func (suite *operatorTestSuite) newTestRegion(regionID uint64, leaderPeer uint64, peers ...[2]uint64) *core.RegionInfo { +func newTestRegion(regionID uint64, leaderPeer uint64, peers ...[2]uint64) *core.RegionInfo { var ( region metapb.Region leader *metapb.Peer @@ -87,7 +87,7 @@ func (suite *operatorTestSuite) newTestRegion(regionID uint64, leaderPeer uint64 func (suite *operatorTestSuite) TestOperatorStep() { re := suite.Require() - region := suite.newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) + region := newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) re.False(TransferLeader{FromStore: 1, ToStore: 2}.IsFinish(region)) re.True(TransferLeader{FromStore: 2, ToStore: 1}.IsFinish(region)) re.False(AddPeer{ToStore: 3, PeerID: 3}.IsFinish(region)) @@ -96,11 +96,7 @@ func (suite *operatorTestSuite) TestOperatorStep() { re.True(RemovePeer{FromStore: 3}.IsFinish(region)) } -func (suite *operatorTestSuite) newTestOperator(regionID uint64, kind OpKind, steps ...OpStep) *Operator { - return NewTestOperator(regionID, &metapb.RegionEpoch{}, kind, steps...) -} - -func (suite *operatorTestSuite) checkSteps(re *require.Assertions, op *Operator, steps []OpStep) { +func checkSteps(re *require.Assertions, op *Operator, steps []OpStep) { re.Len(steps, op.Len()) for i := range steps { re.Equal(steps[i], op.Step(i)) @@ -109,16 +105,16 @@ func (suite *operatorTestSuite) checkSteps(re *require.Assertions, op *Operator, func (suite *operatorTestSuite) TestOperator() { re := suite.Require() - region := suite.newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) + region := newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) // addPeer1, transferLeader1, removePeer3 steps := []OpStep{ AddPeer{ToStore: 1, PeerID: 1}, TransferLeader{FromStore: 3, ToStore: 1}, RemovePeer{FromStore: 3}, } - op := suite.newTestOperator(1, OpAdmin|OpLeader|OpRegion, steps...) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpAdmin|OpLeader|OpRegion, steps...) re.Equal(constant.Urgent, op.GetPriorityLevel()) - suite.checkSteps(re, op, steps) + checkSteps(re, op, steps) op.Start() re.Nil(op.Check(region)) @@ -132,9 +128,9 @@ func (suite *operatorTestSuite) TestOperator() { TransferLeader{FromStore: 2, ToStore: 1}, RemovePeer{FromStore: 2}, } - op = suite.newTestOperator(1, OpLeader|OpRegion, steps...) + op = NewTestOperator(1, &metapb.RegionEpoch{}, OpLeader|OpRegion, steps...) re.Equal(constant.Medium, op.GetPriorityLevel()) - suite.checkSteps(re, op, steps) + checkSteps(re, op, steps) op.Start() re.Equal(RemovePeer{FromStore: 2}, op.Check(region)) re.Equal(int32(2), atomic.LoadInt32(&op.currentStep)) @@ -149,7 +145,7 @@ func (suite *operatorTestSuite) TestOperator() { // check short timeout for transfer leader only operators. steps = []OpStep{TransferLeader{FromStore: 2, ToStore: 1}} - op = suite.newTestOperator(1, OpLeader, steps...) + op = NewTestOperator(1, &metapb.RegionEpoch{}, OpLeader, steps...) op.Start() re.False(op.CheckTimeout()) op.SetStatusReachTime(STARTED, op.GetStartTime().Add(-FastStepWaitTime-time.Second)) @@ -166,7 +162,7 @@ func (suite *operatorTestSuite) TestOperator() { func (suite *operatorTestSuite) TestInfluence() { re := suite.Require() - region := suite.newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) + region := newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) opInfluence := OpInfluence{StoresInfluence: make(map[uint64]*StoreInfluence)} storeOpInfluence := opInfluence.StoresInfluence storeOpInfluence[1] = &StoreInfluence{} @@ -309,7 +305,7 @@ func (suite *operatorTestSuite) TestCheckSuccess() { TransferLeader{FromStore: 2, ToStore: 1}, RemovePeer{FromStore: 2}, } - op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpLeader|OpRegion, steps...) re.Equal(CREATED, op.Status()) re.False(op.CheckSuccess()) re.True(op.Start()) @@ -324,7 +320,7 @@ func (suite *operatorTestSuite) TestCheckSuccess() { TransferLeader{FromStore: 2, ToStore: 1}, RemovePeer{FromStore: 2}, } - op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpLeader|OpRegion, steps...) op.currentStep = int32(len(op.steps)) re.Equal(CREATED, op.Status()) re.False(op.CheckSuccess()) @@ -342,7 +338,7 @@ func (suite *operatorTestSuite) TestCheckTimeout() { TransferLeader{FromStore: 2, ToStore: 1}, RemovePeer{FromStore: 2}, } - op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpLeader|OpRegion, steps...) re.Equal(CREATED, op.Status()) re.True(op.Start()) op.currentStep = int32(len(op.steps)) @@ -355,7 +351,7 @@ func (suite *operatorTestSuite) TestCheckTimeout() { TransferLeader{FromStore: 2, ToStore: 1}, RemovePeer{FromStore: 2}, } - op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpLeader|OpRegion, steps...) re.Equal(CREATED, op.Status()) re.True(op.Start()) op.currentStep = int32(len(op.steps)) @@ -372,7 +368,7 @@ func (suite *operatorTestSuite) TestStart() { TransferLeader{FromStore: 2, ToStore: 1}, RemovePeer{FromStore: 2}, } - op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpLeader|OpRegion, steps...) re.Equal(0, op.GetStartTime().Nanosecond()) re.Equal(CREATED, op.Status()) re.True(op.Start()) @@ -387,7 +383,7 @@ func (suite *operatorTestSuite) TestCheckExpired() { TransferLeader{FromStore: 2, ToStore: 1}, RemovePeer{FromStore: 2}, } - op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpLeader|OpRegion, steps...) re.False(op.CheckExpired()) re.Equal(CREATED, op.Status()) op.SetStatusReachTime(CREATED, time.Now().Add(-OperatorExpireTime)) @@ -398,30 +394,30 @@ func (suite *operatorTestSuite) TestCheckExpired() { func (suite *operatorTestSuite) TestCheck() { re := suite.Require() { - region := suite.newTestRegion(2, 2, [2]uint64{1, 1}, [2]uint64{2, 2}) + region := newTestRegion(2, 2, [2]uint64{1, 1}, [2]uint64{2, 2}) steps := []OpStep{ AddPeer{ToStore: 1, PeerID: 1}, TransferLeader{FromStore: 2, ToStore: 1}, RemovePeer{FromStore: 2}, } - op := suite.newTestOperator(2, OpLeader|OpRegion, steps...) + op := NewTestOperator(2, &metapb.RegionEpoch{}, OpLeader|OpRegion, steps...) re.True(op.Start()) re.NotNil(op.Check(region)) re.Equal(STARTED, op.Status()) - region = suite.newTestRegion(1, 1, [2]uint64{1, 1}) + region = newTestRegion(1, 1, [2]uint64{1, 1}) re.Nil(op.Check(region)) re.Equal(SUCCESS, op.Status()) } { - region := suite.newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) + region := newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) steps := []OpStep{ AddPeer{ToStore: 1, PeerID: 1}, TransferLeader{FromStore: 2, ToStore: 1}, RemovePeer{FromStore: 2}, } - op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpLeader|OpRegion, steps...) re.True(op.Start()) re.NotNil(op.Check(region)) re.Equal(STARTED, op.Status()) @@ -430,18 +426,18 @@ func (suite *operatorTestSuite) TestCheck() { re.Equal(TIMEOUT, op.Status()) } { - region := suite.newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) + region := newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) steps := []OpStep{ AddPeer{ToStore: 1, PeerID: 1}, TransferLeader{FromStore: 2, ToStore: 1}, RemovePeer{FromStore: 2}, } - op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) + op := NewTestOperator(1, &metapb.RegionEpoch{}, OpLeader|OpRegion, steps...) re.True(op.Start()) re.NotNil(op.Check(region)) re.Equal(STARTED, op.Status()) op.status.setTime(STARTED, time.Now().Add(-SlowStepWaitTime)) - region = suite.newTestRegion(1, 1, [2]uint64{1, 1}) + region = newTestRegion(1, 1, [2]uint64{1, 1}) re.Nil(op.Check(region)) re.Equal(SUCCESS, op.Status()) } @@ -454,28 +450,28 @@ func (suite *operatorTestSuite) TestSchedulerKind() { expect OpKind }{ { - op: suite.newTestOperator(1, OpAdmin|OpMerge|OpRegion), + op: NewTestOperator(1, &metapb.RegionEpoch{}, OpAdmin|OpMerge|OpRegion), expect: OpAdmin, }, { - op: suite.newTestOperator(1, OpMerge|OpLeader|OpRegion), + op: NewTestOperator(1, &metapb.RegionEpoch{}, OpMerge|OpLeader|OpRegion), expect: OpMerge, }, { - op: suite.newTestOperator(1, OpReplica|OpRegion), + op: NewTestOperator(1, &metapb.RegionEpoch{}, OpReplica|OpRegion), expect: OpReplica, }, { - op: suite.newTestOperator(1, OpSplit|OpRegion), + op: NewTestOperator(1, &metapb.RegionEpoch{}, OpSplit|OpRegion), expect: OpSplit, }, { - op: suite.newTestOperator(1, OpRange|OpRegion), + op: NewTestOperator(1, &metapb.RegionEpoch{}, OpRange|OpRegion), expect: OpRange, }, { - op: suite.newTestOperator(1, OpHotRegion|OpLeader|OpRegion), + op: NewTestOperator(1, &metapb.RegionEpoch{}, OpHotRegion|OpLeader|OpRegion), expect: OpHotRegion, }, { - op: suite.newTestOperator(1, OpRegion|OpLeader), + op: NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion|OpLeader), expect: OpRegion, }, { - op: suite.newTestOperator(1, OpLeader), + op: NewTestOperator(1, &metapb.RegionEpoch{}, OpLeader), expect: OpLeader, }, } @@ -534,7 +530,7 @@ func (suite *operatorTestSuite) TestOpStepTimeout() { func (suite *operatorTestSuite) TestRecord() { re := suite.Require() - operator := suite.newTestOperator(1, OpLeader, AddLearner{ToStore: 1, PeerID: 1}, RemovePeer{FromStore: 1, PeerID: 1}) + operator := NewTestOperator(1, &metapb.RegionEpoch{}, OpLeader, AddLearner{ToStore: 1, PeerID: 1}, RemovePeer{FromStore: 1, PeerID: 1}) now := time.Now() time.Sleep(time.Second) ob := operator.Record(now) @@ -548,7 +544,7 @@ func (suite *operatorTestSuite) TestToJSONObject() { TransferLeader{FromStore: 3, ToStore: 1}, RemovePeer{FromStore: 3}, } - op := suite.newTestOperator(101, OpLeader|OpRegion, steps...) + op := NewTestOperator(101, &metapb.RegionEpoch{}, OpLeader|OpRegion, steps...) op.Start() obj := op.ToJSONObject() suite.Equal("test", obj.Desc) @@ -559,7 +555,7 @@ func (suite *operatorTestSuite) TestToJSONObject() { suite.Equal(STARTED, obj.Status) // Test SUCCESS status. - region := suite.newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) + region := newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) suite.Nil(op.Check(region)) suite.Equal(SUCCESS, op.Status()) obj = op.ToJSONObject() @@ -567,7 +563,7 @@ func (suite *operatorTestSuite) TestToJSONObject() { // Test TIMEOUT status. steps = []OpStep{TransferLeader{FromStore: 2, ToStore: 1}} - op = suite.newTestOperator(1, OpLeader, steps...) + op = NewTestOperator(1, &metapb.RegionEpoch{}, OpLeader, steps...) op.Start() op.SetStatusReachTime(STARTED, op.GetStartTime().Add(-FastStepWaitTime-time.Second)) suite.True(op.CheckTimeout()) diff --git a/pkg/schedule/operator/status_tracker.go b/pkg/schedule/operator/status_tracker.go index e103a74ccb34..0ba8135750cd 100644 --- a/pkg/schedule/operator/status_tracker.go +++ b/pkg/schedule/operator/status_tracker.go @@ -64,9 +64,8 @@ func (trk *OpStatusTracker) getTime(s OpStatus) time.Time { return trk.reachTimes[s] } else if trk.current == s { return trk.reachTimes[firstEndStatus] - } else { - return time.Time{} } + return time.Time{} } // To transfer the current status to dst if this transition is valid, diff --git a/pkg/schedule/operator/step.go b/pkg/schedule/operator/step.go index 6f14cbb326b5..04e410288657 100644 --- a/pkg/schedule/operator/step.go +++ b/pkg/schedule/operator/step.go @@ -70,7 +70,7 @@ type TransferLeader struct { } // ConfVerChanged returns the delta value for version increased by this step. -func (tl TransferLeader) ConfVerChanged(_ *core.RegionInfo) uint64 { +func (TransferLeader) ConfVerChanged(_ *core.RegionInfo) uint64 { return 0 // transfer leader never change the conf version } @@ -122,12 +122,12 @@ func (tl TransferLeader) Influence(opInfluence OpInfluence, region *core.RegionI } // Timeout returns duration that current step may take. -func (tl TransferLeader) Timeout(regionSize int64) time.Duration { +func (TransferLeader) Timeout(regionSize int64) time.Duration { return fastStepWaitDuration(regionSize) } // GetCmd returns the schedule command for heartbeat response. -func (tl TransferLeader) GetCmd(region *core.RegionInfo, useConfChangeV2 bool) *hbstream.Operation { +func (tl TransferLeader) GetCmd(region *core.RegionInfo, _ bool) *hbstream.Operation { peers := make([]*metapb.Peer, 0, len(tl.ToStores)) for _, storeID := range tl.ToStores { peers = append(peers, region.GetStorePeer(storeID)) @@ -206,7 +206,7 @@ func (ap AddPeer) CheckInProgress(ci *core.BasicCluster, config config.SharedCon } // Timeout returns duration that current step may take. -func (ap AddPeer) Timeout(regionSize int64) time.Duration { +func (AddPeer) Timeout(regionSize int64) time.Duration { return slowStepWaitDuration(regionSize) } @@ -270,7 +270,7 @@ func (bw BecomeWitness) Influence(opInfluence OpInfluence, region *core.RegionIn } // Timeout returns duration that current step may take. -func (bw BecomeWitness) Timeout(regionSize int64) time.Duration { +func (BecomeWitness) Timeout(regionSize int64) time.Duration { return fastStepWaitDuration(regionSize) } @@ -338,12 +338,12 @@ func (bn BecomeNonWitness) Influence(opInfluence OpInfluence, region *core.Regio } // Timeout returns duration that current step may take. -func (bn BecomeNonWitness) Timeout(regionSize int64) time.Duration { +func (BecomeNonWitness) Timeout(regionSize int64) time.Duration { return slowStepWaitDuration(regionSize) } // GetCmd returns the schedule command for heartbeat response. -func (bn BecomeNonWitness) GetCmd(region *core.RegionInfo, useConfChangeV2 bool) *hbstream.Operation { +func (bn BecomeNonWitness) GetCmd(*core.RegionInfo, bool) *hbstream.Operation { return switchWitness(bn.PeerID, false) } @@ -518,7 +518,7 @@ func (al AddLearner) Influence(opInfluence OpInfluence, region *core.RegionInfo) } // Timeout returns duration that current step may take. -func (al AddLearner) Timeout(regionSize int64) time.Duration { +func (AddLearner) Timeout(regionSize int64) time.Duration { return slowStepWaitDuration(regionSize) } @@ -565,7 +565,7 @@ func (pl PromoteLearner) IsFinish(region *core.RegionInfo) bool { } // CheckInProgress checks if the step is in the progress of advancing. -func (pl PromoteLearner) CheckInProgress(_ *core.BasicCluster, config config.SharedConfigProvider, region *core.RegionInfo) error { +func (pl PromoteLearner) CheckInProgress(_ *core.BasicCluster, _ config.SharedConfigProvider, region *core.RegionInfo) error { peer := region.GetStorePeer(pl.ToStore) if peer.GetId() != pl.PeerID { return errors.New("peer does not exist") @@ -574,10 +574,10 @@ func (pl PromoteLearner) CheckInProgress(_ *core.BasicCluster, config config.Sha } // Influence calculates the store difference that current step makes. -func (pl PromoteLearner) Influence(_ OpInfluence, _ *core.RegionInfo) {} +func (PromoteLearner) Influence(OpInfluence, *core.RegionInfo) {} // Timeout returns duration that current step may take. -func (pl PromoteLearner) Timeout(regionSize int64) time.Duration { +func (PromoteLearner) Timeout(regionSize int64) time.Duration { return fastStepWaitDuration(regionSize) } @@ -617,7 +617,7 @@ func (rp RemovePeer) IsFinish(region *core.RegionInfo) bool { } // CheckInProgress checks if the step is in the progress of advancing. -func (rp RemovePeer) CheckInProgress(_ *core.BasicCluster, config config.SharedConfigProvider, region *core.RegionInfo) error { +func (rp RemovePeer) CheckInProgress(_ *core.BasicCluster, _ config.SharedConfigProvider, region *core.RegionInfo) error { if rp.FromStore == region.GetLeader().GetStoreId() { return errors.New("cannot remove leader peer") } @@ -648,7 +648,7 @@ func (rp RemovePeer) Influence(opInfluence OpInfluence, region *core.RegionInfo) } // Timeout returns duration that current step may take. -func (rp RemovePeer) Timeout(regionSize int64) time.Duration { +func (RemovePeer) Timeout(regionSize int64) time.Duration { return fastStepWaitDuration(regionSize) } @@ -674,7 +674,7 @@ type MergeRegion struct { } // ConfVerChanged returns the delta value for version increased by this step. -func (mr MergeRegion) ConfVerChanged(_ *core.RegionInfo) uint64 { +func (MergeRegion) ConfVerChanged(*core.RegionInfo) uint64 { return 0 } @@ -691,7 +691,7 @@ func (mr MergeRegion) IsFinish(region *core.RegionInfo) bool { } // CheckInProgress checks if the step is in the progress of advancing. -func (mr MergeRegion) CheckInProgress(_ *core.BasicCluster, config config.SharedConfigProvider, _ *core.RegionInfo) error { +func (MergeRegion) CheckInProgress(*core.BasicCluster, config.SharedConfigProvider, *core.RegionInfo) error { return nil } @@ -710,12 +710,12 @@ func (mr MergeRegion) Influence(opInfluence OpInfluence, region *core.RegionInfo // Timeout returns duration that current step may take. // The merge step need more time to finish but less than slow step. -func (mr MergeRegion) Timeout(regionSize int64) time.Duration { +func (MergeRegion) Timeout(regionSize int64) time.Duration { return fastStepWaitDuration(regionSize) * 10 } // GetCmd returns the schedule command for heartbeat response. -func (mr MergeRegion) GetCmd(region *core.RegionInfo, useConfChangeV2 bool) *hbstream.Operation { +func (mr MergeRegion) GetCmd(*core.RegionInfo, bool) *hbstream.Operation { if mr.IsPassive { return nil } @@ -734,7 +734,7 @@ type SplitRegion struct { } // ConfVerChanged returns the delta value for version increased by this step. -func (sr SplitRegion) ConfVerChanged(_ *core.RegionInfo) uint64 { +func (SplitRegion) ConfVerChanged(*core.RegionInfo) uint64 { return 0 } @@ -748,7 +748,7 @@ func (sr SplitRegion) IsFinish(region *core.RegionInfo) bool { } // Influence calculates the store difference that current step makes. -func (sr SplitRegion) Influence(opInfluence OpInfluence, region *core.RegionInfo) { +func (SplitRegion) Influence(opInfluence OpInfluence, region *core.RegionInfo) { for _, peer := range region.GetPeers() { inf := opInfluence.GetStoreInfluence(peer.GetStoreId()) inf.RegionCount++ @@ -759,17 +759,17 @@ func (sr SplitRegion) Influence(opInfluence OpInfluence, region *core.RegionInfo } // CheckInProgress checks if the step is in the progress of advancing. -func (sr SplitRegion) CheckInProgress(_ *core.BasicCluster, config config.SharedConfigProvider, _ *core.RegionInfo) error { +func (SplitRegion) CheckInProgress(*core.BasicCluster, config.SharedConfigProvider, *core.RegionInfo) error { return nil } // Timeout returns duration that current step may take. -func (sr SplitRegion) Timeout(regionSize int64) time.Duration { +func (SplitRegion) Timeout(regionSize int64) time.Duration { return fastStepWaitDuration(regionSize) } // GetCmd returns the schedule command for heartbeat response. -func (sr SplitRegion) GetCmd(region *core.RegionInfo, useConfChangeV2 bool) *hbstream.Operation { +func (sr SplitRegion) GetCmd(*core.RegionInfo, bool) *hbstream.Operation { return &hbstream.Operation{ SplitRegion: &pdpb.SplitRegion{ Policy: sr.Policy, @@ -814,7 +814,7 @@ func (dv DemoteVoter) IsFinish(region *core.RegionInfo) bool { } // Timeout returns duration that current step may take. -func (dv DemoteVoter) Timeout(regionSize int64) time.Duration { +func (DemoteVoter) Timeout(regionSize int64) time.Duration { return fastStepWaitDuration(regionSize) } @@ -884,7 +884,7 @@ func (cpe ChangePeerV2Enter) IsFinish(region *core.RegionInfo) bool { } // CheckInProgress checks if the step is in the progress of advancing. -func (cpe ChangePeerV2Enter) CheckInProgress(_ *core.BasicCluster, config config.SharedConfigProvider, region *core.RegionInfo) error { +func (cpe ChangePeerV2Enter) CheckInProgress(_ *core.BasicCluster, _ config.SharedConfigProvider, region *core.RegionInfo) error { inJointState, notInJointState := false, false for _, pl := range cpe.PromoteLearners { peer := region.GetStorePeer(pl.ToStore) @@ -932,7 +932,7 @@ func (cpe ChangePeerV2Enter) CheckInProgress(_ *core.BasicCluster, config config } // Influence calculates the store difference that current step makes. -func (cpe ChangePeerV2Enter) Influence(_ OpInfluence, _ *core.RegionInfo) {} +func (ChangePeerV2Enter) Influence(OpInfluence, *core.RegionInfo) {} // Timeout returns duration that current step may take. func (cpe ChangePeerV2Enter) Timeout(regionSize int64) time.Duration { @@ -1013,7 +1013,7 @@ func (cpl ChangePeerV2Leave) IsFinish(region *core.RegionInfo) bool { } // CheckInProgress checks if the step is in the progress of advancing. -func (cpl ChangePeerV2Leave) CheckInProgress(_ *core.BasicCluster, config config.SharedConfigProvider, region *core.RegionInfo) error { +func (cpl ChangePeerV2Leave) CheckInProgress(_ *core.BasicCluster, _ config.SharedConfigProvider, region *core.RegionInfo) error { inJointState, notInJointState, demoteLeader := false, false, false leaderStoreID := region.GetLeader().GetStoreId() @@ -1072,7 +1072,7 @@ func (cpl ChangePeerV2Leave) CheckInProgress(_ *core.BasicCluster, config config } // Influence calculates the store difference that current step makes. -func (cpl ChangePeerV2Leave) Influence(_ OpInfluence, _ *core.RegionInfo) {} +func (ChangePeerV2Leave) Influence(OpInfluence, *core.RegionInfo) {} // Timeout returns duration that current step may take. func (cpl ChangePeerV2Leave) Timeout(regionSize int64) time.Duration { @@ -1081,7 +1081,7 @@ func (cpl ChangePeerV2Leave) Timeout(regionSize int64) time.Duration { } // GetCmd returns the schedule command for heartbeat response. -func (cpl ChangePeerV2Leave) GetCmd(region *core.RegionInfo, useConfChangeV2 bool) *hbstream.Operation { +func (ChangePeerV2Leave) GetCmd(_ *core.RegionInfo, useConfChangeV2 bool) *hbstream.Operation { if !useConfChangeV2 { // only supported in ChangePeerV2 return nil diff --git a/pkg/schedule/placement/region_rule_cache_test.go b/pkg/schedule/placement/region_rule_cache_test.go index 835203bed26f..e951ea10cc53 100644 --- a/pkg/schedule/placement/region_rule_cache_test.go +++ b/pkg/schedule/placement/region_rule_cache_test.go @@ -226,7 +226,7 @@ func (manager *RegionRuleFitCacheManager) mockRegionRuleFitCache(region *core.Re } } -// nolint +// nolint:unparam func mockStores(num int) []*core.StoreInfo { stores := make([]*core.StoreInfo, 0, num) now := time.Now() @@ -237,7 +237,6 @@ func mockStores(num int) []*core.StoreInfo { return stores } -// nolint func mockStoresNoHeartbeat(num int) []*core.StoreInfo { stores := make([]*core.StoreInfo, 0, num) for i := 1; i <= num; i++ { diff --git a/pkg/schedule/schedulers/balance_leader.go b/pkg/schedule/schedulers/balance_leader.go index e11e8492765b..b4e6feb332cc 100644 --- a/pkg/schedule/schedulers/balance_leader.go +++ b/pkg/schedule/schedulers/balance_leader.go @@ -164,7 +164,7 @@ func (handler *balanceLeaderHandler) UpdateConfig(w http.ResponseWriter, r *http handler.rd.JSON(w, httpCode, v) } -func (handler *balanceLeaderHandler) ListConfig(w http.ResponseWriter, r *http.Request) { +func (handler *balanceLeaderHandler) ListConfig(w http.ResponseWriter, _ *http.Request) { conf := handler.config.Clone() handler.rd.JSON(w, http.StatusOK, conf) } @@ -219,7 +219,7 @@ func (l *balanceLeaderScheduler) GetName() string { return l.name } -func (l *balanceLeaderScheduler) GetType() string { +func (*balanceLeaderScheduler) GetType() string { return BalanceLeaderType } @@ -553,7 +553,7 @@ func (l *balanceLeaderScheduler) createOperator(solver *solver, collector *plan. } solver.Step++ defer func() { solver.Step-- }() - op, err := operator.CreateTransferLeaderOperator(BalanceLeaderType, solver, solver.Region, solver.Region.GetLeader().GetStoreId(), solver.TargetStoreID(), []uint64{}, operator.OpLeader) + op, err := operator.CreateTransferLeaderOperator(BalanceLeaderType, solver, solver.Region, solver.TargetStoreID(), []uint64{}, operator.OpLeader) if err != nil { log.Debug("fail to create balance leader operator", errs.ZapError(err)) if collector != nil { diff --git a/pkg/schedule/schedulers/balance_region.go b/pkg/schedule/schedulers/balance_region.go index 1cef3a4615b3..98e3be6e08a8 100644 --- a/pkg/schedule/schedulers/balance_region.go +++ b/pkg/schedule/schedulers/balance_region.go @@ -96,7 +96,7 @@ func (s *balanceRegionScheduler) GetName() string { return s.conf.Name } -func (s *balanceRegionScheduler) GetType() string { +func (*balanceRegionScheduler) GetType() string { return BalanceRegionType } diff --git a/pkg/schedule/schedulers/balance_test.go b/pkg/schedule/schedulers/balance_test.go index 68332d7067e7..234acfd6d264 100644 --- a/pkg/schedule/schedulers/balance_test.go +++ b/pkg/schedule/schedulers/balance_test.go @@ -186,7 +186,7 @@ func TestTolerantRatio(t *testing.T) { kind constant.ScheduleKind expectTolerantResource func(constant.ScheduleKind) int64 }{ - {0, constant.ScheduleKind{Resource: constant.LeaderKind, Policy: constant.ByCount}, func(k constant.ScheduleKind) int64 { + {0, constant.ScheduleKind{Resource: constant.LeaderKind, Policy: constant.ByCount}, func(constant.ScheduleKind) int64 { return int64(leaderTolerantSizeRatio) }}, {0, constant.ScheduleKind{Resource: constant.LeaderKind, Policy: constant.BySize}, func(k constant.ScheduleKind) int64 { @@ -198,7 +198,7 @@ func TestTolerantRatio(t *testing.T) { {0, constant.ScheduleKind{Resource: constant.RegionKind, Policy: constant.BySize}, func(k constant.ScheduleKind) int64 { return int64(adjustTolerantRatio(tc, k) * float64(regionSize)) }}, - {10, constant.ScheduleKind{Resource: constant.LeaderKind, Policy: constant.ByCount}, func(k constant.ScheduleKind) int64 { + {10, constant.ScheduleKind{Resource: constant.LeaderKind, Policy: constant.ByCount}, func(constant.ScheduleKind) int64 { return int64(tc.GetScheduleConfig().TolerantSizeRatio) }}, {10, constant.ScheduleKind{Resource: constant.LeaderKind, Policy: constant.BySize}, func(k constant.ScheduleKind) int64 { diff --git a/pkg/schedule/schedulers/balance_witness.go b/pkg/schedule/schedulers/balance_witness.go index aee112c9dc1d..3c4776c4666d 100644 --- a/pkg/schedule/schedulers/balance_witness.go +++ b/pkg/schedule/schedulers/balance_witness.go @@ -150,7 +150,7 @@ func (handler *balanceWitnessHandler) UpdateConfig(w http.ResponseWriter, r *htt handler.rd.JSON(w, httpCode, v) } -func (handler *balanceWitnessHandler) ListConfig(w http.ResponseWriter, r *http.Request) { +func (handler *balanceWitnessHandler) ListConfig(w http.ResponseWriter, _ *http.Request) { conf := handler.config.Clone() handler.rd.JSON(w, http.StatusOK, conf) } @@ -214,7 +214,7 @@ func (b *balanceWitnessScheduler) GetName() string { return b.name } -func (b *balanceWitnessScheduler) GetType() string { +func (*balanceWitnessScheduler) GetType() string { return BalanceWitnessType } diff --git a/pkg/schedule/schedulers/base_scheduler.go b/pkg/schedule/schedulers/base_scheduler.go index f4c8c5777670..f3772757ad30 100644 --- a/pkg/schedule/schedulers/base_scheduler.go +++ b/pkg/schedule/schedulers/base_scheduler.go @@ -68,32 +68,32 @@ func NewBaseScheduler(opController *operator.Controller) *BaseScheduler { return &BaseScheduler{OpController: opController} } -func (s *BaseScheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func (*BaseScheduler) ServeHTTP(w http.ResponseWriter, _ *http.Request) { fmt.Fprintf(w, "not implements") } // GetMinInterval returns the minimal interval for the scheduler -func (s *BaseScheduler) GetMinInterval() time.Duration { +func (*BaseScheduler) GetMinInterval() time.Duration { return MinScheduleInterval } // EncodeConfig encode config for the scheduler -func (s *BaseScheduler) EncodeConfig() ([]byte, error) { +func (*BaseScheduler) EncodeConfig() ([]byte, error) { return EncodeConfig(nil) } // ReloadConfig reloads the config from the storage. // By default, the scheduler does not need to reload the config // if it doesn't support the dynamic configuration. -func (s *BaseScheduler) ReloadConfig() error { return nil } +func (*BaseScheduler) ReloadConfig() error { return nil } // GetNextInterval return the next interval for the scheduler -func (s *BaseScheduler) GetNextInterval(interval time.Duration) time.Duration { +func (*BaseScheduler) GetNextInterval(interval time.Duration) time.Duration { return intervalGrow(interval, MaxScheduleInterval, exponentialGrowth) } // PrepareConfig does some prepare work about config. -func (s *BaseScheduler) PrepareConfig(cluster sche.SchedulerCluster) error { return nil } +func (*BaseScheduler) PrepareConfig(sche.SchedulerCluster) error { return nil } // CleanConfig does some cleanup work about config. -func (s *BaseScheduler) CleanConfig(cluster sche.SchedulerCluster) {} +func (*BaseScheduler) CleanConfig(sche.SchedulerCluster) {} diff --git a/pkg/schedule/schedulers/evict_leader.go b/pkg/schedule/schedulers/evict_leader.go index 5cd59583767b..3750834a82d4 100644 --- a/pkg/schedule/schedulers/evict_leader.go +++ b/pkg/schedule/schedulers/evict_leader.go @@ -118,7 +118,7 @@ func (conf *evictLeaderSchedulerConfig) Persist() error { return conf.storage.SaveSchedulerConfig(name, data) } -func (conf *evictLeaderSchedulerConfig) getSchedulerName() string { +func (*evictLeaderSchedulerConfig) getSchedulerName() string { return EvictLeaderName } @@ -190,11 +190,11 @@ func (s *evictLeaderScheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) s.handler.ServeHTTP(w, r) } -func (s *evictLeaderScheduler) GetName() string { +func (*evictLeaderScheduler) GetName() string { return EvictLeaderName } -func (s *evictLeaderScheduler) GetType() string { +func (*evictLeaderScheduler) GetType() string { return EvictLeaderType } @@ -251,7 +251,7 @@ func (s *evictLeaderScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) return allowed } -func (s *evictLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *evictLeaderScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*operator.Operator, []plan.Plan) { evictLeaderCounter.Inc() return scheduleEvictLeaderBatch(s.GetName(), s.GetType(), cluster, s.conf, EvictLeaderBatchSize), nil } @@ -338,7 +338,7 @@ func scheduleEvictLeaderOnce(name, typ string, cluster sche.SchedulerCluster, co for _, t := range targets { targetIDs = append(targetIDs, t.GetID()) } - op, err := operator.CreateTransferLeaderOperator(typ, cluster, region, region.GetLeader().GetStoreId(), target.GetID(), targetIDs, operator.OpLeader) + op, err := operator.CreateTransferLeaderOperator(typ, cluster, region, target.GetID(), targetIDs, operator.OpLeader) if err != nil { log.Debug("fail to create evict leader operator", errs.ZapError(err)) continue @@ -395,7 +395,7 @@ func (handler *evictLeaderHandler) UpdateConfig(w http.ResponseWriter, r *http.R handler.rd.JSON(w, http.StatusOK, "The scheduler has been applied to the store.") } -func (handler *evictLeaderHandler) ListConfig(w http.ResponseWriter, r *http.Request) { +func (handler *evictLeaderHandler) ListConfig(w http.ResponseWriter, _ *http.Request) { conf := handler.config.Clone() handler.rd.JSON(w, http.StatusOK, conf) } diff --git a/pkg/schedule/schedulers/evict_slow_store.go b/pkg/schedule/schedulers/evict_slow_store.go index aa48d0bc9e92..ab30b2568234 100644 --- a/pkg/schedule/schedulers/evict_slow_store.go +++ b/pkg/schedule/schedulers/evict_slow_store.go @@ -177,7 +177,7 @@ func (handler *evictSlowStoreHandler) UpdateConfig(w http.ResponseWriter, r *htt handler.rd.JSON(w, http.StatusOK, "Config updated.") } -func (handler *evictSlowStoreHandler) ListConfig(w http.ResponseWriter, r *http.Request) { +func (handler *evictSlowStoreHandler) ListConfig(w http.ResponseWriter, _ *http.Request) { conf := handler.config.Clone() handler.rd.JSON(w, http.StatusOK, conf) } @@ -192,11 +192,11 @@ func (s *evictSlowStoreScheduler) ServeHTTP(w http.ResponseWriter, r *http.Reque s.handler.ServeHTTP(w, r) } -func (s *evictSlowStoreScheduler) GetName() string { +func (*evictSlowStoreScheduler) GetName() string { return EvictSlowStoreName } -func (s *evictSlowStoreScheduler) GetType() string { +func (*evictSlowStoreScheduler) GetType() string { return EvictSlowStoreType } @@ -280,7 +280,7 @@ func (s *evictSlowStoreScheduler) IsScheduleAllowed(cluster sche.SchedulerCluste return true } -func (s *evictSlowStoreScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *evictSlowStoreScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*operator.Operator, []plan.Plan) { evictSlowStoreCounter.Inc() var ops []*operator.Operator diff --git a/pkg/schedule/schedulers/evict_slow_trend.go b/pkg/schedule/schedulers/evict_slow_trend.go index d919c1c0f0ab..da3dbc24e958 100644 --- a/pkg/schedule/schedulers/evict_slow_trend.go +++ b/pkg/schedule/schedulers/evict_slow_trend.go @@ -263,7 +263,7 @@ func (handler *evictSlowTrendHandler) UpdateConfig(w http.ResponseWriter, r *htt handler.rd.JSON(w, http.StatusOK, "Config updated.") } -func (handler *evictSlowTrendHandler) ListConfig(w http.ResponseWriter, r *http.Request) { +func (handler *evictSlowTrendHandler) ListConfig(w http.ResponseWriter, _ *http.Request) { conf := handler.config.Clone() handler.rd.JSON(w, http.StatusOK, conf) } @@ -274,7 +274,7 @@ type evictSlowTrendScheduler struct { handler http.Handler } -func (s *evictSlowTrendScheduler) GetNextInterval(interval time.Duration) time.Duration { +func (s *evictSlowTrendScheduler) GetNextInterval(time.Duration) time.Duration { var growthType intervalGrowthType // If it already found a slow node as candidate, the next interval should be shorter // to make the next scheduling as soon as possible. This adjustment will decrease the @@ -291,11 +291,11 @@ func (s *evictSlowTrendScheduler) ServeHTTP(w http.ResponseWriter, r *http.Reque s.handler.ServeHTTP(w, r) } -func (s *evictSlowTrendScheduler) GetName() string { +func (*evictSlowTrendScheduler) GetName() string { return EvictSlowTrendName } -func (s *evictSlowTrendScheduler) GetType() string { +func (*evictSlowTrendScheduler) GetType() string { return EvictSlowTrendType } @@ -384,7 +384,7 @@ func (s *evictSlowTrendScheduler) IsScheduleAllowed(cluster sche.SchedulerCluste return allowed } -func (s *evictSlowTrendScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *evictSlowTrendScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*operator.Operator, []plan.Plan) { schedulerCounter.WithLabelValues(s.GetName(), "schedule").Inc() var ops []*operator.Operator diff --git a/pkg/schedule/schedulers/grant_hot_region.go b/pkg/schedule/schedulers/grant_hot_region.go index 262dfe738735..56ed7cd730e3 100644 --- a/pkg/schedule/schedulers/grant_hot_region.go +++ b/pkg/schedule/schedulers/grant_hot_region.go @@ -108,7 +108,7 @@ func (conf *grantHotRegionSchedulerConfig) Persist() error { return conf.storage.SaveSchedulerConfig(name, data) } -func (conf *grantHotRegionSchedulerConfig) getSchedulerName() string { +func (*grantHotRegionSchedulerConfig) getSchedulerName() string { return GrantHotRegionName } @@ -148,11 +148,11 @@ func newGrantHotRegionScheduler(opController *operator.Controller, conf *grantHo return ret } -func (s *grantHotRegionScheduler) GetName() string { +func (*grantHotRegionScheduler) GetName() string { return GrantHotRegionName } -func (s *grantHotRegionScheduler) GetType() string { +func (*grantHotRegionScheduler) GetType() string { return GrantHotRegionType } @@ -256,7 +256,7 @@ func newGrantHotRegionHandler(config *grantHotRegionSchedulerConfig) http.Handle return router } -func (s *grantHotRegionScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *grantHotRegionScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*operator.Operator, []plan.Plan) { grantHotRegionCounter.Inc() rw := s.randomRWType() s.prepareForBalance(rw, cluster) @@ -352,7 +352,7 @@ func (s *grantHotRegionScheduler) transfer(cluster sche.SchedulerCluster, region dstStore := &metapb.Peer{StoreId: destStoreIDs[i]} if isLeader { - op, err = operator.CreateTransferLeaderOperator(GrantHotRegionType+"-leader", cluster, srcRegion, srcRegion.GetLeader().GetStoreId(), dstStore.StoreId, []uint64{}, operator.OpLeader) + op, err = operator.CreateTransferLeaderOperator(GrantHotRegionType+"-leader", cluster, srcRegion, dstStore.StoreId, []uint64{}, operator.OpLeader) } else { op, err = operator.CreateMovePeerOperator(GrantHotRegionType+"-move", cluster, srcRegion, operator.OpRegion|operator.OpLeader, srcStore.GetID(), dstStore) } diff --git a/pkg/schedule/schedulers/grant_leader.go b/pkg/schedule/schedulers/grant_leader.go index 8d36a5ae1c30..5de898489d97 100644 --- a/pkg/schedule/schedulers/grant_leader.go +++ b/pkg/schedule/schedulers/grant_leader.go @@ -98,7 +98,7 @@ func (conf *grantLeaderSchedulerConfig) Persist() error { return conf.storage.SaveSchedulerConfig(name, data) } -func (conf *grantLeaderSchedulerConfig) getSchedulerName() string { +func (*grantLeaderSchedulerConfig) getSchedulerName() string { return GrantLeaderName } @@ -176,11 +176,11 @@ func (s *grantLeaderScheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) s.handler.ServeHTTP(w, r) } -func (s *grantLeaderScheduler) GetName() string { +func (*grantLeaderScheduler) GetName() string { return GrantLeaderName } -func (s *grantLeaderScheduler) GetType() string { +func (*grantLeaderScheduler) GetType() string { return GrantLeaderType } @@ -235,7 +235,7 @@ func (s *grantLeaderScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) return allowed } -func (s *grantLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *grantLeaderScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*operator.Operator, []plan.Plan) { grantLeaderCounter.Inc() storeIDWithRanges := s.conf.getStoreIDWithRanges() ops := make([]*operator.Operator, 0, len(storeIDWithRanges)) @@ -248,7 +248,7 @@ func (s *grantLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bo continue } - op, err := operator.CreateForceTransferLeaderOperator(GrantLeaderType, cluster, region, region.GetLeader().GetStoreId(), id, operator.OpLeader) + op, err := operator.CreateForceTransferLeaderOperator(GrantLeaderType, cluster, region, id, operator.OpLeader) if err != nil { log.Debug("fail to create grant leader operator", errs.ZapError(err)) continue @@ -306,7 +306,7 @@ func (handler *grantLeaderHandler) UpdateConfig(w http.ResponseWriter, r *http.R handler.rd.JSON(w, http.StatusOK, "The scheduler has been applied to the store.") } -func (handler *grantLeaderHandler) ListConfig(w http.ResponseWriter, r *http.Request) { +func (handler *grantLeaderHandler) ListConfig(w http.ResponseWriter, _ *http.Request) { conf := handler.config.Clone() handler.rd.JSON(w, http.StatusOK, conf) } diff --git a/pkg/schedule/schedulers/hot_region.go b/pkg/schedule/schedulers/hot_region.go index 2a38ef399c84..b6293c2dac91 100644 --- a/pkg/schedule/schedulers/hot_region.go +++ b/pkg/schedule/schedulers/hot_region.go @@ -254,7 +254,7 @@ func (h *hotScheduler) GetName() string { return h.name } -func (h *hotScheduler) GetType() string { +func (*hotScheduler) GetType() string { return HotRegionType } @@ -306,11 +306,11 @@ func (h *hotScheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.conf.ServeHTTP(w, r) } -func (h *hotScheduler) GetMinInterval() time.Duration { +func (*hotScheduler) GetMinInterval() time.Duration { return minHotScheduleInterval } -func (h *hotScheduler) GetNextInterval(interval time.Duration) time.Duration { +func (h *hotScheduler) GetNextInterval(time.Duration) time.Duration { return intervalGrow(h.GetMinInterval(), maxHotScheduleInterval, exponentialGrowth) } @@ -322,7 +322,7 @@ func (h *hotScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { return allowed } -func (h *hotScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (h *hotScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*operator.Operator, []plan.Plan) { hotSchedulerCounter.Inc() rw := h.randomRWType() return h.dispatch(rw, cluster), nil @@ -1193,7 +1193,7 @@ func (bs *balanceSolver) checkHistoryByPriorityAndToleranceAnyOf(loads [][]float }) } -func (bs *balanceSolver) checkByPriorityAndToleranceFirstOnly(loads []float64, f func(int) bool) bool { +func (bs *balanceSolver) checkByPriorityAndToleranceFirstOnly(_ []float64, f func(int) bool) bool { return f(bs.firstPriority) } @@ -1732,7 +1732,6 @@ func (bs *balanceSolver) createReadOperator(region *core.RegionInfo, srcStoreID, "transfer-hot-read-leader", bs, region, - srcStoreID, dstStoreID, []uint64{}, operator.OpHotRegion) @@ -1769,7 +1768,6 @@ func (bs *balanceSolver) createWriteOperator(region *core.RegionInfo, srcStoreID "transfer-hot-write-leader", bs, region, - srcStoreID, dstStoreID, []uint64{}, operator.OpHotRegion) diff --git a/pkg/schedule/schedulers/hot_region_config.go b/pkg/schedule/schedulers/hot_region_config.go index b336438830b5..80d20ca65bb6 100644 --- a/pkg/schedule/schedulers/hot_region_config.go +++ b/pkg/schedule/schedulers/hot_region_config.go @@ -375,7 +375,7 @@ func (conf *hotRegionSchedulerConfig) ServeHTTP(w http.ResponseWriter, r *http.R router.ServeHTTP(w, r) } -func (conf *hotRegionSchedulerConfig) handleGetConfig(w http.ResponseWriter, r *http.Request) { +func (conf *hotRegionSchedulerConfig) handleGetConfig(w http.ResponseWriter, _ *http.Request) { conf.RLock() defer conf.RUnlock() rd := render.New(render.Options{IndentJSON: true}) diff --git a/pkg/schedule/schedulers/hot_region_test.go b/pkg/schedule/schedulers/hot_region_test.go index 8b1893887db7..5f6cca892ee6 100644 --- a/pkg/schedule/schedulers/hot_region_test.go +++ b/pkg/schedule/schedulers/hot_region_test.go @@ -43,11 +43,11 @@ func init() { // TODO: remove this global variable in the future. // And use a function to create hot schduler for test. schedulePeerPr = 1.0 - RegisterScheduler(utils.Write.String(), func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(utils.Write.String(), func(opController *operator.Controller, _ endpoint.ConfigStorage, _ ConfigDecoder, _ ...func(string) error) (Scheduler, error) { cfg := initHotRegionScheduleConfig() return newHotWriteScheduler(opController, cfg), nil }) - RegisterScheduler(utils.Read.String(), func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(utils.Read.String(), func(opController *operator.Controller, _ endpoint.ConfigStorage, _ ConfigDecoder, _ ...func(string) error) (Scheduler, error) { return newHotReadScheduler(opController, initHotRegionScheduleConfig()), nil }) } @@ -138,7 +138,7 @@ func checkGCPendingOpInfos(re *require.Assertions, enablePlacementRules bool) { case movePeer: op, err = operator.CreateMovePeerOperator("move-peer-test", tc, region, operator.OpAdmin, 2, &metapb.Peer{Id: region.GetID()*10000 + 1, StoreId: 4}) case transferLeader: - op, err = operator.CreateTransferLeaderOperator("transfer-leader-test", tc, region, 1, 2, []uint64{}, operator.OpAdmin) + op, err = operator.CreateTransferLeaderOperator("transfer-leader-test", tc, region, 2, []uint64{}, operator.OpAdmin) } re.NoError(err) re.NotNil(op) diff --git a/pkg/schedule/schedulers/hot_region_v2.go b/pkg/schedule/schedulers/hot_region_v2.go index 40cb35cd16b3..50016231cada 100644 --- a/pkg/schedule/schedulers/hot_region_v2.go +++ b/pkg/schedule/schedulers/hot_region_v2.go @@ -457,13 +457,13 @@ func (bs *balanceSolver) betterThanV2(old *solution) bool { if bs.cur.mainPeerStat != old.mainPeerStat { // We will firstly consider ensuring converge faster, secondly reduce oscillation if bs.resourceTy == writeLeader { - return bs.getRkCmpByPriorityV2(bs.firstPriority, bs.cur.firstScore, old.firstScore, + return getRkCmpByPriorityV2(bs.firstPriority, bs.cur.firstScore, old.firstScore, bs.cur.getPeersRateFromCache(bs.firstPriority), old.getPeersRateFromCache(bs.firstPriority)) > 0 } - firstCmp := bs.getRkCmpByPriorityV2(bs.firstPriority, bs.cur.firstScore, old.firstScore, + firstCmp := getRkCmpByPriorityV2(bs.firstPriority, bs.cur.firstScore, old.firstScore, bs.cur.getPeersRateFromCache(bs.firstPriority), old.getPeersRateFromCache(bs.firstPriority)) - secondCmp := bs.getRkCmpByPriorityV2(bs.secondPriority, bs.cur.secondScore, old.secondScore, + secondCmp := getRkCmpByPriorityV2(bs.secondPriority, bs.cur.secondScore, old.secondScore, bs.cur.getPeersRateFromCache(bs.secondPriority), old.getPeersRateFromCache(bs.secondPriority)) switch bs.cur.progressiveRank { case -4, -3, -2: // firstPriority @@ -482,7 +482,7 @@ func (bs *balanceSolver) betterThanV2(old *solution) bool { return false } -func (bs *balanceSolver) getRkCmpByPriorityV2(dim int, curScore, oldScore int, curPeersRate, oldPeersRate float64) int { +func getRkCmpByPriorityV2(dim int, curScore, oldScore int, curPeersRate, oldPeersRate float64) int { switch { case curScore > oldScore: return 1 diff --git a/pkg/schedule/schedulers/init.go b/pkg/schedule/schedulers/init.go index e22037703ccd..6bca686404d1 100644 --- a/pkg/schedule/schedulers/init.go +++ b/pkg/schedule/schedulers/init.go @@ -52,7 +52,7 @@ func schedulersRegister() { } }) - RegisterScheduler(BalanceLeaderType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(BalanceLeaderType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &balanceLeaderSchedulerConfig{storage: storage} if err := decoder(conf); err != nil { return nil, err @@ -80,7 +80,7 @@ func schedulersRegister() { } }) - RegisterScheduler(BalanceRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(BalanceRegionType, func(opController *operator.Controller, _ endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &balanceRegionSchedulerConfig{} if err := decoder(conf); err != nil { return nil, err @@ -105,7 +105,7 @@ func schedulersRegister() { } }) - RegisterScheduler(BalanceWitnessType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(BalanceWitnessType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &balanceWitnessSchedulerConfig{storage: storage} if err := decoder(conf); err != nil { return nil, err @@ -152,13 +152,13 @@ func schedulersRegister() { }) // evict slow store - RegisterSliceDecoderBuilder(EvictSlowStoreType, func(args []string) ConfigDecoder { - return func(v any) error { + RegisterSliceDecoderBuilder(EvictSlowStoreType, func([]string) ConfigDecoder { + return func(any) error { return nil } }) - RegisterScheduler(EvictSlowStoreType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(EvictSlowStoreType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := initEvictSlowStoreSchedulerConfig(storage) if err := decoder(conf); err != nil { return nil, err @@ -198,7 +198,7 @@ func schedulersRegister() { } }) - RegisterScheduler(GrantHotRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(GrantHotRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &grantHotRegionSchedulerConfig{StoreIDs: make([]uint64, 0), storage: storage} conf.cluster = opController.GetCluster() if err := decoder(conf); err != nil { @@ -208,13 +208,13 @@ func schedulersRegister() { }) // hot region - RegisterSliceDecoderBuilder(HotRegionType, func(args []string) ConfigDecoder { - return func(v any) error { + RegisterSliceDecoderBuilder(HotRegionType, func([]string) ConfigDecoder { + return func(any) error { return nil } }) - RegisterScheduler(HotRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(HotRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := initHotRegionScheduleConfig() var data map[string]any if err := decoder(&data); err != nil { @@ -286,7 +286,7 @@ func schedulersRegister() { } }) - RegisterScheduler(LabelType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(LabelType, func(opController *operator.Controller, _ endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &labelSchedulerConfig{} if err := decoder(conf); err != nil { return nil, err @@ -311,7 +311,7 @@ func schedulersRegister() { } }) - RegisterScheduler(RandomMergeType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(RandomMergeType, func(opController *operator.Controller, _ endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &randomMergeSchedulerConfig{} if err := decoder(conf); err != nil { return nil, err @@ -340,7 +340,7 @@ func schedulersRegister() { } }) - RegisterScheduler(ScatterRangeType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(ScatterRangeType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &scatterRangeSchedulerConfig{ storage: storage, } @@ -374,7 +374,7 @@ func schedulersRegister() { } }) - RegisterScheduler(ShuffleHotRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(ShuffleHotRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &shuffleHotRegionSchedulerConfig{Limit: uint64(1)} if err := decoder(conf); err != nil { return nil, err @@ -400,7 +400,7 @@ func schedulersRegister() { } }) - RegisterScheduler(ShuffleLeaderType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(ShuffleLeaderType, func(opController *operator.Controller, _ endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &shuffleLeaderSchedulerConfig{} if err := decoder(conf); err != nil { return nil, err @@ -425,7 +425,7 @@ func schedulersRegister() { } }) - RegisterScheduler(ShuffleRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(ShuffleRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &shuffleRegionSchedulerConfig{storage: storage} if err := decoder(conf); err != nil { return nil, err @@ -434,13 +434,13 @@ func schedulersRegister() { }) // split bucket - RegisterSliceDecoderBuilder(SplitBucketType, func(args []string) ConfigDecoder { - return func(v any) error { + RegisterSliceDecoderBuilder(SplitBucketType, func([]string) ConfigDecoder { + return func(any) error { return nil } }) - RegisterScheduler(SplitBucketType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(SplitBucketType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := initSplitBucketConfig() if err := decoder(conf); err != nil { return nil, err @@ -450,24 +450,24 @@ func schedulersRegister() { }) // transfer witness leader - RegisterSliceDecoderBuilder(TransferWitnessLeaderType, func(args []string) ConfigDecoder { - return func(v any) error { + RegisterSliceDecoderBuilder(TransferWitnessLeaderType, func([]string) ConfigDecoder { + return func(any) error { return nil } }) - RegisterScheduler(TransferWitnessLeaderType, func(opController *operator.Controller, _ endpoint.ConfigStorage, _ ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(TransferWitnessLeaderType, func(opController *operator.Controller, _ endpoint.ConfigStorage, _ ConfigDecoder, _ ...func(string) error) (Scheduler, error) { return newTransferWitnessLeaderScheduler(opController), nil }) // evict slow store by trend - RegisterSliceDecoderBuilder(EvictSlowTrendType, func(args []string) ConfigDecoder { - return func(v any) error { + RegisterSliceDecoderBuilder(EvictSlowTrendType, func([]string) ConfigDecoder { + return func(any) error { return nil } }) - RegisterScheduler(EvictSlowTrendType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(EvictSlowTrendType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := initEvictSlowTrendSchedulerConfig(storage) if err := decoder(conf); err != nil { return nil, err diff --git a/pkg/schedule/schedulers/label.go b/pkg/schedule/schedulers/label.go index 90310bcf10e4..24875e3e26a3 100644 --- a/pkg/schedule/schedulers/label.go +++ b/pkg/schedule/schedulers/label.go @@ -68,7 +68,7 @@ func (s *labelScheduler) GetName() string { return s.conf.Name } -func (s *labelScheduler) GetType() string { +func (*labelScheduler) GetType() string { return LabelType } @@ -84,7 +84,7 @@ func (s *labelScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { return allowed } -func (s *labelScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *labelScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*operator.Operator, []plan.Plan) { labelCounter.Inc() stores := cluster.GetStores() rejectLeaderStores := make(map[uint64]struct{}) @@ -119,7 +119,7 @@ func (s *labelScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([ continue } - op, err := operator.CreateTransferLeaderOperator("label-reject-leader", cluster, region, id, target.GetID(), []uint64{}, operator.OpLeader) + op, err := operator.CreateTransferLeaderOperator("label-reject-leader", cluster, region, target.GetID(), []uint64{}, operator.OpLeader) if err != nil { log.Debug("fail to create transfer label reject leader operator", errs.ZapError(err)) return nil, nil diff --git a/pkg/schedule/schedulers/random_merge.go b/pkg/schedule/schedulers/random_merge.go index 44bb5081ef93..7fec0bd95303 100644 --- a/pkg/schedule/schedulers/random_merge.go +++ b/pkg/schedule/schedulers/random_merge.go @@ -70,7 +70,7 @@ func (s *randomMergeScheduler) GetName() string { return s.conf.Name } -func (s *randomMergeScheduler) GetType() string { +func (*randomMergeScheduler) GetType() string { return RandomMergeType } @@ -86,7 +86,7 @@ func (s *randomMergeScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) return allowed } -func (s *randomMergeScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *randomMergeScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*operator.Operator, []plan.Plan) { randomMergeCounter.Inc() store := filter.NewCandidates(cluster.GetStores()). @@ -113,7 +113,7 @@ func (s *randomMergeScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bo return nil, nil } - if !s.allowMerge(cluster, region, target) { + if !allowMerge(cluster, region, target) { randomMergeNotAllowedCounter.Inc() return nil, nil } @@ -129,7 +129,7 @@ func (s *randomMergeScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bo return ops, nil } -func (s *randomMergeScheduler) allowMerge(cluster sche.SchedulerCluster, region, target *core.RegionInfo) bool { +func allowMerge(cluster sche.SchedulerCluster, region, target *core.RegionInfo) bool { if !filter.IsRegionHealthy(region) || !filter.IsRegionHealthy(target) { return false } diff --git a/pkg/schedule/schedulers/scatter_range.go b/pkg/schedule/schedulers/scatter_range.go index 9ad9e597dfd9..8a2f0a5398bc 100644 --- a/pkg/schedule/schedulers/scatter_range.go +++ b/pkg/schedule/schedulers/scatter_range.go @@ -156,7 +156,7 @@ func (l *scatterRangeScheduler) GetName() string { return l.name } -func (l *scatterRangeScheduler) GetType() string { +func (*scatterRangeScheduler) GetType() string { return ScatterRangeType } @@ -206,7 +206,7 @@ func (l *scatterRangeScheduler) allowBalanceRegion(cluster sche.SchedulerCluster return allowed } -func (l *scatterRangeScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (l *scatterRangeScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*operator.Operator, []plan.Plan) { scatterRangeCounter.Inc() // isolate a new cluster according to the key range c := genRangeCluster(cluster, l.config.GetStartKey(), l.config.GetEndKey()) @@ -282,7 +282,7 @@ func (handler *scatterRangeHandler) UpdateConfig(w http.ResponseWriter, r *http. handler.rd.JSON(w, http.StatusOK, nil) } -func (handler *scatterRangeHandler) ListConfig(w http.ResponseWriter, r *http.Request) { +func (handler *scatterRangeHandler) ListConfig(w http.ResponseWriter, _ *http.Request) { conf := handler.config.Clone() handler.rd.JSON(w, http.StatusOK, conf) } diff --git a/pkg/schedule/schedulers/scheduler.go b/pkg/schedule/schedulers/scheduler.go index b74b72283ecc..abace59a2664 100644 --- a/pkg/schedule/schedulers/scheduler.go +++ b/pkg/schedule/schedulers/scheduler.go @@ -101,7 +101,7 @@ func ConfigJSONDecoder(data []byte) ConfigDecoder { func ConfigSliceDecoder(name string, args []string) ConfigDecoder { builder, ok := schedulerArgsToDecoder[name] if !ok { - return func(v any) error { + return func(any) error { return errors.Errorf("the config decoder do not register for %s", name) } } diff --git a/pkg/schedule/schedulers/shuffle_hot_region.go b/pkg/schedule/schedulers/shuffle_hot_region.go index a1448fbd041d..726138e8f7ae 100644 --- a/pkg/schedule/schedulers/shuffle_hot_region.go +++ b/pkg/schedule/schedulers/shuffle_hot_region.go @@ -114,7 +114,7 @@ func (s *shuffleHotRegionScheduler) GetName() string { return s.conf.Name } -func (s *shuffleHotRegionScheduler) GetType() string { +func (*shuffleHotRegionScheduler) GetType() string { return ShuffleHotRegionType } @@ -157,7 +157,7 @@ func (s *shuffleHotRegionScheduler) IsScheduleAllowed(cluster sche.SchedulerClus return hotRegionAllowed && regionAllowed && leaderAllowed } -func (s *shuffleHotRegionScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *shuffleHotRegionScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*operator.Operator, []plan.Plan) { shuffleHotRegionCounter.Inc() rw := s.randomRWType() s.prepareForBalance(rw, cluster) @@ -250,7 +250,7 @@ func (handler *shuffleHotRegionHandler) UpdateConfig(w http.ResponseWriter, r *h handler.rd.JSON(w, http.StatusOK, nil) } -func (handler *shuffleHotRegionHandler) ListConfig(w http.ResponseWriter, r *http.Request) { +func (handler *shuffleHotRegionHandler) ListConfig(w http.ResponseWriter, _ *http.Request) { conf := handler.config.Clone() handler.rd.JSON(w, http.StatusOK, conf) } diff --git a/pkg/schedule/schedulers/shuffle_leader.go b/pkg/schedule/schedulers/shuffle_leader.go index a6ff4baf65bb..5b3dfd9fd20d 100644 --- a/pkg/schedule/schedulers/shuffle_leader.go +++ b/pkg/schedule/schedulers/shuffle_leader.go @@ -71,7 +71,7 @@ func (s *shuffleLeaderScheduler) GetName() string { return s.conf.Name } -func (s *shuffleLeaderScheduler) GetType() string { +func (*shuffleLeaderScheduler) GetType() string { return ShuffleLeaderType } @@ -87,7 +87,7 @@ func (s *shuffleLeaderScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster return allowed } -func (s *shuffleLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *shuffleLeaderScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*operator.Operator, []plan.Plan) { // We shuffle leaders between stores by: // 1. random select a valid store. // 2. transfer a leader to the store. @@ -106,7 +106,7 @@ func (s *shuffleLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun shuffleLeaderNoFollowerCounter.Inc() return nil, nil } - op, err := operator.CreateTransferLeaderOperator(ShuffleLeaderType, cluster, region, region.GetLeader().GetId(), targetStore.GetID(), []uint64{}, operator.OpAdmin) + op, err := operator.CreateTransferLeaderOperator(ShuffleLeaderType, cluster, region, targetStore.GetID(), []uint64{}, operator.OpAdmin) if err != nil { log.Debug("fail to create shuffle leader operator", errs.ZapError(err)) return nil, nil diff --git a/pkg/schedule/schedulers/shuffle_region.go b/pkg/schedule/schedulers/shuffle_region.go index f9bed18d3fa0..b1a100384ae5 100644 --- a/pkg/schedule/schedulers/shuffle_region.go +++ b/pkg/schedule/schedulers/shuffle_region.go @@ -68,11 +68,11 @@ func (s *shuffleRegionScheduler) ServeHTTP(w http.ResponseWriter, r *http.Reques s.conf.ServeHTTP(w, r) } -func (s *shuffleRegionScheduler) GetName() string { +func (*shuffleRegionScheduler) GetName() string { return ShuffleRegionName } -func (s *shuffleRegionScheduler) GetType() string { +func (*shuffleRegionScheduler) GetType() string { return ShuffleRegionType } @@ -107,7 +107,7 @@ func (s *shuffleRegionScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster return allowed } -func (s *shuffleRegionScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *shuffleRegionScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*operator.Operator, []plan.Plan) { shuffleRegionCounter.Inc() region, oldPeer := s.scheduleRemovePeer(cluster) if region == nil { diff --git a/pkg/schedule/schedulers/shuffle_region_config.go b/pkg/schedule/schedulers/shuffle_region_config.go index 552d7ea8bce5..bce64f743b8d 100644 --- a/pkg/schedule/schedulers/shuffle_region_config.go +++ b/pkg/schedule/schedulers/shuffle_region_config.go @@ -77,7 +77,7 @@ func (conf *shuffleRegionSchedulerConfig) ServeHTTP(w http.ResponseWriter, r *ht router.ServeHTTP(w, r) } -func (conf *shuffleRegionSchedulerConfig) handleGetRoles(w http.ResponseWriter, r *http.Request) { +func (conf *shuffleRegionSchedulerConfig) handleGetRoles(w http.ResponseWriter, _ *http.Request) { rd := render.New(render.Options{IndentJSON: true}) rd.JSON(w, http.StatusOK, conf.GetRoles()) } diff --git a/pkg/schedule/schedulers/split_bucket.go b/pkg/schedule/schedulers/split_bucket.go index 7e276402e49e..609510446c75 100644 --- a/pkg/schedule/schedulers/split_bucket.go +++ b/pkg/schedule/schedulers/split_bucket.go @@ -175,12 +175,12 @@ func newSplitBucketScheduler(opController *operator.Controller, conf *splitBucke } // GetName returns the name of the split bucket scheduler. -func (s *splitBucketScheduler) GetName() string { +func (*splitBucketScheduler) GetName() string { return SplitBucketName } // GetType returns the type of the split bucket scheduler. -func (s *splitBucketScheduler) GetType() string { +func (*splitBucketScheduler) GetType() string { return SplitBucketType } @@ -230,7 +230,7 @@ type splitBucketPlan struct { } // Schedule return operators if some bucket is too hot. -func (s *splitBucketScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *splitBucketScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*operator.Operator, []plan.Plan) { splitBucketScheduleCounter.Inc() conf := s.conf.Clone() plan := &splitBucketPlan{ diff --git a/pkg/schedule/schedulers/transfer_witness_leader.go b/pkg/schedule/schedulers/transfer_witness_leader.go index c651a8ef8721..9ba78985d137 100644 --- a/pkg/schedule/schedulers/transfer_witness_leader.go +++ b/pkg/schedule/schedulers/transfer_witness_leader.go @@ -60,19 +60,19 @@ func newTransferWitnessLeaderScheduler(opController *operator.Controller) Schedu } } -func (s *transferWitnessLeaderScheduler) GetName() string { +func (*transferWitnessLeaderScheduler) GetName() string { return TransferWitnessLeaderName } -func (s *transferWitnessLeaderScheduler) GetType() string { +func (*transferWitnessLeaderScheduler) GetType() string { return TransferWitnessLeaderType } -func (s *transferWitnessLeaderScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { +func (*transferWitnessLeaderScheduler) IsScheduleAllowed(sche.SchedulerCluster) bool { return true } -func (s *transferWitnessLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { +func (s *transferWitnessLeaderScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*operator.Operator, []plan.Plan) { transferWitnessLeaderCounter.Inc() return s.scheduleTransferWitnessLeaderBatch(s.GetName(), s.GetType(), cluster, transferWitnessLeaderBatchSize), nil } @@ -83,7 +83,7 @@ batchLoop: for i := 0; i < batchSize; i++ { select { case region := <-s.regions: - op, err := s.scheduleTransferWitnessLeader(name, typ, cluster, region) + op, err := scheduleTransferWitnessLeader(name, typ, cluster, region) if err != nil { log.Debug("fail to create transfer leader operator", errs.ZapError(err)) continue @@ -100,7 +100,7 @@ batchLoop: return ops } -func (s *transferWitnessLeaderScheduler) scheduleTransferWitnessLeader(name, typ string, cluster sche.SchedulerCluster, region *core.RegionInfo) (*operator.Operator, error) { +func scheduleTransferWitnessLeader(name, typ string, cluster sche.SchedulerCluster, region *core.RegionInfo) (*operator.Operator, error) { var filters []filter.Filter unhealthyPeerStores := make(map[uint64]struct{}) for _, peer := range region.GetDownPeers() { @@ -123,7 +123,7 @@ func (s *transferWitnessLeaderScheduler) scheduleTransferWitnessLeader(name, typ for _, t := range targets { targetIDs = append(targetIDs, t.GetID()) } - return operator.CreateTransferLeaderOperator(typ, cluster, region, region.GetLeader().GetStoreId(), target.GetID(), targetIDs, operator.OpWitnessLeader) + return operator.CreateTransferLeaderOperator(typ, cluster, region, target.GetID(), targetIDs, operator.OpWitnessLeader) } // RecvRegionInfo receives a checked region from coordinator diff --git a/pkg/schedule/splitter/region_splitter.go b/pkg/schedule/splitter/region_splitter.go index f0da8442a2c0..aeab4b70cf09 100644 --- a/pkg/schedule/splitter/region_splitter.go +++ b/pkg/schedule/splitter/region_splitter.go @@ -108,6 +108,7 @@ func (r *RegionSplitter) splitRegionsByKeys(parCtx context.Context, splitKeys [] ticker.Stop() cancel() }() +outerLoop: for { select { case <-ticker.C: @@ -118,7 +119,7 @@ func (r *RegionSplitter) splitRegionsByKeys(parCtx context.Context, splitKeys [] r.handler.ScanRegionsByKeyRange(groupKeys, results) } case <-ctx.Done(): - break + break outerLoop } finished := true for _, groupKeys := range validGroups { diff --git a/pkg/schedule/splitter/region_splitter_test.go b/pkg/schedule/splitter/region_splitter_test.go index ebb8b225a9bb..99fd53df1e59 100644 --- a/pkg/schedule/splitter/region_splitter_test.go +++ b/pkg/schedule/splitter/region_splitter_test.go @@ -37,7 +37,7 @@ func newMockSplitRegionsHandler() *mockSplitRegionsHandler { } // SplitRegionByKeys mock SplitRegionsHandler -func (m *mockSplitRegionsHandler) SplitRegionByKeys(region *core.RegionInfo, splitKeys [][]byte) error { +func (m *mockSplitRegionsHandler) SplitRegionByKeys(region *core.RegionInfo, _ [][]byte) error { m.regions[region.GetID()] = [2][]byte{ region.GetStartKey(), region.GetEndKey(), @@ -76,7 +76,7 @@ func (suite *regionSplitterTestSuite) SetupSuite() { suite.ctx, suite.cancel = context.WithCancel(context.Background()) } -func (suite *regionSplitterTestSuite) TearDownTest() { +func (suite *regionSplitterTestSuite) TearDownSuite() { suite.cancel() } diff --git a/pkg/statistics/buckets/hot_bucket_task.go b/pkg/statistics/buckets/hot_bucket_task.go index d6a43a6f8ae8..ff7c30a7d811 100644 --- a/pkg/statistics/buckets/hot_bucket_task.go +++ b/pkg/statistics/buckets/hot_bucket_task.go @@ -55,7 +55,7 @@ func NewCheckPeerTask(buckets *metapb.Buckets) flowBucketsItemTask { } } -func (t *checkBucketsTask) taskType() flowItemTaskKind { +func (*checkBucketsTask) taskType() flowItemTaskKind { return checkBucketsTaskType } @@ -79,7 +79,7 @@ func NewCollectBucketStatsTask(minDegree int, regionIDs ...uint64) *collectBucke } } -func (t *collectBucketStatsTask) taskType() flowItemTaskKind { +func (*collectBucketStatsTask) taskType() flowItemTaskKind { return collectBucketStatsTaskType } diff --git a/pkg/statistics/collector.go b/pkg/statistics/collector.go index e64b673803dd..88986b93d4bc 100644 --- a/pkg/statistics/collector.go +++ b/pkg/statistics/collector.go @@ -36,11 +36,11 @@ func newTikvCollector() storeCollector { return tikvCollector{} } -func (c tikvCollector) Engine() string { +func (tikvCollector) Engine() string { return core.EngineTiKV } -func (c tikvCollector) Filter(info *StoreSummaryInfo, kind constant.ResourceKind) bool { +func (tikvCollector) Filter(info *StoreSummaryInfo, kind constant.ResourceKind) bool { if info.IsTiFlash() { return false } @@ -53,7 +53,7 @@ func (c tikvCollector) Filter(info *StoreSummaryInfo, kind constant.ResourceKind return false } -func (c tikvCollector) GetLoads(storeLoads, peerLoadSum []float64, rwTy utils.RWType, kind constant.ResourceKind) (loads []float64) { +func (tikvCollector) GetLoads(storeLoads, peerLoadSum []float64, rwTy utils.RWType, kind constant.ResourceKind) (loads []float64) { loads = make([]float64, utils.DimLen) switch rwTy { case utils.Read: @@ -87,11 +87,11 @@ func newTiFlashCollector(isTraceRegionFlow bool) storeCollector { return tiflashCollector{isTraceRegionFlow: isTraceRegionFlow} } -func (c tiflashCollector) Engine() string { +func (tiflashCollector) Engine() string { return core.EngineTiFlash } -func (c tiflashCollector) Filter(info *StoreSummaryInfo, kind constant.ResourceKind) bool { +func (tiflashCollector) Filter(info *StoreSummaryInfo, kind constant.ResourceKind) bool { switch kind { case constant.LeaderKind: return false diff --git a/pkg/statistics/hot_cache_task.go b/pkg/statistics/hot_cache_task.go index c84a292b4e7b..fa224b522ff1 100644 --- a/pkg/statistics/hot_cache_task.go +++ b/pkg/statistics/hot_cache_task.go @@ -146,7 +146,7 @@ func newCollectMetricsTask() *collectMetricsTask { return &collectMetricsTask{} } -func (t *collectMetricsTask) runTask(cache *hotPeerCache) { +func (*collectMetricsTask) runTask(cache *hotPeerCache) { cache.collectMetrics() } diff --git a/pkg/statistics/hot_peer_cache.go b/pkg/statistics/hot_peer_cache.go index 0e35e0e23be0..cd27dcad4c8d 100644 --- a/pkg/statistics/hot_peer_cache.go +++ b/pkg/statistics/hot_peer_cache.go @@ -451,7 +451,7 @@ func (f *hotPeerCache) updateHotPeerStat(region *core.RegionInfo, newItem, oldIt // For write stat, as the stat is send by region heartbeat, the first heartbeat will be skipped. // For read stat, as the stat is send by store heartbeat, the first heartbeat won't be skipped. if f.kind == utils.Write { - f.inheritItem(newItem, oldItem) + inheritItem(newItem, oldItem) return newItem } } else { @@ -465,25 +465,25 @@ func (f *hotPeerCache) updateHotPeerStat(region *core.RegionInfo, newItem, oldIt isFull := newItem.rollingLoads[0].isFull(f.interval()) // The intervals of dims are the same, so it is only necessary to determine whether any of them if !isFull { // not update hot degree and anti count - f.inheritItem(newItem, oldItem) + inheritItem(newItem, oldItem) } else { // If item is inCold, it means the pd didn't recv this item in the store heartbeat, // thus we make it colder if newItem.inCold { - f.coldItem(newItem, oldItem) + coldItem(newItem, oldItem) } else { thresholds := f.calcHotThresholds(newItem.StoreID) if f.isOldColdPeer(oldItem, newItem.StoreID) { if newItem.isHot(thresholds) { - f.initItem(newItem) + initItem(newItem, f.kind.DefaultAntiCount()) } else { newItem.actionType = utils.Remove } } else { if newItem.isHot(thresholds) { - f.hotItem(newItem, oldItem) + hotItem(newItem, oldItem, f.kind.DefaultAntiCount()) } else { - f.coldItem(newItem, oldItem) + coldItem(newItem, oldItem) } } } @@ -496,7 +496,7 @@ func (f *hotPeerCache) updateNewHotPeerStat(newItem *HotPeerStat, deltaLoads []f regionStats := f.kind.RegionStats() // interval is not 0 which is guaranteed by the caller. if interval.Seconds() >= float64(f.kind.ReportInterval()) { - f.initItem(newItem) + initItem(newItem, f.kind.DefaultAntiCount()) } newItem.actionType = utils.Add newItem.rollingLoads = make([]*dimStat, len(regionStats)) @@ -556,7 +556,7 @@ func (f *hotPeerCache) removeAllItem() { } } -func (f *hotPeerCache) coldItem(newItem, oldItem *HotPeerStat) { +func coldItem(newItem, oldItem *HotPeerStat) { newItem.HotDegree = oldItem.HotDegree - 1 newItem.AntiCount = oldItem.AntiCount - 1 if newItem.AntiCount <= 0 { @@ -566,9 +566,9 @@ func (f *hotPeerCache) coldItem(newItem, oldItem *HotPeerStat) { } } -func (f *hotPeerCache) hotItem(newItem, oldItem *HotPeerStat) { +func hotItem(newItem, oldItem *HotPeerStat, defaultAntiCount int) { newItem.HotDegree = oldItem.HotDegree + 1 - if oldItem.AntiCount < f.kind.DefaultAntiCount() { + if oldItem.AntiCount < defaultAntiCount { newItem.AntiCount = oldItem.AntiCount + 1 } else { newItem.AntiCount = oldItem.AntiCount @@ -576,13 +576,13 @@ func (f *hotPeerCache) hotItem(newItem, oldItem *HotPeerStat) { newItem.allowInherited = true } -func (f *hotPeerCache) initItem(item *HotPeerStat) { +func initItem(item *HotPeerStat, defaultAntiCount int) { item.HotDegree = 1 - item.AntiCount = f.kind.DefaultAntiCount() + item.AntiCount = defaultAntiCount item.allowInherited = true } -func (f *hotPeerCache) inheritItem(newItem, oldItem *HotPeerStat) { +func inheritItem(newItem, oldItem *HotPeerStat) { newItem.HotDegree = oldItem.HotDegree newItem.AntiCount = oldItem.AntiCount } diff --git a/pkg/statistics/slow_stat.go b/pkg/statistics/slow_stat.go index 4079043d1546..cc579b3d90bf 100644 --- a/pkg/statistics/slow_stat.go +++ b/pkg/statistics/slow_stat.go @@ -15,8 +15,6 @@ package statistics import ( - "context" - "github.com/tikv/pd/pkg/utils/syncutil" ) @@ -26,7 +24,7 @@ type SlowStat struct { } // NewSlowStat creates the container to hold slow nodes' statistics. -func NewSlowStat(ctx context.Context) *SlowStat { +func NewSlowStat() *SlowStat { return &SlowStat{ SlowStoresStats: NewSlowStoresStats(), } diff --git a/pkg/statistics/store_collection.go b/pkg/statistics/store_collection.go index aacd45338d1d..4f76ffb0b5f7 100644 --- a/pkg/statistics/store_collection.go +++ b/pkg/statistics/store_collection.go @@ -147,7 +147,7 @@ func (s *storeStatistics) Observe(store *core.StoreInfo) { } } -func (s *storeStatistics) ObserveHotStat(store *core.StoreInfo, stats *StoresStats) { +func ObserveHotStat(store *core.StoreInfo, stats *StoresStats) { // Store flows. storeAddress := store.GetAddress() id := strconv.FormatUint(store.GetID(), 10) @@ -309,10 +309,6 @@ func (m *storeStatisticsMap) Observe(store *core.StoreInfo) { m.stats.Observe(store) } -func (m *storeStatisticsMap) ObserveHotStat(store *core.StoreInfo, stats *StoresStats) { - m.stats.ObserveHotStat(store, stats) -} - func (m *storeStatisticsMap) Collect() { m.stats.Collect() } diff --git a/pkg/statistics/store_collection_test.go b/pkg/statistics/store_collection_test.go index 02e6350ffa4d..64a02a54bb4d 100644 --- a/pkg/statistics/store_collection_test.go +++ b/pkg/statistics/store_collection_test.go @@ -68,7 +68,7 @@ func TestStoreStatistics(t *testing.T) { storeStats := NewStoreStatisticsMap(opt) for _, store := range stores { storeStats.Observe(store) - storeStats.ObserveHotStat(store, storesStats) + ObserveHotStat(store, storesStats) } stats := storeStats.stats diff --git a/pkg/storage/endpoint/keyspace.go b/pkg/storage/endpoint/keyspace.go index 77c81b2c8d69..30540e49a2ef 100644 --- a/pkg/storage/endpoint/keyspace.go +++ b/pkg/storage/endpoint/keyspace.go @@ -48,7 +48,7 @@ type KeyspaceStorage interface { var _ KeyspaceStorage = (*StorageEndpoint)(nil) // SaveKeyspaceMeta adds a save keyspace meta operation to target transaction. -func (se *StorageEndpoint) SaveKeyspaceMeta(txn kv.Txn, meta *keyspacepb.KeyspaceMeta) error { +func (*StorageEndpoint) SaveKeyspaceMeta(txn kv.Txn, meta *keyspacepb.KeyspaceMeta) error { metaPath := KeyspaceMetaPath(meta.GetId()) metaVal, err := proto.Marshal(meta) if err != nil { @@ -59,7 +59,7 @@ func (se *StorageEndpoint) SaveKeyspaceMeta(txn kv.Txn, meta *keyspacepb.Keyspac // LoadKeyspaceMeta load and return keyspace meta specified by id. // If keyspace does not exist or error occurs, returned meta will be nil. -func (se *StorageEndpoint) LoadKeyspaceMeta(txn kv.Txn, id uint32) (*keyspacepb.KeyspaceMeta, error) { +func (*StorageEndpoint) LoadKeyspaceMeta(txn kv.Txn, id uint32) (*keyspacepb.KeyspaceMeta, error) { metaPath := KeyspaceMetaPath(id) metaVal, err := txn.Load(metaPath) if err != nil || metaVal == "" { @@ -74,7 +74,7 @@ func (se *StorageEndpoint) LoadKeyspaceMeta(txn kv.Txn, id uint32) (*keyspacepb. } // SaveKeyspaceID saves keyspace ID to the path specified by keyspace name. -func (se *StorageEndpoint) SaveKeyspaceID(txn kv.Txn, id uint32, name string) error { +func (*StorageEndpoint) SaveKeyspaceID(txn kv.Txn, id uint32, name string) error { idPath := KeyspaceIDPath(name) idVal := strconv.FormatUint(uint64(id), SpaceIDBase) return txn.Save(idPath, idVal) @@ -83,7 +83,7 @@ func (se *StorageEndpoint) SaveKeyspaceID(txn kv.Txn, id uint32, name string) er // LoadKeyspaceID loads keyspace ID from the path specified by keyspace name. // An additional boolean is returned to indicate whether target id exists, // it returns false if target id not found, or if error occurred. -func (se *StorageEndpoint) LoadKeyspaceID(txn kv.Txn, name string) (bool, uint32, error) { +func (*StorageEndpoint) LoadKeyspaceID(txn kv.Txn, name string) (bool, uint32, error) { idPath := KeyspaceIDPath(name) idVal, err := txn.Load(idPath) // Failed to load the keyspaceID if loading operation errored, or if keyspace does not exist. @@ -99,7 +99,7 @@ func (se *StorageEndpoint) LoadKeyspaceID(txn kv.Txn, name string) (bool, uint32 // LoadRangeKeyspace loads keyspaces starting at startID. // limit specifies the limit of loaded keyspaces. -func (se *StorageEndpoint) LoadRangeKeyspace(txn kv.Txn, startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) { +func (*StorageEndpoint) LoadRangeKeyspace(txn kv.Txn, startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) { startKey := KeyspaceMetaPath(startID) endKey := clientv3.GetPrefixRangeEnd(KeyspaceMetaPrefix()) keys, values, err := txn.LoadRange(startKey, endKey, limit) diff --git a/pkg/storage/endpoint/meta.go b/pkg/storage/endpoint/meta.go index d83e2b386c86..33482da512f1 100644 --- a/pkg/storage/endpoint/meta.go +++ b/pkg/storage/endpoint/meta.go @@ -236,7 +236,7 @@ func (se *StorageEndpoint) DeleteRegion(region *metapb.Region) error { } // Flush flushes the pending data to the underlying storage backend. -func (se *StorageEndpoint) Flush() error { return nil } +func (*StorageEndpoint) Flush() error { return nil } // Close closes the underlying storage backend. -func (se *StorageEndpoint) Close() error { return nil } +func (*StorageEndpoint) Close() error { return nil } diff --git a/pkg/storage/endpoint/rule.go b/pkg/storage/endpoint/rule.go index d0092e8e303f..84ad6ee1352d 100644 --- a/pkg/storage/endpoint/rule.go +++ b/pkg/storage/endpoint/rule.go @@ -44,12 +44,12 @@ type RuleStorage interface { var _ RuleStorage = (*StorageEndpoint)(nil) // SaveRule stores a rule cfg to the rulesPath. -func (se *StorageEndpoint) SaveRule(txn kv.Txn, ruleKey string, rule any) error { +func (*StorageEndpoint) SaveRule(txn kv.Txn, ruleKey string, rule any) error { return saveJSONInTxn(txn, ruleKeyPath(ruleKey), rule) } // DeleteRule removes a rule from storage. -func (se *StorageEndpoint) DeleteRule(txn kv.Txn, ruleKey string) error { +func (*StorageEndpoint) DeleteRule(txn kv.Txn, ruleKey string) error { return txn.Remove(ruleKeyPath(ruleKey)) } @@ -59,12 +59,12 @@ func (se *StorageEndpoint) LoadRuleGroups(f func(k, v string)) error { } // SaveRuleGroup stores a rule group config to storage. -func (se *StorageEndpoint) SaveRuleGroup(txn kv.Txn, groupID string, group any) error { +func (*StorageEndpoint) SaveRuleGroup(txn kv.Txn, groupID string, group any) error { return saveJSONInTxn(txn, ruleGroupIDPath(groupID), group) } // DeleteRuleGroup removes a rule group from storage. -func (se *StorageEndpoint) DeleteRuleGroup(txn kv.Txn, groupID string) error { +func (*StorageEndpoint) DeleteRuleGroup(txn kv.Txn, groupID string) error { return txn.Remove(ruleGroupIDPath(groupID)) } @@ -74,12 +74,12 @@ func (se *StorageEndpoint) LoadRegionRules(f func(k, v string)) error { } // SaveRegionRule saves a region rule to the storage. -func (se *StorageEndpoint) SaveRegionRule(txn kv.Txn, ruleKey string, rule any) error { +func (*StorageEndpoint) SaveRegionRule(txn kv.Txn, ruleKey string, rule any) error { return saveJSONInTxn(txn, regionLabelKeyPath(ruleKey), rule) } // DeleteRegionRule removes a region rule from storage. -func (se *StorageEndpoint) DeleteRegionRule(txn kv.Txn, ruleKey string) error { +func (*StorageEndpoint) DeleteRegionRule(txn kv.Txn, ruleKey string) error { return txn.Remove(regionLabelKeyPath(ruleKey)) } diff --git a/pkg/storage/endpoint/tso_keyspace_group.go b/pkg/storage/endpoint/tso_keyspace_group.go index 39a08afe9374..ba322336feb4 100644 --- a/pkg/storage/endpoint/tso_keyspace_group.go +++ b/pkg/storage/endpoint/tso_keyspace_group.go @@ -163,7 +163,7 @@ type KeyspaceGroupStorage interface { var _ KeyspaceGroupStorage = (*StorageEndpoint)(nil) // LoadKeyspaceGroup loads the keyspace group by ID. -func (se *StorageEndpoint) LoadKeyspaceGroup(txn kv.Txn, id uint32) (*KeyspaceGroup, error) { +func (*StorageEndpoint) LoadKeyspaceGroup(txn kv.Txn, id uint32) (*KeyspaceGroup, error) { value, err := txn.Load(KeyspaceGroupIDPath(id)) if err != nil || value == "" { return nil, err @@ -176,12 +176,12 @@ func (se *StorageEndpoint) LoadKeyspaceGroup(txn kv.Txn, id uint32) (*KeyspaceGr } // SaveKeyspaceGroup saves the keyspace group. -func (se *StorageEndpoint) SaveKeyspaceGroup(txn kv.Txn, kg *KeyspaceGroup) error { +func (*StorageEndpoint) SaveKeyspaceGroup(txn kv.Txn, kg *KeyspaceGroup) error { return saveJSONInTxn(txn, KeyspaceGroupIDPath(kg.ID), kg) } // DeleteKeyspaceGroup deletes the keyspace group. -func (se *StorageEndpoint) DeleteKeyspaceGroup(txn kv.Txn, id uint32) error { +func (*StorageEndpoint) DeleteKeyspaceGroup(txn kv.Txn, id uint32) error { return txn.Remove(KeyspaceGroupIDPath(id)) } diff --git a/pkg/syncer/client_test.go b/pkg/syncer/client_test.go index 84193ebaffe0..e7be77d2bb04 100644 --- a/pkg/syncer/client_test.go +++ b/pkg/syncer/client_test.go @@ -91,7 +91,7 @@ func (s *mockServer) LoopContext() context.Context { return s.ctx } -func (s *mockServer) ClusterID() uint64 { +func (*mockServer) ClusterID() uint64 { return 1 } @@ -107,7 +107,7 @@ func (s *mockServer) GetStorage() storage.Storage { return s.storage } -func (s *mockServer) Name() string { +func (*mockServer) Name() string { return "mock-server" } @@ -115,7 +115,7 @@ func (s *mockServer) GetRegions() []*core.RegionInfo { return s.bc.GetRegions() } -func (s *mockServer) GetTLSConfig() *grpcutil.TLSConfig { +func (*mockServer) GetTLSConfig() *grpcutil.TLSConfig { return &grpcutil.TLSConfig{} } diff --git a/pkg/tso/global_allocator.go b/pkg/tso/global_allocator.go index a37bcc738813..f90dc5f26fec 100644 --- a/pkg/tso/global_allocator.go +++ b/pkg/tso/global_allocator.go @@ -187,7 +187,7 @@ func (gta *GlobalTSOAllocator) Initialize(int) error { gta.tsoAllocatorRoleGauge.Set(1) // The suffix of a Global TSO should always be 0. gta.timestampOracle.suffix = 0 - return gta.timestampOracle.SyncTimestamp(gta.member.GetLeadership()) + return gta.timestampOracle.SyncTimestamp() } // IsInitialize is used to indicates whether this allocator is initialized. @@ -197,7 +197,7 @@ func (gta *GlobalTSOAllocator) IsInitialize() bool { // UpdateTSO is used to update the TSO in memory and the time window in etcd. func (gta *GlobalTSOAllocator) UpdateTSO() error { - return gta.timestampOracle.UpdateTimestamp(gta.member.GetLeadership()) + return gta.timestampOracle.UpdateTimestamp() } // SetTSO sets the physical part with given TSO. diff --git a/pkg/tso/keyspace_group_manager.go b/pkg/tso/keyspace_group_manager.go index d259ab27a5bf..d1e94d445cc7 100644 --- a/pkg/tso/keyspace_group_manager.go +++ b/pkg/tso/keyspace_group_manager.go @@ -674,7 +674,7 @@ func (kgm *KeyspaceGroupManager) isAssignedToMe(group *endpoint.KeyspaceGroup) b // updateKeyspaceGroup applies the given keyspace group. If the keyspace group is just assigned to // this host/pod, it will join the primary election. func (kgm *KeyspaceGroupManager) updateKeyspaceGroup(group *endpoint.KeyspaceGroup) { - if err := kgm.checkKeySpaceGroupID(group.ID); err != nil { + if err := checkKeySpaceGroupID(group.ID); err != nil { log.Warn("keyspace group ID is invalid, ignore it", zap.Error(err)) return } @@ -751,7 +751,7 @@ func (kgm *KeyspaceGroupManager) updateKeyspaceGroup(group *endpoint.KeyspaceGro kgm.groupUpdateRetryList[group.ID] = group return } - participant.SetCampaignChecker(func(leadership *election.Leadership) bool { + participant.SetCampaignChecker(func(*election.Leadership) bool { return splitSourceAM.GetMember().IsLeader() }) } @@ -997,7 +997,7 @@ func (kgm *KeyspaceGroupManager) exitElectionMembership(group *endpoint.Keyspace // GetAllocatorManager returns the AllocatorManager of the given keyspace group func (kgm *KeyspaceGroupManager) GetAllocatorManager(keyspaceGroupID uint32) (*AllocatorManager, error) { - if err := kgm.checkKeySpaceGroupID(keyspaceGroupID); err != nil { + if err := checkKeySpaceGroupID(keyspaceGroupID); err != nil { return nil, err } if am, _ := kgm.getKeyspaceGroupMeta(keyspaceGroupID); am != nil { @@ -1022,7 +1022,7 @@ func (kgm *KeyspaceGroupManager) FindGroupByKeyspaceID( func (kgm *KeyspaceGroupManager) GetElectionMember( keyspaceID, keyspaceGroupID uint32, ) (ElectionMember, error) { - if err := kgm.checkKeySpaceGroupID(keyspaceGroupID); err != nil { + if err := checkKeySpaceGroupID(keyspaceGroupID); err != nil { return nil, err } am, _, _, err := kgm.getKeyspaceGroupMetaWithCheck(keyspaceID, keyspaceGroupID) @@ -1052,7 +1052,7 @@ func (kgm *KeyspaceGroupManager) HandleTSORequest( keyspaceID, keyspaceGroupID uint32, dcLocation string, count uint32, ) (ts pdpb.Timestamp, curKeyspaceGroupID uint32, err error) { - if err := kgm.checkKeySpaceGroupID(keyspaceGroupID); err != nil { + if err := checkKeySpaceGroupID(keyspaceGroupID); err != nil { return pdpb.Timestamp{}, keyspaceGroupID, err } am, _, curKeyspaceGroupID, err := kgm.getKeyspaceGroupMetaWithCheck(keyspaceID, keyspaceGroupID) @@ -1086,7 +1086,7 @@ func (kgm *KeyspaceGroupManager) HandleTSORequest( return ts, curKeyspaceGroupID, err } -func (kgm *KeyspaceGroupManager) checkKeySpaceGroupID(id uint32) error { +func checkKeySpaceGroupID(id uint32) error { if id < mcsutils.MaxKeyspaceGroupCountInUse { return nil } diff --git a/pkg/tso/local_allocator.go b/pkg/tso/local_allocator.go index 45c200ca566e..e9019bf2bf32 100644 --- a/pkg/tso/local_allocator.go +++ b/pkg/tso/local_allocator.go @@ -101,7 +101,7 @@ func (lta *LocalTSOAllocator) GetDCLocation() string { func (lta *LocalTSOAllocator) Initialize(suffix int) error { lta.tsoAllocatorRoleGauge.Set(1) lta.timestampOracle.suffix = suffix - return lta.timestampOracle.SyncTimestamp(lta.leadership) + return lta.timestampOracle.SyncTimestamp() } // IsInitialize is used to indicates whether this allocator is initialized. @@ -112,7 +112,7 @@ func (lta *LocalTSOAllocator) IsInitialize() bool { // UpdateTSO is used to update the TSO in memory and the time window in etcd // for all local TSO allocators this PD server hold. func (lta *LocalTSOAllocator) UpdateTSO() error { - return lta.timestampOracle.UpdateTimestamp(lta.leadership) + return lta.timestampOracle.UpdateTimestamp() } // SetTSO sets the physical part with given TSO. diff --git a/pkg/tso/tso.go b/pkg/tso/tso.go index 5ad786678c49..bcb3169e73c0 100644 --- a/pkg/tso/tso.go +++ b/pkg/tso/tso.go @@ -156,7 +156,7 @@ func (t *timestampOracle) GetTimestampPath() string { } // SyncTimestamp is used to synchronize the timestamp. -func (t *timestampOracle) SyncTimestamp(leadership *election.Leadership) error { +func (t *timestampOracle) SyncTimestamp() error { log.Info("start to sync timestamp", logutil.CondUint32("keyspace-group-id", t.keyspaceGroupID, t.keyspaceGroupID > 0)) t.metrics.syncEvent.Inc() @@ -311,7 +311,7 @@ func (t *timestampOracle) resetUserTimestampInner(leadership *election.Leadershi // // NOTICE: this function should be called after the TSO in memory has been initialized // and should not be called when the TSO in memory has been reset anymore. -func (t *timestampOracle) UpdateTimestamp(leadership *election.Leadership) error { +func (t *timestampOracle) UpdateTimestamp() error { if !t.isInitialized() { return errs.ErrUpdateTimestamp.FastGenByArgs("timestamp in memory has not been initialized") } diff --git a/pkg/utils/apiutil/apiutil.go b/pkg/utils/apiutil/apiutil.go index c762245321e0..d0745ada2717 100644 --- a/pkg/utils/apiutil/apiutil.go +++ b/pkg/utils/apiutil/apiutil.go @@ -441,16 +441,15 @@ func (p *customReverseProxies) ServeHTTP(w http.ResponseWriter, r *http.Request) log.Error("request failed", errs.ZapError(errs.ErrSendRequest, err)) continue } - defer resp.Body.Close() var reader io.ReadCloser switch resp.Header.Get("Content-Encoding") { case "gzip": reader, err = gzip.NewReader(resp.Body) if err != nil { log.Error("failed to parse response with gzip compress", zap.Error(err)) + resp.Body.Close() continue } - defer reader.Close() default: reader = resp.Body } @@ -474,6 +473,8 @@ func (p *customReverseProxies) ServeHTTP(w http.ResponseWriter, r *http.Request) break } } + resp.Body.Close() + reader.Close() if err != nil { log.Error("write failed", errs.ZapError(errs.ErrWriteHTTPBody, err), zap.String("target-address", url.String())) // try next url. diff --git a/pkg/utils/etcdutil/etcdutil_test.go b/pkg/utils/etcdutil/etcdutil_test.go index e02615b695f9..6ddeafe45738 100644 --- a/pkg/utils/etcdutil/etcdutil_test.go +++ b/pkg/utils/etcdutil/etcdutil_test.go @@ -438,7 +438,7 @@ func (suite *loopWatcherTestSuite) TestLoadNoExistedKey() { cache[string(kv.Key)] = struct{}{} return nil }, - func(kv *mvccpb.KeyValue) error { return nil }, + func(*mvccpb.KeyValue) error { return nil }, func([]*clientv3.Event) error { return nil }, false, /* withPrefix */ ) @@ -466,7 +466,7 @@ func (suite *loopWatcherTestSuite) TestLoadWithLimitChange() { cache[string(kv.Key)] = struct{}{} return nil }, - func(kv *mvccpb.KeyValue) error { return nil }, + func(*mvccpb.KeyValue) error { return nil }, func([]*clientv3.Event) error { return nil }, true, /* withPrefix */ ) @@ -559,7 +559,7 @@ func (suite *loopWatcherTestSuite) TestWatcherLoadLimit() { cache = append(cache, string(kv.Key)) return nil }, - func(kv *mvccpb.KeyValue) error { + func(*mvccpb.KeyValue) error { return nil }, func([]*clientv3.Event) error { @@ -598,7 +598,7 @@ func (suite *loopWatcherTestSuite) TestWatcherLoadLargeKey() { cache = append(cache, string(kv.Key)) return nil }, - func(kv *mvccpb.KeyValue) error { + func(*mvccpb.KeyValue) error { return nil }, func([]*clientv3.Event) error { @@ -641,7 +641,7 @@ func (suite *loopWatcherTestSuite) TestWatcherBreak() { } return nil }, - func(kv *mvccpb.KeyValue) error { return nil }, + func(*mvccpb.KeyValue) error { return nil }, func([]*clientv3.Event) error { return nil }, false, /* withPrefix */ ) @@ -719,8 +719,8 @@ func (suite *loopWatcherTestSuite) TestWatcherRequestProgress() { "test", "TestWatcherChanBlock", func([]*clientv3.Event) error { return nil }, - func(kv *mvccpb.KeyValue) error { return nil }, - func(kv *mvccpb.KeyValue) error { return nil }, + func(*mvccpb.KeyValue) error { return nil }, + func(*mvccpb.KeyValue) error { return nil }, func([]*clientv3.Event) error { return nil }, false, /* withPrefix */ ) diff --git a/pkg/utils/etcdutil/health_checker.go b/pkg/utils/etcdutil/health_checker.go index 51c1808de4a5..44bddd8b183d 100644 --- a/pkg/utils/etcdutil/health_checker.go +++ b/pkg/utils/etcdutil/health_checker.go @@ -146,7 +146,7 @@ func (checker *healthChecker) inspector(ctx context.Context) { } func (checker *healthChecker) close() { - checker.healthyClients.Range(func(key, value any) bool { + checker.healthyClients.Range(func(_, value any) bool { healthyCli := value.(*healthyClient) healthyCli.healthState.Set(0) healthyCli.Client.Close() @@ -382,7 +382,7 @@ func (checker *healthChecker) update() { } } // Clean up the stale clients which are not in the etcd cluster anymore. - checker.healthyClients.Range(func(key, value any) bool { + checker.healthyClients.Range(func(key, _ any) bool { ep := key.(string) if _, ok := epMap[ep]; !ok { log.Info("remove stale etcd client", diff --git a/pkg/utils/logutil/log.go b/pkg/utils/logutil/log.go index 8c0977818fa6..ff6ffa7af9af 100644 --- a/pkg/utils/logutil/log.go +++ b/pkg/utils/logutil/log.go @@ -149,7 +149,7 @@ type stringer struct { } // String implement fmt.Stringer -func (s stringer) String() string { +func (stringer) String() string { return "?" } diff --git a/pkg/utils/metricutil/metricutil_test.go b/pkg/utils/metricutil/metricutil_test.go index acac9ce4d495..a5c183abc20d 100644 --- a/pkg/utils/metricutil/metricutil_test.go +++ b/pkg/utils/metricutil/metricutil_test.go @@ -55,7 +55,7 @@ func TestCamelCaseToSnakeCase(t *testing.T) { } } -func TestCoverage(t *testing.T) { +func TestCoverage(_ *testing.T) { cfgs := []*MetricConfig{ { PushJob: "j1", diff --git a/pkg/utils/tempurl/check_env_dummy.go b/pkg/utils/tempurl/check_env_dummy.go index 85f527ea6feb..58d889bbfd6b 100644 --- a/pkg/utils/tempurl/check_env_dummy.go +++ b/pkg/utils/tempurl/check_env_dummy.go @@ -16,6 +16,6 @@ package tempurl -func environmentCheck(addr string) bool { +func environmentCheck(_ string) bool { return true } diff --git a/pkg/utils/testutil/api_check.go b/pkg/utils/testutil/api_check.go index 5356d18514b9..0b7142045004 100644 --- a/pkg/utils/testutil/api_check.go +++ b/pkg/utils/testutil/api_check.go @@ -100,7 +100,8 @@ func ReadGetJSONWithBody(re *require.Assertions, client *http.Client, url string if err != nil { return err } - return checkResp(resp, StatusOK(re), ExtractJSON(re, data)) + checkOpts = append(checkOpts, StatusOK(re), ExtractJSON(re, data)) + return checkResp(resp, checkOpts...) } // CheckPostJSON is used to do post request and do check options. diff --git a/pkg/utils/tsoutil/tso_dispatcher.go b/pkg/utils/tsoutil/tso_dispatcher.go index 6d1ee2ace28a..9dfb2515dc12 100644 --- a/pkg/utils/tsoutil/tso_dispatcher.go +++ b/pkg/utils/tsoutil/tso_dispatcher.go @@ -128,7 +128,7 @@ func (s *TSODispatcher) dispatch( case <-dispatcherCtx.Done(): return } - err = s.processRequests(forwardStream, requests[:pendingTSOReqCount], tsoProtoFactory) + err = s.processRequests(forwardStream, requests[:pendingTSOReqCount]) close(done) if err != nil { log.Error("proxy forward tso error", @@ -155,7 +155,7 @@ func (s *TSODispatcher) dispatch( } } -func (s *TSODispatcher) processRequests(forwardStream stream, requests []Request, tsoProtoFactory ProtoFactory) error { +func (s *TSODispatcher) processRequests(forwardStream stream, requests []Request) error { // Merge the requests count := uint32(0) for _, request := range requests { @@ -163,7 +163,7 @@ func (s *TSODispatcher) processRequests(forwardStream stream, requests []Request } start := time.Now() - resp, err := requests[0].process(forwardStream, count, tsoProtoFactory) + resp, err := requests[0].process(forwardStream, count) if err != nil { return err } @@ -184,7 +184,7 @@ func addLogical(logical, count int64, suffixBits uint32) int64 { return logical + count<&1 | awk '{ print } END { if (NR > 0) { exit 1 } }' @ echo "golangci-lint ..." @ golangci-lint run -c $(ROOT_PATH)/.golangci.yml --verbose ./... --allow-parallel-runners - @ echo "revive ..." - @ revive -formatter friendly -config $(ROOT_PATH)/revive.toml ./... tidy: @ go mod tidy diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index da4be99638d7..10be418c0294 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -114,7 +114,7 @@ func TestClientLeaderChange(t *testing.T) { for i := range endpointsWithWrongURL { endpointsWithWrongURL[i] = "https://" + strings.TrimPrefix(endpointsWithWrongURL[i], "http://") } - cli := setupCli(re, ctx, endpointsWithWrongURL) + cli := setupCli(ctx, re, endpointsWithWrongURL) defer cli.Close() innerCli, ok := cli.(interface{ GetServiceDiscovery() pd.ServiceDiscovery }) re.True(ok) @@ -175,7 +175,7 @@ func TestLeaderTransferAndMoveCluster(t *testing.T) { }() endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints) + cli := setupCli(ctx, re, endpoints) defer cli.Close() var lastTS uint64 @@ -287,7 +287,7 @@ func TestTSOAllocatorLeader(t *testing.T) { }) allocatorLeaderMap[dcLocation] = pdName } - cli := setupCli(re, ctx, endpoints) + cli := setupCli(ctx, re, endpoints) defer cli.Close() innerCli, ok := cli.(interface{ GetServiceDiscovery() pd.ServiceDiscovery }) re.True(ok) @@ -321,9 +321,9 @@ func TestTSOFollowerProxy(t *testing.T) { defer cluster.Destroy() endpoints := runServer(re, cluster) - cli1 := setupCli(re, ctx, endpoints) + cli1 := setupCli(ctx, re, endpoints) defer cli1.Close() - cli2 := setupCli(re, ctx, endpoints) + cli2 := setupCli(ctx, re, endpoints) defer cli2.Close() cli2.UpdateOption(pd.EnableTSOFollowerProxy, true) @@ -385,7 +385,7 @@ func TestUnavailableTimeAfterLeaderIsReady(t *testing.T) { defer cluster.Destroy() endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints) + cli := setupCli(ctx, re, endpoints) defer cli.Close() var wg sync.WaitGroup @@ -417,7 +417,7 @@ func TestUnavailableTimeAfterLeaderIsReady(t *testing.T) { leader.Stop() re.NotEmpty(cluster.WaitLeader()) leaderReadyTime = time.Now() - cluster.RunServers([]*tests.TestServer{leader}) + tests.RunServers([]*tests.TestServer{leader}) }() wg.Wait() re.Less(maxUnavailableTime.UnixMilli(), leaderReadyTime.Add(1*time.Second).UnixMilli()) @@ -458,14 +458,14 @@ func TestGlobalAndLocalTSO(t *testing.T) { defer cluster.Destroy() endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints) + cli := setupCli(ctx, re, endpoints) defer cli.Close() // Wait for all nodes becoming healthy. time.Sleep(time.Second * 5) // Join a new dc-location - pd4, err := cluster.Join(ctx, func(conf *config.Config, serverName string) { + pd4, err := cluster.Join(ctx, func(conf *config.Config, _ string) { conf.EnableLocalTSO = true conf.Labels[config.ZoneLabel] = "dc-4" }) @@ -586,7 +586,7 @@ func TestCustomTimeout(t *testing.T) { defer cluster.Destroy() endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints, pd.WithCustomTimeoutOption(time.Second)) + cli := setupCli(ctx, re, endpoints, pd.WithCustomTimeoutOption(time.Second)) defer cli.Close() start := time.Now() @@ -647,8 +647,7 @@ func (suite *followerForwardAndHandleTestSuite) SetupSuite() { }) } -func (suite *followerForwardAndHandleTestSuite) TearDownTest() { -} +func (*followerForwardAndHandleTestSuite) TearDownTest() {} func (suite *followerForwardAndHandleTestSuite) TearDownSuite() { suite.cluster.Destroy() @@ -660,7 +659,7 @@ func (suite *followerForwardAndHandleTestSuite) TestGetRegionByFollowerForwardin ctx, cancel := context.WithCancel(suite.ctx) defer cancel() - cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) + cli := setupCli(ctx, re, suite.endpoints, pd.WithForwardingOption(true)) defer cli.Close() re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork1", "return(true)")) time.Sleep(200 * time.Millisecond) @@ -680,7 +679,7 @@ func (suite *followerForwardAndHandleTestSuite) TestGetTsoByFollowerForwarding1( re := suite.Require() ctx, cancel := context.WithCancel(suite.ctx) defer cancel() - cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) + cli := setupCli(ctx, re, suite.endpoints, pd.WithForwardingOption(true)) defer cli.Close() re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork", "return(true)")) @@ -715,7 +714,7 @@ func (suite *followerForwardAndHandleTestSuite) TestGetTsoByFollowerForwarding2( re := suite.Require() ctx, cancel := context.WithCancel(suite.ctx) defer cancel() - cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) + cli := setupCli(ctx, re, suite.endpoints, pd.WithForwardingOption(true)) defer cli.Close() re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork", "return(true)")) @@ -752,7 +751,7 @@ func (suite *followerForwardAndHandleTestSuite) TestGetTsoAndRegionByFollowerFor follower := cluster.GetServer(cluster.GetFollower()) re.NoError(failpoint.Enable("github.com/tikv/pd/client/grpcutil/unreachableNetwork2", fmt.Sprintf("return(\"%s\")", follower.GetAddr()))) - cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) + cli := setupCli(ctx, re, suite.endpoints, pd.WithForwardingOption(true)) defer cli.Close() var lastTS uint64 testutil.Eventually(re, func() bool { @@ -821,7 +820,7 @@ func (suite *followerForwardAndHandleTestSuite) TestGetRegionFromLeaderWhenNetwo follower := cluster.GetServer(cluster.GetFollower()) re.NoError(failpoint.Enable("github.com/tikv/pd/client/grpcutil/unreachableNetwork2", fmt.Sprintf("return(\"%s\")", follower.GetAddr()))) - cli := setupCli(re, ctx, suite.endpoints) + cli := setupCli(ctx, re, suite.endpoints) defer cli.Close() cluster.GetLeaderServer().GetServer().GetMember().ResignEtcdLeader(ctx, leader.GetServer().Name(), follower.GetServer().Name()) @@ -854,7 +853,7 @@ func (suite *followerForwardAndHandleTestSuite) TestGetRegionFromFollower() { defer cancel() cluster := suite.cluster - cli := setupCli(re, ctx, suite.endpoints) + cli := setupCli(ctx, re, suite.endpoints) defer cli.Close() cli.UpdateOption(pd.EnableFollowerHandle, true) re.NotEmpty(cluster.WaitLeader()) @@ -949,7 +948,7 @@ func (suite *followerForwardAndHandleTestSuite) TestGetTSFuture() { re.NoError(failpoint.Enable("github.com/tikv/pd/client/shortDispatcherChannel", "return(true)")) - cli := setupCli(re, ctx, suite.endpoints) + cli := setupCli(ctx, re, suite.endpoints) ctxs := make([]context.Context, 20) cancels := make([]context.CancelFunc, 20) @@ -1015,7 +1014,7 @@ func runServer(re *require.Assertions, cluster *tests.TestCluster) []string { return endpoints } -func setupCli(re *require.Assertions, ctx context.Context, endpoints []string, opts ...pd.ClientOption) pd.Client { +func setupCli(ctx context.Context, re *require.Assertions, endpoints []string, opts ...pd.ClientOption) pd.Client { cli, err := pd.NewClientWithContext(ctx, endpoints, pd.SecurityOption{}, opts...) re.NoError(err) return cli @@ -1083,7 +1082,7 @@ func TestCloseClient(t *testing.T) { re.NoError(err) defer cluster.Destroy() endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints) + cli := setupCli(ctx, re, endpoints) ts := cli.GetTSAsync(context.TODO()) time.Sleep(time.Second) cli.Close() @@ -1171,10 +1170,10 @@ func (suite *clientTestSuite) SetupSuite() { suite.grpcSvr = &server.GrpcServer{Server: suite.srv} server.MustWaitLeader(re, []*server.Server{suite.srv}) - suite.bootstrapServer(re, newHeader(suite.srv), suite.grpcPDClient) + bootstrapServer(re, newHeader(suite.srv), suite.grpcPDClient) suite.ctx, suite.clean = context.WithCancel(context.Background()) - suite.client = setupCli(re, suite.ctx, suite.srv.GetEndpoints()) + suite.client = setupCli(suite.ctx, re, suite.srv.GetEndpoints()) suite.regionHeartbeat, err = suite.grpcPDClient.RegionHeartbeat(suite.ctx) re.NoError(err) @@ -1216,7 +1215,7 @@ func newHeader(srv *server.Server) *pdpb.RequestHeader { } } -func (suite *clientTestSuite) bootstrapServer(re *require.Assertions, header *pdpb.RequestHeader, client pdpb.PDClient) { +func bootstrapServer(re *require.Assertions, header *pdpb.RequestHeader, client pdpb.PDClient) { regionID := regionIDAllocator.alloc() region := &metapb.Region{ Id: regionID, @@ -1781,7 +1780,7 @@ func TestWatch(t *testing.T) { re.NoError(err) defer cluster.Destroy() endpoints := runServer(re, cluster) - client := setupCli(re, ctx, endpoints) + client := setupCli(ctx, re, endpoints) defer client.Close() key := "test" @@ -1824,7 +1823,7 @@ func TestPutGet(t *testing.T) { re.NoError(err) defer cluster.Destroy() endpoints := runServer(re, cluster) - client := setupCli(re, ctx, endpoints) + client := setupCli(ctx, re, endpoints) defer client.Close() key := []byte("test") @@ -1859,7 +1858,7 @@ func TestClientWatchWithRevision(t *testing.T) { re.NoError(err) defer cluster.Destroy() endpoints := runServer(re, cluster) - client := setupCli(re, ctx, endpoints) + client := setupCli(ctx, re, endpoints) defer client.Close() s := cluster.GetLeaderServer() watchPrefix := "watch_test" @@ -1927,7 +1926,7 @@ func (suite *clientTestSuite) TestMemberUpdateBackOff() { defer cluster.Destroy() endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints) + cli := setupCli(ctx, re, endpoints) defer cli.Close() innerCli, ok := cli.(interface{ GetServiceDiscovery() pd.ServiceDiscovery }) re.True(ok) diff --git a/tests/integrations/client/client_tls_test.go b/tests/integrations/client/client_tls_test.go index bdfe050bf45c..a5f0f5b200d8 100644 --- a/tests/integrations/client/client_tls_test.go +++ b/tests/integrations/client/client_tls_test.go @@ -120,18 +120,18 @@ func TestTLSReloadAtomicReplace(t *testing.T) { err = os.Rename(certsDirExp, certsDir) re.NoError(err) } - testTLSReload(re, ctx, cloneFunc, replaceFunc, revertFunc) + testTLSReload(ctx, re, cloneFunc, replaceFunc, revertFunc) } func testTLSReload( - re *require.Assertions, ctx context.Context, + re *require.Assertions, cloneFunc func() transport.TLSInfo, replaceFunc func(), revertFunc func()) { tlsInfo := cloneFunc() // 1. start cluster with valid certs - clus, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, serverName string) { + clus, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, _ string) { conf.Security.TLSConfig = grpcutil.TLSConfig{ KeyPath: tlsInfo.KeyFile, CertPath: tlsInfo.CertFile, diff --git a/tests/integrations/client/gc_client_test.go b/tests/integrations/client/gc_client_test.go index 737fd09a08fa..0913579f47e3 100644 --- a/tests/integrations/client/gc_client_test.go +++ b/tests/integrations/client/gc_client_test.go @@ -135,6 +135,7 @@ func (suite *gcClientTestSuite) TestClientWatchWithRevision() { suite.testClientWatchWithRevision(true) } +// nolint:revive func (suite *gcClientTestSuite) testClientWatchWithRevision(fromNewRevision bool) { re := suite.Require() testKeyspaceID := uint32(100) diff --git a/tests/integrations/client/global_config_test.go b/tests/integrations/client/global_config_test.go index c52a35159b07..d813ec99676a 100644 --- a/tests/integrations/client/global_config_test.go +++ b/tests/integrations/client/global_config_test.go @@ -89,7 +89,7 @@ func (suite *globalConfigTestSuite) TearDownSuite() { suite.client.Close() } -func (suite *globalConfigTestSuite) GetEtcdPath(configPath string) string { +func getEtcdPath(configPath string) string { return globalConfigPath + configPath } @@ -97,10 +97,10 @@ func (suite *globalConfigTestSuite) TestLoadWithoutNames() { re := suite.Require() defer func() { // clean up - _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath("test")) + _, err := suite.server.GetClient().Delete(suite.server.Context(), getEtcdPath("test")) re.NoError(err) }() - r, err := suite.server.GetClient().Put(suite.server.Context(), suite.GetEtcdPath("test"), "test") + r, err := suite.server.GetClient().Put(suite.server.Context(), getEtcdPath("test"), "test") re.NoError(err) res, err := suite.server.LoadGlobalConfig(suite.server.Context(), &pdpb.LoadGlobalConfigRequest{ ConfigPath: globalConfigPath, @@ -115,10 +115,10 @@ func (suite *globalConfigTestSuite) TestLoadWithoutConfigPath() { re := suite.Require() defer func() { // clean up - _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath("source_id")) + _, err := suite.server.GetClient().Delete(suite.server.Context(), getEtcdPath("source_id")) re.NoError(err) }() - _, err := suite.server.GetClient().Put(suite.server.Context(), suite.GetEtcdPath("source_id"), "1") + _, err := suite.server.GetClient().Put(suite.server.Context(), getEtcdPath("source_id"), "1") re.NoError(err) res, err := suite.server.LoadGlobalConfig(suite.server.Context(), &pdpb.LoadGlobalConfigRequest{ Names: []string{"source_id"}, @@ -132,7 +132,7 @@ func (suite *globalConfigTestSuite) TestLoadOtherConfigPath() { re := suite.Require() defer func() { for i := 0; i < 3; i++ { - _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) + _, err := suite.server.GetClient().Delete(suite.server.Context(), getEtcdPath(strconv.Itoa(i))) re.NoError(err) } }() @@ -155,7 +155,7 @@ func (suite *globalConfigTestSuite) TestLoadAndStore() { re := suite.Require() defer func() { for i := 0; i < 3; i++ { - _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath("test")) + _, err := suite.server.GetClient().Delete(suite.server.Context(), getEtcdPath("test")) re.NoError(err) } }() @@ -171,7 +171,7 @@ func (suite *globalConfigTestSuite) TestLoadAndStore() { re.Len(res.Items, 3) re.NoError(err) for i, item := range res.Items { - re.Equal(&pdpb.GlobalConfigItem{Kind: pdpb.EventType_PUT, Name: suite.GetEtcdPath(strconv.Itoa(i)), Payload: []byte(strconv.Itoa(i))}, item) + re.Equal(&pdpb.GlobalConfigItem{Kind: pdpb.EventType_PUT, Name: getEtcdPath(strconv.Itoa(i)), Payload: []byte(strconv.Itoa(i))}, item) } } @@ -179,7 +179,7 @@ func (suite *globalConfigTestSuite) TestStore() { re := suite.Require() defer func() { for i := 0; i < 3; i++ { - _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath("test")) + _, err := suite.server.GetClient().Delete(suite.server.Context(), getEtcdPath("test")) re.NoError(err) } }() @@ -190,9 +190,9 @@ func (suite *globalConfigTestSuite) TestStore() { }) re.NoError(err) for i := 0; i < 3; i++ { - res, err := suite.server.GetClient().Get(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) + res, err := suite.server.GetClient().Get(suite.server.Context(), getEtcdPath(strconv.Itoa(i))) re.NoError(err) - re.Equal(suite.GetEtcdPath(string(res.Kvs[0].Value)), string(res.Kvs[0].Key)) + re.Equal(getEtcdPath(string(res.Kvs[0].Value)), string(res.Kvs[0].Key)) } } @@ -201,7 +201,7 @@ func (suite *globalConfigTestSuite) TestWatch() { defer func() { for i := 0; i < 3; i++ { // clean up - _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) + _, err := suite.server.GetClient().Delete(suite.server.Context(), getEtcdPath(strconv.Itoa(i))) re.NoError(err) } }() @@ -213,11 +213,11 @@ func (suite *globalConfigTestSuite) TestWatch() { Revision: 0, }, server) for i := 0; i < 6; i++ { - _, err := suite.server.GetClient().Put(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i)), strconv.Itoa(i)) + _, err := suite.server.GetClient().Put(suite.server.Context(), getEtcdPath(strconv.Itoa(i)), strconv.Itoa(i)) re.NoError(err) } for i := 3; i < 6; i++ { - _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) + _, err := suite.server.GetClient().Delete(suite.server.Context(), getEtcdPath(strconv.Itoa(i))) re.NoError(err) } res, err := suite.server.LoadGlobalConfig(suite.server.Context(), &pdpb.LoadGlobalConfigRequest{ @@ -231,29 +231,29 @@ func (suite *globalConfigTestSuite) TestClientLoadWithoutNames() { re := suite.Require() defer func() { for i := 0; i < 3; i++ { - _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) + _, err := suite.server.GetClient().Delete(suite.server.Context(), getEtcdPath(strconv.Itoa(i))) re.NoError(err) } }() for i := 0; i < 3; i++ { - _, err := suite.server.GetClient().Put(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i)), strconv.Itoa(i)) + _, err := suite.server.GetClient().Put(suite.server.Context(), getEtcdPath(strconv.Itoa(i)), strconv.Itoa(i)) re.NoError(err) } res, _, err := suite.client.LoadGlobalConfig(suite.server.Context(), nil, globalConfigPath) re.NoError(err) re.Len(res, 3) for i, item := range res { - re.Equal(pd.GlobalConfigItem{EventType: pdpb.EventType_PUT, Name: suite.GetEtcdPath(strconv.Itoa(i)), PayLoad: []byte(strconv.Itoa(i)), Value: strconv.Itoa(i)}, item) + re.Equal(pd.GlobalConfigItem{EventType: pdpb.EventType_PUT, Name: getEtcdPath(strconv.Itoa(i)), PayLoad: []byte(strconv.Itoa(i)), Value: strconv.Itoa(i)}, item) } } func (suite *globalConfigTestSuite) TestClientLoadWithoutConfigPath() { re := suite.Require() defer func() { - _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath("source_id")) + _, err := suite.server.GetClient().Delete(suite.server.Context(), getEtcdPath("source_id")) re.NoError(err) }() - _, err := suite.server.GetClient().Put(suite.server.Context(), suite.GetEtcdPath("source_id"), "1") + _, err := suite.server.GetClient().Put(suite.server.Context(), getEtcdPath("source_id"), "1") re.NoError(err) res, _, err := suite.client.LoadGlobalConfig(suite.server.Context(), []string{"source_id"}, "") re.NoError(err) @@ -265,7 +265,7 @@ func (suite *globalConfigTestSuite) TestClientLoadOtherConfigPath() { re := suite.Require() defer func() { for i := 0; i < 3; i++ { - _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) + _, err := suite.server.GetClient().Delete(suite.server.Context(), getEtcdPath(strconv.Itoa(i))) re.NoError(err) } }() @@ -285,7 +285,7 @@ func (suite *globalConfigTestSuite) TestClientStore() { re := suite.Require() defer func() { for i := 0; i < 3; i++ { - _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) + _, err := suite.server.GetClient().Delete(suite.server.Context(), getEtcdPath(strconv.Itoa(i))) re.NoError(err) } }() @@ -293,9 +293,9 @@ func (suite *globalConfigTestSuite) TestClientStore() { []pd.GlobalConfigItem{{Name: "0", Value: "0"}, {Name: "1", Value: "1"}, {Name: "2", Value: "2"}}) re.NoError(err) for i := 0; i < 3; i++ { - res, err := suite.server.GetClient().Get(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) + res, err := suite.server.GetClient().Get(suite.server.Context(), getEtcdPath(strconv.Itoa(i))) re.NoError(err) - re.Equal(suite.GetEtcdPath(string(res.Kvs[0].Value)), string(res.Kvs[0].Key)) + re.Equal(getEtcdPath(string(res.Kvs[0].Value)), string(res.Kvs[0].Key)) } } @@ -303,25 +303,25 @@ func (suite *globalConfigTestSuite) TestClientWatchWithRevision() { re := suite.Require() ctx := suite.server.Context() defer func() { - _, err := suite.server.GetClient().Delete(ctx, suite.GetEtcdPath("test")) + _, err := suite.server.GetClient().Delete(ctx, getEtcdPath("test")) re.NoError(err) for i := 3; i < 9; i++ { - _, err := suite.server.GetClient().Delete(ctx, suite.GetEtcdPath(strconv.Itoa(i))) + _, err := suite.server.GetClient().Delete(ctx, getEtcdPath(strconv.Itoa(i))) re.NoError(err) } }() // Mock get revision by loading - r, err := suite.server.GetClient().Put(ctx, suite.GetEtcdPath("test"), "test") + r, err := suite.server.GetClient().Put(ctx, getEtcdPath("test"), "test") re.NoError(err) res, revision, err := suite.client.LoadGlobalConfig(ctx, nil, globalConfigPath) re.NoError(err) re.Len(res, 1) suite.LessOrEqual(r.Header.GetRevision(), revision) - re.Equal(pd.GlobalConfigItem{EventType: pdpb.EventType_PUT, Name: suite.GetEtcdPath("test"), PayLoad: []byte("test"), Value: "test"}, res[0]) + re.Equal(pd.GlobalConfigItem{EventType: pdpb.EventType_PUT, Name: getEtcdPath("test"), PayLoad: []byte("test"), Value: "test"}, res[0]) // Mock when start watcher there are existed some keys, will load firstly for i := 0; i < 6; i++ { - _, err = suite.server.GetClient().Put(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i)), strconv.Itoa(i)) + _, err = suite.server.GetClient().Put(suite.server.Context(), getEtcdPath(strconv.Itoa(i)), strconv.Itoa(i)) re.NoError(err) } // Start watcher at next revision @@ -329,12 +329,12 @@ func (suite *globalConfigTestSuite) TestClientWatchWithRevision() { re.NoError(err) // Mock delete for i := 0; i < 3; i++ { - _, err = suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) + _, err = suite.server.GetClient().Delete(suite.server.Context(), getEtcdPath(strconv.Itoa(i))) re.NoError(err) } // Mock put for i := 6; i < 9; i++ { - _, err = suite.server.GetClient().Put(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i)), strconv.Itoa(i)) + _, err = suite.server.GetClient().Put(suite.server.Context(), getEtcdPath(strconv.Itoa(i)), strconv.Itoa(i)) re.NoError(err) } timer := time.NewTimer(time.Second) @@ -347,7 +347,7 @@ func (suite *globalConfigTestSuite) TestClientWatchWithRevision() { return case res := <-configChan: for _, r := range res { - re.Equal(suite.GetEtcdPath(r.Value), r.Name) + re.Equal(getEtcdPath(r.Value), r.Name) } runTest = true } diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index 9efbc5878476..d35b7f005848 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -121,7 +121,7 @@ func (suite *httpClientTestSuite) TearDownSuite() { // RunTestInTwoModes is to run test in two modes. func (suite *httpClientTestSuite) RunTestInTwoModes(test func(mode mode, client pd.Client)) { // Run test with specific service discovery. - cli := setupCli(suite.Require(), suite.env[specificServiceDiscovery].ctx, suite.env[specificServiceDiscovery].endpoints) + cli := setupCli(suite.env[specificServiceDiscovery].ctx, suite.Require(), suite.env[specificServiceDiscovery].endpoints) sd := cli.GetServiceDiscovery() client := pd.NewClientWithServiceDiscovery("pd-http-client-it-grpc", sd) test(specificServiceDiscovery, client) @@ -268,7 +268,7 @@ func (suite *httpClientTestSuite) checkRule(mode mode, client pd.Client) { re.NoError(err) re.Equal(bundles[0], bundle) // Check if we have the default rule. - suite.checkRuleResult(re, env, client, &pd.Rule{ + checkRuleResult(re, env, client, &pd.Rule{ GroupID: placement.DefaultGroupID, ID: placement.DefaultRuleID, Role: pd.Voter, @@ -277,7 +277,7 @@ func (suite *httpClientTestSuite) checkRule(mode mode, client pd.Client) { EndKey: []byte{}, }, 1, true) // Should be the same as the rules in the bundle. - suite.checkRuleResult(re, env, client, bundle.Rules[0], 1, true) + checkRuleResult(re, env, client, bundle.Rules[0], 1, true) testRule := &pd.Rule{ GroupID: placement.DefaultGroupID, ID: "test", @@ -288,24 +288,24 @@ func (suite *httpClientTestSuite) checkRule(mode mode, client pd.Client) { } err = client.SetPlacementRule(env.ctx, testRule) re.NoError(err) - suite.checkRuleResult(re, env, client, testRule, 2, true) + checkRuleResult(re, env, client, testRule, 2, true) err = client.DeletePlacementRule(env.ctx, placement.DefaultGroupID, "test") re.NoError(err) - suite.checkRuleResult(re, env, client, testRule, 1, false) + checkRuleResult(re, env, client, testRule, 1, false) testRuleOp := &pd.RuleOp{ Rule: testRule, Action: pd.RuleOpAdd, } err = client.SetPlacementRuleInBatch(env.ctx, []*pd.RuleOp{testRuleOp}) re.NoError(err) - suite.checkRuleResult(re, env, client, testRule, 2, true) + checkRuleResult(re, env, client, testRule, 2, true) testRuleOp = &pd.RuleOp{ Rule: testRule, Action: pd.RuleOpDel, } err = client.SetPlacementRuleInBatch(env.ctx, []*pd.RuleOp{testRuleOp}) re.NoError(err) - suite.checkRuleResult(re, env, client, testRule, 1, false) + checkRuleResult(re, env, client, testRule, 1, false) err = client.SetPlacementRuleBundles(env.ctx, []*pd.GroupBundle{ { ID: placement.DefaultGroupID, @@ -313,7 +313,7 @@ func (suite *httpClientTestSuite) checkRule(mode mode, client pd.Client) { }, }, true) re.NoError(err) - suite.checkRuleResult(re, env, client, testRule, 1, true) + checkRuleResult(re, env, client, testRule, 1, true) ruleGroups, err := client.GetAllPlacementRuleGroups(env.ctx) re.NoError(err) re.Len(ruleGroups, 1) @@ -347,10 +347,10 @@ func (suite *httpClientTestSuite) checkRule(mode mode, client pd.Client) { } err = client.SetPlacementRule(env.ctx, testRule) re.NoError(err) - suite.checkRuleResult(re, env, client, testRule, 1, true) + checkRuleResult(re, env, client, testRule, 1, true) } -func (suite *httpClientTestSuite) checkRuleResult( +func checkRuleResult( re *require.Assertions, env *httpClientTestEnv, client pd.Client, @@ -724,7 +724,7 @@ func (suite *httpClientTestSuite) TestRedirectWithMetrics() { re := suite.Require() env := suite.env[defaultServiceDiscovery] - cli := setupCli(suite.Require(), env.ctx, env.endpoints) + cli := setupCli(env.ctx, suite.Require(), env.endpoints) defer cli.Close() sd := cli.GetServiceDiscovery() diff --git a/tests/integrations/mcs/discovery/register_test.go b/tests/integrations/mcs/discovery/register_test.go index 1b61a2642320..69d534638189 100644 --- a/tests/integrations/mcs/discovery/register_test.go +++ b/tests/integrations/mcs/discovery/register_test.go @@ -124,9 +124,15 @@ func (suite *serverRegisterTestSuite) checkServerPrimaryChange(serviceName strin re.Empty(primary) serverMap := make(map[string]bs.Server) + var cleanups []func() + defer func() { + for _, cleanup := range cleanups { + cleanup() + } + }() for i := 0; i < serverNum; i++ { s, cleanup := suite.addServer(serviceName) - defer cleanup() + cleanups = append(cleanups, cleanup) serverMap[s.GetAddr()] = s } diff --git a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go index ccec0a7cdc0b..160eea167d63 100644 --- a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go +++ b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go @@ -89,9 +89,15 @@ func (suite *keyspaceGroupTestSuite) TestAllocNodesUpdate() { re := suite.Require() // add three nodes. nodes := make(map[string]bs.Server) + var cleanups []func() + defer func() { + for _, cleanup := range cleanups { + cleanup() + } + }() for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount+1; i++ { s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) - defer cleanup() + cleanups = append(cleanups, cleanup) nodes[s.GetAddr()] = s } tests.WaitForPrimaryServing(re, nodes) @@ -139,9 +145,15 @@ func (suite *keyspaceGroupTestSuite) TestAllocNodesUpdate() { func (suite *keyspaceGroupTestSuite) TestAllocReplica() { re := suite.Require() nodes := make(map[string]bs.Server) + var cleanups []func() + defer func() { + for _, cleanup := range cleanups { + cleanup() + } + }() for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount; i++ { s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) - defer cleanup() + cleanups = append(cleanups, cleanup) nodes[s.GetAddr()] = s } tests.WaitForPrimaryServing(re, nodes) @@ -233,9 +245,15 @@ func (suite *keyspaceGroupTestSuite) TestSetNodes() { re := suite.Require() nodes := make(map[string]bs.Server) nodesList := []string{} + var cleanups []func() + defer func() { + for _, cleanup := range cleanups { + cleanup() + } + }() for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount; i++ { s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) - defer cleanup() + cleanups = append(cleanups, cleanup) nodes[s.GetAddr()] = s nodesList = append(nodesList, s.GetAddr()) } @@ -294,9 +312,15 @@ func (suite *keyspaceGroupTestSuite) TestSetNodes() { func (suite *keyspaceGroupTestSuite) TestDefaultKeyspaceGroup() { re := suite.Require() nodes := make(map[string]bs.Server) + var cleanups []func() + defer func() { + for _, cleanup := range cleanups { + cleanup() + } + }() for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount; i++ { s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) - defer cleanup() + cleanups = append(cleanups, cleanup) nodes[s.GetAddr()] = s } tests.WaitForPrimaryServing(re, nodes) diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go index f1ea8736fda2..17673213a978 100644 --- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go +++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go @@ -78,7 +78,7 @@ func (suite *resourceManagerClientTestSuite) SetupSuite() { suite.client, err = pd.NewClientWithContext(suite.ctx, suite.cluster.GetConfig().GetClientURLs(), pd.SecurityOption{}) re.NoError(err) leader := suite.cluster.GetServer(suite.cluster.WaitLeader()) - suite.waitLeader(re, suite.client, leader.GetAddr()) + waitLeader(re, suite.client, leader.GetAddr()) suite.initGroups = []*rmpb.ResourceGroup{ { @@ -135,7 +135,7 @@ func (suite *resourceManagerClientTestSuite) SetupSuite() { } } -func (suite *resourceManagerClientTestSuite) waitLeader(re *require.Assertions, cli pd.Client, leaderAddr string) { +func waitLeader(re *require.Assertions, cli pd.Client, leaderAddr string) { innerCli, ok := cli.(interface{ GetServiceDiscovery() pd.ServiceDiscovery }) re.True(ok) re.NotNil(innerCli) @@ -177,7 +177,7 @@ func (suite *resourceManagerClientTestSuite) resignAndWaitLeader(re *require.Ass re.NoError(suite.cluster.ResignLeader()) newLeader := suite.cluster.GetServer(suite.cluster.WaitLeader()) re.NotNil(newLeader) - suite.waitLeader(re, suite.client, newLeader.GetAddr()) + waitLeader(re, suite.client, newLeader.GetAddr()) } func (suite *resourceManagerClientTestSuite) TestWatchResourceGroup() { @@ -349,7 +349,7 @@ type tokenConsumptionPerSecond struct { waitDuration time.Duration } -func (t tokenConsumptionPerSecond) makeReadRequest() *controller.TestRequestInfo { +func (tokenConsumptionPerSecond) makeReadRequest() *controller.TestRequestInfo { return controller.NewTestRequestInfo(false, 0, 0) } @@ -365,7 +365,7 @@ func (t tokenConsumptionPerSecond) makeReadResponse() *controller.TestResponseIn ) } -func (t tokenConsumptionPerSecond) makeWriteResponse() *controller.TestResponseInfo { +func (tokenConsumptionPerSecond) makeWriteResponse() *controller.TestResponseInfo { return controller.NewTestResponseInfo( 0, time.Duration(0), @@ -706,7 +706,6 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { c.Stop() } -// nolint:gosec func (suite *resourceManagerClientTestSuite) TestAcquireTokenBucket() { re := suite.Require() cli := suite.client @@ -960,7 +959,7 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { re.NoError(err) resp, err := http.Post(getAddr(i)+"/resource-manager/api/v1/config/group", "application/json", strings.NewReader(string(createJSON))) re.NoError(err) - defer resp.Body.Close() + resp.Body.Close() re.Equal(http.StatusOK, resp.StatusCode) if tcase.isNewGroup { finalNum++ @@ -975,7 +974,7 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { req.Header.Set("Content-Type", "application/json") resp, err = http.DefaultClient.Do(req) re.NoError(err) - defer resp.Body.Close() + resp.Body.Close() if tcase.modifySuccess { re.Equal(http.StatusOK, resp.StatusCode) } else { @@ -985,9 +984,9 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { // Get Resource Group resp, err = http.Get(getAddr(i) + "/resource-manager/api/v1/config/group/" + tcase.name) re.NoError(err) - defer resp.Body.Close() re.Equal(http.StatusOK, resp.StatusCode) respString, err := io.ReadAll(resp.Body) + resp.Body.Close() re.NoError(err) re.Contains(string(respString), tcase.name) if tcase.modifySuccess { @@ -998,9 +997,9 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { if i == len(testCasesSet1)-1 { resp, err := http.Get(getAddr(i) + "/resource-manager/api/v1/config/groups") re.NoError(err) - defer resp.Body.Close() re.Equal(http.StatusOK, resp.StatusCode) respString, err := io.ReadAll(resp.Body) + resp.Body.Close() re.NoError(err) groups := make([]*server.ResourceGroup, 0) json.Unmarshal(respString, &groups) @@ -1012,8 +1011,8 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { re.NoError(err) resp, err := http.DefaultClient.Do(req) re.NoError(err) - defer resp.Body.Close() respString, err := io.ReadAll(resp.Body) + resp.Body.Close() re.NoError(err) if g.Name == "default" { re.Contains(string(respString), "cannot delete reserved group") @@ -1026,9 +1025,9 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { // verify again resp1, err := http.Get(getAddr(i) + "/resource-manager/api/v1/config/groups") re.NoError(err) - defer resp1.Body.Close() re.Equal(http.StatusOK, resp1.StatusCode) respString1, err := io.ReadAll(resp1.Body) + resp1.Body.Close() re.NoError(err) groups1 := make([]server.ResourceGroup, 0) json.Unmarshal(respString1, &groups1) @@ -1046,7 +1045,7 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { for _, s := range servers { serverList = append(serverList, s) } - re.NoError(suite.cluster.RunServers(serverList)) + re.NoError(tests.RunServers(serverList)) suite.cluster.WaitLeader() // re-connect client as well suite.client, err = pd.NewClientWithContext(suite.ctx, suite.cluster.GetConfig().GetClientURLs(), pd.SecurityOption{}) @@ -1314,9 +1313,8 @@ func (suite *resourceManagerClientTestSuite) TestCheckBackgroundJobs() { enableBackgroundGroup := func(enable bool) string { if enable { return "background_enable" - } else { - return "background_unable" } + return "background_unable" } // Mock add resource group. group := &rmpb.ResourceGroup{ diff --git a/tests/integrations/mcs/tso/api_test.go b/tests/integrations/mcs/tso/api_test.go index 327254184620..dc9bfa1e291d 100644 --- a/tests/integrations/mcs/tso/api_test.go +++ b/tests/integrations/mcs/tso/api_test.go @@ -141,7 +141,7 @@ func TestTSOServerStartFirst(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - apiCluster, err := tests.NewTestAPICluster(ctx, 1, func(conf *config.Config, serverName string) { + apiCluster, err := tests.NewTestAPICluster(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = []string{"k1", "k2"} }) defer apiCluster.Destroy() diff --git a/tests/integrations/mcs/tso/proxy_test.go b/tests/integrations/mcs/tso/proxy_test.go index 7ed329610f25..43877f262e22 100644 --- a/tests/integrations/mcs/tso/proxy_test.go +++ b/tests/integrations/mcs/tso/proxy_test.go @@ -84,7 +84,7 @@ func (s *tsoProxyTestSuite) SetupSuite() { } func (s *tsoProxyTestSuite) TearDownSuite() { - s.cleanupGRPCStreams(s.cleanupFuncs) + cleanupGRPCStreams(s.cleanupFuncs) s.tsoCluster.Destroy() s.apiCluster.Destroy() s.cancel() @@ -112,7 +112,7 @@ func (s *tsoProxyTestSuite) TestTSOProxyWorksWithCancellation() { for j := 0; j < 10; j++ { s.verifyTSOProxy(s.ctx, streams, cleanupFuncs, 10, true) } - s.cleanupGRPCStreams(cleanupFuncs) + cleanupGRPCStreams(cleanupFuncs) } }() for i := 0; i < 10; i++ { @@ -125,7 +125,7 @@ func (s *tsoProxyTestSuite) TestTSOProxyWorksWithCancellation() { // TestTSOProxyStress tests the TSO Proxy can work correctly under the stress. gPRC and TSO failures are allowed, // but the TSO Proxy should not panic, blocked or deadlocked, and if it returns a timestamp, it should be a valid // timestamp monotonic increasing. After the stress, the TSO Proxy should still work correctly. -func TestTSOProxyStress(t *testing.T) { +func TestTSOProxyStress(_ *testing.T) { s := new(tsoProxyTestSuite) s.SetT(&testing.T{}) s.SetupSuite() @@ -154,7 +154,7 @@ func TestTSOProxyStress(t *testing.T) { cleanupFuncs = append(cleanupFuncs, cleanupFuncsTemp...) s.verifyTSOProxy(ctxTimeout, streams, cleanupFuncs, 50, false) } - s.cleanupGRPCStreams(cleanupFuncs) + cleanupGRPCStreams(cleanupFuncs) log.Info("the stress test completed.") // Verify the TSO Proxy can still work correctly after the stress. @@ -192,7 +192,7 @@ func (s *tsoProxyTestSuite) TestTSOProxyClientsWithSameContext() { } s.verifyTSOProxy(ctx, streams, cleanupFuncs, 100, true) - s.cleanupGRPCStreams(cleanupFuncs) + cleanupGRPCStreams(cleanupFuncs) } // TestTSOProxyRecvFromClientTimeout tests the TSO Proxy can properly close the grpc stream on the server side @@ -207,7 +207,7 @@ func (s *tsoProxyTestSuite) TestTSOProxyRecvFromClientTimeout() { time.Sleep(2 * time.Second) err := streams[0].Send(s.defaultReq) re.Error(err) - s.cleanupGRPCStreams(cleanupFuncs) + cleanupGRPCStreams(cleanupFuncs) re.NoError(failpoint.Disable("github.com/tikv/pd/server/tsoProxyRecvFromClientTimeout")) // Verify the streams with no fault injection can work correctly. @@ -226,7 +226,7 @@ func (s *tsoProxyTestSuite) TestTSOProxyFailToSendToClient() { re.NoError(err) _, err = streams[0].Recv() re.Error(err) - s.cleanupGRPCStreams(cleanupFuncs) + cleanupGRPCStreams(cleanupFuncs) re.NoError(failpoint.Disable("github.com/tikv/pd/server/tsoProxyFailToSendToClient")) s.verifyTSOProxy(s.ctx, s.streams, s.cleanupFuncs, 1, true) @@ -244,7 +244,7 @@ func (s *tsoProxyTestSuite) TestTSOProxySendToTSOTimeout() { re.NoError(err) _, err = streams[0].Recv() re.Error(err) - s.cleanupGRPCStreams(cleanupFuncs) + cleanupGRPCStreams(cleanupFuncs) re.NoError(failpoint.Disable("github.com/tikv/pd/server/tsoProxySendToTSOTimeout")) s.verifyTSOProxy(s.ctx, s.streams, s.cleanupFuncs, 1, true) @@ -262,13 +262,13 @@ func (s *tsoProxyTestSuite) TestTSOProxyRecvFromTSOTimeout() { re.NoError(err) _, err = streams[0].Recv() re.Error(err) - s.cleanupGRPCStreams(cleanupFuncs) + cleanupGRPCStreams(cleanupFuncs) re.NoError(failpoint.Disable("github.com/tikv/pd/server/tsoProxyRecvFromTSOTimeout")) s.verifyTSOProxy(s.ctx, s.streams, s.cleanupFuncs, 1, true) } -func (s *tsoProxyTestSuite) cleanupGRPCStreams(cleanupFuncs []testutil.CleanupFunc) { +func cleanupGRPCStreams(cleanupFuncs []testutil.CleanupFunc) { for i := 0; i < len(cleanupFuncs); i++ { if cleanupFuncs[i] != nil { cleanupFuncs[i]() @@ -277,7 +277,7 @@ func (s *tsoProxyTestSuite) cleanupGRPCStreams(cleanupFuncs []testutil.CleanupFu } } -func (s *tsoProxyTestSuite) cleanupGRPCStream( +func cleanupGRPCStream( streams []pdpb.PD_TsoClient, cleanupFuncs []testutil.CleanupFunc, index int, ) { if cleanupFuncs[index] != nil { @@ -318,7 +318,7 @@ func (s *tsoProxyTestSuite) verifyTSOProxy( for j := 0; j < requestsPerClient; j++ { select { case <-ctx.Done(): - s.cleanupGRPCStream(streams, cleanupFuncs, i) + cleanupGRPCStream(streams, cleanupFuncs, i) return default: } @@ -327,14 +327,14 @@ func (s *tsoProxyTestSuite) verifyTSOProxy( err := streams[i].Send(req) if err != nil && !mustReliable { respErr.Store(err) - s.cleanupGRPCStream(streams, cleanupFuncs, i) + cleanupGRPCStream(streams, cleanupFuncs, i) return } re.NoError(err) resp, err := streams[i].Recv() if err != nil && !mustReliable { respErr.Store(err) - s.cleanupGRPCStream(streams, cleanupFuncs, i) + cleanupGRPCStream(streams, cleanupFuncs, i) return } re.NoError(err) @@ -495,7 +495,7 @@ func benchmarkTSOProxyNClients(clientCount int, b *testing.B) { } b.StopTimer() - suite.cleanupGRPCStreams(cleanupFuncs) + cleanupGRPCStreams(cleanupFuncs) suite.TearDownSuite() } diff --git a/tests/integrations/realcluster/Makefile b/tests/integrations/realcluster/Makefile index 278f585feaac..e161d52a86e4 100644 --- a/tests/integrations/realcluster/Makefile +++ b/tests/integrations/realcluster/Makefile @@ -22,8 +22,6 @@ static: install-tools @ gofmt -s -l -d . 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }' @ echo "golangci-lint ..." @ golangci-lint run -c $(ROOT_PATH)/.golangci.yml --verbose ./... --allow-parallel-runners - @ echo "revive ..." - @ revive -formatter friendly -config $(ROOT_PATH)/revive.toml ./... tidy: @ go mod tidy diff --git a/tests/registry/registry_test.go b/tests/registry/registry_test.go index dab2ccae6831..416a7420d2eb 100644 --- a/tests/registry/registry_test.go +++ b/tests/registry/registry_test.go @@ -41,18 +41,18 @@ func TestMain(m *testing.M) { type testServiceRegistry struct { } -func (t *testServiceRegistry) RegisterGRPCService(g *grpc.Server) { +func (*testServiceRegistry) RegisterGRPCService(g *grpc.Server) { grpc_testing.RegisterTestServiceServer(g, &grpc_testing.UnimplementedTestServiceServer{}) } -func (t *testServiceRegistry) RegisterRESTHandler(userDefineHandlers map[string]http.Handler) { +func (*testServiceRegistry) RegisterRESTHandler(userDefineHandlers map[string]http.Handler) { group := apiutil.APIServiceGroup{ Name: "my-http-service", Version: "v1alpha1", IsCore: false, PathPrefix: "/my-service", } - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte("Hello World!")) }) diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go index b70c688993d6..98e458f6d179 100644 --- a/tests/server/api/api_test.go +++ b/tests/server/api/api_test.go @@ -51,7 +51,7 @@ func TestReconnect(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, serverName string) { + cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, _ string) { conf.TickInterval = typeutil.Duration{Duration: 50 * time.Millisecond} conf.ElectionInterval = typeutil.Duration{Duration: 250 * time.Millisecond} }) @@ -577,7 +577,7 @@ func (suite *redirectorTestSuite) SetupSuite() { re := suite.Require() ctx, cancel := context.WithCancel(context.Background()) suite.cleanup = cancel - cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, serverName string) { + cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, _ string) { conf.TickInterval = typeutil.Duration{Duration: 50 * time.Millisecond} conf.ElectionInterval = typeutil.Duration{Duration: 250 * time.Millisecond} }) @@ -703,7 +703,7 @@ func TestRemovingProgress(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, serverName string) { + cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, _ string) { conf.Replication.MaxReplicas = 1 }) re.NoError(err) @@ -820,7 +820,7 @@ func TestSendApiWhenRestartRaftCluster(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, serverName string) { + cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, _ string) { conf.Replication.MaxReplicas = 1 }) re.NoError(err) @@ -862,7 +862,7 @@ func TestPreparingProgress(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, serverName string) { + cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, _ string) { conf.Replication.MaxReplicas = 1 }) re.NoError(err) diff --git a/tests/server/api/checker_test.go b/tests/server/api/checker_test.go index 198cfca216f0..0304d7fd369b 100644 --- a/tests/server/api/checker_test.go +++ b/tests/server/api/checker_test.go @@ -49,7 +49,7 @@ func (suite *checkerTestSuite) TestAPI() { func (suite *checkerTestSuite) checkAPI(cluster *tests.TestCluster) { re := suite.Require() - suite.testErrCases(re, cluster) + testErrCases(re, cluster) testCases := []struct { name string @@ -62,12 +62,12 @@ func (suite *checkerTestSuite) checkAPI(cluster *tests.TestCluster) { {name: "joint-state"}, } for _, testCase := range testCases { - suite.testGetStatus(re, cluster, testCase.name) - suite.testPauseOrResume(re, cluster, testCase.name) + testGetStatus(re, cluster, testCase.name) + testPauseOrResume(re, cluster, testCase.name) } } -func (suite *checkerTestSuite) testErrCases(re *require.Assertions, cluster *tests.TestCluster) { +func testErrCases(re *require.Assertions, cluster *tests.TestCluster) { urlPrefix := fmt.Sprintf("%s/pd/api/v1/checker", cluster.GetLeaderServer().GetAddr()) // missing args input := make(map[string]any) @@ -97,7 +97,7 @@ func (suite *checkerTestSuite) testErrCases(re *require.Assertions, cluster *tes re.NoError(err) } -func (suite *checkerTestSuite) testGetStatus(re *require.Assertions, cluster *tests.TestCluster, name string) { +func testGetStatus(re *require.Assertions, cluster *tests.TestCluster, name string) { input := make(map[string]any) urlPrefix := fmt.Sprintf("%s/pd/api/v1/checker", cluster.GetLeaderServer().GetAddr()) // normal run @@ -128,7 +128,7 @@ func (suite *checkerTestSuite) testGetStatus(re *require.Assertions, cluster *te re.False(resp["paused"].(bool)) } -func (suite *checkerTestSuite) testPauseOrResume(re *require.Assertions, cluster *tests.TestCluster, name string) { +func testPauseOrResume(re *require.Assertions, cluster *tests.TestCluster, name string) { input := make(map[string]any) urlPrefix := fmt.Sprintf("%s/pd/api/v1/checker", cluster.GetLeaderServer().GetAddr()) resp := make(map[string]any) diff --git a/tests/server/api/operator_test.go b/tests/server/api/operator_test.go index 32ca4ea300d8..a5cd865b454d 100644 --- a/tests/server/api/operator_test.go +++ b/tests/server/api/operator_test.go @@ -26,7 +26,6 @@ import ( "time" "github.com/pingcap/kvproto/pkg/metapb" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/operator" @@ -56,7 +55,7 @@ func TestOperatorTestSuite(t *testing.T) { func (suite *operatorTestSuite) SetupSuite() { suite.env = tests.NewSchedulingTestEnvironment(suite.T(), - func(conf *config.Config, serverName string) { + func(conf *config.Config, _ string) { conf.Replication.MaxReplicas = 1 }) } @@ -71,7 +70,7 @@ func (suite *operatorTestSuite) TestAddRemovePeer() { func (suite *operatorTestSuite) checkAddRemovePeer(cluster *tests.TestCluster) { re := suite.Require() - suite.pauseRuleChecker(re, cluster) + pauseRuleChecker(re, cluster) stores := []*metapb.Store{ { Id: 1, @@ -206,7 +205,7 @@ func (suite *operatorTestSuite) checkMergeRegionOperator(cluster *tests.TestClus tests.MustPutStore(re, cluster, store) } - suite.pauseRuleChecker(re, cluster) + pauseRuleChecker(re, cluster) r1 := core.NewTestRegionInfo(10, 1, []byte(""), []byte("b"), core.SetWrittenBytes(1000), core.SetReadBytes(1000), core.SetRegionConfVer(1), core.SetRegionVersion(1)) tests.MustPutRegionInfo(re, cluster, r1) r2 := core.NewTestRegionInfo(20, 1, []byte("b"), []byte("c"), core.SetWrittenBytes(2000), core.SetReadBytes(0), core.SetRegionConfVer(2), core.SetRegionVersion(3)) @@ -233,7 +232,7 @@ func (suite *operatorTestSuite) checkMergeRegionOperator(cluster *tests.TestClus func (suite *operatorTestSuite) TestTransferRegionWithPlacementRule() { // use a new environment to avoid affecting other tests env := tests.NewSchedulingTestEnvironment(suite.T(), - func(conf *config.Config, serverName string) { + func(conf *config.Config, _ string) { conf.Replication.MaxReplicas = 3 }) env.RunTestInTwoModes(suite.checkTransferRegionWithPlacementRule) @@ -242,7 +241,7 @@ func (suite *operatorTestSuite) TestTransferRegionWithPlacementRule() { func (suite *operatorTestSuite) checkTransferRegionWithPlacementRule(cluster *tests.TestCluster) { re := suite.Require() - suite.pauseRuleChecker(re, cluster) + pauseRuleChecker(re, cluster) stores := []*metapb.Store{ { Id: 1, @@ -513,7 +512,7 @@ func (suite *operatorTestSuite) checkTransferRegionWithPlacementRule(cluster *te func (suite *operatorTestSuite) TestGetOperatorsAsObject() { // use a new environment to avoid being affected by other tests env := tests.NewSchedulingTestEnvironment(suite.T(), - func(conf *config.Config, serverName string) { + func(conf *config.Config, _ string) { conf.Replication.MaxReplicas = 1 }) env.RunTestInTwoModes(suite.checkGetOperatorsAsObject) @@ -522,7 +521,7 @@ func (suite *operatorTestSuite) TestGetOperatorsAsObject() { func (suite *operatorTestSuite) checkGetOperatorsAsObject(cluster *tests.TestCluster) { re := suite.Require() - suite.pauseRuleChecker(re, cluster) + pauseRuleChecker(re, cluster) stores := []*metapb.Store{ { Id: 1, @@ -612,19 +611,6 @@ func (suite *operatorTestSuite) checkGetOperatorsAsObject(cluster *tests.TestClu re.Equal("admin-add-peer", resp[2].Desc) } -// pauseRuleChecker will pause rule checker to avoid unexpected operator. -func (suite *operatorTestSuite) pauseRuleChecker(re *require.Assertions, cluster *tests.TestCluster) { - checkerName := "rule" - addr := cluster.GetLeaderServer().GetAddr() - resp := make(map[string]any) - url := fmt.Sprintf("%s/pd/api/v1/checker/%s", addr, checkerName) - err := tu.CheckPostJSON(testDialClient, url, []byte(`{"delay":1000}`), tu.StatusOK(re)) - re.NoError(err) - err = tu.ReadGetJSON(re, testDialClient, url, &resp) - re.NoError(err) - re.True(resp["paused"].(bool)) -} - func (suite *operatorTestSuite) TestRemoveOperators() { suite.env.RunTestInTwoModes(suite.checkRemoveOperators) } @@ -656,7 +642,7 @@ func (suite *operatorTestSuite) checkRemoveOperators(cluster *tests.TestCluster) tests.MustPutStore(re, cluster, store) } - suite.pauseRuleChecker(re, cluster) + pauseRuleChecker(re, cluster) r1 := core.NewTestRegionInfo(10, 1, []byte(""), []byte("b"), core.SetWrittenBytes(1000), core.SetReadBytes(1000), core.SetRegionConfVer(1), core.SetRegionVersion(1)) tests.MustPutRegionInfo(re, cluster, r1) r2 := core.NewTestRegionInfo(20, 1, []byte("b"), []byte("c"), core.SetWrittenBytes(2000), core.SetReadBytes(0), core.SetRegionConfVer(2), core.SetRegionVersion(3)) diff --git a/tests/server/api/region_test.go b/tests/server/api/region_test.go index 8c286dc12e2c..b233ce94a993 100644 --- a/tests/server/api/region_test.go +++ b/tests/server/api/region_test.go @@ -114,14 +114,14 @@ func (suite *regionTestSuite) checkSplitRegions(cluster *tests.TestCluster) { r1 := core.NewTestRegionInfo(601, 13, []byte("aaa"), []byte("ggg")) r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 5, StoreId: 14}, &metapb.Peer{Id: 6, StoreId: 15}) tests.MustPutRegionInfo(re, cluster, r1) - suite.checkRegionCount(re, cluster, 1) + checkRegionCount(re, cluster, 1) newRegionID := uint64(11) body := fmt.Sprintf(`{"retry_limit":%v, "split_keys": ["%s","%s","%s"]}`, 3, hex.EncodeToString([]byte("bbb")), hex.EncodeToString([]byte("ccc")), hex.EncodeToString([]byte("ddd"))) - checkOpt := func(res []byte, code int, _ http.Header) { + checkOpt := func(res []byte, _ int, _ http.Header) { s := &struct { ProcessedPercentage int `json:"processed-percentage"` NewRegionsID []uint64 `json:"regions-id"` @@ -159,7 +159,7 @@ func (suite *regionTestSuite) checkAccelerateRegionsScheduleInRange(cluster *tes r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 100 + i, StoreId: (i + 1) % regionCount}, &metapb.Peer{Id: 200 + i, StoreId: (i + 2) % regionCount}) tests.MustPutRegionInfo(re, cluster, r1) } - suite.checkRegionCount(re, cluster, regionCount) + checkRegionCount(re, cluster, regionCount) body := fmt.Sprintf(`{"start_key":"%s", "end_key": "%s"}`, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a3"))) err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/accelerate-schedule", urlPrefix), []byte(body), @@ -194,7 +194,7 @@ func (suite *regionTestSuite) checkAccelerateRegionsScheduleInRanges(cluster *te r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 100 + i, StoreId: (i + 1) % regionCount}, &metapb.Peer{Id: 200 + i, StoreId: (i + 2) % regionCount}) tests.MustPutRegionInfo(re, cluster, r1) } - suite.checkRegionCount(re, cluster, regionCount) + checkRegionCount(re, cluster, regionCount) body := fmt.Sprintf(`[{"start_key":"%s", "end_key": "%s"}, {"start_key":"%s", "end_key": "%s"}]`, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a3")), hex.EncodeToString([]byte("a4")), hex.EncodeToString([]byte("a6"))) @@ -236,7 +236,7 @@ func (suite *regionTestSuite) checkScatterRegions(cluster *tests.TestCluster) { tests.MustPutRegionInfo(re, cluster, r1) tests.MustPutRegionInfo(re, cluster, r2) tests.MustPutRegionInfo(re, cluster, r3) - suite.checkRegionCount(re, cluster, 3) + checkRegionCount(re, cluster, 3) body := fmt.Sprintf(`{"start_key":"%s", "end_key": "%s"}`, hex.EncodeToString([]byte("b1")), hex.EncodeToString([]byte("b3"))) err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/scatter", urlPrefix), []byte(body), tu.StatusOK(re)) @@ -263,7 +263,7 @@ func (suite *regionTestSuite) TestCheckRegionsReplicated() { func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) { re := suite.Require() - suite.pauseRuleChecker(re, cluster) + pauseRuleChecker(re, cluster) leader := cluster.GetLeaderServer() urlPrefix := leader.GetAddr() + "/pd/api/v1" @@ -276,7 +276,7 @@ func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) tests.MustPutStore(re, cluster, s1) r1 := core.NewTestRegionInfo(2, 1, []byte("a"), []byte("b")) tests.MustPutRegionInfo(re, cluster, r1) - suite.checkRegionCount(re, cluster, 1) + checkRegionCount(re, cluster, 1) // set the bundle bundle := []placement.GroupBundle{ @@ -404,7 +404,7 @@ func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) }) } -func (suite *regionTestSuite) checkRegionCount(re *require.Assertions, cluster *tests.TestCluster, count uint64) { +func checkRegionCount(re *require.Assertions, cluster *tests.TestCluster, count uint64) { leader := cluster.GetLeaderServer() tu.Eventually(re, func() bool { return leader.GetRaftCluster().GetRegionCount([]byte{}, []byte{}).Count == int(count) @@ -417,7 +417,7 @@ func (suite *regionTestSuite) checkRegionCount(re *require.Assertions, cluster * } // pauseRuleChecker will pause rule checker to avoid unexpected operator. -func (suite *regionTestSuite) pauseRuleChecker(re *require.Assertions, cluster *tests.TestCluster) { +func pauseRuleChecker(re *require.Assertions, cluster *tests.TestCluster) { checkerName := "rule" addr := cluster.GetLeaderServer().GetAddr() resp := make(map[string]any) diff --git a/tests/server/api/rule_test.go b/tests/server/api/rule_test.go index 83ab0f1cebb9..4f60b5cfb281 100644 --- a/tests/server/api/rule_test.go +++ b/tests/server/api/rule_test.go @@ -49,7 +49,7 @@ func TestRuleTestSuite(t *testing.T) { } func (suite *ruleTestSuite) SetupSuite() { - suite.env = tests.NewSchedulingTestEnvironment(suite.T(), func(conf *config.Config, serverName string) { + suite.env = tests.NewSchedulingTestEnvironment(suite.T(), func(conf *config.Config, _ string) { conf.PDServerCfg.KeyType = "raw" conf.Replication.EnablePlacementRules = true }) @@ -235,7 +235,7 @@ func (suite *ruleTestSuite) checkGet(cluster *tests.TestCluster) { if testCase.found { tu.Eventually(re, func() bool { err = tu.ReadGetJSON(re, testDialClient, url, &resp) - return suite.compareRule(&resp, &testCase.rule) + return compareRule(&resp, &testCase.rule) }) } else { err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, testCase.code)) @@ -432,7 +432,7 @@ func (suite *ruleTestSuite) checkGetAllByGroup(cluster *tests.TestCluster) { return false } if testCase.count == 2 { - return suite.compareRule(resp[0], &rule) && suite.compareRule(resp[1], &rule1) + return compareRule(resp[0], &rule) && compareRule(resp[1], &rule1) } return true }) @@ -492,7 +492,7 @@ func (suite *ruleTestSuite) checkGetAllByRegion(cluster *tests.TestCluster) { err = tu.ReadGetJSON(re, testDialClient, url, &resp) for _, r := range resp { if r.GroupID == "e" { - return suite.compareRule(r, &rule) + return compareRule(r, &rule) } } return true @@ -780,7 +780,7 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { }, }, } - suite.assertBundlesEqual(re, urlPrefix+"/placement-rule", []placement.GroupBundle{b1}, 1) + assertBundlesEqual(re, urlPrefix+"/placement-rule", []placement.GroupBundle{b1}, 1) // Set b2 := placement.GroupBundle{ @@ -797,17 +797,17 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { re.NoError(err) // Get - suite.assertBundleEqual(re, urlPrefix+"/placement-rule/foo", b2) + assertBundleEqual(re, urlPrefix+"/placement-rule/foo", b2) // GetAll again - suite.assertBundlesEqual(re, urlPrefix+"/placement-rule", []placement.GroupBundle{b1, b2}, 2) + assertBundlesEqual(re, urlPrefix+"/placement-rule", []placement.GroupBundle{b1, b2}, 2) // Delete err = tu.CheckDelete(testDialClient, urlPrefix+"/placement-rule/pd", tu.StatusOK(re)) re.NoError(err) // GetAll again - suite.assertBundlesEqual(re, urlPrefix+"/placement-rule", []placement.GroupBundle{b2}, 1) + assertBundlesEqual(re, urlPrefix+"/placement-rule", []placement.GroupBundle{b2}, 1) // SetAll b2.Rules = append(b2.Rules, &placement.Rule{GroupID: "foo", ID: "baz", Index: 2, Role: placement.Follower, Count: 1}) @@ -819,14 +819,14 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { re.NoError(err) // GetAll again - suite.assertBundlesEqual(re, urlPrefix+"/placement-rule", []placement.GroupBundle{b1, b2, b3}, 3) + assertBundlesEqual(re, urlPrefix+"/placement-rule", []placement.GroupBundle{b1, b2, b3}, 3) // Delete using regexp err = tu.CheckDelete(testDialClient, urlPrefix+"/placement-rule/"+url.PathEscape("foo.*")+"?regexp", tu.StatusOK(re)) re.NoError(err) // GetAll again - suite.assertBundlesEqual(re, urlPrefix+"/placement-rule", []placement.GroupBundle{b1}, 1) + assertBundlesEqual(re, urlPrefix+"/placement-rule", []placement.GroupBundle{b1}, 1) // Set id := "rule-without-group-id" @@ -844,10 +844,10 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { b4.ID = id b4.Rules[0].GroupID = b4.ID // Get - suite.assertBundleEqual(re, urlPrefix+"/placement-rule/"+id, b4) + assertBundleEqual(re, urlPrefix+"/placement-rule/"+id, b4) // GetAll again - suite.assertBundlesEqual(re, urlPrefix+"/placement-rule", []placement.GroupBundle{b1, b4}, 2) + assertBundlesEqual(re, urlPrefix+"/placement-rule", []placement.GroupBundle{b1, b4}, 2) // SetAll b5 := placement.GroupBundle{ @@ -865,7 +865,7 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { b5.Rules[0].GroupID = b5.ID // GetAll again - suite.assertBundlesEqual(re, urlPrefix+"/placement-rule", []placement.GroupBundle{b1, b4, b5}, 3) + assertBundlesEqual(re, urlPrefix+"/placement-rule", []placement.GroupBundle{b1, b4, b5}, 3) } func (suite *ruleTestSuite) TestBundleBadRequest() { @@ -1194,18 +1194,18 @@ func (suite *ruleTestSuite) checkLargeRules(cluster *tests.TestCluster) { suite.postAndCheckRuleBundle(urlPrefix, genBundlesWithRulesNum(etcdutil.MaxEtcdTxnOps*2)) } -func (suite *ruleTestSuite) assertBundleEqual(re *require.Assertions, url string, expectedBundle placement.GroupBundle) { +func assertBundleEqual(re *require.Assertions, url string, expectedBundle placement.GroupBundle) { var bundle placement.GroupBundle tu.Eventually(re, func() bool { err := tu.ReadGetJSON(re, testDialClient, url, &bundle) if err != nil { return false } - return suite.compareBundle(bundle, expectedBundle) + return compareBundle(bundle, expectedBundle) }) } -func (suite *ruleTestSuite) assertBundlesEqual(re *require.Assertions, url string, expectedBundles []placement.GroupBundle, expectedLen int) { +func assertBundlesEqual(re *require.Assertions, url string, expectedBundles []placement.GroupBundle, expectedLen int) { var bundles []placement.GroupBundle tu.Eventually(re, func() bool { err := tu.ReadGetJSON(re, testDialClient, url, &bundles) @@ -1218,7 +1218,7 @@ func (suite *ruleTestSuite) assertBundlesEqual(re *require.Assertions, url strin sort.Slice(bundles, func(i, j int) bool { return bundles[i].ID < bundles[j].ID }) sort.Slice(expectedBundles, func(i, j int) bool { return expectedBundles[i].ID < expectedBundles[j].ID }) for i := range bundles { - if !suite.compareBundle(bundles[i], expectedBundles[i]) { + if !compareBundle(bundles[i], expectedBundles[i]) { return false } } @@ -1226,21 +1226,21 @@ func (suite *ruleTestSuite) assertBundlesEqual(re *require.Assertions, url strin }) } -func (suite *ruleTestSuite) compareBundle(b1, b2 placement.GroupBundle) bool { +func compareBundle(b1, b2 placement.GroupBundle) bool { if b2.ID != b1.ID || b2.Index != b1.Index || b2.Override != b1.Override || len(b2.Rules) != len(b1.Rules) { return false } sort.Slice(b1.Rules, func(i, j int) bool { return b1.Rules[i].ID < b1.Rules[j].ID }) sort.Slice(b2.Rules, func(i, j int) bool { return b2.Rules[i].ID < b2.Rules[j].ID }) for i := range b1.Rules { - if !suite.compareRule(b1.Rules[i], b2.Rules[i]) { + if !compareRule(b1.Rules[i], b2.Rules[i]) { return false } } return true } -func (suite *ruleTestSuite) compareRule(r1 *placement.Rule, r2 *placement.Rule) bool { +func compareRule(r1 *placement.Rule, r2 *placement.Rule) bool { return r2.GroupID == r1.GroupID && r2.ID == r1.ID && r2.StartKeyHex == r1.StartKeyHex && @@ -1267,7 +1267,7 @@ func (suite *ruleTestSuite) postAndCheckRuleBundle(urlPrefix string, bundle []pl sort.Slice(respBundle, func(i, j int) bool { return respBundle[i].ID < respBundle[j].ID }) sort.Slice(bundle, func(i, j int) bool { return bundle[i].ID < bundle[j].ID }) for i := range respBundle { - if !suite.compareBundle(respBundle[i], bundle[i]) { + if !compareBundle(respBundle[i], bundle[i]) { return false } } @@ -1285,7 +1285,7 @@ func TestRegionRuleTestSuite(t *testing.T) { } func (suite *regionRuleTestSuite) SetupSuite() { - suite.env = tests.NewSchedulingTestEnvironment(suite.T(), func(conf *config.Config, serverName string) { + suite.env = tests.NewSchedulingTestEnvironment(suite.T(), func(conf *config.Config, _ string) { conf.Replication.EnablePlacementRules = true conf.Replication.MaxReplicas = 1 }) @@ -1396,14 +1396,14 @@ func (suite *regionRuleTestSuite) checkRegionPlacementRule(cluster *tests.TestCl re.Equal("keyspaces/0", labels[0].ID) u = fmt.Sprintf("%s/config/region-label/rules/ids", urlPrefix) - err = tu.CheckGetJSON(testDialClient, u, []byte(`["rule1", "rule3"]`), func(resp []byte, statusCode int, _ http.Header) { + err = tu.CheckGetJSON(testDialClient, u, []byte(`["rule1", "rule3"]`), func(resp []byte, _ int, _ http.Header) { err := json.Unmarshal(resp, &labels) re.NoError(err) re.Empty(labels) }) re.NoError(err) - err = tu.CheckGetJSON(testDialClient, u, []byte(`["keyspaces/0"]`), func(resp []byte, statusCode int, _ http.Header) { + err = tu.CheckGetJSON(testDialClient, u, []byte(`["keyspaces/0"]`), func(resp []byte, _ int, _ http.Header) { err := json.Unmarshal(resp, &labels) re.NoError(err) re.Len(labels, 1) diff --git a/tests/server/api/scheduler_test.go b/tests/server/api/scheduler_test.go index 2329077209dd..4f71315803a1 100644 --- a/tests/server/api/scheduler_test.go +++ b/tests/server/api/scheduler_test.go @@ -123,7 +123,7 @@ func (suite *scheduleTestSuite) checkOriginAPI(cluster *tests.TestCluster) { re.NoError(failpoint.Disable("github.com/tikv/pd/server/config/persistFail")) err = tu.CheckDelete(testDialClient, deleteURL, tu.StatusOK(re)) re.NoError(err) - suite.assertNoScheduler(re, urlPrefix, "evict-leader-scheduler") + assertNoScheduler(re, urlPrefix, "evict-leader-scheduler") re.NoError(tu.CheckGetJSON(testDialClient, listURL, nil, tu.Status(re, http.StatusNotFound))) err = tu.CheckDelete(testDialClient, deleteURL, tu.Status(re, http.StatusNotFound)) re.NoError(err) @@ -531,8 +531,8 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { if testCase.extraTestFunc != nil { testCase.extraTestFunc(testCase.createdName) } - suite.deleteScheduler(re, urlPrefix, testCase.createdName) - suite.assertNoScheduler(re, urlPrefix, testCase.createdName) + deleteScheduler(re, urlPrefix, testCase.createdName) + assertNoScheduler(re, urlPrefix, testCase.createdName) } // test pause and resume all schedulers. @@ -546,7 +546,7 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { } body, err := json.Marshal(input) re.NoError(err) - suite.addScheduler(re, urlPrefix, body) + addScheduler(re, urlPrefix, body) suite.assertSchedulerExists(urlPrefix, testCase.createdName) // wait for scheduler to be synced. if testCase.extraTestFunc != nil { testCase.extraTestFunc(testCase.createdName) @@ -566,7 +566,7 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { if createdName == "" { createdName = testCase.name } - isPaused := suite.isSchedulerPaused(re, urlPrefix, createdName) + isPaused := isSchedulerPaused(re, urlPrefix, createdName) re.True(isPaused) } input["delay"] = 1 @@ -580,7 +580,7 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { if createdName == "" { createdName = testCase.name } - isPaused := suite.isSchedulerPaused(re, urlPrefix, createdName) + isPaused := isSchedulerPaused(re, urlPrefix, createdName) re.False(isPaused) } @@ -600,7 +600,7 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { if createdName == "" { createdName = testCase.name } - isPaused := suite.isSchedulerPaused(re, urlPrefix, createdName) + isPaused := isSchedulerPaused(re, urlPrefix, createdName) re.False(isPaused) } @@ -610,8 +610,8 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { if createdName == "" { createdName = testCase.name } - suite.deleteScheduler(re, urlPrefix, createdName) - suite.assertNoScheduler(re, urlPrefix, createdName) + deleteScheduler(re, urlPrefix, createdName) + assertNoScheduler(re, urlPrefix, createdName) } } @@ -638,7 +638,7 @@ func (suite *scheduleTestSuite) checkDisable(cluster *tests.TestCluster) { input["name"] = name body, err := json.Marshal(input) re.NoError(err) - suite.addScheduler(re, urlPrefix, body) + addScheduler(re, urlPrefix, body) u := fmt.Sprintf("%s%s/api/v1/config/schedule", leaderAddr, apiPrefix) var scheduleConfig sc.ScheduleConfig @@ -652,7 +652,7 @@ func (suite *scheduleTestSuite) checkDisable(cluster *tests.TestCluster) { err = tu.CheckPostJSON(testDialClient, u, body, tu.StatusOK(re)) re.NoError(err) - suite.assertNoScheduler(re, urlPrefix, name) + assertNoScheduler(re, urlPrefix, name) suite.assertSchedulerExists(fmt.Sprintf("%s?status=disabled", urlPrefix), name) // reset schedule config @@ -662,16 +662,16 @@ func (suite *scheduleTestSuite) checkDisable(cluster *tests.TestCluster) { err = tu.CheckPostJSON(testDialClient, u, body, tu.StatusOK(re)) re.NoError(err) - suite.deleteScheduler(re, urlPrefix, name) - suite.assertNoScheduler(re, urlPrefix, name) + deleteScheduler(re, urlPrefix, name) + assertNoScheduler(re, urlPrefix, name) } -func (suite *scheduleTestSuite) addScheduler(re *require.Assertions, urlPrefix string, body []byte) { +func addScheduler(re *require.Assertions, urlPrefix string, body []byte) { err := tu.CheckPostJSON(testDialClient, urlPrefix, body, tu.StatusOK(re)) re.NoError(err) } -func (suite *scheduleTestSuite) deleteScheduler(re *require.Assertions, urlPrefix string, createdName string) { +func deleteScheduler(re *require.Assertions, urlPrefix string, createdName string) { deleteURL := fmt.Sprintf("%s/%s", urlPrefix, createdName) err := tu.CheckDelete(testDialClient, deleteURL, tu.StatusOK(re)) re.NoError(err) @@ -696,7 +696,7 @@ func (suite *scheduleTestSuite) testPauseOrResume(re *require.Assertions, urlPre re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/"+createdName, pauseArgs, tu.StatusOK(re)) re.NoError(err) - isPaused := suite.isSchedulerPaused(re, urlPrefix, createdName) + isPaused := isSchedulerPaused(re, urlPrefix, createdName) re.True(isPaused) input["delay"] = 1 pauseArgs, err = json.Marshal(input) @@ -704,7 +704,7 @@ func (suite *scheduleTestSuite) testPauseOrResume(re *require.Assertions, urlPre err = tu.CheckPostJSON(testDialClient, urlPrefix+"/"+createdName, pauseArgs, tu.StatusOK(re)) re.NoError(err) time.Sleep(time.Second * 2) - isPaused = suite.isSchedulerPaused(re, urlPrefix, createdName) + isPaused = isSchedulerPaused(re, urlPrefix, createdName) re.False(isPaused) // test resume. @@ -719,7 +719,7 @@ func (suite *scheduleTestSuite) testPauseOrResume(re *require.Assertions, urlPre re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/"+createdName, pauseArgs, tu.StatusOK(re)) re.NoError(err) - isPaused = suite.isSchedulerPaused(re, urlPrefix, createdName) + isPaused = isSchedulerPaused(re, urlPrefix, createdName) re.False(isPaused) } @@ -749,9 +749,9 @@ func (suite *scheduleTestSuite) checkEmptySchedulers(cluster *tests.TestCluster) input["name"] = scheduler body, err := json.Marshal(input) re.NoError(err) - suite.addScheduler(re, urlPrefix, body) + addScheduler(re, urlPrefix, body) } else { - suite.deleteScheduler(re, urlPrefix, scheduler) + deleteScheduler(re, urlPrefix, scheduler) } } tu.Eventually(re, func() bool { @@ -777,7 +777,7 @@ func (suite *scheduleTestSuite) assertSchedulerExists(urlPrefix string, schedule }) } -func (suite *scheduleTestSuite) assertNoScheduler(re *require.Assertions, urlPrefix string, scheduler string) { +func assertNoScheduler(re *require.Assertions, urlPrefix string, scheduler string) { var schedulers []string tu.Eventually(re, func() bool { err := tu.ReadGetJSON(re, testDialClient, urlPrefix, &schedulers, @@ -787,7 +787,7 @@ func (suite *scheduleTestSuite) assertNoScheduler(re *require.Assertions, urlPre }) } -func (suite *scheduleTestSuite) isSchedulerPaused(re *require.Assertions, urlPrefix, name string) bool { +func isSchedulerPaused(re *require.Assertions, urlPrefix, name string) bool { var schedulers []string err := tu.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s?status=paused", urlPrefix), &schedulers, tu.StatusOK(re)) diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index 3415c22a77b2..aea5ff739683 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -824,7 +824,7 @@ func TestSetScheduleOpt(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() // TODO: enable placementrules - tc, err := tests.NewTestCluster(ctx, 1, func(cfg *config.Config, svr string) { cfg.Replication.EnablePlacementRules = false }) + tc, err := tests.NewTestCluster(ctx, 1, func(cfg *config.Config, _ string) { cfg.Replication.EnablePlacementRules = false }) defer tc.Destroy() re.NoError(err) @@ -985,7 +985,7 @@ func TestTiFlashWithPlacementRules(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tc, err := tests.NewTestCluster(ctx, 1, func(cfg *config.Config, name string) { cfg.Replication.EnablePlacementRules = false }) + tc, err := tests.NewTestCluster(ctx, 1, func(cfg *config.Config, _ string) { cfg.Replication.EnablePlacementRules = false }) defer tc.Destroy() re.NoError(err) err = tc.RunInitialServers() @@ -1035,7 +1035,7 @@ func TestReplicationModeStatus(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tc, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, serverName string) { + tc, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, _ string) { conf.ReplicationMode.ReplicationMode = "dr-auto-sync" }) diff --git a/tests/server/config/config_test.go b/tests/server/config/config_test.go index 108bc5fc7539..b6fcecbd47b1 100644 --- a/tests/server/config/config_test.go +++ b/tests/server/config/config_test.go @@ -451,7 +451,7 @@ type ttlConfigInterface interface { IsTikvRegionSplitEnabled() bool } -func (suite *configTestSuite) assertTTLConfig( +func assertTTLConfig( re *require.Assertions, cluster *tests.TestCluster, expectedEqual bool, @@ -488,7 +488,7 @@ func (suite *configTestSuite) assertTTLConfig( } } -func (suite *configTestSuite) assertTTLConfigItemEqual( +func assertTTLConfigItemEqual( re *require.Assertions, cluster *tests.TestCluster, item string, @@ -532,22 +532,22 @@ func (suite *configTestSuite) checkConfigTTL(cluster *tests.TestCluster) { // test no config and cleaning up err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 0), postData, tu.StatusOK(re)) re.NoError(err) - suite.assertTTLConfig(re, cluster, false) + assertTTLConfig(re, cluster, false) // test time goes by err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 5), postData, tu.StatusOK(re)) re.NoError(err) - suite.assertTTLConfig(re, cluster, true) + assertTTLConfig(re, cluster, true) time.Sleep(5 * time.Second) - suite.assertTTLConfig(re, cluster, false) + assertTTLConfig(re, cluster, false) // test cleaning up err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 5), postData, tu.StatusOK(re)) re.NoError(err) - suite.assertTTLConfig(re, cluster, true) + assertTTLConfig(re, cluster, true) err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 0), postData, tu.StatusOK(re)) re.NoError(err) - suite.assertTTLConfig(re, cluster, false) + assertTTLConfig(re, cluster, false) postData, err = json.Marshal(invalidTTLConfig) re.NoError(err) @@ -564,9 +564,9 @@ func (suite *configTestSuite) checkConfigTTL(cluster *tests.TestCluster) { err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 1), postData, tu.StatusOK(re)) re.NoError(err) - suite.assertTTLConfigItemEqual(re, cluster, "max-merge-region-size", uint64(999)) + assertTTLConfigItemEqual(re, cluster, "max-merge-region-size", uint64(999)) // max-merge-region-keys should keep consistence with max-merge-region-size. - suite.assertTTLConfigItemEqual(re, cluster, "max-merge-region-keys", uint64(999*10000)) + assertTTLConfigItemEqual(re, cluster, "max-merge-region-keys", uint64(999*10000)) // on invalid value, we use default config mergeConfig = map[string]any{ @@ -576,7 +576,7 @@ func (suite *configTestSuite) checkConfigTTL(cluster *tests.TestCluster) { re.NoError(err) err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 10), postData, tu.StatusOK(re)) re.NoError(err) - suite.assertTTLConfigItemEqual(re, cluster, "enable-tikv-split-region", true) + assertTTLConfigItemEqual(re, cluster, "enable-tikv-split-region", true) } func (suite *configTestSuite) TestTTLConflict() { @@ -592,7 +592,7 @@ func (suite *configTestSuite) checkTTLConflict(cluster *tests.TestCluster) { re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) re.NoError(err) - suite.assertTTLConfig(re, cluster, true) + assertTTLConfig(re, cluster, true) cfg := map[string]any{"max-snapshot-count": 30} postData, err = json.Marshal(cfg) diff --git a/tests/server/join/join_test.go b/tests/server/join/join_test.go index 5cdcbc090b8d..32e66c275898 100644 --- a/tests/server/join/join_test.go +++ b/tests/server/join/join_test.go @@ -105,7 +105,7 @@ func TestFailedAndDeletedPDJoinsPreviousCluster(t *testing.T) { re.NoError(err) // The server should not successfully start. - res := cluster.RunServer(pd3) + res := tests.RunServer(pd3) re.Error(<-res) members, err := etcdutil.ListEtcdMembers(ctx, client) @@ -138,7 +138,7 @@ func TestDeletedPDJoinsPreviousCluster(t *testing.T) { re.NoError(err) // The server should not successfully start. - res := cluster.RunServer(pd3) + res := tests.RunServer(pd3) re.Error(<-res) members, err := etcdutil.ListEtcdMembers(ctx, client) diff --git a/tests/server/keyspace/keyspace_test.go b/tests/server/keyspace/keyspace_test.go index aa2e89296bb7..d6e188359ce2 100644 --- a/tests/server/keyspace/keyspace_test.go +++ b/tests/server/keyspace/keyspace_test.go @@ -53,7 +53,7 @@ func (suite *keyspaceTestSuite) SetupTest() { re := suite.Require() ctx, cancel := context.WithCancel(context.Background()) suite.cancel = cancel - cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, serverName string) { + cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = preAllocKeyspace }) suite.cluster = cluster diff --git a/tests/server/member/member_test.go b/tests/server/member/member_test.go index 7aadc2772e8a..92ed11a75cee 100644 --- a/tests/server/member/member_test.go +++ b/tests/server/member/member_test.go @@ -152,7 +152,7 @@ func TestLeaderPriority(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, serverName string) { + cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, _ string) { conf.LeaderPriorityCheckInterval = typeutil.NewDuration(time.Second) }) defer cluster.Destroy() diff --git a/tests/server/region_syncer/region_syncer_test.go b/tests/server/region_syncer/region_syncer_test.go index 1470173e0ed7..f82346571ef1 100644 --- a/tests/server/region_syncer/region_syncer_test.go +++ b/tests/server/region_syncer/region_syncer_test.go @@ -40,7 +40,7 @@ func TestRegionSyncer(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/syncer/noFastExitSync", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/syncer/disableClientStreaming", `return(true)`)) - cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, serverName string) { conf.PDServerCfg.UseRegionStorage = true }) + cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, _ string) { conf.PDServerCfg.UseRegionStorage = true }) defer func() { cluster.Destroy() cancel() @@ -163,7 +163,7 @@ func TestFullSyncWithAddMember(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, serverName string) { conf.PDServerCfg.UseRegionStorage = true }) + cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, _ string) { conf.PDServerCfg.UseRegionStorage = true }) defer cluster.Destroy() re.NoError(err) @@ -207,7 +207,7 @@ func TestPrepareChecker(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/changeCoordinatorTicker", `return(true)`)) - cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, serverName string) { conf.PDServerCfg.UseRegionStorage = true }) + cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, _ string) { conf.PDServerCfg.UseRegionStorage = true }) defer cluster.Destroy() re.NoError(err) @@ -256,7 +256,7 @@ func TestPrepareCheckerWithTransferLeader(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/changeCoordinatorTicker", `return(true)`)) - cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, serverName string) { conf.PDServerCfg.UseRegionStorage = true }) + cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, _ string) { conf.PDServerCfg.UseRegionStorage = true }) defer cluster.Destroy() re.NoError(err) diff --git a/tests/server/server_test.go b/tests/server/server_test.go index 3b85cd3cf0d6..adf7202454b5 100644 --- a/tests/server/server_test.go +++ b/tests/server/server_test.go @@ -98,7 +98,7 @@ func TestClusterID(t *testing.T) { re.Equal(clusterID, s.GetClusterID()) } - cluster2, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, serverName string) { conf.InitialClusterToken = "foobar" }) + cluster2, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, _ string) { conf.InitialClusterToken = "foobar" }) defer cluster2.Destroy() re.NoError(err) err = cluster2.RunInitialServers() diff --git a/tests/server/storage/hot_region_storage_test.go b/tests/server/storage/hot_region_storage_test.go index 12110be0249e..b63b533bc0f5 100644 --- a/tests/server/storage/hot_region_storage_test.go +++ b/tests/server/storage/hot_region_storage_test.go @@ -37,7 +37,7 @@ func TestHotRegionStorage(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() cluster, err := tests.NewTestCluster(ctx, 1, - func(cfg *config.Config, serverName string) { + func(cfg *config.Config, _ string) { cfg.Schedule.HotRegionCacheHitsThreshold = 0 cfg.Schedule.HotRegionsWriteInterval.Duration = 1000 * time.Millisecond cfg.Schedule.HotRegionsReservedDays = 1 @@ -145,7 +145,7 @@ func TestHotRegionStorageReservedDayConfigChange(t *testing.T) { interval := 100 * time.Millisecond defer cancel() cluster, err := tests.NewTestCluster(ctx, 1, - func(cfg *config.Config, serverName string) { + func(cfg *config.Config, _ string) { cfg.Schedule.HotRegionCacheHitsThreshold = 0 cfg.Schedule.HotRegionsWriteInterval.Duration = interval cfg.Schedule.HotRegionsReservedDays = 1 @@ -237,7 +237,7 @@ func TestHotRegionStorageWriteIntervalConfigChange(t *testing.T) { interval := 100 * time.Millisecond defer cancel() cluster, err := tests.NewTestCluster(ctx, 1, - func(cfg *config.Config, serverName string) { + func(cfg *config.Config, _ string) { cfg.Schedule.HotRegionCacheHitsThreshold = 0 cfg.Schedule.HotRegionsWriteInterval.Duration = interval cfg.Schedule.HotRegionsReservedDays = 1 diff --git a/tests/server/tso/allocator_test.go b/tests/server/tso/allocator_test.go index 3bc4d56ac581..692aec490eb2 100644 --- a/tests/server/tso/allocator_test.go +++ b/tests/server/tso/allocator_test.go @@ -132,7 +132,7 @@ func TestPriorityAndDifferentLocalTSO(t *testing.T) { time.Sleep(time.Second * 5) // Join a new dc-location - pd4, err := cluster.Join(ctx, func(conf *config.Config, serverName string) { + pd4, err := cluster.Join(ctx, func(conf *config.Config, _ string) { conf.EnableLocalTSO = true conf.Labels[config.ZoneLabel] = "dc-4" }) diff --git a/tests/server/tso/consistency_test.go b/tests/server/tso/consistency_test.go index d1c45df7f17e..1bf20cce20d6 100644 --- a/tests/server/tso/consistency_test.go +++ b/tests/server/tso/consistency_test.go @@ -275,7 +275,7 @@ func (suite *tsoConsistencyTestSuite) TestLocalTSOAfterMemberChanged() { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/systemTimeSlow", `return(true)`)) // Join a new dc-location - pd4, err := cluster.Join(suite.ctx, func(conf *config.Config, serverName string) { + pd4, err := cluster.Join(suite.ctx, func(conf *config.Config, _ string) { conf.EnableLocalTSO = true conf.Labels[config.ZoneLabel] = "dc-4" }) diff --git a/tests/server/tso/global_tso_test.go b/tests/server/tso/global_tso_test.go index 5ae2e6e0f675..f705bdf12b55 100644 --- a/tests/server/tso/global_tso_test.go +++ b/tests/server/tso/global_tso_test.go @@ -137,7 +137,7 @@ func TestLogicalOverflow(t *testing.T) { runCase := func(updateInterval time.Duration) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, serverName string) { + cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, _ string) { conf.TSOUpdatePhysicalInterval = typeutil.Duration{Duration: updateInterval} }) defer cluster.Destroy() diff --git a/tests/server/watch/leader_watch_test.go b/tests/server/watch/leader_watch_test.go index f77652970236..84e16398677b 100644 --- a/tests/server/watch/leader_watch_test.go +++ b/tests/server/watch/leader_watch_test.go @@ -35,7 +35,7 @@ func TestWatcher(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, serverName string) { conf.AutoCompactionRetention = "1s" }) + cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, _ string) { conf.AutoCompactionRetention = "1s" }) defer cluster.Destroy() re.NoError(err) @@ -73,7 +73,7 @@ func TestWatcherCompacted(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, serverName string) { conf.AutoCompactionRetention = "1s" }) + cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, _ string) { conf.AutoCompactionRetention = "1s" }) defer cluster.Destroy() re.NoError(err) diff --git a/tests/testutil.go b/tests/testutil.go index 106cddc9dfb2..5d9905af64c9 100644 --- a/tests/testutil.go +++ b/tests/testutil.go @@ -273,14 +273,14 @@ func (s *SchedulingTestEnvironment) RunTestInTwoModes(test func(*TestCluster)) { // RunTestInPDMode is to run test in pd mode. func (s *SchedulingTestEnvironment) RunTestInPDMode(test func(*TestCluster)) { - s.t.Logf("start test %s in pd mode", s.getTestName()) + s.t.Logf("start test %s in pd mode", getTestName()) if _, ok := s.clusters[pdMode]; !ok { s.startCluster(pdMode) } test(s.clusters[pdMode]) } -func (s *SchedulingTestEnvironment) getTestName() string { +func getTestName() string { pc, _, _, _ := runtime.Caller(2) caller := runtime.FuncForPC(pc) if caller == nil || strings.Contains(caller.Name(), "RunTestInTwoModes") { @@ -303,7 +303,7 @@ func (s *SchedulingTestEnvironment) RunTestInAPIMode(test func(*TestCluster)) { re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/scheduling/server/fastUpdateMember")) re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) }() - s.t.Logf("start test %s in api mode", s.getTestName()) + s.t.Logf("start test %s in api mode", getTestName()) if _, ok := s.clusters[apiMode]; !ok { s.startCluster(apiMode) } diff --git a/tests/tso_cluster.go b/tests/tso_cluster.go index 4021613df2aa..e1fdb6d69cad 100644 --- a/tests/tso_cluster.go +++ b/tests/tso_cluster.go @@ -76,7 +76,7 @@ func RestartTestTSOCluster( defer wg.Done() clean() serverCfg := cluster.servers[addr].GetConfig() - newServer, newCleanup, err := NewTSOTestServer(newCluster.ctx, serverCfg) + newServer, newCleanup, err := NewTSOTestServer(ctx, serverCfg) serverMap.Store(addr, newServer) cleanupMap.Store(addr, newCleanup) errorMap.Store(addr, err) diff --git a/tools.go b/tools.go index 909f42ab9b54..e5298de2827c 100644 --- a/tools.go +++ b/tools.go @@ -20,7 +20,6 @@ package tools import ( _ "github.com/AlekSi/gocov-xml" _ "github.com/axw/gocov/gocov" - _ "github.com/mgechev/revive" _ "github.com/pingcap/errors/errdoc-gen" _ "github.com/pingcap/failpoint/failpoint-ctl" _ "github.com/swaggo/swag/cmd/swag" diff --git a/tools/Makefile b/tools/Makefile index 336cc536949e..4195160aff6d 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -25,8 +25,6 @@ static: install-tools @ gofmt -s -l -d . 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }' @ echo "golangci-lint ..." @ golangci-lint run -c $(ROOT_PATH)/.golangci.yml --verbose ./... --allow-parallel-runners - @ echo "revive ..." - @ revive -formatter friendly -config $(ROOT_PATH)/revive.toml ./... tidy: @ go mod tidy diff --git a/tools/pd-analysis/analysis/parse_log.go b/tools/pd-analysis/analysis/parse_log.go index 44ae617284f9..f096e3fe380a 100644 --- a/tools/pd-analysis/analysis/parse_log.go +++ b/tools/pd-analysis/analysis/parse_log.go @@ -42,7 +42,7 @@ type Interpreter interface { } // CompileRegex is to provide regexp for transfer counter. -func (c *TransferCounter) CompileRegex(operator string) (*regexp.Regexp, error) { +func (*TransferCounter) CompileRegex(operator string) (*regexp.Regexp, error) { var r *regexp.Regexp var err error @@ -64,7 +64,7 @@ func (c *TransferCounter) CompileRegex(operator string) (*regexp.Regexp, error) return r, err } -func (c *TransferCounter) parseLine(content string, r *regexp.Regexp) ([]uint64, error) { +func parseLine(content string, r *regexp.Regexp) ([]uint64, error) { results := make([]uint64, 0, 4) subStrings := r.FindStringSubmatch(content) if len(subStrings) == 0 { @@ -78,9 +78,8 @@ func (c *TransferCounter) parseLine(content string, r *regexp.Regexp) ([]uint64, results = append(results, uint64(num)) } return results, nil - } else { - return results, errors.New("Can't parse Log, with " + content) } + return results, errors.New("Can't parse Log, with " + content) } func forEachLine(filename string, solve func(string) error) error { @@ -116,7 +115,7 @@ func forEachLine(filename string, solve func(string) error) error { func isExpectTime(expect, layout string, isBeforeThanExpect bool) func(time.Time) bool { expectTime, err := time.Parse(layout, expect) if err != nil { - return func(current time.Time) bool { + return func(_ time.Time) bool { return true } } @@ -142,14 +141,13 @@ func currentTime(layout string) func(content string) (time.Time, error) { return time.Parse(layout, result[1]) } else if len(result) == 0 { return time.Time{}, nil - } else { - return time.Time{}, errors.New("There is no valid time in log with " + content) } + return time.Time{}, errors.New("There is no valid time in log with " + content) } } // ParseLog is to parse log for transfer counter. -func (c *TransferCounter) ParseLog(filename, start, end, layout string, r *regexp.Regexp) error { +func (*TransferCounter) ParseLog(filename, start, end, layout string, r *regexp.Regexp) error { afterStart := isExpectTime(start, layout, false) beforeEnd := isExpectTime(end, layout, true) getCurrent := currentTime(layout) @@ -161,7 +159,7 @@ func (c *TransferCounter) ParseLog(filename, start, end, layout string, r *regex } // if current line time between start and end if afterStart(current) && beforeEnd(current) { - results, err := c.parseLine(content, r) + results, err := parseLine(content, r) if err != nil { return err } diff --git a/tools/pd-analysis/analysis/parse_log_test.go b/tools/pd-analysis/analysis/parse_log_test.go index ffdcb2137c0e..345f70959f8a 100644 --- a/tools/pd-analysis/analysis/parse_log_test.go +++ b/tools/pd-analysis/analysis/parse_log_test.go @@ -23,7 +23,7 @@ import ( func transferCounterParseLog(operator, content string, expect []uint64) bool { r, _ := GetTransferCounter().CompileRegex(operator) - results, _ := GetTransferCounter().parseLine(content, r) + results, _ := parseLine(content, r) if len(results) != len(expect) { return false } diff --git a/tools/pd-api-bench/cases/cases.go b/tools/pd-api-bench/cases/cases.go index 473a11d749ad..72986df5ed8d 100644 --- a/tools/pd-api-bench/cases/cases.go +++ b/tools/pd-api-bench/cases/cases.go @@ -37,6 +37,8 @@ var ( storesID []uint64 ) +const defaultKeyLen = 56 + // InitCluster initializes the cluster. func InitCluster(ctx context.Context, cli pd.Client, httpCli pdHttp.Client) error { statsResp, err := httpCli.GetRegionStatusByKeyRange(ctx, pdHttp.NewKeyRange([]byte(""), []byte("")), false) @@ -221,7 +223,7 @@ func (c *regionsStats) Do(ctx context.Context, cli pdHttp.Client) error { startID := c.regionSample*random*4 + 1 endID := c.regionSample*(random+1)*4 + 1 regionStats, err := cli.GetRegionStatusByKeyRange(ctx, - pdHttp.NewKeyRange(generateKeyForSimulator(startID, 56), generateKeyForSimulator(endID, 56)), false) + pdHttp.NewKeyRange(generateKeyForSimulator(startID), generateKeyForSimulator(endID)), false) if Debug { log.Info("do HTTP case", zap.String("case", c.name), zap.Any("region-stats", regionStats), zap.Error(err)) } @@ -246,7 +248,7 @@ func newUpdateGCSafePoint() func() GRPCCase { } } -func (c *updateGCSafePoint) Unary(ctx context.Context, cli pd.Client) error { +func (*updateGCSafePoint) Unary(ctx context.Context, cli pd.Client) error { s := time.Now().Unix() _, err := cli.UpdateGCSafePoint(ctx, uint64(s)) if err != nil { @@ -270,7 +272,7 @@ func newUpdateServiceGCSafePoint() func() GRPCCase { } } -func (c *updateServiceGCSafePoint) Unary(ctx context.Context, cli pd.Client) error { +func (*updateServiceGCSafePoint) Unary(ctx context.Context, cli pd.Client) error { s := time.Now().Unix() id := rand.Int63n(100) + 1 _, err := cli.UpdateServiceGCSafePoint(ctx, strconv.FormatInt(id, 10), id, uint64(s)) @@ -295,9 +297,9 @@ func newGetRegion() func() GRPCCase { } } -func (c *getRegion) Unary(ctx context.Context, cli pd.Client) error { +func (*getRegion) Unary(ctx context.Context, cli pd.Client) error { id := rand.Intn(totalRegion)*4 + 1 - _, err := cli.GetRegion(ctx, generateKeyForSimulator(id, 56)) + _, err := cli.GetRegion(ctx, generateKeyForSimulator(id)) if err != nil { return err } @@ -319,9 +321,9 @@ func newGetRegionEnableFollower() func() GRPCCase { } } -func (c *getRegionEnableFollower) Unary(ctx context.Context, cli pd.Client) error { +func (*getRegionEnableFollower) Unary(ctx context.Context, cli pd.Client) error { id := rand.Intn(totalRegion)*4 + 1 - _, err := cli.GetRegion(ctx, generateKeyForSimulator(id, 56), pd.WithAllowFollowerHandle()) + _, err := cli.GetRegion(ctx, generateKeyForSimulator(id), pd.WithAllowFollowerHandle()) if err != nil { return err } @@ -350,7 +352,7 @@ func (c *scanRegions) Unary(ctx context.Context, cli pd.Client) error { random := rand.Intn(upperBound) startID := c.regionSample*random*4 + 1 endID := c.regionSample*(random+1)*4 + 1 - _, err := cli.ScanRegions(ctx, generateKeyForSimulator(startID, 56), generateKeyForSimulator(endID, 56), c.regionSample) + _, err := cli.ScanRegions(ctx, generateKeyForSimulator(startID), generateKeyForSimulator(endID), c.regionSample) if err != nil { return err } @@ -372,7 +374,7 @@ func newTso() func() GRPCCase { } } -func (c *tso) Unary(ctx context.Context, cli pd.Client) error { +func (*tso) Unary(ctx context.Context, cli pd.Client) error { _, _, err := cli.GetTS(ctx) if err != nil { return err @@ -395,7 +397,7 @@ func newGetStore() func() GRPCCase { } } -func (c *getStore) Unary(ctx context.Context, cli pd.Client) error { +func (*getStore) Unary(ctx context.Context, cli pd.Client) error { storeIdx := rand.Intn(totalStore) _, err := cli.GetStore(ctx, storesID[storeIdx]) if err != nil { @@ -419,7 +421,7 @@ func newGetStores() func() GRPCCase { } } -func (c *getStores) Unary(ctx context.Context, cli pd.Client) error { +func (*getStores) Unary(ctx context.Context, cli pd.Client) error { _, err := cli.GetAllStores(ctx) if err != nil { return err @@ -427,9 +429,8 @@ func (c *getStores) Unary(ctx context.Context, cli pd.Client) error { return nil } -// nolint -func generateKeyForSimulator(id int, keyLen int) []byte { - k := make([]byte, keyLen) +func generateKeyForSimulator(id int) []byte { + k := make([]byte, defaultKeyLen) copy(k, fmt.Sprintf("%010d", id)) return k } @@ -449,7 +450,7 @@ func newGetKV() func() ETCDCase { } } -func (c *getKV) Init(ctx context.Context, cli *clientv3.Client) error { +func (*getKV) Init(ctx context.Context, cli *clientv3.Client) error { for i := 0; i < 100; i++ { _, err := cli.Put(ctx, fmt.Sprintf("/test/0001/%4d", i), fmt.Sprintf("%4d", i)) if err != nil { @@ -459,7 +460,7 @@ func (c *getKV) Init(ctx context.Context, cli *clientv3.Client) error { return nil } -func (c *getKV) Unary(ctx context.Context, cli *clientv3.Client) error { +func (*getKV) Unary(ctx context.Context, cli *clientv3.Client) error { _, err := cli.Get(ctx, "/test/0001", clientv3.WithPrefix()) return err } @@ -479,9 +480,9 @@ func newPutKV() func() ETCDCase { } } -func (c *putKV) Init(ctx context.Context, cli *clientv3.Client) error { return nil } +func (*putKV) Init(context.Context, *clientv3.Client) error { return nil } -func (c *putKV) Unary(ctx context.Context, cli *clientv3.Client) error { +func (*putKV) Unary(ctx context.Context, cli *clientv3.Client) error { _, err := cli.Put(ctx, "/test/0001/0000", "test") return err } @@ -501,9 +502,9 @@ func newDeleteKV() func() ETCDCase { } } -func (c *deleteKV) Init(ctx context.Context, cli *clientv3.Client) error { return nil } +func (*deleteKV) Init(context.Context, *clientv3.Client) error { return nil } -func (c *deleteKV) Unary(ctx context.Context, cli *clientv3.Client) error { +func (*deleteKV) Unary(ctx context.Context, cli *clientv3.Client) error { _, err := cli.Delete(ctx, "/test/0001/0000") return err } @@ -523,9 +524,9 @@ func newTxnKV() func() ETCDCase { } } -func (c *txnKV) Init(ctx context.Context, cli *clientv3.Client) error { return nil } +func (*txnKV) Init(context.Context, *clientv3.Client) error { return nil } -func (c *txnKV) Unary(ctx context.Context, cli *clientv3.Client) error { +func (*txnKV) Unary(ctx context.Context, cli *clientv3.Client) error { txn := cli.Txn(ctx) txn = txn.If(clientv3.Compare(clientv3.Value("/test/0001/0000"), "=", "test")) txn = txn.Then(clientv3.OpPut("/test/0001/0000", "test2")) diff --git a/tools/pd-api-bench/cases/controller.go b/tools/pd-api-bench/cases/controller.go index d6679cad1d92..42eeafe4597f 100644 --- a/tools/pd-api-bench/cases/controller.go +++ b/tools/pd-api-bench/cases/controller.go @@ -64,7 +64,7 @@ func (c *Coordinator) GetHTTPCase(name string) (*Config, error) { if controller, ok := c.http[name]; ok { return controller.GetConfig(), nil } - return nil, errors.Errorf("case %v does not exist.", name) + return nil, errors.Errorf("case %v does not exist", name) } // GetGRPCCase returns the gRPC case config. @@ -74,7 +74,7 @@ func (c *Coordinator) GetGRPCCase(name string) (*Config, error) { if controller, ok := c.grpc[name]; ok { return controller.GetConfig(), nil } - return nil, errors.Errorf("case %v does not exist.", name) + return nil, errors.Errorf("case %v does not exist", name) } // GetETCDCase returns the etcd case config. @@ -84,7 +84,7 @@ func (c *Coordinator) GetETCDCase(name string) (*Config, error) { if controller, ok := c.etcd[name]; ok { return controller.GetConfig(), nil } - return nil, errors.Errorf("case %v does not exist.", name) + return nil, errors.Errorf("case %v does not exist", name) } // GetAllHTTPCases returns the all HTTP case configs. diff --git a/tools/pd-api-bench/config/config.go b/tools/pd-api-bench/config/config.go index 675e665ab0a9..d1048c0da727 100644 --- a/tools/pd-api-bench/config/config.go +++ b/tools/pd-api-bench/config/config.go @@ -15,7 +15,6 @@ package config import ( - "github.com/BurntSushi/toml" "github.com/pingcap/log" "github.com/pkg/errors" flag "github.com/spf13/pflag" @@ -73,14 +72,13 @@ func (c *Config) Parse(arguments []string) error { } // Load config file if specified. - var meta *toml.MetaData if c.configFile != "" { - meta, err = configutil.ConfigFromFile(c, c.configFile) + _, err = configutil.ConfigFromFile(c, c.configFile) if err != nil { return err } } - c.Adjust(meta) + c.Adjust() // Parse again to replace with command line options. err = c.flagSet.Parse(arguments) @@ -118,7 +116,7 @@ func (c *Config) InitCoordinator(co *cases.Coordinator) { } // Adjust is used to adjust configurations -func (c *Config) Adjust(meta *toml.MetaData) { +func (c *Config) Adjust() { if len(c.Log.Format) == 0 { c.Log.Format = "text" } diff --git a/tools/pd-api-bench/main.go b/tools/pd-api-bench/main.go index dff40555fd69..f9feeeea5807 100644 --- a/tools/pd-api-bench/main.go +++ b/tools/pd-api-bench/main.go @@ -341,7 +341,6 @@ func runHTTPServer(cfg *config.Config, co *cases.Coordinator) { } c.IndentedJSON(http.StatusOK, cfg) }) - // nolint engine.Run(cfg.StatusAddr) } diff --git a/tools/pd-backup/pdbackup/backup_test.go b/tools/pd-backup/pdbackup/backup_test.go index b35bf1e8a70d..0ab9116ddbe0 100644 --- a/tools/pd-backup/pdbackup/backup_test.go +++ b/tools/pd-backup/pdbackup/backup_test.go @@ -83,7 +83,7 @@ func setupServer() (*httptest.Server, *config.Config) { }, } - server := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + server := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, _ *http.Request) { b, err := json.Marshal(serverConfig) if err != nil { res.WriteHeader(http.StatusInternalServerError) @@ -98,7 +98,7 @@ func setupServer() (*httptest.Server, *config.Config) { return server, serverConfig } -func (s *backupTestSuite) BeforeTest(suiteName, testName string) { +func (s *backupTestSuite) BeforeTest(string, string) { re := s.Require() ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) defer cancel() @@ -124,7 +124,7 @@ func (s *backupTestSuite) BeforeTest(suiteName, testName string) { re.NoError(err) } -func (s *backupTestSuite) AfterTest(suiteName, testName string) { +func (s *backupTestSuite) AfterTest(string, string) { s.etcd.Close() } diff --git a/tools/pd-ctl/pdctl/command/config_command.go b/tools/pd-ctl/pdctl/command/config_command.go index c70c33e26c32..0c3851350ccf 100644 --- a/tools/pd-ctl/pdctl/command/config_command.go +++ b/tools/pd-ctl/pdctl/command/config_command.go @@ -212,7 +212,7 @@ func NewDeleteLabelPropertyConfigCommand() *cobra.Command { return sc } -func showConfigCommandFunc(cmd *cobra.Command, args []string) { +func showConfigCommandFunc(cmd *cobra.Command, _ []string) { header := buildHeader(cmd) allR, err := doRequest(cmd, configPrefix, http.MethodGet, header) if err != nil { @@ -268,7 +268,7 @@ var hideConfig = []string{ "scheduler-max-waiting-operator", } -func showScheduleConfigCommandFunc(cmd *cobra.Command, args []string) { +func showScheduleConfigCommandFunc(cmd *cobra.Command, _ []string) { header := buildHeader(cmd) r, err := doRequest(cmd, schedulePrefix, http.MethodGet, header) if err != nil { @@ -278,7 +278,7 @@ func showScheduleConfigCommandFunc(cmd *cobra.Command, args []string) { cmd.Println(r) } -func showReplicationConfigCommandFunc(cmd *cobra.Command, args []string) { +func showReplicationConfigCommandFunc(cmd *cobra.Command, _ []string) { header := buildHeader(cmd) r, err := doRequest(cmd, replicatePrefix, http.MethodGet, header) if err != nil { @@ -288,7 +288,7 @@ func showReplicationConfigCommandFunc(cmd *cobra.Command, args []string) { cmd.Println(r) } -func showLabelPropertyConfigCommandFunc(cmd *cobra.Command, args []string) { +func showLabelPropertyConfigCommandFunc(cmd *cobra.Command, _ []string) { r, err := doRequest(cmd, labelPropertyPrefix, http.MethodGet, http.Header{}) if err != nil { cmd.Printf("Failed to get config: %s\n", err) @@ -297,7 +297,7 @@ func showLabelPropertyConfigCommandFunc(cmd *cobra.Command, args []string) { cmd.Println(r) } -func showAllConfigCommandFunc(cmd *cobra.Command, args []string) { +func showAllConfigCommandFunc(cmd *cobra.Command, _ []string) { header := buildHeader(cmd) r, err := doRequest(cmd, configPrefix, http.MethodGet, header) if err != nil { @@ -307,7 +307,7 @@ func showAllConfigCommandFunc(cmd *cobra.Command, args []string) { cmd.Println(r) } -func showClusterVersionCommandFunc(cmd *cobra.Command, args []string) { +func showClusterVersionCommandFunc(cmd *cobra.Command, _ []string) { r, err := doRequest(cmd, clusterVersionPrefix, http.MethodGet, http.Header{}) if err != nil { cmd.Printf("Failed to get cluster version: %s\n", err) @@ -316,7 +316,7 @@ func showClusterVersionCommandFunc(cmd *cobra.Command, args []string) { cmd.Println(r) } -func showReplicationModeCommandFunc(cmd *cobra.Command, args []string) { +func showReplicationModeCommandFunc(cmd *cobra.Command, _ []string) { r, err := doRequest(cmd, replicationModePrefix, http.MethodGet, http.Header{}) if err != nil { cmd.Printf("Failed to get replication mode config: %s\n", err) @@ -325,7 +325,7 @@ func showReplicationModeCommandFunc(cmd *cobra.Command, args []string) { cmd.Println(r) } -func showServerCommandFunc(cmd *cobra.Command, args []string) { +func showServerCommandFunc(cmd *cobra.Command, _ []string) { r, err := doRequest(cmd, pdServerPrefix, http.MethodGet, http.Header{}) if err != nil { cmd.Printf("Failed to get server config: %s\n", err) @@ -529,7 +529,7 @@ func NewPlacementRulesCommand() *cobra.Command { return c } -func enablePlacementRulesFunc(cmd *cobra.Command, args []string) { +func enablePlacementRulesFunc(cmd *cobra.Command, _ []string) { err := postConfigDataWithPath(cmd, "enable-placement-rules", "true", configPrefix) if err != nil { cmd.Printf("Failed to set config: %s\n", err) @@ -538,7 +538,7 @@ func enablePlacementRulesFunc(cmd *cobra.Command, args []string) { cmd.Println("Success!") } -func disablePlacementRulesFunc(cmd *cobra.Command, args []string) { +func disablePlacementRulesFunc(cmd *cobra.Command, _ []string) { err := postConfigDataWithPath(cmd, "enable-placement-rules", "false", configPrefix) if err != nil { cmd.Printf("Failed to set config: %s\n", err) @@ -547,7 +547,7 @@ func disablePlacementRulesFunc(cmd *cobra.Command, args []string) { cmd.Println("Success!") } -func getPlacementRulesFunc(cmd *cobra.Command, args []string) { +func getPlacementRulesFunc(cmd *cobra.Command, _ []string) { getFlag := func(key string) string { if f := cmd.Flag(key); f != nil { return f.Value.String() @@ -598,7 +598,7 @@ func getPlacementRulesFunc(cmd *cobra.Command, args []string) { cmd.Println("rules saved to file " + file) } -func putPlacementRulesFunc(cmd *cobra.Command, args []string) { +func putPlacementRulesFunc(cmd *cobra.Command, _ []string) { var file string if f := cmd.Flag("in"); f != nil { file = f.Value.String() @@ -712,7 +712,7 @@ func getRuleBundle(cmd *cobra.Command, args []string) { cmd.Printf("rule group saved to file %s\n", file) } -func setRuleBundle(cmd *cobra.Command, args []string) { +func setRuleBundle(cmd *cobra.Command, _ []string) { var file string if f := cmd.Flag("in"); f != nil { file = f.Value.String() @@ -763,7 +763,7 @@ func delRuleBundle(cmd *cobra.Command, args []string) { cmd.Println(res) } -func loadRuleBundle(cmd *cobra.Command, args []string) { +func loadRuleBundle(cmd *cobra.Command, _ []string) { header := buildHeader(cmd) res, err := doRequest(cmd, ruleBundlePrefix, http.MethodGet, header) if err != nil { @@ -788,7 +788,7 @@ func loadRuleBundle(cmd *cobra.Command, args []string) { cmd.Printf("rule group saved to file %s\n", file) } -func saveRuleBundle(cmd *cobra.Command, args []string) { +func saveRuleBundle(cmd *cobra.Command, _ []string) { var file string if f := cmd.Flag("in"); f != nil { file = f.Value.String() diff --git a/tools/pd-ctl/pdctl/command/exit_command.go b/tools/pd-ctl/pdctl/command/exit_command.go index a3d38be97bd0..3ead7e54e8e1 100644 --- a/tools/pd-ctl/pdctl/command/exit_command.go +++ b/tools/pd-ctl/pdctl/command/exit_command.go @@ -30,6 +30,6 @@ func NewExitCommand() *cobra.Command { return conf } -func exitCommandFunc(cmd *cobra.Command, args []string) { +func exitCommandFunc(*cobra.Command, []string) { os.Exit(0) } diff --git a/tools/pd-ctl/pdctl/command/gc_safepoint_command.go b/tools/pd-ctl/pdctl/command/gc_safepoint_command.go index 80c6328e9555..f4a6b6fcfd03 100644 --- a/tools/pd-ctl/pdctl/command/gc_safepoint_command.go +++ b/tools/pd-ctl/pdctl/command/gc_safepoint_command.go @@ -49,7 +49,7 @@ func NewDeleteServiceGCSafepointCommand() *cobra.Command { return l } -func showSSPs(cmd *cobra.Command, args []string) { +func showSSPs(cmd *cobra.Command, _ []string) { r, err := doRequest(cmd, serviceGCSafepointPrefix, http.MethodGet, http.Header{}) if err != nil { cmd.Printf("Failed to get service GC safepoint: %s\n", err) diff --git a/tools/pd-ctl/pdctl/command/global.go b/tools/pd-ctl/pdctl/command/global.go index 4f20b0b35b41..fa77df6a1013 100644 --- a/tools/pd-ctl/pdctl/command/global.go +++ b/tools/pd-ctl/pdctl/command/global.go @@ -126,7 +126,7 @@ var dialClient = &http.Client{ } // RequireHTTPSClient creates a HTTPS client if the related flags are set -func RequireHTTPSClient(cmd *cobra.Command, args []string) error { +func RequireHTTPSClient(cmd *cobra.Command, _ []string) error { caPath, err := cmd.Flags().GetString("cacert") if err == nil && len(caPath) != 0 { certPath, err := cmd.Flags().GetString("cert") diff --git a/tools/pd-ctl/pdctl/command/health_command.go b/tools/pd-ctl/pdctl/command/health_command.go index 1bae871285dd..50ac7763d28d 100644 --- a/tools/pd-ctl/pdctl/command/health_command.go +++ b/tools/pd-ctl/pdctl/command/health_command.go @@ -34,7 +34,7 @@ func NewHealthCommand() *cobra.Command { return m } -func showHealthCommandFunc(cmd *cobra.Command, args []string) { +func showHealthCommandFunc(cmd *cobra.Command, _ []string) { r, err := doRequest(cmd, healthPrefix, http.MethodGet, http.Header{}) if err != nil { cmd.Println(err) diff --git a/tools/pd-ctl/pdctl/command/hot_command.go b/tools/pd-ctl/pdctl/command/hot_command.go index f6be9c7176b0..77c0ee4d7de3 100644 --- a/tools/pd-ctl/pdctl/command/hot_command.go +++ b/tools/pd-ctl/pdctl/command/hot_command.go @@ -107,7 +107,7 @@ func NewHotStoreCommand() *cobra.Command { return cmd } -func showHotStoresCommandFunc(cmd *cobra.Command, args []string) { +func showHotStoresCommandFunc(cmd *cobra.Command, _ []string) { r, err := doRequest(cmd, hotStoresPrefix, http.MethodGet, http.Header{}) if err != nil { cmd.Printf("Failed to get store hotspot: %s\n", err) diff --git a/tools/pd-ctl/pdctl/command/label_command.go b/tools/pd-ctl/pdctl/command/label_command.go index c0ae3135210a..6d95465392fc 100644 --- a/tools/pd-ctl/pdctl/command/label_command.go +++ b/tools/pd-ctl/pdctl/command/label_command.go @@ -53,7 +53,7 @@ func NewLabelListStoresCommand() *cobra.Command { return l } -func showLabelsCommandFunc(cmd *cobra.Command, args []string) { +func showLabelsCommandFunc(cmd *cobra.Command, _ []string) { r, err := doRequest(cmd, labelsPrefix, http.MethodGet, http.Header{}) if err != nil { cmd.Printf("Failed to get labels: %s\n", err) diff --git a/tools/pd-ctl/pdctl/command/member_command.go b/tools/pd-ctl/pdctl/command/member_command.go index c16a879429c5..b939935cfb96 100644 --- a/tools/pd-ctl/pdctl/command/member_command.go +++ b/tools/pd-ctl/pdctl/command/member_command.go @@ -89,7 +89,7 @@ func NewLeaderMemberCommand() *cobra.Command { return d } -func showMemberCommandFunc(cmd *cobra.Command, args []string) { +func showMemberCommandFunc(cmd *cobra.Command, _ []string) { r, err := doRequest(cmd, membersPrefix, http.MethodGet, http.Header{}) if err != nil { cmd.Printf("Failed to get pd members: %s\n", err) @@ -126,7 +126,7 @@ func deleteMemberByIDCommandFunc(cmd *cobra.Command, args []string) { cmd.Println("Success!") } -func getLeaderMemberCommandFunc(cmd *cobra.Command, args []string) { +func getLeaderMemberCommandFunc(cmd *cobra.Command, _ []string) { r, err := doRequest(cmd, leaderMemberPrefix, http.MethodGet, http.Header{}) if err != nil { cmd.Printf("Failed to get the leader of pd members: %s\n", err) @@ -135,7 +135,7 @@ func getLeaderMemberCommandFunc(cmd *cobra.Command, args []string) { cmd.Println(r) } -func resignLeaderCommandFunc(cmd *cobra.Command, args []string) { +func resignLeaderCommandFunc(cmd *cobra.Command, _ []string) { prefix := leaderMemberPrefix + "/resign" _, err := doRequest(cmd, prefix, http.MethodPost, http.Header{}) if err != nil { diff --git a/tools/pd-ctl/pdctl/command/min_resolved_ts.go b/tools/pd-ctl/pdctl/command/min_resolved_ts.go index dbf0c47b2de7..904f880d82dd 100644 --- a/tools/pd-ctl/pdctl/command/min_resolved_ts.go +++ b/tools/pd-ctl/pdctl/command/min_resolved_ts.go @@ -35,7 +35,7 @@ func NewMinResolvedTSCommand() *cobra.Command { } // ShowMinResolvedTS show min resolved ts -func ShowMinResolvedTS(cmd *cobra.Command, args []string) { +func ShowMinResolvedTS(cmd *cobra.Command, _ []string) { r, err := doRequest(cmd, minResolvedTSPrefix, http.MethodGet, http.Header{}) if err != nil { cmd.Printf("Failed to get min resolved ts: %s\n", err) diff --git a/tools/pd-ctl/pdctl/command/operator.go b/tools/pd-ctl/pdctl/command/operator.go index c57e07db75ab..4e7771580de2 100644 --- a/tools/pd-ctl/pdctl/command/operator.go +++ b/tools/pd-ctl/pdctl/command/operator.go @@ -375,7 +375,6 @@ func splitRegionCommandFunc(cmd *cobra.Command, args []string) { policy := cmd.Flags().Lookup("policy").Value.String() switch policy { case "scan", "approximate", "usekey": - break default: cmd.Println("Error: unknown policy") return diff --git a/tools/pd-ctl/pdctl/command/ping_command.go b/tools/pd-ctl/pdctl/command/ping_command.go index 6622b079d479..7efa46180d11 100644 --- a/tools/pd-ctl/pdctl/command/ping_command.go +++ b/tools/pd-ctl/pdctl/command/ping_command.go @@ -33,7 +33,7 @@ func NewPingCommand() *cobra.Command { return m } -func showPingCommandFunc(cmd *cobra.Command, args []string) { +func showPingCommandFunc(cmd *cobra.Command, _ []string) { start := time.Now() _, err := doRequest(cmd, pingPrefix, http.MethodGet, http.Header{}) if err != nil { diff --git a/tools/pd-ctl/pdctl/command/region_command.go b/tools/pd-ctl/pdctl/command/region_command.go index 33191bbe12b6..e03de1c62ac4 100644 --- a/tools/pd-ctl/pdctl/command/region_command.go +++ b/tools/pd-ctl/pdctl/command/region_command.go @@ -156,7 +156,7 @@ func showRegionCommandFunc(cmd *cobra.Command, args []string) { cmd.Println(r) } -func scanRegionCommandFunc(cmd *cobra.Command, args []string) { +func scanRegionCommandFunc(cmd *cobra.Command, _ []string) { const limit = 1024 var key []byte for { @@ -533,7 +533,7 @@ func NewRangesWithRangeHolesCommand() *cobra.Command { return r } -func showRangesWithRangeHolesCommandFunc(cmd *cobra.Command, args []string) { +func showRangesWithRangeHolesCommandFunc(cmd *cobra.Command, _ []string) { r, err := doRequest(cmd, regionsRangeHolesPrefix, http.MethodGet, http.Header{}) if err != nil { cmd.Printf("Failed to get range holes: %s\n", err) diff --git a/tools/pd-ctl/pdctl/command/scheduler.go b/tools/pd-ctl/pdctl/command/scheduler.go index 3799c4a820ea..d5deba670ad2 100644 --- a/tools/pd-ctl/pdctl/command/scheduler.go +++ b/tools/pd-ctl/pdctl/command/scheduler.go @@ -391,7 +391,7 @@ func NewSlowTrendEvictLeaderSchedulerCommand() *cobra.Command { return c } -func addSchedulerForSplitBucketCommandFunc(cmd *cobra.Command, args []string) { +func addSchedulerForSplitBucketCommandFunc(cmd *cobra.Command, _ []string) { input := make(map[string]any) input["name"] = cmd.Name() postJSON(cmd, schedulersPrefix, input) diff --git a/tools/pd-ctl/pdctl/command/store_command.go b/tools/pd-ctl/pdctl/command/store_command.go index 085483cc5df4..bc024d5a2e6b 100644 --- a/tools/pd-ctl/pdctl/command/store_command.go +++ b/tools/pd-ctl/pdctl/command/store_command.go @@ -675,7 +675,7 @@ func storeCheckCommandFunc(cmd *cobra.Command, args []string) { cmd.Println(r) } -func showStoresCommandFunc(cmd *cobra.Command, args []string) { +func showStoresCommandFunc(cmd *cobra.Command, _ []string) { prefix := storesPrefix r, err := doRequest(cmd, prefix, http.MethodGet, http.Header{}) if err != nil { @@ -706,7 +706,7 @@ func showAllStoresLimitCommandFunc(cmd *cobra.Command, args []string) { cmd.Println(r) } -func removeTombStoneCommandFunc(cmd *cobra.Command, args []string) { +func removeTombStoneCommandFunc(cmd *cobra.Command, _ []string) { prefix := path.Join(storesPrefix, "remove-tombstone") _, err := doRequest(cmd, prefix, http.MethodDelete, http.Header{}) if err != nil { diff --git a/tools/pd-ctl/pdctl/command/unsafe_command.go b/tools/pd-ctl/pdctl/command/unsafe_command.go index 66ef8e6c934c..04d272385e70 100644 --- a/tools/pd-ctl/pdctl/command/unsafe_command.go +++ b/tools/pd-ctl/pdctl/command/unsafe_command.go @@ -106,7 +106,7 @@ func removeFailedStoresCommandFunc(cmd *cobra.Command, args []string) { postJSON(cmd, prefix, postInput) } -func removeFailedStoresShowCommandFunc(cmd *cobra.Command, args []string) { +func removeFailedStoresShowCommandFunc(cmd *cobra.Command, _ []string) { var resp string var err error prefix := fmt.Sprintf("%s/remove-failed-stores/show", unsafePrefix) diff --git a/tools/pd-ctl/pdctl/ctl.go b/tools/pd-ctl/pdctl/ctl.go index 5790911d79f1..f8eaff5e76e4 100644 --- a/tools/pd-ctl/pdctl/ctl.go +++ b/tools/pd-ctl/pdctl/ctl.go @@ -86,7 +86,7 @@ func MainStart(args []string) { // TODO: deprecated rootCmd.Flags().BoolP("detach", "d", true, "Run pdctl without readline.") - rootCmd.Run = func(cmd *cobra.Command, args []string) { + rootCmd.Run = func(cmd *cobra.Command, _ []string) { if v, err := cmd.Flags().GetBool("version"); err == nil && v { versioninfo.Print() return diff --git a/tools/pd-ctl/tests/config/config_test.go b/tools/pd-ctl/tests/config/config_test.go index 6776c9851b36..07a7c2aa9905 100644 --- a/tools/pd-ctl/tests/config/config_test.go +++ b/tools/pd-ctl/tests/config/config_test.go @@ -568,7 +568,7 @@ func (suite *configTestSuite) checkPlacementRules(cluster *pdTests.TestCluster) re.Contains(string(output), "Success!") // test show - suite.checkShowRuleKey(re, pdAddr, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}}) + checkShowRuleKey(re, pdAddr, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}}) f, _ := os.CreateTemp("/tmp", "pd_tests") fname := f.Name() @@ -576,7 +576,7 @@ func (suite *configTestSuite) checkPlacementRules(cluster *pdTests.TestCluster) defer os.RemoveAll(fname) // test load - rules := suite.checkLoadRule(re, pdAddr, fname, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}}) + rules := checkLoadRule(re, pdAddr, fname, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}}) // test save rules = append(rules, placement.Rule{ @@ -596,11 +596,11 @@ func (suite *configTestSuite) checkPlacementRules(cluster *pdTests.TestCluster) re.NoError(err) // test show group - suite.checkShowRuleKey(re, pdAddr, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}, {placement.DefaultGroupID, "test1"}}, "--group=pd") + checkShowRuleKey(re, pdAddr, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}, {placement.DefaultGroupID, "test1"}}, "--group=pd") // test rule region detail pdTests.MustPutRegion(re, cluster, 1, 1, []byte("a"), []byte("b")) - suite.checkShowRuleKey(re, pdAddr, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}}, "--region=1", "--detail") + checkShowRuleKey(re, pdAddr, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}}, "--region=1", "--detail") // test delete // need clear up args, so create new a cobra.Command. Otherwise gourp still exists. @@ -609,7 +609,7 @@ func (suite *configTestSuite) checkPlacementRules(cluster *pdTests.TestCluster) os.WriteFile(fname, b, 0600) _, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "save", "--in="+fname) re.NoError(err) - suite.checkShowRuleKey(re, pdAddr, [][2]string{{placement.DefaultGroupID, "test1"}}, "--group=pd") + checkShowRuleKey(re, pdAddr, [][2]string{{placement.DefaultGroupID, "test1"}}, "--group=pd") } func (suite *configTestSuite) TestPlacementRuleGroups() { @@ -724,7 +724,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *pdTests.TestClus defer os.RemoveAll(fname) // test load - suite.checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ + checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ {ID: placement.DefaultGroupID, Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: placement.DefaultGroupID, ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) @@ -736,7 +736,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *pdTests.TestClus re.NoError(os.WriteFile(fname, b, 0600)) _, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "set", "--in="+fname) re.NoError(err) - suite.checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ + checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ {ID: placement.DefaultGroupID, Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: placement.DefaultGroupID, ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) @@ -745,7 +745,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *pdTests.TestClus _, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "delete", placement.DefaultGroupID) re.NoError(err) - suite.checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ + checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) @@ -757,7 +757,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *pdTests.TestClus re.NoError(os.WriteFile(fname, b, 0600)) _, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "set", "--in="+fname) re.NoError(err) - suite.checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ + checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) @@ -768,7 +768,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *pdTests.TestClus bundles := []placement.GroupBundle{ {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, } - suite.checkLoadRuleBundle(re, pdAddr, fname, bundles) + checkLoadRuleBundle(re, pdAddr, fname, bundles) // test save bundle.Rules = []*placement.Rule{{GroupID: "pf", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}} @@ -778,7 +778,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *pdTests.TestClus re.NoError(os.WriteFile(fname, b, 0600)) _, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "save", "--in="+fname) re.NoError(err) - suite.checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ + checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) @@ -791,7 +791,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *pdTests.TestClus _, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "save", "--in="+fname, "--partial") re.NoError(err) - suite.checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ + checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) @@ -810,12 +810,12 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *pdTests.TestClus _, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "delete", "--regexp", ".*f") re.NoError(err) - suite.checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ + checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ {ID: "pd", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pd", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) } -func (suite *configTestSuite) checkLoadRuleBundle(re *require.Assertions, pdAddr string, fname string, expectValues []placement.GroupBundle) { +func checkLoadRuleBundle(re *require.Assertions, pdAddr string, fname string, expectValues []placement.GroupBundle) { var bundles []placement.GroupBundle cmd := ctl.GetRootCmd() testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server @@ -828,7 +828,7 @@ func (suite *configTestSuite) checkLoadRuleBundle(re *require.Assertions, pdAddr assertBundles(re, bundles, expectValues) } -func (suite *configTestSuite) checkLoadRule(re *require.Assertions, pdAddr string, fname string, expectValues [][2]string) []placement.Rule { +func checkLoadRule(re *require.Assertions, pdAddr string, fname string, expectValues [][2]string) []placement.Rule { var rules []placement.Rule cmd := ctl.GetRootCmd() testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server @@ -844,7 +844,7 @@ func (suite *configTestSuite) checkLoadRule(re *require.Assertions, pdAddr strin return rules } -func (suite *configTestSuite) checkShowRuleKey(re *require.Assertions, pdAddr string, expectValues [][2]string, opts ...string) { +func checkShowRuleKey(re *require.Assertions, pdAddr string, expectValues [][2]string, opts ...string) { var ( rules []placement.Rule fit placement.RegionFit diff --git a/tools/pd-ctl/tests/global_test.go b/tools/pd-ctl/tests/global_test.go index 14b7aafdccd5..f4f55e2af894 100644 --- a/tools/pd-ctl/tests/global_test.go +++ b/tools/pd-ctl/tests/global_test.go @@ -34,7 +34,7 @@ const pdControlCallerID = "pd-ctl" func TestSendAndGetComponent(t *testing.T) { re := require.New(t) - handler := func(ctx context.Context, s *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { + handler := func(context.Context, *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { mux := http.NewServeMux() mux.HandleFunc("/pd/api/v1/health", func(w http.ResponseWriter, r *http.Request) { callerID := apiutil.GetCallerIDOnHTTP(r) diff --git a/tools/pd-ctl/tests/hot/hot_test.go b/tools/pd-ctl/tests/hot/hot_test.go index 9d8dbbd123a3..7661704aa41d 100644 --- a/tools/pd-ctl/tests/hot/hot_test.go +++ b/tools/pd-ctl/tests/hot/hot_test.go @@ -51,7 +51,7 @@ func TestHotTestSuite(t *testing.T) { func (suite *hotTestSuite) SetupSuite() { suite.env = pdTests.NewSchedulingTestEnvironment(suite.T(), - func(conf *config.Config, serverName string) { + func(conf *config.Config, _ string) { conf.Schedule.MaxStoreDownTime.Duration = time.Hour conf.Schedule.HotRegionCacheHitsThreshold = 0 }, @@ -398,7 +398,7 @@ func TestHistoryHotRegions(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() cluster, err := pdTests.NewTestCluster(ctx, 1, - func(cfg *config.Config, serverName string) { + func(cfg *config.Config, _ string) { cfg.Schedule.HotRegionCacheHitsThreshold = 0 cfg.Schedule.HotRegionsWriteInterval.Duration = 1000 * time.Millisecond cfg.Schedule.HotRegionsReservedDays = 1 @@ -520,7 +520,7 @@ func TestBuckets(t *testing.T) { statistics.Denoising = false ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cluster, err := pdTests.NewTestCluster(ctx, 1, func(cfg *config.Config, serverName string) { cfg.Schedule.HotRegionCacheHitsThreshold = 0 }) + cluster, err := pdTests.NewTestCluster(ctx, 1, func(cfg *config.Config, _ string) { cfg.Schedule.HotRegionCacheHitsThreshold = 0 }) re.NoError(err) defer cluster.Destroy() err = cluster.RunInitialServers() diff --git a/tools/pd-ctl/tests/keyspace/keyspace_group_test.go b/tools/pd-ctl/tests/keyspace/keyspace_group_test.go index 5d85f35dacf4..87fd17a97d44 100644 --- a/tools/pd-ctl/tests/keyspace/keyspace_group_test.go +++ b/tools/pd-ctl/tests/keyspace/keyspace_group_test.go @@ -100,7 +100,7 @@ func TestSplitKeyspaceGroup(t *testing.T) { for i := 0; i < 129; i++ { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestAPICluster(ctx, 3, func(conf *config.Config, serverName string) { + tc, err := pdTests.NewTestAPICluster(ctx, 3, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -155,7 +155,7 @@ func TestExternalAllocNodeWhenStart(t *testing.T) { for i := 0; i < 10; i++ { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestAPICluster(ctx, 1, func(conf *config.Config, serverName string) { + tc, err := pdTests.NewTestAPICluster(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -195,7 +195,7 @@ func TestSetNodeAndPriorityKeyspaceGroup(t *testing.T) { for i := 0; i < 10; i++ { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestAPICluster(ctx, 3, func(conf *config.Config, serverName string) { + tc, err := pdTests.NewTestAPICluster(ctx, 3, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -299,7 +299,7 @@ func TestMergeKeyspaceGroup(t *testing.T) { for i := 0; i < 129; i++ { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestAPICluster(ctx, 1, func(conf *config.Config, serverName string) { + tc, err := pdTests.NewTestAPICluster(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -418,7 +418,7 @@ func TestKeyspaceGroupState(t *testing.T) { for i := 0; i < 10; i++ { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestAPICluster(ctx, 1, func(conf *config.Config, serverName string) { + tc, err := pdTests.NewTestAPICluster(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -509,7 +509,7 @@ func TestShowKeyspaceGroupPrimary(t *testing.T) { for i := 0; i < 10; i++ { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestAPICluster(ctx, 1, func(conf *config.Config, serverName string) { + tc, err := pdTests.NewTestAPICluster(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) diff --git a/tools/pd-ctl/tests/keyspace/keyspace_test.go b/tools/pd-ctl/tests/keyspace/keyspace_test.go index 4c1fb2aadd5c..54c25fc20998 100644 --- a/tools/pd-ctl/tests/keyspace/keyspace_test.go +++ b/tools/pd-ctl/tests/keyspace/keyspace_test.go @@ -47,7 +47,7 @@ func TestKeyspace(t *testing.T) { for i := 1; i < 10; i++ { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestAPICluster(ctx, 3, func(conf *config.Config, serverName string) { + tc, err := pdTests.NewTestAPICluster(ctx, 3, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) diff --git a/tools/pd-ctl/tests/label/label_test.go b/tools/pd-ctl/tests/label/label_test.go index 9ba6f267ae18..f7370a718721 100644 --- a/tools/pd-ctl/tests/label/label_test.go +++ b/tools/pd-ctl/tests/label/label_test.go @@ -34,7 +34,7 @@ func TestLabel(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cluster, err := pdTests.NewTestCluster(ctx, 1, func(cfg *config.Config, serverName string) { cfg.Replication.StrictlyMatchLabel = false }) + cluster, err := pdTests.NewTestCluster(ctx, 1, func(cfg *config.Config, _ string) { cfg.Replication.StrictlyMatchLabel = false }) re.NoError(err) defer cluster.Destroy() err = cluster.RunInitialServers() diff --git a/tools/pd-ctl/tests/operator/operator_test.go b/tools/pd-ctl/tests/operator/operator_test.go index 5af731840764..7e5d390c4ce4 100644 --- a/tools/pd-ctl/tests/operator/operator_test.go +++ b/tools/pd-ctl/tests/operator/operator_test.go @@ -43,7 +43,7 @@ func TestOperatorTestSuite(t *testing.T) { func (suite *operatorTestSuite) SetupSuite() { suite.env = pdTests.NewSchedulingTestEnvironment(suite.T(), - func(conf *config.Config, serverName string) { + func(conf *config.Config, _ string) { // TODO: enable placement rules conf.Replication.MaxReplicas = 2 conf.Replication.EnablePlacementRules = false diff --git a/tools/pd-ctl/tests/resourcemanager/resource_manager_command_test.go b/tools/pd-ctl/tests/resourcemanager/resource_manager_command_test.go index 5cfc16ffb020..d387a2b87ae0 100644 --- a/tools/pd-ctl/tests/resourcemanager/resource_manager_command_test.go +++ b/tools/pd-ctl/tests/resourcemanager/resource_manager_command_test.go @@ -58,7 +58,7 @@ func (s *testResourceManagerSuite) TearDownSuite() { func (s *testResourceManagerSuite) TestConfigController() { re := s.Require() - expectCfg := server.ControllerConfig{} + expectCfg := server.Config{} expectCfg.Adjust(nil) // Show controller config checkShow := func() { @@ -69,7 +69,7 @@ func (s *testResourceManagerSuite) TestConfigController() { actualCfg := server.ControllerConfig{} err = json.Unmarshal(output, &actualCfg) re.NoError(err, string(output)) - re.Equal(expectCfg, actualCfg) + re.Equal(expectCfg.Controller, actualCfg) } // Check default config @@ -80,20 +80,20 @@ func (s *testResourceManagerSuite) TestConfigController() { output, err := tests.ExecuteCommand(ctl.GetRootCmd(), args...) re.NoError(err) re.Contains(string(output), "Success!") - expectCfg.LTBMaxWaitDuration = typeutil.Duration{Duration: 1 * time.Hour} + expectCfg.Controller.LTBMaxWaitDuration = typeutil.Duration{Duration: 1 * time.Hour} checkShow() args = []string{"-u", s.pdAddr, "resource-manager", "config", "controller", "set", "enable-controller-trace-log", "true"} output, err = tests.ExecuteCommand(ctl.GetRootCmd(), args...) re.NoError(err) re.Contains(string(output), "Success!") - expectCfg.EnableControllerTraceLog = true + expectCfg.Controller.EnableControllerTraceLog = true checkShow() args = []string{"-u", s.pdAddr, "resource-manager", "config", "controller", "set", "write-base-cost", "2"} output, err = tests.ExecuteCommand(ctl.GetRootCmd(), args...) re.NoError(err) re.Contains(string(output), "Success!") - expectCfg.RequestUnit.WriteBaseCost = 2 + expectCfg.Controller.RequestUnit.WriteBaseCost = 2 checkShow() } diff --git a/tools/pd-ctl/tests/store/store_test.go b/tools/pd-ctl/tests/store/store_test.go index b8cd5c13a799..afb974011687 100644 --- a/tools/pd-ctl/tests/store/store_test.go +++ b/tools/pd-ctl/tests/store/store_test.go @@ -581,7 +581,7 @@ func TestStoreTLS(t *testing.T) { CertFile: filepath.Join(certPath, "pd-server.pem"), TrustedCAFile: filepath.Join(certPath, "ca.pem"), } - cluster, err := pdTests.NewTestCluster(ctx, 1, func(conf *config.Config, serverName string) { + cluster, err := pdTests.NewTestCluster(ctx, 1, func(conf *config.Config, _ string) { conf.Security.TLSConfig = grpcutil.TLSConfig{ KeyPath: tlsInfo.KeyFile, CertPath: tlsInfo.CertFile, diff --git a/tools/pd-heartbeat-bench/main.go b/tools/pd-heartbeat-bench/main.go index 44d1b001269d..ec5e2506e6b9 100644 --- a/tools/pd-heartbeat-bench/main.go +++ b/tools/pd-heartbeat-bench/main.go @@ -192,7 +192,7 @@ type Regions struct { updateFlow []int } -func (rs *Regions) init(cfg *config.Config, options *config.Options) { +func (rs *Regions) init(cfg *config.Config) { rs.regions = make([]*pdpb.RegionHeartbeatRequest, 0, cfg.RegionCount) rs.updateRound = 0 @@ -507,7 +507,7 @@ func main() { initClusterID(ctx, cli) go runHTTPServer(cfg, options) regions := new(Regions) - regions.init(cfg, options) + regions.init(cfg) log.Info("finish init regions") stores := newStores(cfg.StoreCount) stores.update(regions) diff --git a/tools/pd-simulator/main.go b/tools/pd-simulator/main.go index 5d781757b39d..73f4a0bba12b 100644 --- a/tools/pd-simulator/main.go +++ b/tools/pd-simulator/main.go @@ -128,8 +128,11 @@ func runHTTPServer() { http.Handle("/pprof/allocs", pprof.Handler("allocs")) http.Handle("/pprof/block", pprof.Handler("block")) http.Handle("/pprof/goroutine", pprof.Handler("goroutine")) - // nolint - http.ListenAndServe(*statusAddress, nil) + server := &http.Server{ + Addr: *statusAddress, + ReadHeaderTimeout: 3 * time.Second, + } + server.ListenAndServe() } // NewSingleServer creates a pd server for simulator. diff --git a/tools/pd-simulator/simulator/cases/add_nodes.go b/tools/pd-simulator/simulator/cases/add_nodes.go index 833ead89f53a..241b34a94735 100644 --- a/tools/pd-simulator/simulator/cases/add_nodes.go +++ b/tools/pd-simulator/simulator/cases/add_nodes.go @@ -55,7 +55,7 @@ func newAddNodes() *Case { } threshold := 0.05 - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(regions *core.RegionsInfo, _ []info.StoreStats) bool { res := true leaderCounts := make([]int, 0, storeNum) regionCounts := make([]int, 0, storeNum) diff --git a/tools/pd-simulator/simulator/cases/add_nodes_dynamic.go b/tools/pd-simulator/simulator/cases/add_nodes_dynamic.go index 410d5e984c75..59b0b54e1ca4 100644 --- a/tools/pd-simulator/simulator/cases/add_nodes_dynamic.go +++ b/tools/pd-simulator/simulator/cases/add_nodes_dynamic.go @@ -73,7 +73,7 @@ func newAddNodesDynamic() *Case { simCase.Events = []EventDescriptor{e} threshold := 0.05 - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(regions *core.RegionsInfo, _ []info.StoreStats) bool { res := numNodes == storeNum leaderCounts := make([]int, 0, numNodes) regionCounts := make([]int, 0, numNodes) diff --git a/tools/pd-simulator/simulator/cases/balance_leader.go b/tools/pd-simulator/simulator/cases/balance_leader.go index 8f2b87e31800..bbc7ce97f687 100644 --- a/tools/pd-simulator/simulator/cases/balance_leader.go +++ b/tools/pd-simulator/simulator/cases/balance_leader.go @@ -51,7 +51,7 @@ func newBalanceLeader() *Case { } threshold := 0.05 - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(regions *core.RegionsInfo, _ []info.StoreStats) bool { res := true leaderCounts := make([]int, 0, storeNum) for i := 1; i <= storeNum; i++ { diff --git a/tools/pd-simulator/simulator/cases/balance_region.go b/tools/pd-simulator/simulator/cases/balance_region.go index 0a013cf38765..3b0c46f1670a 100644 --- a/tools/pd-simulator/simulator/cases/balance_region.go +++ b/tools/pd-simulator/simulator/cases/balance_region.go @@ -59,7 +59,7 @@ func newRedundantBalanceRegion() *Case { storesLastUpdateTime := make([]int64, storeNum+1) storeLastAvailable := make([]uint64, storeNum+1) - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(_ *core.RegionsInfo, stats []info.StoreStats) bool { res := true curTime := time.Now().Unix() storesAvailable := make([]uint64, 0, storeNum+1) diff --git a/tools/pd-simulator/simulator/cases/delete_nodes.go b/tools/pd-simulator/simulator/cases/delete_nodes.go index 33f7ada14a05..4ba8e5064a4c 100644 --- a/tools/pd-simulator/simulator/cases/delete_nodes.go +++ b/tools/pd-simulator/simulator/cases/delete_nodes.go @@ -72,7 +72,7 @@ func newDeleteNodes() *Case { simCase.Events = []EventDescriptor{e} threshold := 0.05 - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(regions *core.RegionsInfo, _ []info.StoreStats) bool { res := numNodes == noEmptyStoreNum leaderCounts := make([]int, 0, numNodes) regionCounts := make([]int, 0, numNodes) diff --git a/tools/pd-simulator/simulator/cases/diagnose_label_isolation.go b/tools/pd-simulator/simulator/cases/diagnose_label_isolation.go index bd056bdf9c12..7fa50e56197b 100644 --- a/tools/pd-simulator/simulator/cases/diagnose_label_isolation.go +++ b/tools/pd-simulator/simulator/cases/diagnose_label_isolation.go @@ -62,7 +62,7 @@ func newLabelNotMatch1() *Case { storesLastUpdateTime := make([]int64, storeNum+1) storeLastAvailable := make([]uint64, storeNum+1) - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(_ *core.RegionsInfo, stats []info.StoreStats) bool { res := true curTime := time.Now().Unix() storesAvailable := make([]uint64, 0, storeNum+1) @@ -128,7 +128,7 @@ func newLabelIsolation1() *Case { storesLastUpdateTime := make([]int64, storeNum+1) storeLastAvailable := make([]uint64, storeNum+1) - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(_ *core.RegionsInfo, stats []info.StoreStats) bool { res := true curTime := time.Now().Unix() storesAvailable := make([]uint64, 0, storeNum+1) @@ -189,7 +189,7 @@ func newLabelIsolation2() *Case { storesLastUpdateTime := make([]int64, storeNum+1) storeLastAvailable := make([]uint64, storeNum+1) - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(_ *core.RegionsInfo, stats []info.StoreStats) bool { res := true curTime := time.Now().Unix() storesAvailable := make([]uint64, 0, storeNum+1) diff --git a/tools/pd-simulator/simulator/cases/diagnose_rule.go b/tools/pd-simulator/simulator/cases/diagnose_rule.go index 6cd76c854b74..15c5942d810e 100644 --- a/tools/pd-simulator/simulator/cases/diagnose_rule.go +++ b/tools/pd-simulator/simulator/cases/diagnose_rule.go @@ -100,7 +100,7 @@ func newRule1() *Case { storesLastUpdateTime := make([]int64, storeNum+1) storeLastAvailable := make([]uint64, storeNum+1) - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(_ *core.RegionsInfo, stats []info.StoreStats) bool { res := true curTime := time.Now().Unix() storesAvailable := make([]uint64, 0, storeNum+1) @@ -179,7 +179,7 @@ func newRule2() *Case { storesLastUpdateTime := make([]int64, storeNum+1) storeLastAvailable := make([]uint64, storeNum+1) - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(_ *core.RegionsInfo, stats []info.StoreStats) bool { res := true curTime := time.Now().Unix() storesAvailable := make([]uint64, 0, storeNum+1) diff --git a/tools/pd-simulator/simulator/cases/event_inner.go b/tools/pd-simulator/simulator/cases/event_inner.go index 3edf26b72a50..72521584e888 100644 --- a/tools/pd-simulator/simulator/cases/event_inner.go +++ b/tools/pd-simulator/simulator/cases/event_inner.go @@ -25,7 +25,7 @@ type WriteFlowOnSpotDescriptor struct { } // Type implements the EventDescriptor interface. -func (w *WriteFlowOnSpotDescriptor) Type() string { +func (*WriteFlowOnSpotDescriptor) Type() string { return "write-flow-on-spot" } @@ -35,7 +35,7 @@ type WriteFlowOnRegionDescriptor struct { } // Type implements the EventDescriptor interface. -func (w *WriteFlowOnRegionDescriptor) Type() string { +func (*WriteFlowOnRegionDescriptor) Type() string { return "write-flow-on-region" } @@ -45,7 +45,7 @@ type ReadFlowOnRegionDescriptor struct { } // Type implements the EventDescriptor interface. -func (w *ReadFlowOnRegionDescriptor) Type() string { +func (*ReadFlowOnRegionDescriptor) Type() string { return "read-flow-on-region" } @@ -55,7 +55,7 @@ type AddNodesDescriptor struct { } // Type implements the EventDescriptor interface. -func (w *AddNodesDescriptor) Type() string { +func (*AddNodesDescriptor) Type() string { return "add-nodes" } @@ -65,6 +65,6 @@ type DeleteNodesDescriptor struct { } // Type implements the EventDescriptor interface. -func (w *DeleteNodesDescriptor) Type() string { +func (*DeleteNodesDescriptor) Type() string { return "delete-nodes" } diff --git a/tools/pd-simulator/simulator/cases/hot_read.go b/tools/pd-simulator/simulator/cases/hot_read.go index 9df4f8796e85..d4ec6831d95d 100644 --- a/tools/pd-simulator/simulator/cases/hot_read.go +++ b/tools/pd-simulator/simulator/cases/hot_read.go @@ -67,12 +67,12 @@ func newHotRead() *Case { } } e := &ReadFlowOnRegionDescriptor{} - e.Step = func(tick int64) map[uint64]int64 { + e.Step = func(int64) map[uint64]int64 { return readFlow } simCase.Events = []EventDescriptor{e} // Checker description - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(regions *core.RegionsInfo, _ []info.StoreStats) bool { leaderCount := make([]int, storeNum) for id := range readFlow { leaderStore := regions.GetRegion(id).GetLeader().GetStoreId() diff --git a/tools/pd-simulator/simulator/cases/hot_write.go b/tools/pd-simulator/simulator/cases/hot_write.go index 8efe32c56574..8428afa75b58 100644 --- a/tools/pd-simulator/simulator/cases/hot_write.go +++ b/tools/pd-simulator/simulator/cases/hot_write.go @@ -66,14 +66,14 @@ func newHotWrite() *Case { } } e := &WriteFlowOnRegionDescriptor{} - e.Step = func(tick int64) map[uint64]int64 { + e.Step = func(int64) map[uint64]int64 { return writeFlow } simCase.Events = []EventDescriptor{e} // Checker description - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(regions *core.RegionsInfo, _ []info.StoreStats) bool { leaderCount := make([]int, storeNum) peerCount := make([]int, storeNum) for id := range writeFlow { diff --git a/tools/pd-simulator/simulator/cases/import_data.go b/tools/pd-simulator/simulator/cases/import_data.go index 0e7f7770a485..6cf3b79a736f 100644 --- a/tools/pd-simulator/simulator/cases/import_data.go +++ b/tools/pd-simulator/simulator/cases/import_data.go @@ -78,7 +78,7 @@ func newImportData() *Case { checkCount := uint64(0) var newRegionCount [][3]int var allRegionCount [][3]int - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(regions *core.RegionsInfo, _ []info.StoreStats) bool { leaderDist := make(map[uint64]int) peerDist := make(map[uint64]int) leaderTotal := 0 diff --git a/tools/pd-simulator/simulator/cases/makeup_down_replica.go b/tools/pd-simulator/simulator/cases/makeup_down_replica.go index 57eb2dd1f53d..86c9b4cac1da 100644 --- a/tools/pd-simulator/simulator/cases/makeup_down_replica.go +++ b/tools/pd-simulator/simulator/cases/makeup_down_replica.go @@ -64,7 +64,7 @@ func newMakeupDownReplicas() *Case { } simCase.Events = []EventDescriptor{e} - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(regions *core.RegionsInfo, _ []info.StoreStats) bool { sum := 0 regionCounts := make([]int, 0, storeNum) for i := 1; i <= storeNum; i++ { diff --git a/tools/pd-simulator/simulator/cases/region_merge.go b/tools/pd-simulator/simulator/cases/region_merge.go index 501803d439ea..3d5d57f804fc 100644 --- a/tools/pd-simulator/simulator/cases/region_merge.go +++ b/tools/pd-simulator/simulator/cases/region_merge.go @@ -54,7 +54,7 @@ func newRegionMerge() *Case { // Checker description threshold := 0.05 mergeRatio := 4 // when max-merge-region-size is 20, per region will reach 40MB - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(regions *core.RegionsInfo, _ []info.StoreStats) bool { sum := 0 regionCounts := make([]int, 0, storeNum) for i := 1; i <= storeNum; i++ { diff --git a/tools/pd-simulator/simulator/cases/region_split.go b/tools/pd-simulator/simulator/cases/region_split.go index 6a69386cb6bd..b85cd319494f 100644 --- a/tools/pd-simulator/simulator/cases/region_split.go +++ b/tools/pd-simulator/simulator/cases/region_split.go @@ -48,7 +48,7 @@ func newRegionSplit() *Case { simCase.RegionSplitKeys = 10000 // Events description e := &WriteFlowOnSpotDescriptor{} - e.Step = func(tick int64) map[string]int64 { + e.Step = func(int64) map[string]int64 { return map[string]int64{ "foobar": 8 * units.MiB, } @@ -56,7 +56,7 @@ func newRegionSplit() *Case { simCase.Events = []EventDescriptor{e} // Checker description - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(regions *core.RegionsInfo, _ []info.StoreStats) bool { res := true regionCounts := make([]int, 0, storeNum) for i := 1; i <= storeNum; i++ { diff --git a/tools/pd-simulator/simulator/client.go b/tools/pd-simulator/simulator/client.go index 81453307afab..808c991e97f6 100644 --- a/tools/pd-simulator/simulator/client.go +++ b/tools/pd-simulator/simulator/client.go @@ -380,7 +380,7 @@ func (c *client) StoreHeartbeat(ctx context.Context, stats *pdpb.StoreStats) err return nil } -func (c *client) RegionHeartbeat(ctx context.Context, region *core.RegionInfo) error { +func (c *client) RegionHeartbeat(_ context.Context, region *core.RegionInfo) error { c.reportRegionHeartbeatCh <- region return nil } diff --git a/tools/pd-simulator/simulator/task.go b/tools/pd-simulator/simulator/task.go index b1c609b503dd..a19854b53ba3 100644 --- a/tools/pd-simulator/simulator/task.go +++ b/tools/pd-simulator/simulator/task.go @@ -261,7 +261,7 @@ type transferLeader struct { toPeers []*metapb.Peer } -func (t *transferLeader) tick(engine *RaftEngine, region *core.RegionInfo) (newRegion *core.RegionInfo, isFinished bool) { +func (t *transferLeader) tick(_ *RaftEngine, region *core.RegionInfo) (newRegion *core.RegionInfo, isFinished bool) { isFinished = true toPeer := t.toPeers[0] // TODO: Support selection logic if peer := region.GetPeer(toPeer.GetId()); peer == nil || peer.GetRole() != toPeer.GetRole() || core.IsLearner(peer) { @@ -313,7 +313,7 @@ type promoteLearner struct { peer *metapb.Peer } -func (pl *promoteLearner) tick(engine *RaftEngine, region *core.RegionInfo) (newRegion *core.RegionInfo, isFinished bool) { +func (pl *promoteLearner) tick(_ *RaftEngine, region *core.RegionInfo) (newRegion *core.RegionInfo, isFinished bool) { isFinished = true peer := region.GetPeer(pl.peer.GetId()) opts := checkAndCreateChangePeerOption(region, peer, metapb.PeerRole_Learner, metapb.PeerRole_Voter) @@ -327,7 +327,7 @@ type demoteVoter struct { peer *metapb.Peer } -func (dv *demoteVoter) tick(engine *RaftEngine, region *core.RegionInfo) (newRegion *core.RegionInfo, isFinished bool) { +func (dv *demoteVoter) tick(_ *RaftEngine, region *core.RegionInfo) (newRegion *core.RegionInfo, isFinished bool) { isFinished = true peer := region.GetPeer(dv.peer.GetId()) opts := checkAndCreateChangePeerOption(region, peer, metapb.PeerRole_Voter, metapb.PeerRole_Learner) @@ -342,7 +342,7 @@ type changePeerV2Enter struct { demoteVoters []*metapb.Peer } -func (ce *changePeerV2Enter) tick(engine *RaftEngine, region *core.RegionInfo) (newRegion *core.RegionInfo, isFinished bool) { +func (ce *changePeerV2Enter) tick(_ *RaftEngine, region *core.RegionInfo) (newRegion *core.RegionInfo, isFinished bool) { isFinished = true var opts []core.RegionCreateOption for _, pl := range ce.promoteLearners { @@ -367,7 +367,7 @@ func (ce *changePeerV2Enter) tick(engine *RaftEngine, region *core.RegionInfo) ( type changePeerV2Leave struct{} -func (cl *changePeerV2Leave) tick(engine *RaftEngine, region *core.RegionInfo) (newRegion *core.RegionInfo, isFinished bool) { +func (*changePeerV2Leave) tick(_ *RaftEngine, region *core.RegionInfo) (newRegion *core.RegionInfo, isFinished bool) { isFinished = true var opts []core.RegionCreateOption for _, peer := range region.GetPeers() { diff --git a/tools/pd-tso-bench/main.go b/tools/pd-tso-bench/main.go index 236e78c78088..b4101bda2702 100644 --- a/tools/pd-tso-bench/main.go +++ b/tools/pd-tso-bench/main.go @@ -382,10 +382,10 @@ func reqWorker(ctx context.Context, pdClients []pd.Client, clientIdx int, durCh i := 0 for ; i < maxRetryTime; i++ { + var ticker *time.Ticker if *maxTSOSendIntervalMilliseconds > 0 { sleepBeforeGetTS := time.Duration(rand.Intn(*maxTSOSendIntervalMilliseconds)) * time.Millisecond - ticker := time.NewTicker(sleepBeforeGetTS) - defer ticker.Stop() + ticker = time.NewTicker(sleepBeforeGetTS) select { case <-reqCtx.Done(): case <-ticker.C: @@ -394,9 +394,11 @@ func reqWorker(ctx context.Context, pdClients []pd.Client, clientIdx int, durCh } _, _, err = pdCli.GetLocalTS(reqCtx, *dcLocation) if errors.Cause(err) == context.Canceled { + ticker.Stop() return } if err == nil { + ticker.Stop() break } log.Error(fmt.Sprintf("%v", err)) diff --git a/tools/pd-ut/ut.go b/tools/pd-ut/ut.go index 5d50bbd51a85..69a83f007b6f 100644 --- a/tools/pd-ut/ut.go +++ b/tools/pd-ut/ut.go @@ -562,7 +562,7 @@ func failureCases(input []JUnitTestCase) int { return sum } -func (n *numa) testCommand(pkg string, fn string) *exec.Cmd { +func (*numa) testCommand(pkg string, fn string) *exec.Cmd { args := make([]string, 0, 10) exe := "./" + testFileName(pkg) args = append(args, "-test.cpu", "1") From 39f571220bfa5100f746e131de4990c346d7e5a1 Mon Sep 17 00:00:00 2001 From: Hu# Date: Thu, 11 Apr 2024 17:43:52 +0800 Subject: [PATCH 06/50] tests/api: refine TestPreparingProgress to make test stable (#8014) ref tikv/pd#7969 Signed-off-by: husharp --- tests/server/api/api_test.go | 199 ++++++++++++++++++++++++----------- 1 file changed, 138 insertions(+), 61 deletions(-) diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go index 98e458f6d179..9e1636df0458 100644 --- a/tests/server/api/api_test.go +++ b/tests/server/api/api_test.go @@ -755,9 +755,9 @@ func TestRemovingProgress(t *testing.T) { // no store removing output := sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores/progress?action=removing", http.MethodGet, http.StatusNotFound) - re.Contains((string(output)), "no progress found for the action") + re.Contains(string(output), "no progress found for the action") output = sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores/progress?id=2", http.MethodGet, http.StatusNotFound) - re.Contains((string(output)), "no progress found for the given store ID") + re.Contains(string(output), "no progress found for the given store ID") // remove store 1 and store 2 _ = sendRequest(re, leader.GetAddr()+"/pd/api/v1/store/1", http.MethodDelete, http.StatusOK) @@ -776,32 +776,69 @@ func TestRemovingProgress(t *testing.T) { tests.MustPutRegion(re, cluster, 1000, 1, []byte("a"), []byte("b"), core.SetApproximateSize(20)) tests.MustPutRegion(re, cluster, 1001, 2, []byte("c"), []byte("d"), core.SetApproximateSize(10)) - // is not prepared - time.Sleep(2 * time.Second) - output = sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores/progress?action=removing", http.MethodGet, http.StatusOK) - re.NoError(json.Unmarshal(output, &p)) - re.Equal("removing", p.Action) - re.Equal(0.0, p.Progress) - re.Equal(0.0, p.CurrentSpeed) - re.Equal(math.MaxFloat64, p.LeftSeconds) + if !leader.GetRaftCluster().IsPrepared() { + testutil.Eventually(re, func() bool { + if leader.GetRaftCluster().IsPrepared() { + return true + } + url := leader.GetAddr() + "/pd/api/v1/stores/progress?action=removing" + req, _ := http.NewRequest(http.MethodGet, url, http.NoBody) + resp, err := dialClient.Do(req) + re.NoError(err) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return false + } + // is not prepared + re.NoError(json.Unmarshal(output, &p)) + re.Equal("removing", p.Action) + re.Equal(0.0, p.Progress) + re.Equal(0.0, p.CurrentSpeed) + re.Equal(math.MaxFloat64, p.LeftSeconds) + return true + }) + } - leader.GetRaftCluster().SetPrepared() - time.Sleep(2 * time.Second) - output = sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores/progress?action=removing", http.MethodGet, http.StatusOK) - re.NoError(json.Unmarshal(output, &p)) - re.Equal("removing", p.Action) - // store 1: (60-20)/(60+50) ~= 0.36 - // store 2: (30-10)/(30+40) ~= 0.28 - // average progress ~= (0.36+0.28)/2 = 0.32 - re.Equal("0.32", fmt.Sprintf("%.2f", p.Progress)) - // store 1: 40/10s = 4 - // store 2: 20/10s = 2 - // average speed = (2+4)/2 = 33 - re.Equal(3.0, p.CurrentSpeed) - // store 1: (20+50)/4 = 17.5s - // store 2: (10+40)/2 = 25s - // average time = (17.5+25)/2 = 21.25s - re.Equal(21.25, p.LeftSeconds) + testutil.Eventually(re, func() bool { + // wait for cluster prepare + if !leader.GetRaftCluster().IsPrepared() { + leader.GetRaftCluster().SetPrepared() + return false + } + url := leader.GetAddr() + "/pd/api/v1/stores/progress?action=removing" + req, _ := http.NewRequest(http.MethodGet, url, http.NoBody) + resp, err := dialClient.Do(req) + re.NoError(err) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return false + } + output, err := io.ReadAll(resp.Body) + re.NoError(err) + re.NoError(json.Unmarshal(output, &p)) + if p.Action != "removing" { + return false + } + // store 1: (60-20)/(60+50) ~= 0.36 + // store 2: (30-10)/(30+40) ~= 0.28 + // average progress ~= (0.36+0.28)/2 = 0.32 + if fmt.Sprintf("%.2f", p.Progress) != "0.32" { + return false + } + // store 1: 40/10s = 4 + // store 2: 20/10s = 2 + // average speed = (2+4)/2 = 33 + if p.CurrentSpeed != 3.0 { + return false + } + // store 1: (20+50)/4 = 17.5s + // store 2: (10+40)/2 = 25s + // average time = (17.5+25)/2 = 21.25s + if p.LeftSeconds != 21.25 { + return false + } + return true + }) output = sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores/progress?id=2", http.MethodGet, http.StatusOK) re.NoError(json.Unmarshal(output, &p)) @@ -929,47 +966,87 @@ func TestPreparingProgress(t *testing.T) { } // no store preparing output := sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores/progress?action=preparing", http.MethodGet, http.StatusNotFound) - re.Contains((string(output)), "no progress found for the action") + re.Contains(string(output), "no progress found for the action") output = sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores/progress?id=4", http.MethodGet, http.StatusNotFound) - re.Contains((string(output)), "no progress found for the given store ID") - - // is not prepared - time.Sleep(2 * time.Second) - output = sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores/progress?action=preparing", http.MethodGet, http.StatusNotFound) - re.Contains((string(output)), "no progress found for the action") - output = sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores/progress?id=4", http.MethodGet, http.StatusNotFound) - re.Contains((string(output)), "no progress found for the given store ID") + re.Contains(string(output), "no progress found for the given store ID") + + if !leader.GetRaftCluster().IsPrepared() { + testutil.Eventually(re, func() bool { + if leader.GetRaftCluster().IsPrepared() { + return true + } + url := leader.GetAddr() + "/pd/api/v1/stores/progress?action=preparing" + req, _ := http.NewRequest(http.MethodGet, url, http.NoBody) + resp, err := dialClient.Do(req) + re.NoError(err) + defer resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + return false + } + // is not prepared + output, err := io.ReadAll(resp.Body) + re.NoError(err) + re.Contains(string(output), "no progress found for the action") + output = sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores/progress?id=4", http.MethodGet, http.StatusNotFound) + re.Contains(string(output), "no progress found for the given store ID") + return true + }) + } - // size is not changed. - leader.GetRaftCluster().SetPrepared() - time.Sleep(2 * time.Second) - output = sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores/progress?action=preparing", http.MethodGet, http.StatusOK) var p api.Progress - re.NoError(json.Unmarshal(output, &p)) - re.Equal("preparing", p.Action) - re.Equal(0.0, p.Progress) - re.Equal(0.0, p.CurrentSpeed) - re.Equal(math.MaxFloat64, p.LeftSeconds) + testutil.Eventually(re, func() bool { + // wait for cluster prepare + if !leader.GetRaftCluster().IsPrepared() { + leader.GetRaftCluster().SetPrepared() + return false + } + url := leader.GetAddr() + "/pd/api/v1/stores/progress?action=preparing" + req, _ := http.NewRequest(http.MethodGet, url, http.NoBody) + resp, err := dialClient.Do(req) + re.NoError(err) + defer resp.Body.Close() + output, err := io.ReadAll(resp.Body) + re.NoError(err) + if resp.StatusCode != http.StatusOK { + return false + } + re.NoError(json.Unmarshal(output, &p)) + re.Equal("preparing", p.Action) + re.Equal(0.0, p.Progress) + re.Equal(0.0, p.CurrentSpeed) + re.Equal(math.MaxFloat64, p.LeftSeconds) + return true + }) // update size tests.MustPutRegion(re, cluster, 1000, 4, []byte(fmt.Sprintf("%20d", 1000)), []byte(fmt.Sprintf("%20d", 1001)), core.SetApproximateSize(10)) tests.MustPutRegion(re, cluster, 1001, 5, []byte(fmt.Sprintf("%20d", 1001)), []byte(fmt.Sprintf("%20d", 1002)), core.SetApproximateSize(40)) - time.Sleep(2 * time.Second) - output = sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores/progress?action=preparing", http.MethodGet, http.StatusOK) - re.NoError(json.Unmarshal(output, &p)) - re.Equal("preparing", p.Action) - // store 4: 10/(210*0.9) ~= 0.05 - // store 5: 40/(210*0.9) ~= 0.21 - // average progress ~= (0.05+0.21)/2 = 0.13 - re.Equal("0.13", fmt.Sprintf("%.2f", p.Progress)) - // store 4: 10/10s = 1 - // store 5: 40/10s = 4 - // average speed = (1+4)/2 = 2.5 - re.Equal(2.5, p.CurrentSpeed) - // store 4: 179/1 ~= 179 - // store 5: 149/4 ~= 37.25 - // average time ~= (179+37.25)/2 = 108.125 - re.Equal(108.125, p.LeftSeconds) + testutil.Eventually(re, func() bool { + output := sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores/progress?action=preparing", http.MethodGet, http.StatusOK) + re.NoError(json.Unmarshal(output, &p)) + if p.Action != "preparing" { + return false + } + // store 4: 10/(210*0.9) ~= 0.05 + // store 5: 40/(210*0.9) ~= 0.21 + // average progress ~= (0.05+0.21)/2 = 0.13 + if fmt.Sprintf("%.2f", p.Progress) != "0.13" { + return false + } + // store 4: 10/10s = 1 + // store 5: 40/10s = 4 + // average speed = (1+4)/2 = 2.5 + if p.CurrentSpeed != 2.5 { + return false + } + // store 4: 179/1 ~= 179 + // store 5: 149/4 ~= 37.25 + // average time ~= (179+37.25)/2 = 108.125 + if p.LeftSeconds != 108.125 { + return false + } + return true + }) output = sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores/progress?id=4", http.MethodGet, http.StatusOK) re.NoError(json.Unmarshal(output, &p)) From 8f7999e311fe6113a65822337f9ff0d7e3e3e601 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 11 Apr 2024 19:10:52 +0800 Subject: [PATCH 07/50] *: fix `TestKeyspaceGroupMergeIntoDefault` (#8056) close tikv/pd#6991 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tests/integrations/mcs/tso/keyspace_group_manager_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go index af33b85b1a1a..f678e9014193 100644 --- a/tests/integrations/mcs/tso/keyspace_group_manager_test.go +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -793,9 +793,11 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestKeyspaceGroupMergeIntoDefault // Check if the first keyspace group is served. svr := suite.tsoCluster.WaitForDefaultPrimaryServing(re) re.NotNil(svr) - // Check if the last keyspace group is served. - svr = suite.tsoCluster.WaitForPrimaryServing(re, uint32(keyspaceGroupNum), uint32(keyspaceGroupNum)) - re.NotNil(svr) + for i := 1; i < keyspaceGroupNum; i++ { + // Check if the keyspace group is served. + svr = suite.tsoCluster.WaitForPrimaryServing(re, uint32(i), uint32(i)) + re.NotNil(svr) + } // Merge all the keyspace groups into the default keyspace group. handlersutil.MustMergeKeyspaceGroup(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID, &handlers.MergeKeyspaceGroupsParams{ MergeAllIntoDefault: true, From 1cd99f1aff6509b767514ce51c36e4ba50b16f60 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 12 Apr 2024 11:45:23 +0800 Subject: [PATCH 08/50] pdctl: support top query in pdctl (#7843) close tikv/pd#7369 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/core/region_option.go | 36 ++++++++- server/api/region.go | 26 ++++++ server/api/router.go | 2 + tools/pd-ctl/pdctl/command/region_command.go | 51 ++++++++++-- tools/pd-ctl/tests/config/config_test.go | 28 +++++++ tools/pd-ctl/tests/helper.go | 9 +++ tools/pd-ctl/tests/region/region_test.go | 85 ++++++++++++++------ 7 files changed, 204 insertions(+), 33 deletions(-) diff --git a/pkg/core/region_option.go b/pkg/core/region_option.go index 36db7cf34604..e973a1e7c1f6 100644 --- a/pkg/core/region_option.go +++ b/pkg/core/region_option.go @@ -248,18 +248,23 @@ func SetReadKeys(v uint64) RegionCreateOption { // SetReadQuery sets the read query for the region, only used for unit test. func SetReadQuery(v uint64) RegionCreateOption { - q := RandomKindReadQuery(v) - return SetQueryStats(q) + return func(region *RegionInfo) { + resetReadQuery(region.queryStats) + region.queryStats = mergeQueryStat(region.queryStats, RandomKindReadQuery(v)) + } } // SetWrittenQuery sets the write query for the region, only used for unit test. func SetWrittenQuery(v uint64) RegionCreateOption { - q := RandomKindWriteQuery(v) - return SetQueryStats(q) + return func(region *RegionInfo) { + resetWriteQuery(region.queryStats) + region.queryStats = mergeQueryStat(region.queryStats, RandomKindWriteQuery(v)) + } } // SetQueryStats sets the query stats for the region, it will cover previous statistic. // This func is only used for unit test. +// It will cover previous statistic. func SetQueryStats(v *pdpb.QueryStats) RegionCreateOption { return func(region *RegionInfo) { region.queryStats = v @@ -268,6 +273,7 @@ func SetQueryStats(v *pdpb.QueryStats) RegionCreateOption { // AddQueryStats sets the query stats for the region, it will preserve previous statistic. // This func is only used for test and simulator. +// It will preserve previous statistic. func AddQueryStats(v *pdpb.QueryStats) RegionCreateOption { return func(region *RegionInfo) { q := mergeQueryStat(region.queryStats, v) @@ -469,3 +475,25 @@ func mergeQueryStat(q1, q2 *pdpb.QueryStats) *pdpb.QueryStats { q2.Rollback += q1.Rollback return q2 } + +func resetReadQuery(q *pdpb.QueryStats) { + if q == nil { + return + } + q.Get = 0 + q.Scan = 0 + q.Coprocessor = 0 +} + +func resetWriteQuery(q *pdpb.QueryStats) { + if q == nil { + return + } + q.Put = 0 + q.Delete = 0 + q.DeleteRange = 0 + q.AcquirePessimisticLock = 0 + q.Rollback = 0 + q.Prewrite = 0 + q.Commit = 0 +} diff --git a/server/api/region.go b/server/api/region.go index cf3157f8834b..dac92f247ca9 100644 --- a/server/api/region.go +++ b/server/api/region.go @@ -545,6 +545,19 @@ func (h *regionsHandler) GetTopWriteFlowRegions(w http.ResponseWriter, r *http.R h.GetTopNRegions(w, r, func(a, b *core.RegionInfo) bool { return a.GetBytesWritten() < b.GetBytesWritten() }) } +// @Tags region +// @Summary List regions with the highest write flow. +// @Param limit query integer false "Limit count" default(16) +// @Produce json +// @Success 200 {object} response.RegionsInfo +// @Failure 400 {string} string "The input is invalid." +// @Router /regions/writequery [get] +func (h *regionsHandler) GetTopWriteQueryRegions(w http.ResponseWriter, r *http.Request) { + h.GetTopNRegions(w, r, func(a, b *core.RegionInfo) bool { + return a.GetWriteQueryNum() < b.GetWriteQueryNum() + }) +} + // @Tags region // @Summary List regions with the highest read flow. // @Param limit query integer false "Limit count" default(16) @@ -556,6 +569,19 @@ func (h *regionsHandler) GetTopReadFlowRegions(w http.ResponseWriter, r *http.Re h.GetTopNRegions(w, r, func(a, b *core.RegionInfo) bool { return a.GetBytesRead() < b.GetBytesRead() }) } +// @Tags region +// @Summary List regions with the highest write flow. +// @Param limit query integer false "Limit count" default(16) +// @Produce json +// @Success 200 {object} response.RegionsInfo +// @Failure 400 {string} string "The input is invalid." +// @Router /regions/readquery [get] +func (h *regionsHandler) GetTopReadQueryRegions(w http.ResponseWriter, r *http.Request) { + h.GetTopNRegions(w, r, func(a, b *core.RegionInfo) bool { + return a.GetReadQueryNum() < b.GetReadQueryNum() + }) +} + // @Tags region // @Summary List regions with the largest conf version. // @Param limit query integer false "Limit count" default(16) diff --git a/server/api/router.go b/server/api/router.go index 5ca8aac582df..553332e96af2 100644 --- a/server/api/router.go +++ b/server/api/router.go @@ -257,7 +257,9 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { registerFunc(clusterRouter, "/regions/store/{id}", regionsHandler.GetStoreRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(clusterRouter, "/regions/keyspace/id/{id}", regionsHandler.GetKeyspaceRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(clusterRouter, "/regions/writeflow", regionsHandler.GetTopWriteFlowRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/writequery", regionsHandler.GetTopWriteQueryRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(clusterRouter, "/regions/readflow", regionsHandler.GetTopReadFlowRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/regions/readquery", regionsHandler.GetTopReadQueryRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(clusterRouter, "/regions/confver", regionsHandler.GetTopConfVerRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(clusterRouter, "/regions/version", regionsHandler.GetTopVersionRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(clusterRouter, "/regions/size", regionsHandler.GetTopSizeRegions, setMethods(http.MethodGet), setAuditBackend(prometheus)) diff --git a/tools/pd-ctl/pdctl/command/region_command.go b/tools/pd-ctl/pdctl/command/region_command.go index e03de1c62ac4..d7e19967c7a0 100644 --- a/tools/pd-ctl/pdctl/command/region_command.go +++ b/tools/pd-ctl/pdctl/command/region_command.go @@ -37,6 +37,8 @@ var ( regionsCheckPrefix = "pd/api/v1/regions/check" regionsWriteFlowPrefix = "pd/api/v1/regions/writeflow" regionsReadFlowPrefix = "pd/api/v1/regions/readflow" + regionsWriteQueryPrefix = "pd/api/v1/regions/writequery" + regionsReadQueryPrefix = "pd/api/v1/regions/readquery" regionsConfVerPrefix = "pd/api/v1/regions/confver" regionsVersionPrefix = "pd/api/v1/regions/version" regionsSizePrefix = "pd/api/v1/regions/size" @@ -66,17 +68,17 @@ func NewRegionCommand() *cobra.Command { r.AddCommand(NewRangesWithRangeHolesCommand()) topRead := &cobra.Command{ - Use: `topread [--jq=""]`, - Short: "show regions with top read flow", - Run: showRegionsTopCommand(regionsReadFlowPrefix), + Use: `topread [byte|query] [--jq=""]`, + Short: "show regions with top read flow or query", + Run: showTopReadRegions, } topRead.Flags().String("jq", "", "jq query") r.AddCommand(topRead) topWrite := &cobra.Command{ - Use: `topwrite [--jq=""]`, - Short: "show regions with top write flow", - Run: showRegionsTopCommand(regionsWriteFlowPrefix), + Use: `topwrite [byte|query] [--jq=""]`, + Short: "show regions with top write flow or query", + Run: showTopWriteRegions, } topWrite.Flags().String("jq", "", "jq query") r.AddCommand(topWrite) @@ -212,6 +214,9 @@ func showRegionsTopCommand(prefix string) run { return } prefix += "?limit=" + args[0] + } else if len(args) > 1 { + cmd.Println(cmd.UsageString()) + return } r, err := doRequest(cmd, prefix, http.MethodGet, http.Header{}) if err != nil { @@ -226,6 +231,40 @@ func showRegionsTopCommand(prefix string) run { } } +func showTopReadRegions(cmd *cobra.Command, args []string) { + // default to show top read flow + if len(args) == 0 { + showRegionsTopCommand(regionsReadFlowPrefix)(cmd, args) + return + } + // default to show top read flow with limit + switch args[0] { + case "query": + showRegionsTopCommand(regionsReadQueryPrefix)(cmd, args[1:]) + case "byte": + showRegionsTopCommand(regionsReadFlowPrefix)(cmd, args[1:]) + default: + showRegionsTopCommand(regionsReadFlowPrefix)(cmd, args) + } +} + +func showTopWriteRegions(cmd *cobra.Command, args []string) { + // default to show top write flow + if len(args) == 0 { + showRegionsTopCommand(regionsWriteFlowPrefix)(cmd, args) + return + } + // default to show top write flow with limit + switch args[0] { + case "query": + showRegionsTopCommand(regionsWriteQueryPrefix)(cmd, args[1:]) + case "byte": + showRegionsTopCommand(regionsWriteFlowPrefix)(cmd, args[1:]) + default: + showRegionsTopCommand(regionsWriteFlowPrefix)(cmd, args) + } +} + // NewRegionWithKeyCommand return a region with key subcommand of regionCmd func NewRegionWithKeyCommand() *cobra.Command { r := &cobra.Command{ diff --git a/tools/pd-ctl/tests/config/config_test.go b/tools/pd-ctl/tests/config/config_test.go index 07a7c2aa9905..c6430789cfc8 100644 --- a/tools/pd-ctl/tests/config/config_test.go +++ b/tools/pd-ctl/tests/config/config_test.go @@ -1144,6 +1144,34 @@ func (suite *configTestSuite) checkMicroServiceConfig(cluster *pdTests.TestClust re.False(svr.GetMicroServiceConfig().EnableSchedulingFallback) } +func (suite *configTestSuite) TestRegionRules() { + suite.env.RunTestInTwoModes(suite.checkRegionRules) +} + +func (suite *configTestSuite) checkRegionRules(cluster *pdTests.TestCluster) { + re := suite.Require() + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + cmd := ctl.GetRootCmd() + + storeID, regionID := uint64(1), uint64(2) + store := &metapb.Store{ + Id: storeID, + State: metapb.StoreState_Up, + } + pdTests.MustPutStore(re, cluster, store) + pdTests.MustPutRegion(re, cluster, regionID, storeID, []byte{}, []byte{}) + + args := []string{"-u", pdAddr, "config", "placement-rules", "show", "--region=" + strconv.Itoa(int(regionID)), "--detail"} + output, err := tests.ExecuteCommand(cmd, args...) + re.NoError(err) + fit := &placement.RegionFit{} + re.NoError(json.Unmarshal(output, fit)) + re.Len(fit.RuleFits, 1) + re.Equal(placement.DefaultGroupID, fit.RuleFits[0].Rule.GroupID) + re.Equal(placement.DefaultRuleID, fit.RuleFits[0].Rule.ID) +} + func assertBundles(re *require.Assertions, a, b []placement.GroupBundle) { re.Len(b, len(a)) for i := 0; i < len(a); i++ { diff --git a/tools/pd-ctl/tests/helper.go b/tools/pd-ctl/tests/helper.go index 90626afe5c02..bdacae48c228 100644 --- a/tools/pd-ctl/tests/helper.go +++ b/tools/pd-ctl/tests/helper.go @@ -82,3 +82,12 @@ func CheckRegionsInfo(re *require.Assertions, output *response.RegionsInfo, expe CheckRegionInfo(re, &got[i], region) } } + +// CheckRegionsInfoWithoutSort is used to check the test results without sort. +func CheckRegionsInfoWithoutSort(re *require.Assertions, output *response.RegionsInfo, expected []*core.RegionInfo) { + re.Len(expected, output.Count) + got := output.Regions + for i, region := range expected { + CheckRegionInfo(re, &got[i], region) + } +} diff --git a/tools/pd-ctl/tests/region/region_test.go b/tools/pd-ctl/tests/region/region_test.go index 5df305201151..b328fd882864 100644 --- a/tools/pd-ctl/tests/region/region_test.go +++ b/tools/pd-ctl/tests/region/region_test.go @@ -83,6 +83,7 @@ func TestRegion(t *testing.T) { r1 := pdTests.MustPutRegion(re, cluster, 1, 1, []byte("a"), []byte("b"), core.SetWrittenBytes(1000), core.SetReadBytes(1000), core.SetRegionConfVer(1), core.SetRegionVersion(1), core.SetApproximateSize(1), core.SetApproximateKeys(100), + core.SetReadQuery(100), core.SetWrittenQuery(100), core.SetPeers([]*metapb.Peer{ {Id: 1, StoreId: 1}, {Id: 5, StoreId: 2}, @@ -92,15 +93,18 @@ func TestRegion(t *testing.T) { r2 := pdTests.MustPutRegion(re, cluster, 2, 1, []byte("b"), []byte("c"), core.SetWrittenBytes(2000), core.SetReadBytes(0), core.SetRegionConfVer(2), core.SetRegionVersion(3), core.SetApproximateSize(144), core.SetApproximateKeys(14400), + core.SetReadQuery(200), core.SetWrittenQuery(200), ) r3 := pdTests.MustPutRegion(re, cluster, 3, 1, []byte("c"), []byte("d"), core.SetWrittenBytes(500), core.SetReadBytes(800), core.SetRegionConfVer(3), core.SetRegionVersion(2), core.SetApproximateSize(30), core.SetApproximateKeys(3000), + core.SetReadQuery(300), core.SetWrittenQuery(300), core.WithDownPeers([]*pdpb.PeerStats{{Peer: downPeer, DownSeconds: 3600}}), core.WithPendingPeers([]*metapb.Peer{downPeer}), core.WithLearners([]*metapb.Peer{{Id: 3, StoreId: 1}})) r4 := pdTests.MustPutRegion(re, cluster, 4, 1, []byte("d"), []byte("e"), - core.SetWrittenBytes(100), core.SetReadBytes(100), core.SetRegionConfVer(1), - core.SetRegionVersion(1), core.SetApproximateSize(10), core.SetApproximateKeys(1000), + core.SetWrittenBytes(100), core.SetReadBytes(100), core.SetRegionConfVer(4), + core.SetRegionVersion(4), core.SetApproximateSize(10), core.SetApproximateKeys(1000), + core.SetReadQuery(400), core.SetWrittenQuery(400), ) defer cluster.Destroy() @@ -115,26 +119,6 @@ func TestRegion(t *testing.T) { // region store command {[]string{"region", "store", "1"}, leaderServer.GetStoreRegions(1)}, {[]string{"region", "store", "1"}, []*core.RegionInfo{r1, r2, r3, r4}}, - // region topread [limit] command - {[]string{"region", "topread", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { return a.GetBytesRead() < b.GetBytesRead() }, 2)}, - // region topwrite [limit] command - {[]string{"region", "topwrite", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { return a.GetBytesWritten() < b.GetBytesWritten() }, 2)}, - // region topconfver [limit] command - {[]string{"region", "topconfver", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { - return a.GetMeta().GetRegionEpoch().GetConfVer() < b.GetMeta().GetRegionEpoch().GetConfVer() - }, 2)}, - // region topversion [limit] command - {[]string{"region", "topversion", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { - return a.GetMeta().GetRegionEpoch().GetVersion() < b.GetMeta().GetRegionEpoch().GetVersion() - }, 2)}, - // region topsize [limit] command - {[]string{"region", "topsize", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { - return a.GetApproximateSize() < b.GetApproximateSize() - }, 2)}, - // region topkeys [limit] command - {[]string{"region", "topkeys", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { - return a.GetApproximateKeys() < b.GetApproximateKeys() - }, 2)}, // region check extra-peer command {[]string{"region", "check", "extra-peer"}, []*core.RegionInfo{r1}}, // region check miss-peer command @@ -172,10 +156,65 @@ func TestRegion(t *testing.T) { output, err := tests.ExecuteCommand(cmd, args...) re.NoError(err) regions := &response.RegionsInfo{} - re.NoError(json.Unmarshal(output, regions)) + re.NoError(json.Unmarshal(output, regions), string(output)) tests.CheckRegionsInfo(re, regions, testCase.expect) } + testRegionsCases = []struct { + args []string + expect []*core.RegionInfo + }{ + // region topread [limit] command + {[]string{"region", "topread"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { return a.GetBytesRead() < b.GetBytesRead() }, 4)}, + // region topwrite [limit] command + {[]string{"region", "topwrite"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { return a.GetBytesWritten() < b.GetBytesWritten() }, 4)}, + // region topread [limit] command + {[]string{"region", "topread", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { return a.GetBytesRead() < b.GetBytesRead() }, 2)}, + // region topwrite [limit] command + {[]string{"region", "topwrite", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { return a.GetBytesWritten() < b.GetBytesWritten() }, 2)}, + // region topread byte [limit] command + {[]string{"region", "topread", "byte"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { return a.GetBytesRead() < b.GetBytesRead() }, 4)}, + // region topwrite byte [limit] command + {[]string{"region", "topwrite", "byte"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { return a.GetBytesWritten() < b.GetBytesWritten() }, 4)}, + // region topread byte [limit] command + {[]string{"region", "topread", "query"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { return a.GetReadQueryNum() < b.GetReadQueryNum() }, 4)}, + // region topwrite byte [limit] command + {[]string{"region", "topwrite", "query"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { return a.GetWriteQueryNum() < b.GetWriteQueryNum() }, 4)}, + // region topread byte [limit] command + {[]string{"region", "topread", "byte", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { return a.GetBytesRead() < b.GetBytesRead() }, 2)}, + // region topwrite byte [limit] command + {[]string{"region", "topwrite", "byte", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { return a.GetBytesWritten() < b.GetBytesWritten() }, 2)}, + // region topread byte [limit] command + {[]string{"region", "topread", "query", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { return a.GetReadQueryNum() < b.GetReadQueryNum() }, 2)}, + // region topwrite byte [limit] command + {[]string{"region", "topwrite", "query", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { return a.GetWriteQueryNum() < b.GetWriteQueryNum() }, 2)}, + // region topconfver [limit] command + {[]string{"region", "topconfver", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { + return a.GetMeta().GetRegionEpoch().GetConfVer() < b.GetMeta().GetRegionEpoch().GetConfVer() + }, 2)}, + // region topversion [limit] command + {[]string{"region", "topversion", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { + return a.GetMeta().GetRegionEpoch().GetVersion() < b.GetMeta().GetRegionEpoch().GetVersion() + }, 2)}, + // region topsize [limit] command + {[]string{"region", "topsize", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { + return a.GetApproximateSize() < b.GetApproximateSize() + }, 2)}, + // region topkeys [limit] command + {[]string{"region", "topkeys", "2"}, api.TopNRegions(leaderServer.GetRegions(), func(a, b *core.RegionInfo) bool { + return a.GetApproximateKeys() < b.GetApproximateKeys() + }, 2)}, + } + + for _, testCase := range testRegionsCases { + args := append([]string{"-u", pdAddr}, testCase.args...) + output, err := tests.ExecuteCommand(cmd, args...) + re.NoError(err) + regions := &response.RegionsInfo{} + re.NoError(json.Unmarshal(output, regions), string(output)) + tests.CheckRegionsInfoWithoutSort(re, regions, testCase.expect) + } + var testRegionCases = []struct { args []string expect *core.RegionInfo From 33ae3b614e46fe0b3a1a94a62b3944883129c829 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 12 Apr 2024 12:34:53 +0800 Subject: [PATCH 09/50] scheduler: use move-hot-write-leader operator (#7852) close tikv/pd#7848 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- metrics/grafana/pd.json | 15 +++++ pkg/schedule/schedulers/hot_region.go | 45 ++------------- pkg/schedule/schedulers/hot_region_test.go | 57 +++++++++++-------- pkg/utils/operatorutil/operator_check.go | 66 ++++++++++++++-------- 4 files changed, 96 insertions(+), 87 deletions(-) diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index 89f2828757f4..9941004e0c23 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -5881,6 +5881,21 @@ "intervalFactor": 1, "legendFormat": "store-{{store}}-in", "refId": "B" + }, + { + "expr": "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"out\",rw=\"write\"}[1m]))by (store)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "store-{{store}}-out", + "refId": "C", + "step": 4 + }, + { + "expr": "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"in\",rw=\"write\"}[1m]))by (store)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "store-{{store}}-in", + "refId": "D" } ], "thresholds": [], diff --git a/pkg/schedule/schedulers/hot_region.go b/pkg/schedule/schedulers/hot_region.go index b6293c2dac91..b4e904c14812 100644 --- a/pkg/schedule/schedulers/hot_region.go +++ b/pkg/schedule/schedulers/hot_region.go @@ -1546,20 +1546,12 @@ func (bs *balanceSolver) buildOperators() (ops []*operator.Operator) { targetLabel := strconv.FormatUint(dstStoreID, 10) dim := bs.rankToDimString() - var createOperator func(region *core.RegionInfo, srcStoreID, dstStoreID uint64) (op *operator.Operator, typ string, err error) - switch bs.rwTy { - case utils.Read: - createOperator = bs.createReadOperator - case utils.Write: - createOperator = bs.createWriteOperator - } - - currentOp, typ, err := createOperator(bs.cur.region, srcStoreID, dstStoreID) + currentOp, typ, err := bs.createOperator(bs.cur.region, srcStoreID, dstStoreID) if err == nil { bs.decorateOperator(currentOp, false, sourceLabel, targetLabel, typ, dim) ops = []*operator.Operator{currentOp} if bs.cur.revertRegion != nil { - currentOp, typ, err = createOperator(bs.cur.revertRegion, dstStoreID, srcStoreID) + currentOp, typ, err = bs.createOperator(bs.cur.revertRegion, dstStoreID, srcStoreID) if err == nil { bs.decorateOperator(currentOp, true, targetLabel, sourceLabel, typ, dim) ops = append(ops, currentOp) @@ -1725,11 +1717,11 @@ func (bs *balanceSolver) createSplitOperator(regions []*core.RegionInfo, strateg return operators } -func (bs *balanceSolver) createReadOperator(region *core.RegionInfo, srcStoreID, dstStoreID uint64) (op *operator.Operator, typ string, err error) { +func (bs *balanceSolver) createOperator(region *core.RegionInfo, srcStoreID, dstStoreID uint64) (op *operator.Operator, typ string, err error) { if region.GetStorePeer(dstStoreID) != nil { typ = "transfer-leader" op, err = operator.CreateTransferLeaderOperator( - "transfer-hot-read-leader", + "transfer-hot-"+bs.rwTy.String()+"-leader", bs, region, dstStoreID, @@ -1741,7 +1733,7 @@ func (bs *balanceSolver) createReadOperator(region *core.RegionInfo, srcStoreID, if region.GetLeader().GetStoreId() == srcStoreID { typ = "move-leader" op, err = operator.CreateMoveLeaderOperator( - "move-hot-read-leader", + "move-hot-"+bs.rwTy.String()+"-leader", bs, region, operator.OpHotRegion, @@ -1750,7 +1742,7 @@ func (bs *balanceSolver) createReadOperator(region *core.RegionInfo, srcStoreID, } else { typ = "move-peer" op, err = operator.CreateMovePeerOperator( - "move-hot-read-peer", + "move-hot-"+bs.rwTy.String()+"-peer", bs, region, operator.OpHotRegion, @@ -1761,31 +1753,6 @@ func (bs *balanceSolver) createReadOperator(region *core.RegionInfo, srcStoreID, return } -func (bs *balanceSolver) createWriteOperator(region *core.RegionInfo, srcStoreID, dstStoreID uint64) (op *operator.Operator, typ string, err error) { - if region.GetStorePeer(dstStoreID) != nil { - typ = "transfer-leader" - op, err = operator.CreateTransferLeaderOperator( - "transfer-hot-write-leader", - bs, - region, - dstStoreID, - []uint64{}, - operator.OpHotRegion) - } else { - srcPeer := region.GetStorePeer(srcStoreID) // checked in `filterHotPeers` - dstPeer := &metapb.Peer{StoreId: dstStoreID, Role: srcPeer.Role} - typ = "move-peer" - op, err = operator.CreateMovePeerOperator( - "move-hot-write-peer", - bs, - region, - operator.OpHotRegion, - srcStoreID, - dstPeer) - } - return -} - func (bs *balanceSolver) decorateOperator(op *operator.Operator, isRevert bool, sourceLabel, targetLabel, typ, dim string) { op.SetPriorityLevel(constant.High) op.FinishedCounters = append(op.FinishedCounters, diff --git a/pkg/schedule/schedulers/hot_region_test.go b/pkg/schedule/schedulers/hot_region_test.go index 5f6cca892ee6..cfc5196909f1 100644 --- a/pkg/schedule/schedulers/hot_region_test.go +++ b/pkg/schedule/schedulers/hot_region_test.go @@ -70,6 +70,11 @@ func clearPendingInfluence(h *hotScheduler) { h.regionPendings = make(map[uint64]*pendingInfluence) } +func newTestRegion(id uint64) *core.RegionInfo { + peers := []*metapb.Peer{{Id: id*100 + 1, StoreId: 1}, {Id: id*100 + 2, StoreId: 2}, {Id: id*100 + 3, StoreId: 3}} + return core.NewRegionInfo(&metapb.Region{Id: id, Peers: peers}, peers[0]) +} + func TestUpgrade(t *testing.T) { re := require.New(t) cancel, _, _, oc := prepareSchedulersTest() @@ -193,20 +198,6 @@ func checkGCPendingOpInfos(re *require.Assertions, enablePlacementRules bool) { } } -func newTestRegion(id uint64) *core.RegionInfo { - peers := []*metapb.Peer{{Id: id*100 + 1, StoreId: 1}, {Id: id*100 + 2, StoreId: 2}, {Id: id*100 + 3, StoreId: 3}} - return core.NewRegionInfo(&metapb.Region{Id: id, Peers: peers}, peers[0]) -} - -func TestHotWriteRegionScheduleByteRateOnly(t *testing.T) { - re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 - checkHotWriteRegionScheduleByteRateOnly(re, false /* disable placement rules */) - checkHotWriteRegionScheduleByteRateOnly(re, true /* enable placement rules */) - checkHotWriteRegionPlacement(re, true) -} - func TestSplitIfRegionTooHot(t *testing.T) { re := require.New(t) statistics.Denoising = false @@ -395,6 +386,15 @@ func TestSplitBucketsByLoad(t *testing.T) { } } +func TestHotWriteRegionScheduleByteRateOnly(t *testing.T) { + re := require.New(t) + statistics.Denoising = false + statisticsInterval = 0 + checkHotWriteRegionScheduleByteRateOnly(re, false /* disable placement rules */) + checkHotWriteRegionScheduleByteRateOnly(re, true /* enable placement rules */) + checkHotWriteRegionPlacement(re, true) +} + func checkHotWriteRegionPlacement(re *require.Assertions, enablePlacementRules bool) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() @@ -446,14 +446,13 @@ func checkHotWriteRegionPlacement(re *require.Assertions, enablePlacementRules b tc.RuleManager.DeleteRule("pd", "follower") ops, _ = hb.Schedule(tc, false) re.NotEmpty(ops) - // TODO: fix the test - // re.NotContains(ops[0].Step(1).String(), "transfer leader") + re.NotContains(ops[0].Step(1).String(), "transfer leader") } func checkHotWriteRegionScheduleByteRateOnly(re *require.Assertions, enablePlacementRules bool) { cancel, opt, tc, oc := prepareSchedulersTest() defer cancel() - tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) + tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.ConfChangeV2)) tc.SetEnablePlacementRules(enablePlacementRules) labels := []string{"zone", "host"} tc.SetMaxReplicasWithLabel(enablePlacementRules, 3, labels...) @@ -511,12 +510,14 @@ func checkHotWriteRegionScheduleByteRateOnly(re *require.Assertions, enablePlace switch op.Len() { case 1: // balance by leader selected + re.Equal("transfer-hot-write-leader", op.Desc()) operatorutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 1) - case 4: + case 5: // balance by peer selected + re.Equal("move-hot-write-leader", op.Desc()) if op.RegionID() == 2 { // peer in store 1 of the region 2 can transfer to store 5 or store 6 because of the label - operatorutil.CheckTransferPeerWithLeaderTransferFrom(re, op, operator.OpHotRegion, 1) + operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 0) } else { // peer in store 1 of the region 1,3 can only transfer to store 6 operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 6) @@ -536,10 +537,10 @@ func checkHotWriteRegionScheduleByteRateOnly(re *require.Assertions, enablePlace ops, _ := hb.Schedule(tc, false) op := ops[0] clearPendingInfluence(hb.(*hotScheduler)) - re.Equal(4, op.Len()) + re.Equal(5, op.Len()) if op.RegionID() == 2 { // peer in store 1 of the region 2 can transfer to store 5 or store 6 because of the label - operatorutil.CheckTransferPeerWithLeaderTransferFrom(re, op, operator.OpHotRegion, 1) + operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 0) } else { // peer in store 1 of the region 1,3 can only transfer to store 6 operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 6) @@ -636,7 +637,7 @@ func TestHotWriteRegionScheduleByteRateOnlyWithTiFlash(t *testing.T) { statisticsInterval = 0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) + tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.ConfChangeV2)) tc.SetHotRegionCacheHitsThreshold(0) re.NoError(tc.RuleManager.SetRules([]*placement.Rule{ { @@ -730,9 +731,11 @@ func TestHotWriteRegionScheduleByteRateOnlyWithTiFlash(t *testing.T) { switch op.Len() { case 1: // balance by leader selected + re.Equal("transfer-hot-write-leader", op.Desc()) operatorutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 1) case 2: // balance by peer selected + re.Equal("move-hot-write-leader", op.Desc()) operatorutil.CheckTransferLearner(re, op, operator.OpHotRegion, 8, 10) default: re.FailNow("wrong op: " + op.String()) @@ -823,12 +826,14 @@ func TestHotWriteRegionScheduleByteRateOnlyWithTiFlash(t *testing.T) { switch op.Len() { case 1: // balance by leader selected + re.Equal("transfer-hot-write-leader", op.Desc()) operatorutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 1) - case 4: + case 5: // balance by peer selected + re.Equal("move-hot-write-leader", op.Desc()) if op.RegionID() == 2 { // peer in store 1 of the region 2 can transfer to store 5 or store 6 because of the label - operatorutil.CheckTransferPeerWithLeaderTransferFrom(re, op, operator.OpHotRegion, 1) + operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 0) } else { // peer in store 1 of the region 1,3 can only transfer to store 6 operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 6) @@ -1164,9 +1169,11 @@ func checkHotWriteRegionScheduleWithPendingInfluence(re *require.Assertions, dim switch op.Len() { case 1: // balance by leader selected + re.Equal("transfer-hot-write-leader", op.Desc()) operatorutil.CheckTransferLeaderFrom(re, op, operator.OpHotRegion, 1) - case 4: + case 5: // balance by peer selected + re.Equal("move-hot-write-leader", op.Desc()) operatorutil.CheckTransferPeerWithLeaderTransfer(re, op, operator.OpHotRegion, 1, 4) cnt++ if cnt == 3 { diff --git a/pkg/utils/operatorutil/operator_check.go b/pkg/utils/operatorutil/operator_check.go index 5d366cf8fb95..61efd84ef1a1 100644 --- a/pkg/utils/operatorutil/operator_check.go +++ b/pkg/utils/operatorutil/operator_check.go @@ -65,11 +65,27 @@ func trimTransferLeaders(op *operator.Operator) (steps []operator.OpStep, lastLe // CheckTransferPeer checks if the operator is to transfer peer between the specified source and target stores. func CheckTransferPeer(re *require.Assertions, op *operator.Operator, kind operator.OpKind, sourceID, targetID uint64) { re.NotNil(op) + var addLearnerTo, removePeerFrom uint64 steps, _ := trimTransferLeaders(op) - re.Len(steps, 3) - re.Equal(targetID, steps[0].(operator.AddLearner).ToStore) - re.IsType(operator.PromoteLearner{}, steps[1]) - re.Equal(sourceID, steps[2].(operator.RemovePeer).FromStore) + switch len(steps) { + case 3: // without joint consensus + re.IsType(operator.AddLearner{}, steps[0]) + re.IsType(operator.PromoteLearner{}, steps[1]) + re.IsType(operator.RemovePeer{}, steps[2]) + addLearnerTo = steps[0].(operator.AddLearner).ToStore + removePeerFrom = steps[2].(operator.RemovePeer).FromStore + case 4: // with joint consensus + re.IsType(operator.AddLearner{}, steps[0]) + re.IsType(operator.ChangePeerV2Enter{}, steps[1]) + re.IsType(operator.ChangePeerV2Leave{}, steps[2]) + re.IsType(operator.RemovePeer{}, steps[3]) + addLearnerTo = steps[0].(operator.AddLearner).ToStore + removePeerFrom = steps[3].(operator.RemovePeer).FromStore + default: + re.FailNow("unexpected operator steps") + } + re.Equal(sourceID, removePeerFrom) + re.Equal(targetID, addLearnerTo) kind |= operator.OpRegion re.Equal(kind, op.Kind()&kind) } @@ -88,32 +104,36 @@ func CheckTransferLearner(re *require.Assertions, op *operator.Operator, kind op // CheckTransferPeerWithLeaderTransfer checks if the operator is to transfer // peer between the specified source and target stores and it meanwhile // transfers the leader out of source store. +// If targetID is 0, it means the operator is to transfer peer to any store. func CheckTransferPeerWithLeaderTransfer(re *require.Assertions, op *operator.Operator, kind operator.OpKind, sourceID, targetID uint64) { re.NotNil(op) + var addLearnerTo, removePeerFrom uint64 steps, lastLeader := trimTransferLeaders(op) - re.Len(steps, 3) - re.Equal(targetID, steps[0].(operator.AddLearner).ToStore) - re.IsType(operator.PromoteLearner{}, steps[1]) - re.Equal(sourceID, steps[2].(operator.RemovePeer).FromStore) + switch len(steps) { + case 3: // without joint consensus + re.IsType(operator.AddLearner{}, steps[0]) + re.IsType(operator.PromoteLearner{}, steps[1]) + re.IsType(operator.RemovePeer{}, steps[2]) + addLearnerTo = steps[0].(operator.AddLearner).ToStore + removePeerFrom = steps[2].(operator.RemovePeer).FromStore + case 4: // with joint consensus + re.IsType(operator.AddLearner{}, steps[0]) + re.IsType(operator.ChangePeerV2Enter{}, steps[1]) + re.IsType(operator.ChangePeerV2Leave{}, steps[2]) + re.IsType(operator.RemovePeer{}, steps[3]) + addLearnerTo = steps[0].(operator.AddLearner).ToStore + removePeerFrom = steps[3].(operator.RemovePeer).FromStore + default: + re.FailNow("unexpected operator steps") + } re.NotZero(lastLeader) re.NotEqual(sourceID, lastLeader) kind |= operator.OpRegion re.Equal(kind, op.Kind()&kind) -} - -// CheckTransferPeerWithLeaderTransferFrom checks if the operator is to transfer -// peer out of the specified store and it meanwhile transfers the leader out of -// the store. -func CheckTransferPeerWithLeaderTransferFrom(re *require.Assertions, op *operator.Operator, kind operator.OpKind, sourceID uint64) { - re.NotNil(op) - steps, lastLeader := trimTransferLeaders(op) - re.IsType(operator.AddLearner{}, steps[0]) - re.IsType(operator.PromoteLearner{}, steps[1]) - re.Equal(sourceID, steps[2].(operator.RemovePeer).FromStore) - re.NotZero(lastLeader) - re.NotEqual(sourceID, lastLeader) - kind |= operator.OpRegion | operator.OpLeader - re.Equal(kind, op.Kind()&kind) + re.Equal(sourceID, removePeerFrom) + if targetID != 0 { + re.Equal(targetID, addLearnerTo) + } } // CheckAddPeer checks if the operator is to add peer on specified store. From 5da6def96a3094385fdfd6d037a74845aaf51e4e Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 12 Apr 2024 12:51:23 +0800 Subject: [PATCH 10/50] pd-ctl: hidden some hot scheduler config (#7892) ref tikv/pd#5691 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tools/pd-ctl/pdctl/command/scheduler.go | 29 +++++++++++++++ .../pd-ctl/tests/scheduler/scheduler_test.go | 35 +++++++------------ 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/tools/pd-ctl/pdctl/command/scheduler.go b/tools/pd-ctl/pdctl/command/scheduler.go index d5deba670ad2..c1db24cc176e 100644 --- a/tools/pd-ctl/pdctl/command/scheduler.go +++ b/tools/pd-ctl/pdctl/command/scheduler.go @@ -649,6 +649,18 @@ func addStoreToSchedulerConfig(cmd *cobra.Command, schedulerName string, args [] postJSON(cmd, path.Join(schedulerConfigPrefix, schedulerName, "config"), input) } +var hiddenHotConfig = []string{ + "max-zombie-rounds", + "max-peer-number", + "byte-rate-rank-step-ratio", + "key-rate-rank-step-ratio", + "query-rate-rank-step-ratio", + "count-rank-step-ratio", + "great-dec-ratio", + "minor-dec-ratio", + "enable-for-tiflash", +} + func listSchedulerConfigCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 0 { cmd.Println(cmd.UsageString()) @@ -667,6 +679,23 @@ func listSchedulerConfigCommandFunc(cmd *cobra.Command, args []string) { cmd.Println(err) return } + if p == "balance-hot-region-scheduler" { + schedulerConfig := make(map[string]any) + err := json.Unmarshal([]byte(r), &schedulerConfig) + if err != nil { + cmd.Println(err) + return + } + for _, config := range hiddenHotConfig { + delete(schedulerConfig, config) + } + b, err := json.MarshalIndent(schedulerConfig, "", " ") + if err != nil { + cmd.Println(err) + return + } + r = string(b) + } cmd.Println(r) } diff --git a/tools/pd-ctl/tests/scheduler/scheduler_test.go b/tools/pd-ctl/tests/scheduler/scheduler_test.go index b5a2128752b0..00fab12b99b0 100644 --- a/tools/pd-ctl/tests/scheduler/scheduler_test.go +++ b/tools/pd-ctl/tests/scheduler/scheduler_test.go @@ -386,28 +386,19 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { // test hot region config expected1 := map[string]any{ - "min-hot-byte-rate": float64(100), - "min-hot-key-rate": float64(10), - "min-hot-query-rate": float64(10), - "max-zombie-rounds": float64(3), - "max-peer-number": float64(1000), - "byte-rate-rank-step-ratio": 0.05, - "key-rate-rank-step-ratio": 0.05, - "query-rate-rank-step-ratio": 0.05, - "count-rank-step-ratio": 0.01, - "great-dec-ratio": 0.95, - "minor-dec-ratio": 0.99, - "src-tolerance-ratio": 1.05, - "dst-tolerance-ratio": 1.05, - "read-priorities": []any{"byte", "key"}, - "write-leader-priorities": []any{"key", "byte"}, - "write-peer-priorities": []any{"byte", "key"}, - "strict-picking-store": "true", - "enable-for-tiflash": "true", - "rank-formula-version": "v2", - "split-thresholds": 0.2, - "history-sample-duration": "5m0s", - "history-sample-interval": "30s", + "min-hot-byte-rate": float64(100), + "min-hot-key-rate": float64(10), + "min-hot-query-rate": float64(10), + "src-tolerance-ratio": 1.05, + "dst-tolerance-ratio": 1.05, + "read-priorities": []any{"byte", "key"}, + "write-leader-priorities": []any{"key", "byte"}, + "write-peer-priorities": []any{"byte", "key"}, + "strict-picking-store": "true", + "rank-formula-version": "v2", + "split-thresholds": 0.2, + "history-sample-duration": "5m0s", + "history-sample-interval": "30s", } checkHotSchedulerConfig := func(expect map[string]any) { testutil.Eventually(re, func() bool { From c01d8fd76091d5591bd72282a82ead7204a258ba Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 12 Apr 2024 16:27:24 +0800 Subject: [PATCH 11/50] *: make `TestTSOKeyspaceGroupSplit` stable (#8059) close tikv/pd#7038 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tests/integrations/mcs/tso/keyspace_group_manager_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go index f678e9014193..9194811cd379 100644 --- a/tests/integrations/mcs/tso/keyspace_group_manager_test.go +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -277,7 +277,10 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupSplit() { }) // Wait for the split to complete automatically even there is no TSO request from the outside. testutil.Eventually(re, func() bool { - kg2 := handlersutil.MustLoadKeyspaceGroupByID(re, suite.pdLeaderServer, 2) + kg2, code := handlersutil.TryLoadKeyspaceGroupByID(re, suite.pdLeaderServer, 2) + if code != http.StatusOK { + return false + } re.Equal(uint32(2), kg2.ID) re.Equal([]uint32{222, 333}, kg2.Keyspaces) return !kg2.IsSplitting() From 340c58c02d15fbcf0a529da30c214dea3ed3a3dc Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 15 Apr 2024 10:58:35 +0800 Subject: [PATCH 12/50] *: fix client log (#8060) ref tikv/pd#4399 Signed-off-by: Ryan Leung --- client/tso_service_discovery.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/tso_service_discovery.go b/client/tso_service_discovery.go index 34ef16f88b0b..f88403bb3220 100644 --- a/client/tso_service_discovery.go +++ b/client/tso_service_discovery.go @@ -465,7 +465,7 @@ func (c *tsoServiceDiscovery) updateMember() error { oldGroupID := c.GetKeyspaceGroupID() if oldGroupID != keyspaceGroup.Id { log.Info("[tso] the keyspace group changed", - zap.Uint32("keyspace-id", keyspaceGroup.Id), + zap.Uint32("keyspace-id", keyspaceID), zap.Uint32("new-keyspace-group-id", keyspaceGroup.Id), zap.Uint32("old-keyspace-group-id", oldGroupID)) } From 02147789a144930f465a0142dd33da8f48c0d16b Mon Sep 17 00:00:00 2001 From: ShuNing Date: Mon, 15 Apr 2024 17:13:37 +0800 Subject: [PATCH 13/50] *: optimize heartbeat process with concurrent runner - part 1 (#8053) ref tikv/pd#7897 *: Optimize the heartbeat process with the concurrent runner - add a switch about the runner - move some tasks out, not block the heartbeat process Signed-off-by: nolouch --- pkg/cluster/cluster.go | 14 ++- pkg/core/context.go | 40 +++++++ pkg/core/region.go | 82 +++++++++++---- pkg/core/region_test.go | 6 +- pkg/core/region_tree.go | 5 + pkg/core/region_tree_test.go | 2 + pkg/mcs/scheduling/server/cluster.go | 87 +++++++++++++-- pkg/ratelimit/runner.go | 32 +++--- pkg/ratelimit/runner_test.go | 8 +- pkg/schedule/config/config.go | 8 ++ pkg/statistics/region_collection.go | 12 +++ pkg/syncer/client.go | 12 ++- server/cluster/cluster.go | 152 ++++++++++++++++++--------- server/cluster/cluster_test.go | 102 +++++++++--------- server/cluster/cluster_worker.go | 14 ++- tests/cluster.go | 2 + 16 files changed, 424 insertions(+), 154 deletions(-) create mode 100644 pkg/core/context.go diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 916200bfa3ec..8bd2616f41f7 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -28,6 +28,7 @@ type Cluster interface { GetLabelStats() *statistics.LabelStatistics GetCoordinator() *schedule.Coordinator GetRuleManager() *placement.RuleManager + GetBasicCluster() *core.BasicCluster } // HandleStatsAsync handles the flow asynchronously. @@ -55,8 +56,17 @@ func HandleOverlaps(c Cluster, overlaps []*core.RegionInfo) { } // Collect collects the cluster information. -func Collect(c Cluster, region *core.RegionInfo, stores []*core.StoreInfo, hasRegionStats bool) { +func Collect(c Cluster, region *core.RegionInfo, hasRegionStats bool) { if hasRegionStats { - c.GetRegionStats().Observe(region, stores) + // get region again from root tree. make sure the observed region is the latest. + bc := c.GetBasicCluster() + if bc == nil { + return + } + region = bc.GetRegion(region.GetID()) + if region == nil { + return + } + c.GetRegionStats().Observe(region, c.GetBasicCluster().GetRegionStores(region)) } } diff --git a/pkg/core/context.go b/pkg/core/context.go new file mode 100644 index 000000000000..ab149378b1d1 --- /dev/null +++ b/pkg/core/context.go @@ -0,0 +1,40 @@ +// Copyright 2024 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "context" + + "github.com/tikv/pd/pkg/ratelimit" +) + +// MetaProcessContext is a context for meta process. +type MetaProcessContext struct { + context.Context + Tracer RegionHeartbeatProcessTracer + TaskRunner ratelimit.Runner + Limiter *ratelimit.ConcurrencyLimiter +} + +// NewMetaProcessContext creates a new MetaProcessContext. +// used in tests, can be changed if no need to test concurrency. +func ContextTODO() *MetaProcessContext { + return &MetaProcessContext{ + Context: context.TODO(), + Tracer: NewNoopHeartbeatProcessTracer(), + TaskRunner: ratelimit.NewSyncRunner(), + // Limit default is nil + } +} diff --git a/pkg/core/region.go b/pkg/core/region.go index baabafa1fa97..41bfb4d31add 100644 --- a/pkg/core/region.go +++ b/pkg/core/region.go @@ -16,6 +16,7 @@ package core import ( "bytes" + "context" "encoding/hex" "fmt" "math" @@ -35,6 +36,7 @@ import ( "github.com/pingcap/kvproto/pkg/replication_modepb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/typeutil" @@ -711,20 +713,50 @@ func (r *RegionInfo) isRegionRecreated() bool { // RegionGuideFunc is a function that determines which follow-up operations need to be performed based on the origin // and new region information. -type RegionGuideFunc func(region, origin *RegionInfo) (saveKV, saveCache, needSync bool) +type RegionGuideFunc func(ctx *MetaProcessContext, region, origin *RegionInfo) (saveKV, saveCache, needSync bool) // GenerateRegionGuideFunc is used to generate a RegionGuideFunc. Control the log output by specifying the log function. // nil means do not print the log. func GenerateRegionGuideFunc(enableLog bool) RegionGuideFunc { noLog := func(string, ...zap.Field) {} - debug, info := noLog, noLog + d, i := noLog, noLog if enableLog { - debug = log.Debug - info = log.Info + d = log.Debug + i = log.Info } // Save to storage if meta is updated. // Save to cache if meta or leader is updated, or contains any down/pending peer. - return func(region, origin *RegionInfo) (saveKV, saveCache, needSync bool) { + return func(ctx *MetaProcessContext, region, origin *RegionInfo) (saveKV, saveCache, needSync bool) { + taskRunner := ctx.TaskRunner + limiter := ctx.Limiter + // print log asynchronously + debug, info := d, i + if taskRunner != nil { + debug = func(msg string, fields ...zap.Field) { + taskRunner.RunTask( + ctx.Context, + ratelimit.TaskOpts{ + TaskName: "Log", + Limit: limiter, + }, + func(_ context.Context) { + d(msg, fields...) + }, + ) + } + info = func(msg string, fields ...zap.Field) { + taskRunner.RunTask( + ctx.Context, + ratelimit.TaskOpts{ + TaskName: "Log", + Limit: limiter, + }, + func(_ context.Context) { + i(msg, fields...) + }, + ) + } + } if origin == nil { if log.GetLevel() <= zap.DebugLevel { debug("insert new region", @@ -789,7 +821,7 @@ func GenerateRegionGuideFunc(enableLog bool) RegionGuideFunc { } if !SortedPeersStatsEqual(region.GetDownPeers(), origin.GetDownPeers()) { if log.GetLevel() <= zap.DebugLevel { - debug("down-peers changed", zap.Uint64("region-id", region.GetID())) + debug("down-peers changed", zap.Uint64("region-id", region.GetID()), zap.Reflect("before", origin.GetDownPeers()), zap.Reflect("after", region.GetDownPeers())) } saveCache, needSync = true, true return @@ -912,7 +944,7 @@ func (r *RegionsInfo) CheckAndPutRegion(region *RegionInfo) []*RegionInfo { if origin == nil || !bytes.Equal(origin.GetStartKey(), region.GetStartKey()) || !bytes.Equal(origin.GetEndKey(), region.GetEndKey()) { ols = r.tree.overlaps(®ionItem{RegionInfo: region}) } - err := check(region, origin, ols) + err := check(region, origin, convertItemsToRegions(ols)) if err != nil { log.Debug("region is stale", zap.Stringer("origin", origin.GetMeta()), errs.ZapError(err)) // return the state region to delete. @@ -933,48 +965,59 @@ func (r *RegionsInfo) PutRegion(region *RegionInfo) []*RegionInfo { } // PreCheckPutRegion checks if the region is valid to put. -func (r *RegionsInfo) PreCheckPutRegion(region *RegionInfo, trace RegionHeartbeatProcessTracer) (*RegionInfo, []*regionItem, error) { - origin, overlaps := r.GetRelevantRegions(region, trace) +func (r *RegionsInfo) PreCheckPutRegion(region *RegionInfo) (*RegionInfo, []*RegionInfo, error) { + origin, overlaps := r.GetRelevantRegions(region) err := check(region, origin, overlaps) return origin, overlaps, err } +func convertItemsToRegions(items []*regionItem) []*RegionInfo { + regions := make([]*RegionInfo, 0, len(items)) + for _, item := range items { + regions = append(regions, item.RegionInfo) + } + return regions +} + // AtomicCheckAndPutRegion checks if the region is valid to put, if valid then put. -func (r *RegionsInfo) AtomicCheckAndPutRegion(region *RegionInfo, trace RegionHeartbeatProcessTracer) ([]*RegionInfo, error) { +func (r *RegionsInfo) AtomicCheckAndPutRegion(ctx *MetaProcessContext, region *RegionInfo) ([]*RegionInfo, error) { + tracer := ctx.Tracer r.t.Lock() var ols []*regionItem origin := r.getRegionLocked(region.GetID()) if origin == nil || !bytes.Equal(origin.GetStartKey(), region.GetStartKey()) || !bytes.Equal(origin.GetEndKey(), region.GetEndKey()) { ols = r.tree.overlaps(®ionItem{RegionInfo: region}) } - trace.OnCheckOverlapsFinished() - err := check(region, origin, ols) + tracer.OnCheckOverlapsFinished() + err := check(region, origin, convertItemsToRegions(ols)) if err != nil { r.t.Unlock() - trace.OnValidateRegionFinished() + tracer.OnValidateRegionFinished() return nil, err } - trace.OnValidateRegionFinished() + tracer.OnValidateRegionFinished() origin, overlaps, rangeChanged := r.setRegionLocked(region, true, ols...) r.t.Unlock() - trace.OnSetRegionFinished() + tracer.OnSetRegionFinished() r.UpdateSubTree(region, origin, overlaps, rangeChanged) - trace.OnUpdateSubTreeFinished() + tracer.OnUpdateSubTreeFinished() return overlaps, nil } // GetRelevantRegions returns the relevant regions for a given region. -func (r *RegionsInfo) GetRelevantRegions(region *RegionInfo, _ RegionHeartbeatProcessTracer) (origin *RegionInfo, overlaps []*regionItem) { +func (r *RegionsInfo) GetRelevantRegions(region *RegionInfo) (origin *RegionInfo, overlaps []*RegionInfo) { r.t.RLock() defer r.t.RUnlock() origin = r.getRegionLocked(region.GetID()) if origin == nil || !bytes.Equal(origin.GetStartKey(), region.GetStartKey()) || !bytes.Equal(origin.GetEndKey(), region.GetEndKey()) { - overlaps = r.tree.overlaps(®ionItem{RegionInfo: region}) + for _, item := range r.tree.overlaps(®ionItem{RegionInfo: region}) { + overlaps = append(overlaps, item.RegionInfo) + } } return } -func check(region, origin *RegionInfo, overlaps []*regionItem) error { +func check(region, origin *RegionInfo, overlaps []*RegionInfo) error { for _, item := range overlaps { // PD ignores stale regions' heartbeats, unless it is recreated recently by unsafe recover operation. if region.GetRegionEpoch().GetVersion() < item.GetRegionEpoch().GetVersion() && !region.isRegionRecreated() { @@ -1043,7 +1086,6 @@ func (r *RegionsInfo) setRegionLocked(region *RegionInfo, withOverlaps bool, ol item = ®ionItem{RegionInfo: region} r.regions[region.GetID()] = item } - var overlaps []*RegionInfo if rangeChanged { overlaps = r.tree.update(item, withOverlaps, ol...) diff --git a/pkg/core/region_test.go b/pkg/core/region_test.go index 3c6536a6a773..ae91886369f9 100644 --- a/pkg/core/region_test.go +++ b/pkg/core/region_test.go @@ -363,7 +363,7 @@ func TestNeedSync(t *testing.T) { for _, testCase := range testCases { regionA := region.Clone(testCase.optionsA...) regionB := region.Clone(testCase.optionsB...) - _, _, needSync := RegionGuide(regionA, regionB) + _, _, needSync := RegionGuide(ContextTODO(), regionA, regionB) re.Equal(testCase.needSync, needSync) } } @@ -459,9 +459,9 @@ func TestSetRegionConcurrence(t *testing.T) { regions := NewRegionsInfo() region := NewTestRegionInfo(1, 1, []byte("a"), []byte("b")) go func() { - regions.AtomicCheckAndPutRegion(region, NewNoopHeartbeatProcessTracer()) + regions.AtomicCheckAndPutRegion(ContextTODO(), region) }() - regions.AtomicCheckAndPutRegion(region, NewNoopHeartbeatProcessTracer()) + regions.AtomicCheckAndPutRegion(ContextTODO(), region) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/core/UpdateSubTree")) } diff --git a/pkg/core/region_tree.go b/pkg/core/region_tree.go index 333e1730ec8a..8c928f391eb3 100644 --- a/pkg/core/region_tree.go +++ b/pkg/core/region_tree.go @@ -35,6 +35,11 @@ func (r *regionItem) GetStartKey() []byte { return r.meta.StartKey } +// GetID returns the ID of the region. +func (r *regionItem) GetID() uint64 { + return r.meta.GetId() +} + // GetEndKey returns the end key of the region. func (r *regionItem) GetEndKey() []byte { return r.meta.EndKey diff --git a/pkg/core/region_tree_test.go b/pkg/core/region_tree_test.go index 4e002fb8157b..f4ef6cb67b3b 100644 --- a/pkg/core/region_tree_test.go +++ b/pkg/core/region_tree_test.go @@ -158,6 +158,8 @@ func TestRegionTree(t *testing.T) { updateNewItem(tree, regionA) updateNewItem(tree, regionC) + re.Nil(tree.overlaps(newRegionItem([]byte("b"), []byte("c")))) + re.Equal(regionC, tree.overlaps(newRegionItem([]byte("a"), []byte("cc")))[1].RegionInfo) re.Nil(tree.search([]byte{})) re.Equal(regionA, tree.search([]byte("a"))) re.Nil(tree.search([]byte("b"))) diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index 7ee7ae88cd1d..42e8c3a35cb7 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -2,6 +2,7 @@ package server import ( "context" + "runtime" "sync" "sync/atomic" "time" @@ -15,6 +16,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/mcs/scheduling/server/config" + "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/pkg/schedule" sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/hbstream" @@ -51,14 +53,22 @@ type Cluster struct { apiServerLeader atomic.Value clusterID uint64 running atomic.Bool + + taskRunner ratelimit.Runner + hbConcurrencyLimiter *ratelimit.ConcurrencyLimiter } const ( regionLabelGCInterval = time.Hour requestTimeout = 3 * time.Second collectWaitTime = time.Minute + + // heartbeat relative const + hbConcurrentRunner = "heartbeat-concurrent-task-runner" ) +var syncRunner = ratelimit.NewSyncRunner() + // NewCluster creates a new cluster. func NewCluster(parentCtx context.Context, persistConfig *config.PersistConfig, storage storage.Storage, basicCluster *core.BasicCluster, hbStreams *hbstream.HeartbeatStreams, clusterID uint64, checkMembershipCh chan struct{}) (*Cluster, error) { ctx, cancel := context.WithCancel(parentCtx) @@ -81,6 +91,9 @@ func NewCluster(parentCtx context.Context, persistConfig *config.PersistConfig, storage: storage, clusterID: clusterID, checkMembershipCh: checkMembershipCh, + + taskRunner: ratelimit.NewConcurrentRunner(hbConcurrentRunner, time.Minute), + hbConcurrencyLimiter: ratelimit.NewConcurrencyLimiter(uint64(runtime.NumCPU() * 2)), } c.coordinator = schedule.NewCoordinator(ctx, c, hbStreams) err = c.ruleManager.Initialize(persistConfig.GetMaxReplicas(), persistConfig.GetLocationLabels(), persistConfig.GetIsolationLevel()) @@ -517,6 +530,7 @@ func (c *Cluster) StartBackgroundJobs() { go c.runUpdateStoreStats() go c.runCoordinator() go c.runMetricsCollectionJob() + c.taskRunner.Start() c.running.Store(true) } @@ -527,6 +541,7 @@ func (c *Cluster) StopBackgroundJobs() { } c.running.Store(false) c.coordinator.Stop() + c.taskRunner.Stop() c.cancel() c.wg.Wait() } @@ -542,8 +557,19 @@ func (c *Cluster) HandleRegionHeartbeat(region *core.RegionInfo) error { if c.persistConfig.GetScheduleConfig().EnableHeartbeatBreakdownMetrics { tracer = core.NewHeartbeatProcessTracer() } + var runner ratelimit.Runner + runner = syncRunner + if c.persistConfig.GetScheduleConfig().EnableHeartbeatConcurrentRunner { + runner = c.taskRunner + } + ctx := &core.MetaProcessContext{ + Context: c.ctx, + Limiter: c.hbConcurrencyLimiter, + Tracer: tracer, + TaskRunner: runner, + } tracer.Begin() - if err := c.processRegionHeartbeat(region, tracer); err != nil { + if err := c.processRegionHeartbeat(ctx, region); err != nil { tracer.OnAllStageFinished() return err } @@ -553,26 +579,47 @@ func (c *Cluster) HandleRegionHeartbeat(region *core.RegionInfo) error { } // processRegionHeartbeat updates the region information. -func (c *Cluster) processRegionHeartbeat(region *core.RegionInfo, tracer core.RegionHeartbeatProcessTracer) error { - origin, _, err := c.PreCheckPutRegion(region, tracer) +func (c *Cluster) processRegionHeartbeat(ctx *core.MetaProcessContext, region *core.RegionInfo) error { + tracer := ctx.Tracer + origin, _, err := c.PreCheckPutRegion(region) tracer.OnPreCheckFinished() if err != nil { return err } region.Inherit(origin, c.GetStoreConfig().IsEnableRegionBucket()) - cluster.HandleStatsAsync(c, region) + ctx.TaskRunner.RunTask( + ctx, + ratelimit.TaskOpts{ + TaskName: "HandleStatsAsync", + Limit: ctx.Limiter, + }, + func(_ context.Context) { + cluster.HandleStatsAsync(c, region) + }, + ) tracer.OnAsyncHotStatsFinished() hasRegionStats := c.regionStats != nil // Save to storage if meta is updated, except for flashback. // Save to cache if meta or leader is updated, or contains any down/pending peer. - _, saveCache, _ := core.GenerateRegionGuideFunc(true)(region, origin) + _, saveCache, _ := core.GenerateRegionGuideFunc(true)(ctx, region, origin) if !saveCache { // Due to some config changes need to update the region stats as well, // so we do some extra checks here. if hasRegionStats && c.regionStats.RegionStatsNeedUpdate(region) { - c.regionStats.Observe(region, c.GetRegionStores(region)) + ctx.TaskRunner.RunTask( + ctx, + ratelimit.TaskOpts{ + TaskName: "ObserveRegionStatsAsync", + Limit: ctx.Limiter, + }, + func(_ context.Context) { + if c.regionStats.RegionStatsNeedUpdate(region) { + cluster.Collect(c, region, hasRegionStats) + } + }, + ) } return nil } @@ -583,15 +630,35 @@ func (c *Cluster) processRegionHeartbeat(region *core.RegionInfo, tracer core.Re // check its validation again here. // // However, it can't solve the race condition of concurrent heartbeats from the same region. - if overlaps, err = c.AtomicCheckAndPutRegion(region, tracer); err != nil { + + // Async task in next PR. + if overlaps, err = c.AtomicCheckAndPutRegion(ctx, region); err != nil { tracer.OnSaveCacheFinished() return err } - - cluster.HandleOverlaps(c, overlaps) + ctx.TaskRunner.RunTask( + ctx, + ratelimit.TaskOpts{ + TaskName: "HandleOverlaps", + Limit: ctx.Limiter, + }, + func(_ context.Context) { + cluster.HandleOverlaps(c, overlaps) + }, + ) } tracer.OnSaveCacheFinished() - cluster.Collect(c, region, c.GetRegionStores(region), hasRegionStats) + // handle region stats + ctx.TaskRunner.RunTask( + ctx, + ratelimit.TaskOpts{ + TaskName: "CollectRegionStatsAsync", + Limit: ctx.Limiter, + }, + func(_ context.Context) { + cluster.Collect(c, region, hasRegionStats) + }, + ) tracer.OnCollectRegionStatsFinished() return nil } diff --git a/pkg/ratelimit/runner.go b/pkg/ratelimit/runner.go index dd92a10179dc..7f0ef21f7916 100644 --- a/pkg/ratelimit/runner.go +++ b/pkg/ratelimit/runner.go @@ -29,6 +29,8 @@ const initialCapacity = 100 // Runner is the interface for running tasks. type Runner interface { RunTask(ctx context.Context, opt TaskOpts, f func(context.Context)) error + Start() + Stop() } // Task is a task to be run. @@ -42,8 +44,8 @@ type Task struct { // ErrMaxWaitingTasksExceeded is returned when the number of waiting tasks exceeds the maximum. var ErrMaxWaitingTasksExceeded = errors.New("max waiting tasks exceeded") -// AsyncRunner is a simple task runner that limits the number of concurrent tasks. -type AsyncRunner struct { +// ConcurrentRunner is a simple task runner that limits the number of concurrent tasks. +type ConcurrentRunner struct { name string maxPendingDuration time.Duration taskChan chan *Task @@ -53,16 +55,14 @@ type AsyncRunner struct { wg sync.WaitGroup } -// NewAsyncRunner creates a new AsyncRunner. -func NewAsyncRunner(name string, maxPendingDuration time.Duration) *AsyncRunner { - s := &AsyncRunner{ +// NewConcurrentRunner creates a new ConcurrentRunner. +func NewConcurrentRunner(name string, maxPendingDuration time.Duration) *ConcurrentRunner { + s := &ConcurrentRunner{ name: name, maxPendingDuration: maxPendingDuration, taskChan: make(chan *Task), pendingTasks: make([]*Task, 0, initialCapacity), - stopChan: make(chan struct{}), } - s.Start() return s } @@ -74,7 +74,8 @@ type TaskOpts struct { } // Start starts the runner. -func (s *AsyncRunner) Start() { +func (s *ConcurrentRunner) Start() { + s.stopChan = make(chan struct{}) s.wg.Add(1) go func() { defer s.wg.Done() @@ -84,7 +85,6 @@ func (s *AsyncRunner) Start() { if task.Opts.Limit != nil { token, err := task.Opts.Limit.Acquire(context.Background()) if err != nil { - log.Error("failed to acquire semaphore", zap.String("task-name", task.Opts.TaskName), zap.Error(err)) continue } go s.run(task.Ctx, task.f, token) @@ -99,7 +99,7 @@ func (s *AsyncRunner) Start() { }() } -func (s *AsyncRunner) run(ctx context.Context, task func(context.Context), token *TaskToken) { +func (s *ConcurrentRunner) run(ctx context.Context, task func(context.Context), token *TaskToken) { task(ctx) if token != nil { token.Release() @@ -107,7 +107,7 @@ func (s *AsyncRunner) run(ctx context.Context, task func(context.Context), token } } -func (s *AsyncRunner) processPendingTasks() { +func (s *ConcurrentRunner) processPendingTasks() { s.pendingMu.Lock() defer s.pendingMu.Unlock() for len(s.pendingTasks) > 0 { @@ -123,13 +123,13 @@ func (s *AsyncRunner) processPendingTasks() { } // Stop stops the runner. -func (s *AsyncRunner) Stop() { +func (s *ConcurrentRunner) Stop() { close(s.stopChan) s.wg.Wait() } // RunTask runs the task asynchronously. -func (s *AsyncRunner) RunTask(ctx context.Context, opt TaskOpts, f func(context.Context)) error { +func (s *ConcurrentRunner) RunTask(ctx context.Context, opt TaskOpts, f func(context.Context)) error { task := &Task{ Ctx: ctx, Opts: opt, @@ -166,3 +166,9 @@ func (*SyncRunner) RunTask(ctx context.Context, _ TaskOpts, f func(context.Conte f(ctx) return nil } + +// Start starts the runner. +func (*SyncRunner) Start() {} + +// Stop stops the runner. +func (*SyncRunner) Stop() {} diff --git a/pkg/ratelimit/runner_test.go b/pkg/ratelimit/runner_test.go index 9b8dca231d11..507f8cf4ee8a 100644 --- a/pkg/ratelimit/runner_test.go +++ b/pkg/ratelimit/runner_test.go @@ -23,10 +23,11 @@ import ( "github.com/stretchr/testify/require" ) -func TestAsyncRunner(t *testing.T) { +func TestConcurrentRunner(t *testing.T) { t.Run("RunTask", func(t *testing.T) { limiter := NewConcurrencyLimiter(1) - runner := NewAsyncRunner("test", time.Second) + runner := NewConcurrentRunner("test", time.Second) + runner.Start() defer runner.Stop() var wg sync.WaitGroup @@ -47,7 +48,8 @@ func TestAsyncRunner(t *testing.T) { t.Run("MaxPendingDuration", func(t *testing.T) { limiter := NewConcurrencyLimiter(1) - runner := NewAsyncRunner("test", 2*time.Millisecond) + runner := NewConcurrentRunner("test", 2*time.Millisecond) + runner.Start() defer runner.Stop() var wg sync.WaitGroup for i := 0; i < 10; i++ { diff --git a/pkg/schedule/config/config.go b/pkg/schedule/config/config.go index abf4c776f8a5..1f3701763837 100644 --- a/pkg/schedule/config/config.go +++ b/pkg/schedule/config/config.go @@ -52,6 +52,7 @@ const ( defaultEnableJointConsensus = true defaultEnableTiKVSplitRegion = true defaultEnableHeartbeatBreakdownMetrics = true + defaultEnableHeartbeatConcurrentRunner = false defaultEnableCrossTableMerge = true defaultEnableDiagnostic = true defaultStrictlyMatchLabel = false @@ -267,6 +268,9 @@ type ScheduleConfig struct { // EnableHeartbeatBreakdownMetrics is the option to enable heartbeat stats metrics. EnableHeartbeatBreakdownMetrics bool `toml:"enable-heartbeat-breakdown-metrics" json:"enable-heartbeat-breakdown-metrics,string"` + // EnableHeartbeatConcurrentRunner is the option to enable heartbeat concurrent runner. + EnableHeartbeatConcurrentRunner bool `toml:"enable-heartbeat-concurrent-runner" json:"enable-heartbeat-concurrent-runner,string"` + // Schedulers support for loading customized schedulers Schedulers SchedulerConfigs `toml:"schedulers" json:"schedulers-v2"` // json v2 is for the sake of compatible upgrade @@ -382,6 +386,10 @@ func (c *ScheduleConfig) Adjust(meta *configutil.ConfigMetaData, reloading bool) c.EnableHeartbeatBreakdownMetrics = defaultEnableHeartbeatBreakdownMetrics } + if !meta.IsDefined("enable-heartbeat-concurrent-runner") { + c.EnableHeartbeatConcurrentRunner = defaultEnableHeartbeatConcurrentRunner + } + if !meta.IsDefined("enable-cross-table-merge") { c.EnableCrossTableMerge = defaultEnableCrossTableMerge } diff --git a/pkg/statistics/region_collection.go b/pkg/statistics/region_collection.go index 52655b093a21..488763142e1c 100644 --- a/pkg/statistics/region_collection.go +++ b/pkg/statistics/region_collection.go @@ -157,6 +157,18 @@ func (r *RegionStatistics) RegionStatsNeedUpdate(region *core.RegionInfo) bool { region.IsOversized(int64(r.conf.GetRegionMaxSize()), int64(r.conf.GetRegionMaxKeys())) { return true } + // expected to be zero for below type + if r.IsRegionStatsType(regionID, PendingPeer) && len(region.GetPendingPeers()) == 0 { + return true + } + if r.IsRegionStatsType(regionID, DownPeer) && len(region.GetDownPeers()) == 0 { + return true + } + if r.IsRegionStatsType(regionID, LearnerPeer) && len(region.GetLearners()) == 0 { + return true + } + + // merge return r.IsRegionStatsType(regionID, UndersizedRegion) != region.NeedMerge(int64(r.conf.GetMaxMergeRegionSize()), int64(r.conf.GetMaxMergeRegionKeys())) } diff --git a/pkg/syncer/client.go b/pkg/syncer/client.go index ffbd71d2f1ea..8a2e757d5cde 100644 --- a/pkg/syncer/client.go +++ b/pkg/syncer/client.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/logutil" @@ -200,13 +201,18 @@ func (s *RegionSyncer) StartSyncWithLeader(addr string) { region = core.NewRegionInfo(r, regionLeader, core.SetSource(core.Sync)) } - tracer := core.NewNoopHeartbeatProcessTracer() - origin, _, err := bc.PreCheckPutRegion(region, tracer) + origin, _, err := bc.PreCheckPutRegion(region) if err != nil { log.Debug("region is stale", zap.Stringer("origin", origin.GetMeta()), errs.ZapError(err)) continue } - saveKV, _, _ := regionGuide(region, origin) + ctx := &core.MetaProcessContext{ + Context: ctx, + TaskRunner: ratelimit.NewSyncRunner(), + Tracer: core.NewNoopHeartbeatProcessTracer(), + // no limit for followers. + } + saveKV, _, _ := regionGuide(ctx, region, origin) overlaps := bc.PutRegion(region) if hasBuckets { diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index a8116a5e78fd..dd59b63240ff 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -21,6 +21,7 @@ import ( "io" "math" "net/http" + "runtime" "strconv" "strings" "sync" @@ -44,6 +45,7 @@ import ( mcsutils "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/memory" "github.com/tikv/pd/pkg/progress" + "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/pkg/replication" sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/hbstream" @@ -103,6 +105,9 @@ const ( // minSnapshotDurationSec is the minimum duration that a store can tolerate. // It should enlarge the limiter if the snapshot's duration is less than this value. minSnapshotDurationSec = 5 + + // heartbeat relative const + hbConcurrentRunner = "heartbeat-async-task-runner" ) // Server is the interface for cluster. @@ -166,6 +171,9 @@ type RaftCluster struct { keyspaceGroupManager *keyspace.GroupManager independentServices sync.Map hbstreams *hbstream.HeartbeatStreams + + taskRunner ratelimit.Runner + hbConcurrencyLimiter *ratelimit.ConcurrencyLimiter } // Status saves some state information. @@ -182,13 +190,15 @@ type Status struct { func NewRaftCluster(ctx context.Context, clusterID uint64, basicCluster *core.BasicCluster, storage storage.Storage, regionSyncer *syncer.RegionSyncer, etcdClient *clientv3.Client, httpClient *http.Client) *RaftCluster { return &RaftCluster{ - serverCtx: ctx, - clusterID: clusterID, - regionSyncer: regionSyncer, - httpClient: httpClient, - etcdClient: etcdClient, - core: basicCluster, - storage: storage, + serverCtx: ctx, + clusterID: clusterID, + regionSyncer: regionSyncer, + httpClient: httpClient, + etcdClient: etcdClient, + core: basicCluster, + storage: storage, + taskRunner: ratelimit.NewConcurrentRunner(hbConcurrentRunner, time.Minute), + hbConcurrencyLimiter: ratelimit.NewConcurrencyLimiter(uint64(runtime.NumCPU() * 2)), } } @@ -291,7 +301,6 @@ func (c *RaftCluster) Start(s Server) error { log.Warn("raft cluster has already been started") return nil } - c.isAPIServiceMode = s.IsAPIServiceMode() err := c.InitCluster(s.GetAllocator(), s.GetPersistOptions(), s.GetHBStreams(), s.GetKeyspaceGroupManager()) if err != nil { @@ -347,6 +356,7 @@ func (c *RaftCluster) Start(s Server) error { go c.startGCTuner() c.running = true + c.taskRunner.Start() return nil } @@ -740,6 +750,7 @@ func (c *RaftCluster) Stop() { if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { c.stopSchedulingJobs() } + c.taskRunner.Stop() c.Unlock() c.wg.Wait() @@ -988,10 +999,12 @@ func (c *RaftCluster) processReportBuckets(buckets *metapb.Buckets) error { } var regionGuide = core.GenerateRegionGuideFunc(true) +var syncRunner = ratelimit.NewSyncRunner() // processRegionHeartbeat updates the region information. -func (c *RaftCluster) processRegionHeartbeat(region *core.RegionInfo, tracer core.RegionHeartbeatProcessTracer) error { - origin, _, err := c.core.PreCheckPutRegion(region, tracer) +func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, region *core.RegionInfo) error { + tracer := ctx.Tracer + origin, _, err := c.core.PreCheckPutRegion(region) tracer.OnPreCheckFinished() if err != nil { return err @@ -1000,13 +1013,22 @@ func (c *RaftCluster) processRegionHeartbeat(region *core.RegionInfo, tracer cor region.Inherit(origin, c.GetStoreConfig().IsEnableRegionBucket()) if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { - cluster.HandleStatsAsync(c, region) + ctx.TaskRunner.RunTask( + ctx.Context, + ratelimit.TaskOpts{ + TaskName: "HandleStatsAsync", + Limit: ctx.Limiter, + }, + func(_ context.Context) { + cluster.HandleStatsAsync(c, region) + }, + ) } tracer.OnAsyncHotStatsFinished() hasRegionStats := c.regionStats != nil // Save to storage if meta is updated, except for flashback. // Save to cache if meta or leader is updated, or contains any down/pending peer. - saveKV, saveCache, needSync := regionGuide(region, origin) + saveKV, saveCache, needSync := regionGuide(ctx, region, origin) tracer.OnRegionGuideFinished() if !saveKV && !saveCache { // Due to some config changes need to update the region stats as well, @@ -1015,7 +1037,18 @@ func (c *RaftCluster) processRegionHeartbeat(region *core.RegionInfo, tracer cor // region stats needs to be collected in API mode. // We need to think of a better way to reduce this part of the cost in the future. if hasRegionStats && c.regionStats.RegionStatsNeedUpdate(region) { - c.regionStats.Observe(region, c.getRegionStoresLocked(region)) + ctx.TaskRunner.RunTask( + ctx.Context, + ratelimit.TaskOpts{ + TaskName: "ObserveRegionStatsAsync", + Limit: ctx.Limiter, + }, + func(_ context.Context) { + if c.regionStats.RegionStatsNeedUpdate(region) { + cluster.Collect(c, region, hasRegionStats) + } + }, + ) } return nil } @@ -1032,43 +1065,72 @@ func (c *RaftCluster) processRegionHeartbeat(region *core.RegionInfo, tracer cor // check its validation again here. // // However, it can't solve the race condition of concurrent heartbeats from the same region. - if overlaps, err = c.core.AtomicCheckAndPutRegion(region, tracer); err != nil { + if overlaps, err = c.core.AtomicCheckAndPutRegion(ctx, region); err != nil { tracer.OnSaveCacheFinished() return err } if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { - cluster.HandleOverlaps(c, overlaps) + ctx.TaskRunner.RunTask( + ctx.Context, + ratelimit.TaskOpts{ + TaskName: "HandleOverlaps", + Limit: ctx.Limiter, + }, + func(_ context.Context) { + cluster.HandleOverlaps(c, overlaps) + }, + ) } regionUpdateCacheEventCounter.Inc() } tracer.OnSaveCacheFinished() - // TODO: Due to the accuracy requirements of the API "/regions/check/xxx", - // region stats needs to be collected in API mode. - // We need to think of a better way to reduce this part of the cost in the future. - cluster.Collect(c, region, c.GetRegionStores(region), hasRegionStats) + // handle region stats + ctx.TaskRunner.RunTask( + ctx.Context, + ratelimit.TaskOpts{ + TaskName: "CollectRegionStatsAsync", + Limit: ctx.Limiter, + }, + func(_ context.Context) { + // TODO: Due to the accuracy requirements of the API "/regions/check/xxx", + // region stats needs to be collected in API mode. + // We need to think of a better way to reduce this part of the cost in the future. + cluster.Collect(c, region, hasRegionStats) + }, + ) + tracer.OnCollectRegionStatsFinished() if c.storage != nil { - // If there are concurrent heartbeats from the same region, the last write will win even if - // writes to storage in the critical area. So don't use mutex to protect it. - // Not successfully saved to storage is not fatal, it only leads to longer warm-up - // after restart. Here we only log the error then go on updating cache. - for _, item := range overlaps { - if err := c.storage.DeleteRegion(item.GetMeta()); err != nil { - log.Error("failed to delete region from storage", - zap.Uint64("region-id", item.GetID()), - logutil.ZapRedactStringer("region-meta", core.RegionToHexMeta(item.GetMeta())), - errs.ZapError(err)) - } - } if saveKV { - if err := c.storage.SaveRegion(region.GetMeta()); err != nil { - log.Error("failed to save region to storage", - zap.Uint64("region-id", region.GetID()), - logutil.ZapRedactStringer("region-meta", core.RegionToHexMeta(region.GetMeta())), - errs.ZapError(err)) - } - regionUpdateKVEventCounter.Inc() + ctx.TaskRunner.RunTask( + ctx.Context, + ratelimit.TaskOpts{ + TaskName: "SaveRegionToKV", + Limit: ctx.Limiter, + }, + func(_ context.Context) { + // If there are concurrent heartbeats from the same region, the last write will win even if + // writes to storage in the critical area. So don't use mutex to protect it. + // Not successfully saved to storage is not fatal, it only leads to longer warm-up + // after restart. Here we only log the error then go on updating cache. + for _, item := range overlaps { + if err := c.storage.DeleteRegion(item.GetMeta()); err != nil { + log.Error("failed to delete region from storage", + zap.Uint64("region-id", item.GetID()), + logutil.ZapRedactStringer("region-meta", core.RegionToHexMeta(item.GetMeta())), + errs.ZapError(err)) + } + } + if err := c.storage.SaveRegion(region.GetMeta()); err != nil { + log.Error("failed to save region to storage", + zap.Uint64("region-id", region.GetID()), + logutil.ZapRedactStringer("region-meta", core.RegionToHexMeta(region.GetMeta())), + errs.ZapError(err)) + } + regionUpdateKVEventCounter.Inc() + }, + ) } } @@ -2039,7 +2101,7 @@ func (c *RaftCluster) collectMetrics() { } func (c *RaftCluster) resetMetrics() { - resetHealthStatus() + c.resetHealthStatus() c.resetProgressIndicator() } @@ -2058,7 +2120,7 @@ func (c *RaftCluster) collectHealthStatus() { } } -func resetHealthStatus() { +func (*RaftCluster) resetHealthStatus() { healthStatusGauge.Reset() } @@ -2069,16 +2131,6 @@ func (c *RaftCluster) resetProgressIndicator() { storesETAGauge.Reset() } -func (c *RaftCluster) getRegionStoresLocked(region *core.RegionInfo) []*core.StoreInfo { - stores := make([]*core.StoreInfo, 0, len(region.GetPeers())) - for _, p := range region.GetPeers() { - if store := c.core.GetStore(p.StoreId); store != nil { - stores = append(stores, store) - } - } - return stores -} - // OnStoreVersionChange changes the version of the cluster when needed. func (c *RaftCluster) OnStoreVersionChange() { c.RLock() diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index d3da21ab3ddf..d01446ba1434 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -631,7 +631,7 @@ func TestRegionHeartbeatHotStat(t *testing.T) { region := core.NewRegionInfo(regionMeta, leader, core.WithInterval(&pdpb.TimeInterval{StartTimestamp: 0, EndTimestamp: utils.RegionHeartBeatReportInterval}), core.SetWrittenBytes(30000*10), core.SetWrittenKeys(300000*10)) - err = cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer()) + err = cluster.processRegionHeartbeat(core.ContextTODO(), region) re.NoError(err) // wait HotStat to update items time.Sleep(time.Second) @@ -644,7 +644,7 @@ func TestRegionHeartbeatHotStat(t *testing.T) { StoreId: 4, } region = region.Clone(core.WithRemoveStorePeer(2), core.WithAddPeer(newPeer)) - err = cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer()) + err = cluster.processRegionHeartbeat(core.ContextTODO(), region) re.NoError(err) // wait HotStat to update items time.Sleep(time.Second) @@ -681,8 +681,8 @@ func TestBucketHeartbeat(t *testing.T) { re.NoError(cluster.putStoreLocked(store)) } - re.NoError(cluster.processRegionHeartbeat(regions[0], core.NewNoopHeartbeatProcessTracer())) - re.NoError(cluster.processRegionHeartbeat(regions[1], core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), regions[0])) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), regions[1])) re.Nil(cluster.GetRegion(uint64(1)).GetBuckets()) re.NoError(cluster.processReportBuckets(buckets)) re.Equal(buckets, cluster.GetRegion(uint64(1)).GetBuckets()) @@ -701,13 +701,13 @@ func TestBucketHeartbeat(t *testing.T) { // case5: region update should inherit buckets. newRegion := regions[1].Clone(core.WithIncConfVer(), core.SetBuckets(nil)) opt.SetRegionBucketEnabled(true) - re.NoError(cluster.processRegionHeartbeat(newRegion, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), newRegion)) re.Len(cluster.GetRegion(uint64(1)).GetBuckets().GetKeys(), 2) // case6: disable region bucket in opt.SetRegionBucketEnabled(false) newRegion2 := regions[1].Clone(core.WithIncConfVer(), core.SetBuckets(nil)) - re.NoError(cluster.processRegionHeartbeat(newRegion2, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), newRegion2)) re.Nil(cluster.GetRegion(uint64(1)).GetBuckets()) re.Empty(cluster.GetRegion(uint64(1)).GetBuckets().GetKeys()) } @@ -733,25 +733,25 @@ func TestRegionHeartbeat(t *testing.T) { for i, region := range regions { // region does not exist. - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) checkRegionsKV(re, cluster.storage, regions[:i+1]) // region is the same, not updated. - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) checkRegionsKV(re, cluster.storage, regions[:i+1]) origin := region // region is updated. region = origin.Clone(core.WithIncVersion()) regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) checkRegionsKV(re, cluster.storage, regions[:i+1]) // region is stale (Version). stale := origin.Clone(core.WithIncConfVer()) - re.Error(cluster.processRegionHeartbeat(stale, core.NewNoopHeartbeatProcessTracer())) + re.Error(cluster.processRegionHeartbeat(core.ContextTODO(), stale)) checkRegions(re, cluster.core, regions[:i+1]) checkRegionsKV(re, cluster.storage, regions[:i+1]) @@ -761,13 +761,13 @@ func TestRegionHeartbeat(t *testing.T) { core.WithIncConfVer(), ) regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) checkRegionsKV(re, cluster.storage, regions[:i+1]) // region is stale (ConfVer). stale = origin.Clone(core.WithIncConfVer()) - re.Error(cluster.processRegionHeartbeat(stale, core.NewNoopHeartbeatProcessTracer())) + re.Error(cluster.processRegionHeartbeat(core.ContextTODO(), stale)) checkRegions(re, cluster.core, regions[:i+1]) checkRegionsKV(re, cluster.storage, regions[:i+1]) @@ -779,38 +779,38 @@ func TestRegionHeartbeat(t *testing.T) { }, })) regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) // Add a pending peer. region = region.Clone(core.WithPendingPeers([]*metapb.Peer{region.GetPeers()[rand.Intn(len(region.GetPeers()))]})) regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) // Clear down peers. region = region.Clone(core.WithDownPeers(nil)) regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) // Clear pending peers. region = region.Clone(core.WithPendingPeers(nil)) regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) // Remove peers. origin = region region = origin.Clone(core.SetPeers(region.GetPeers()[:1])) regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) checkRegionsKV(re, cluster.storage, regions[:i+1]) // Add peers. region = origin regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) checkRegionsKV(re, cluster.storage, regions[:i+1]) @@ -820,47 +820,47 @@ func TestRegionHeartbeat(t *testing.T) { core.WithIncConfVer(), ) regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) // Change leader. region = region.Clone(core.WithLeader(region.GetPeers()[1])) regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) // Change ApproximateSize. region = region.Clone(core.SetApproximateSize(144)) regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) // Change ApproximateKeys. region = region.Clone(core.SetApproximateKeys(144000)) regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) // Change bytes written. region = region.Clone(core.SetWrittenBytes(24000)) regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) // Change bytes read. region = region.Clone(core.SetReadBytes(1080000)) regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) // Flashback region = region.Clone(core.WithFlashback(true, 1)) regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) region = region.Clone(core.WithFlashback(false, 0)) regions[i] = region - re.NoError(cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region)) checkRegions(re, cluster.core, regions[:i+1]) } @@ -916,8 +916,7 @@ func TestRegionHeartbeat(t *testing.T) { core.WithNewRegionID(10000), core.WithDecVersion(), ) - tracer := core.NewHeartbeatProcessTracer() - re.Error(cluster.processRegionHeartbeat(overlapRegion, tracer)) + re.Error(cluster.processRegionHeartbeat(core.ContextTODO(), overlapRegion)) region := &metapb.Region{} ok, err := storage.LoadRegion(regions[n-1].GetID(), region) re.True(ok) @@ -941,9 +940,11 @@ func TestRegionHeartbeat(t *testing.T) { core.WithStartKey(regions[n-2].GetStartKey()), core.WithNewRegionID(regions[n-1].GetID()+1), ) - tracer = core.NewHeartbeatProcessTracer() + tracer := core.NewHeartbeatProcessTracer() tracer.Begin() - re.NoError(cluster.processRegionHeartbeat(overlapRegion, tracer)) + ctx := core.ContextTODO() + ctx.Tracer = tracer + re.NoError(cluster.processRegionHeartbeat(ctx, overlapRegion)) tracer.OnAllStageFinished() re.Condition(func() bool { fileds := tracer.LogFields() @@ -977,7 +978,9 @@ func TestRegionFlowChanged(t *testing.T) { regions := []*core.RegionInfo{core.NewTestRegionInfo(1, 1, []byte{}, []byte{})} processRegions := func(regions []*core.RegionInfo) { for _, r := range regions { - cluster.processRegionHeartbeat(r, core.NewNoopHeartbeatProcessTracer()) + mctx := core.ContextTODO() + mctx.Context = ctx + cluster.processRegionHeartbeat(mctx, r) } } regions = core.SplitRegions(regions) @@ -1013,7 +1016,7 @@ func TestRegionSizeChanged(t *testing.T) { core.SetApproximateKeys(curMaxMergeKeys-1), core.SetSource(core.Heartbeat), ) - cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer()) + cluster.processRegionHeartbeat(core.ContextTODO(), region) regionID := region.GetID() re.True(cluster.regionStats.IsRegionStatsType(regionID, statistics.UndersizedRegion)) // Test ApproximateSize and ApproximateKeys change. @@ -1023,16 +1026,16 @@ func TestRegionSizeChanged(t *testing.T) { core.SetApproximateKeys(curMaxMergeKeys+1), core.SetSource(core.Heartbeat), ) - cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer()) + cluster.processRegionHeartbeat(core.ContextTODO(), region) re.False(cluster.regionStats.IsRegionStatsType(regionID, statistics.UndersizedRegion)) // Test MaxMergeRegionSize and MaxMergeRegionKeys change. cluster.opt.SetMaxMergeRegionSize(uint64(curMaxMergeSize + 2)) cluster.opt.SetMaxMergeRegionKeys(uint64(curMaxMergeKeys + 2)) - cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer()) + cluster.processRegionHeartbeat(core.ContextTODO(), region) re.True(cluster.regionStats.IsRegionStatsType(regionID, statistics.UndersizedRegion)) cluster.opt.SetMaxMergeRegionSize(uint64(curMaxMergeSize)) cluster.opt.SetMaxMergeRegionKeys(uint64(curMaxMergeKeys)) - cluster.processRegionHeartbeat(region, core.NewNoopHeartbeatProcessTracer()) + cluster.processRegionHeartbeat(core.ContextTODO(), region) re.False(cluster.regionStats.IsRegionStatsType(regionID, statistics.UndersizedRegion)) } @@ -1095,11 +1098,11 @@ func TestConcurrentRegionHeartbeat(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/concurrentRegionHeartbeat", "return(true)")) go func() { defer wg.Done() - cluster.processRegionHeartbeat(source, core.NewNoopHeartbeatProcessTracer()) + cluster.processRegionHeartbeat(core.ContextTODO(), source) }() time.Sleep(100 * time.Millisecond) re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/concurrentRegionHeartbeat")) - re.NoError(cluster.processRegionHeartbeat(target, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), target)) wg.Wait() checkRegion(re, cluster.GetRegionByKey([]byte{}), target) } @@ -1161,7 +1164,7 @@ func TestRegionLabelIsolationLevel(t *testing.T) { func heartbeatRegions(re *require.Assertions, cluster *RaftCluster, regions []*core.RegionInfo) { // Heartbeat and check region one by one. for _, r := range regions { - re.NoError(cluster.processRegionHeartbeat(r, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), r)) checkRegion(re, cluster.GetRegion(r.GetID()), r) checkRegion(re, cluster.GetRegionByKey(r.GetStartKey()), r) @@ -1198,7 +1201,7 @@ func TestHeartbeatSplit(t *testing.T) { // 1: [nil, nil) region1 := core.NewRegionInfo(&metapb.Region{Id: 1, RegionEpoch: &metapb.RegionEpoch{Version: 1, ConfVer: 1}}, nil) - re.NoError(cluster.processRegionHeartbeat(region1, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region1)) checkRegion(re, cluster.GetRegionByKey([]byte("foo")), region1) // split 1 to 2: [nil, m) 1: [m, nil), sync 2 first. @@ -1207,12 +1210,12 @@ func TestHeartbeatSplit(t *testing.T) { core.WithIncVersion(), ) region2 := core.NewRegionInfo(&metapb.Region{Id: 2, EndKey: []byte("m"), RegionEpoch: &metapb.RegionEpoch{Version: 1, ConfVer: 1}}, nil) - re.NoError(cluster.processRegionHeartbeat(region2, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region2)) checkRegion(re, cluster.GetRegionByKey([]byte("a")), region2) // [m, nil) is missing before r1's heartbeat. re.Nil(cluster.GetRegionByKey([]byte("z"))) - re.NoError(cluster.processRegionHeartbeat(region1, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region1)) checkRegion(re, cluster.GetRegionByKey([]byte("z")), region1) // split 1 to 3: [m, q) 1: [q, nil), sync 1 first. @@ -1221,12 +1224,12 @@ func TestHeartbeatSplit(t *testing.T) { core.WithIncVersion(), ) region3 := core.NewRegionInfo(&metapb.Region{Id: 3, StartKey: []byte("m"), EndKey: []byte("q"), RegionEpoch: &metapb.RegionEpoch{Version: 1, ConfVer: 1}}, nil) - re.NoError(cluster.processRegionHeartbeat(region1, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region1)) checkRegion(re, cluster.GetRegionByKey([]byte("z")), region1) checkRegion(re, cluster.GetRegionByKey([]byte("a")), region2) // [m, q) is missing before r3's heartbeat. re.Nil(cluster.GetRegionByKey([]byte("n"))) - re.NoError(cluster.processRegionHeartbeat(region3, core.NewNoopHeartbeatProcessTracer())) + re.NoError(cluster.processRegionHeartbeat(core.ContextTODO(), region3)) checkRegion(re, cluster.GetRegionByKey([]byte("n")), region3) } @@ -1522,11 +1525,11 @@ func TestUpdateStorePendingPeerCount(t *testing.T) { }, } origin := core.NewRegionInfo(&metapb.Region{Id: 1, Peers: peers[:3]}, peers[0], core.WithPendingPeers(peers[1:3])) - re.NoError(tc.processRegionHeartbeat(origin, core.NewNoopHeartbeatProcessTracer())) + re.NoError(tc.processRegionHeartbeat(core.ContextTODO(), origin)) time.Sleep(50 * time.Millisecond) checkPendingPeerCount([]int{0, 1, 1, 0}, tc.RaftCluster, re) newRegion := core.NewRegionInfo(&metapb.Region{Id: 1, Peers: peers[1:]}, peers[1], core.WithPendingPeers(peers[3:4])) - re.NoError(tc.processRegionHeartbeat(newRegion, core.NewNoopHeartbeatProcessTracer())) + re.NoError(tc.processRegionHeartbeat(core.ContextTODO(), newRegion)) time.Sleep(50 * time.Millisecond) checkPendingPeerCount([]int{0, 0, 0, 1}, tc.RaftCluster, re) } @@ -2137,6 +2140,7 @@ func newTestRaftCluster( opt *config.PersistOptions, s storage.Storage, ) *RaftCluster { + opt.GetScheduleConfig().EnableHeartbeatConcurrentRunner = false rc := &RaftCluster{serverCtx: ctx, core: core.NewBasicCluster(), storage: s} rc.InitCluster(id, opt, nil, nil) rc.ruleManager = placement.NewRuleManager(ctx, storage.NewStorageWithMemoryBackend(), rc, opt) @@ -2955,12 +2959,12 @@ func TestShouldRun(t *testing.T) { for _, testCase := range testCases { r := tc.GetRegion(testCase.regionID) nr := r.Clone(core.WithLeader(r.GetPeers()[0]), core.SetSource(core.Heartbeat)) - re.NoError(tc.processRegionHeartbeat(nr, core.NewNoopHeartbeatProcessTracer())) + re.NoError(tc.processRegionHeartbeat(core.ContextTODO(), nr)) re.Equal(testCase.ShouldRun, co.ShouldRun()) } nr := &metapb.Region{Id: 6, Peers: []*metapb.Peer{}} newRegion := core.NewRegionInfo(nr, nil, core.SetSource(core.Heartbeat)) - re.Error(tc.processRegionHeartbeat(newRegion, core.NewNoopHeartbeatProcessTracer())) + re.Error(tc.processRegionHeartbeat(core.ContextTODO(), newRegion)) re.Equal(7, tc.core.GetClusterNotFromStorageRegionsCnt()) } @@ -2998,12 +3002,12 @@ func TestShouldRunWithNonLeaderRegions(t *testing.T) { for _, testCase := range testCases { r := tc.GetRegion(testCase.regionID) nr := r.Clone(core.WithLeader(r.GetPeers()[0]), core.SetSource(core.Heartbeat)) - re.NoError(tc.processRegionHeartbeat(nr, core.NewNoopHeartbeatProcessTracer())) + re.NoError(tc.processRegionHeartbeat(core.ContextTODO(), nr)) re.Equal(testCase.ShouldRun, co.ShouldRun()) } nr := &metapb.Region{Id: 9, Peers: []*metapb.Peer{}} newRegion := core.NewRegionInfo(nr, nil, core.SetSource(core.Heartbeat)) - re.Error(tc.processRegionHeartbeat(newRegion, core.NewNoopHeartbeatProcessTracer())) + re.Error(tc.processRegionHeartbeat(core.ContextTODO(), newRegion)) re.Equal(9, tc.core.GetClusterNotFromStorageRegionsCnt()) // Now, after server is prepared, there exist some regions with no leader. diff --git a/server/cluster/cluster_worker.go b/server/cluster/cluster_worker.go index e23ae2100bdd..5c2bb9502970 100644 --- a/server/cluster/cluster_worker.go +++ b/server/cluster/cluster_worker.go @@ -24,6 +24,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/utils/logutil" @@ -38,8 +39,19 @@ func (c *RaftCluster) HandleRegionHeartbeat(region *core.RegionInfo) error { if c.GetScheduleConfig().EnableHeartbeatBreakdownMetrics { tracer = core.NewHeartbeatProcessTracer() } + var runner ratelimit.Runner + runner = syncRunner + if c.GetScheduleConfig().EnableHeartbeatConcurrentRunner { + runner = c.taskRunner + } + ctx := &core.MetaProcessContext{ + Context: c.ctx, + Limiter: c.hbConcurrencyLimiter, + Tracer: tracer, + TaskRunner: runner, + } tracer.Begin() - if err := c.processRegionHeartbeat(region, tracer); err != nil { + if err := c.processRegionHeartbeat(ctx, region); err != nil { tracer.OnAllStageFinished() return err } diff --git a/tests/cluster.go b/tests/cluster.go index cd90a6f2b456..c7368fe3c3a4 100644 --- a/tests/cluster.go +++ b/tests/cluster.go @@ -88,6 +88,8 @@ func NewTestAPIServer(ctx context.Context, cfg *config.Config) (*TestServer, err } func createTestServer(ctx context.Context, cfg *config.Config, services []string) (*TestServer, error) { + // disable the heartbeat async runner in test + cfg.Schedule.EnableHeartbeatConcurrentRunner = false err := logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) if err != nil { return nil, err From a109f9c9ee63b7ba19d2a83d38bf8725f350e93a Mon Sep 17 00:00:00 2001 From: Hu# Date: Mon, 15 Apr 2024 18:55:35 +0800 Subject: [PATCH 14/50] cluster: make TestStoreOverloadedWithReplace stable (#8068) ref tikv/pd#7969 Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/cluster/cluster_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index d01446ba1434..ecd579d88817 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -3480,10 +3480,11 @@ func TestStoreOverloadedWithReplace(t *testing.T) { re.False(oc.AddOperator(op3)) ops, _ := lb.Schedule(tc, false /* dryRun */) re.Empty(ops) - // sleep 2 seconds to make sure that token is filled up - time.Sleep(2 * time.Second) - ops, _ = lb.Schedule(tc, false /* dryRun */) - re.NotEmpty(ops) + // make sure that token is filled up + testutil.Eventually(re, func() bool { + ops, _ = lb.Schedule(tc, false /* dryRun */) + return len(ops) != 0 + }) } func TestDownStoreLimit(t *testing.T) { From a767a5f5fd8c892d4e7b430d372fc18c1356791b Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 16 Apr 2024 10:25:35 +0800 Subject: [PATCH 15/50] *: make `TestEtcdScaleInAndOut` stable (#8067) close tikv/pd#6892 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/utils/etcdutil/testutil.go | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/pkg/utils/etcdutil/testutil.go b/pkg/utils/etcdutil/testutil.go index 3ea4d0576452..d9464eeceeb9 100644 --- a/pkg/utils/etcdutil/testutil.go +++ b/pkg/utils/etcdutil/testutil.go @@ -122,18 +122,27 @@ func MustAddEtcdMember(t *testing.T, cfg1 *embed.Config, client *clientv3.Client func checkMembers(re *require.Assertions, client *clientv3.Client, etcds []*embed.Etcd) { // Check the client can get the new member. - listResp, err := ListEtcdMembers(client.Ctx(), client) - re.NoError(err) - re.Len(listResp.Members, len(etcds)) - inList := func(m *etcdserverpb.Member) bool { - for _, etcd := range etcds { - if m.ID == uint64(etcd.Server.ID()) { - return true + testutil.Eventually(re, func() bool { + listResp, err := ListEtcdMembers(client.Ctx(), client) + if err != nil { + return false + } + if len(etcds) != len(listResp.Members) { + return false + } + inList := func(m *etcdserverpb.Member) bool { + for _, etcd := range etcds { + if m.ID == uint64(etcd.Server.ID()) { + return true + } } + return false } - return false - } - for _, m := range listResp.Members { - re.True(inList(m)) - } + for _, m := range listResp.Members { + if !inList(m) { + return false + } + } + return true + }) } From 2a4a7b76123e241ec98f79aac60b3769e756654c Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 16 Apr 2024 14:22:36 +0800 Subject: [PATCH 16/50] *: make `TestEtcdWithHangLeaderEnableCheck` stable (#8072) close tikv/pd#7292 Signed-off-by: Ryan Leung --- pkg/utils/etcdutil/etcdutil_test.go | 6 +++--- tests/integrations/mcs/scheduling/api_test.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/utils/etcdutil/etcdutil_test.go b/pkg/utils/etcdutil/etcdutil_test.go index 6ddeafe45738..4c1e20fa73c3 100644 --- a/pkg/utils/etcdutil/etcdutil_test.go +++ b/pkg/utils/etcdutil/etcdutil_test.go @@ -300,11 +300,10 @@ func checkEtcdWithHangLeader(t *testing.T) error { etcd2 := MustAddEtcdMember(t, &cfg1, client1) defer etcd2.Close() checkMembers(re, client1, []*embed.Etcd{etcd1, etcd2}) - time.Sleep(1 * time.Second) // wait for etcd client sync endpoints // Hang the etcd1 and wait for the client to connect to etcd2. enableDiscard.Store(true) - time.Sleep(time.Second) + time.Sleep(3 * time.Second) _, err = EtcdKVGet(client1, "test/key1") return err } @@ -366,7 +365,8 @@ func ioCopy(ctx context.Context, dst io.Writer, src io.Reader, enableDiscard *at return nil default: if enableDiscard.Load() { - io.Copy(io.Discard, src) + _, err := io.Copy(io.Discard, src) + return err } readNum, errRead := src.Read(buffer) if readNum > 0 { diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index be51123532d9..f615e879e403 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -647,11 +647,11 @@ func (suite *apiTestSuite) checkStores(cluster *tests.TestCluster) { Version: "2.0.0", }, } + // prevent the offline store from changing to tombstone + tests.MustPutRegion(re, cluster, 3, 6, []byte("a"), []byte("b")) for _, store := range stores { tests.MustPutStore(re, cluster, store) } - // prevent the offline store from changing to tombstone - tests.MustPutRegion(re, cluster, 3, 6, []byte("a"), []byte("b")) // Test /stores apiServerAddr := cluster.GetLeaderServer().GetAddr() urlPrefix := fmt.Sprintf("%s/pd/api/v1/stores", apiServerAddr) From 41ff34bafec95b45968f1e74f14ba6cdcb62531d Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 16 Apr 2024 16:25:36 +0800 Subject: [PATCH 17/50] *: make `TestLogicalOverflow` stable (#8075) close tikv/pd#7017 Signed-off-by: Ryan Leung --- tests/server/tso/global_tso_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/server/tso/global_tso_test.go b/tests/server/tso/global_tso_test.go index f705bdf12b55..8dd98b1d6287 100644 --- a/tests/server/tso/global_tso_test.go +++ b/tests/server/tso/global_tso_test.go @@ -165,7 +165,7 @@ func TestLogicalOverflow(t *testing.T) { re.NoError(err) if i == 1 { // the 2nd request may (but not must) overflow, as max logical interval is 262144 - re.Less(time.Since(begin), updateInterval+20*time.Millisecond) // additional 20ms for gRPC latency + re.Less(time.Since(begin), updateInterval+50*time.Millisecond) // additional 50ms for gRPC latency } } // the 3rd request must overflow From f5fd36f259a9078f083b35862b774a53179a774c Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 16 Apr 2024 16:36:05 +0800 Subject: [PATCH 18/50] schedule: enlarge waiting list cache size (#7559) ref tikv/pd#7963 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/schedule/checker/checker_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/schedule/checker/checker_controller.go b/pkg/schedule/checker/checker_controller.go index 355226cd2d86..cdc826a1dda0 100644 --- a/pkg/schedule/checker/checker_controller.go +++ b/pkg/schedule/checker/checker_controller.go @@ -31,7 +31,7 @@ import ( ) // DefaultCacheSize is the default length of waiting list. -const DefaultCacheSize = 1000 +const DefaultCacheSize = 100000 var denyCheckersByLabelerCounter = labeler.LabelerEventCounter.WithLabelValues("checkers", "deny") From 22543a9cc3dea19ce19ea9d3354db0c2330c0e57 Mon Sep 17 00:00:00 2001 From: Hu# Date: Tue, 16 Apr 2024 16:46:35 +0800 Subject: [PATCH 19/50] tools/pd-ut: accelarate build by `compile-without-link` (#8069) ref tikv/pd#7969 Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tools/pd-ut/go-compile-without-link.sh | 20 ++++++++++++++++++++ tools/pd-ut/ut.go | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100755 tools/pd-ut/go-compile-without-link.sh diff --git a/tools/pd-ut/go-compile-without-link.sh b/tools/pd-ut/go-compile-without-link.sh new file mode 100755 index 000000000000..88e6282b076e --- /dev/null +++ b/tools/pd-ut/go-compile-without-link.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# See https://gist.github.com/howardjohn/c0f5d0bc293ef7d7fada533a2c9ffaf4 +# Usage: go test -exec=true -toolexec=go-compile-without-link -vet=off ./... +# Preferably as an alias like `alias go-test-compile='go test -exec=true -toolexec=go-compile-without-link -vet=off'` +# This will compile all tests, but not link them (which is the least cacheable part) + +if [[ "${2}" == "-V=full" ]]; then + "$@" + exit 0 +fi +case "$(basename ${1})" in + link) + # Output a dummy file + touch "${3}" + ;; + # We could skip vet as well, but it can be done with -vet=off if desired + *) + "$@" +esac diff --git a/tools/pd-ut/ut.go b/tools/pd-ut/ut.go index 69a83f007b6f..7fc96ee11cff 100644 --- a/tools/pd-ut/ut.go +++ b/tools/pd-ut/ut.go @@ -589,8 +589,28 @@ func skipDIR(pkg string) bool { return false } +func generateBuildCache() error { + // cd cmd/pd-server && go test -tags=tso_function_test,deadlock -exec-=true -vet=off -toolexec=go-compile-without-link + cmd := exec.Command("go", "test", "-exec=true", "-vet", "off", "--tags=tso_function_test,deadlock") + goCompileWithoutLink := fmt.Sprintf("-toolexec=%s/tools/pd-ut/go-compile-without-link.sh", workDir) + cmd.Args = append(cmd.Args, goCompileWithoutLink) + cmd.Dir = fmt.Sprintf("%s/cmd/pd-server", workDir) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return withTrace(err) + } + return nil +} + // buildTestBinaryMulti is much faster than build the test packages one by one. func buildTestBinaryMulti(pkgs []string) error { + // staged build, generate the build cache for all the tests first, then generate the test binary. + // This way is faster than generating test binaries directly, because the cache can be used. + if err := generateBuildCache(); err != nil { + return withTrace(err) + } + // go test --exec=xprog --tags=tso_function_test,deadlock -vet=off --count=0 $(pkgs) xprogPath := path.Join(workDir, "bin/xprog") packages := make([]string, 0, len(pkgs)) From acbb9afaae4a534d77e71d8aa9a9af57e21c0567 Mon Sep 17 00:00:00 2001 From: Hu# Date: Tue, 16 Apr 2024 18:03:06 +0800 Subject: [PATCH 20/50] ci: transitioning action version from node 16 to node 20 (#8071) close tikv/pd#8070 Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- .github/workflows/check.yaml | 16 +++---------- .github/workflows/label.yaml | 2 +- .github/workflows/pd-docker-image.yaml | 6 ++--- .github/workflows/pd-tests.yaml | 26 +++++++-------------- .github/workflows/tso-consistency-test.yaml | 6 ++--- .github/workflows/tso-function-test.yaml | 6 ++--- 6 files changed, 22 insertions(+), 40 deletions(-) diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index e2bf99c026f2..3ccb2635e9f8 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -8,21 +8,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/setup-go@v3 - with: - go-version: '1.21' - name: Checkout code - uses: actions/checkout@v3 - - name: Restore cache - uses: actions/cache@v3 + uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - **/.dashboard_download_cache - key: ${{ runner.os }}-golang-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-golang + go-version: '1.21' - name: Make Check run: | SWAGGER=1 make build diff --git a/.github/workflows/label.yaml b/.github/workflows/label.yaml index 5ff2b895528e..00438d26b63f 100644 --- a/.github/workflows/label.yaml +++ b/.github/workflows/label.yaml @@ -7,7 +7,7 @@ jobs: add_labels: runs-on: ubuntu-latest steps: - - uses: actions/github-script@v4 + - uses: actions/github-script@v7 name: Add labels with: script: | diff --git a/.github/workflows/pd-docker-image.yaml b/.github/workflows/pd-docker-image.yaml index 2a04c030016f..5beaa66c1568 100644 --- a/.github/workflows/pd-docker-image.yaml +++ b/.github/workflows/pd-docker-image.yaml @@ -15,10 +15,10 @@ jobs: strategy: fail-fast: true steps: - - uses: actions/setup-go@v3 + - name: Checkout code + uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: '1.21' - - name: Checkout code - uses: actions/checkout@v3 - name: Make run: make docker-image diff --git a/.github/workflows/pd-tests.yaml b/.github/workflows/pd-tests.yaml index 1508c1a14573..9084c7545a84 100644 --- a/.github/workflows/pd-tests.yaml +++ b/.github/workflows/pd-tests.yaml @@ -29,20 +29,11 @@ jobs: outputs: job-total: 13 steps: - - uses: actions/setup-go@v3 - with: - go-version: '1.21' - name: Checkout code - uses: actions/checkout@v3 - - name: Restore cache - uses: actions/cache@v3 + uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - **/.tools - **/.dashboard_download_cache - key: ${{ runner.os }}-go-${{ matrix.worker_id }}-${{ hashFiles('**/go.sum') }} + go-version: '1.21' - name: Make Test env: WORKER_ID: ${{ matrix.worker_id }} @@ -53,20 +44,21 @@ jobs: mv covprofile covprofile_$WORKER_ID sed -i "/failpoint_binding/d" covprofile_$WORKER_ID - name: Upload coverage result ${{ matrix.worker_id }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: - name: cover-reports + name: cover-reports-${{ matrix.worker_id }} path: covprofile_${{ matrix.worker_id }} report-coverage: needs: chunks runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download chunk report - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: - name: cover-reports + pattern: cover-reports-* + merge-multiple: true - name: Merge env: TOTAL_JOBS: ${{needs.chunks.outputs.job-total}} diff --git a/.github/workflows/tso-consistency-test.yaml b/.github/workflows/tso-consistency-test.yaml index 570cbbc5da8f..3cb24898a100 100644 --- a/.github/workflows/tso-consistency-test.yaml +++ b/.github/workflows/tso-consistency-test.yaml @@ -8,10 +8,10 @@ jobs: tso-consistency-test: runs-on: ubuntu-latest steps: - - uses: actions/setup-go@v3 + - name: Checkout code + uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: '1.21' - - name: Checkout code - uses: actions/checkout@v3 - name: Make TSO Consistency Test run: make test-tso-consistency diff --git a/.github/workflows/tso-function-test.yaml b/.github/workflows/tso-function-test.yaml index d7780425d304..13fd6fe7df68 100644 --- a/.github/workflows/tso-function-test.yaml +++ b/.github/workflows/tso-function-test.yaml @@ -21,10 +21,10 @@ jobs: tso-function-test: runs-on: ubuntu-latest steps: - - uses: actions/setup-go@v3 + - name: Checkout code + uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: '1.21' - - name: Checkout code - uses: actions/checkout@v3 - name: Make TSO Function Test run: make test-tso-function From 4c0a862440206c387ebf1589f1fcf657fb626cd4 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 16 Apr 2024 20:49:35 +0800 Subject: [PATCH 21/50] *: make `TestStores` stable (#8078) close tikv/pd#7954 Signed-off-by: Ryan Leung Co-authored-by: Hu# --- tests/integrations/mcs/scheduling/api_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index f615e879e403..e9033e5016a9 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -651,6 +651,9 @@ func (suite *apiTestSuite) checkStores(cluster *tests.TestCluster) { tests.MustPutRegion(re, cluster, 3, 6, []byte("a"), []byte("b")) for _, store := range stores { tests.MustPutStore(re, cluster, store) + if store.GetId() == 6 { + cluster.GetLeaderServer().GetRaftCluster().GetBasicCluster().UpdateStoreStatus(6) + } } // Test /stores apiServerAddr := cluster.GetLeaderServer().GetAddr() From 2dd230101346c430397947bbe1028fb46411c98b Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 17 Apr 2024 16:51:06 +0800 Subject: [PATCH 22/50] *: make `TestKeyspaceGroupMergeIntoDefault` stable (#8080) close tikv/pd#6991 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tests/integrations/mcs/tso/keyspace_group_manager_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go index 9194811cd379..909972f03154 100644 --- a/tests/integrations/mcs/tso/keyspace_group_manager_test.go +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -781,13 +781,12 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestKeyspaceGroupMergeIntoDefault Keyspaces: []uint32{uint32(i)}, }) keyspaces = append(keyspaces, uint32(i)) - if len(keyspaceGroups) < etcdutil.MaxEtcdTxnOps/2 && i != keyspaceGroupNum { + if i != keyspaceGroupNum { continue } handlersutil.MustCreateKeyspaceGroup(re, suite.pdLeaderServer, &handlers.CreateKeyspaceGroupParams{ KeyspaceGroups: keyspaceGroups, }) - keyspaceGroups = keyspaceGroups[:0] } // Check if all the keyspace groups are created. groups := handlersutil.MustLoadKeyspaceGroups(re, suite.pdLeaderServer, "0", "0") From bf234b0e4bc4b49f665aea413237d4292393bcfb Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 17 Apr 2024 17:49:36 +0800 Subject: [PATCH 23/50] tests: reduce duplicate configurations in hot scheduler tests (#8081) ref tikv/pd#8073 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/schedule/schedulers/hot_region_test.go | 69 +------------------ pkg/schedule/schedulers/hot_region_v2_test.go | 22 ------ pkg/schedule/schedulers/scheduler_test.go | 5 +- 3 files changed, 4 insertions(+), 92 deletions(-) diff --git a/pkg/schedule/schedulers/hot_region_test.go b/pkg/schedule/schedulers/hot_region_test.go index cfc5196909f1..e5b722a488d4 100644 --- a/pkg/schedule/schedulers/hot_region_test.go +++ b/pkg/schedule/schedulers/hot_region_test.go @@ -43,6 +43,9 @@ func init() { // TODO: remove this global variable in the future. // And use a function to create hot schduler for test. schedulePeerPr = 1.0 + // disable denoising in test. + statistics.Denoising = false + statisticsInterval = 0 RegisterScheduler(utils.Write.String(), func(opController *operator.Controller, _ endpoint.ConfigStorage, _ ConfigDecoder, _ ...func(string) error) (Scheduler, error) { cfg := initHotRegionScheduleConfig() return newHotWriteScheduler(opController, cfg), nil @@ -200,10 +203,8 @@ func checkGCPendingOpInfos(re *require.Assertions, enablePlacementRules bool) { func TestSplitIfRegionTooHot(t *testing.T) { re := require.New(t) - statistics.Denoising = false cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - tc.SetHotRegionCacheHitsThreshold(1) hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) b := &metapb.Buckets{ @@ -274,9 +275,7 @@ func TestSplitIfRegionTooHot(t *testing.T) { func TestSplitBucketsBySize(t *testing.T) { re := require.New(t) - statistics.Denoising = false cancel, _, tc, oc := prepareSchedulersTest() - tc.SetHotRegionCacheHitsThreshold(1) tc.SetRegionBucketEnabled(true) defer cancel() hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) @@ -327,9 +326,7 @@ func TestSplitBucketsBySize(t *testing.T) { func TestSplitBucketsByLoad(t *testing.T) { re := require.New(t) - statistics.Denoising = false cancel, _, tc, oc := prepareSchedulersTest() - tc.SetHotRegionCacheHitsThreshold(1) tc.SetRegionBucketEnabled(true) defer cancel() hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) @@ -388,8 +385,6 @@ func TestSplitBucketsByLoad(t *testing.T) { func TestHotWriteRegionScheduleByteRateOnly(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 checkHotWriteRegionScheduleByteRateOnly(re, false /* disable placement rules */) checkHotWriteRegionScheduleByteRateOnly(re, true /* enable placement rules */) checkHotWriteRegionPlacement(re, true) @@ -406,7 +401,6 @@ func checkHotWriteRegionPlacement(re *require.Assertions, enablePlacementRules b hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.SetHistorySampleDuration(0) - tc.SetHotRegionCacheHitsThreshold(0) tc.AddLabelsStore(1, 2, map[string]string{"zone": "z1", "host": "h1"}) tc.AddLabelsStore(2, 2, map[string]string{"zone": "z1", "host": "h2"}) @@ -633,12 +627,9 @@ func checkHotWriteRegionScheduleByteRateOnly(re *require.Assertions, enablePlace func TestHotWriteRegionScheduleByteRateOnlyWithTiFlash(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.ConfChangeV2)) - tc.SetHotRegionCacheHitsThreshold(0) re.NoError(tc.RuleManager.SetRules([]*placement.Rule{ { GroupID: placement.DefaultGroupID, @@ -853,8 +844,6 @@ func TestHotWriteRegionScheduleWithQuery(t *testing.T) { }() cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - statistics.Denoising = false - statisticsInterval = 0 hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) @@ -863,7 +852,6 @@ func TestHotWriteRegionScheduleWithQuery(t *testing.T) { hb.(*hotScheduler).conf.WriteLeaderPriorities = []string{utils.QueryPriority, utils.BytePriority} hb.(*hotScheduler).conf.SetHistorySampleDuration(0) - tc.SetHotRegionCacheHitsThreshold(0) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) tc.AddRegionStore(3, 20) @@ -888,8 +876,6 @@ func TestHotWriteRegionScheduleWithQuery(t *testing.T) { func TestHotWriteRegionScheduleWithKeyRate(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() @@ -901,7 +887,6 @@ func TestHotWriteRegionScheduleWithKeyRate(t *testing.T) { hb.(*hotScheduler).conf.RankFormulaVersion = "v1" hb.(*hotScheduler).conf.SetHistorySampleDuration(0) - tc.SetHotRegionCacheHitsThreshold(0) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) @@ -948,8 +933,6 @@ func TestHotWriteRegionScheduleWithKeyRate(t *testing.T) { func TestHotWriteRegionScheduleUnhealthyStore(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() @@ -958,7 +941,6 @@ func TestHotWriteRegionScheduleUnhealthyStore(t *testing.T) { hb.(*hotScheduler).conf.SetDstToleranceRatio(1) hb.(*hotScheduler).conf.SetSrcToleranceRatio(1) - tc.SetHotRegionCacheHitsThreshold(0) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) @@ -996,8 +978,6 @@ func TestHotWriteRegionScheduleUnhealthyStore(t *testing.T) { func TestHotWriteRegionScheduleCheckHot(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() @@ -1006,7 +986,6 @@ func TestHotWriteRegionScheduleCheckHot(t *testing.T) { hb.(*hotScheduler).conf.SetDstToleranceRatio(1) hb.(*hotScheduler).conf.SetSrcToleranceRatio(1) - tc.SetHotRegionCacheHitsThreshold(0) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) @@ -1031,8 +1010,6 @@ func TestHotWriteRegionScheduleCheckHot(t *testing.T) { func TestHotWriteRegionScheduleWithLeader(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() @@ -1041,7 +1018,6 @@ func TestHotWriteRegionScheduleWithLeader(t *testing.T) { hb.(*hotScheduler).conf.SetHistorySampleDuration(0) re.NoError(err) - tc.SetHotRegionCacheHitsThreshold(0) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) tc.AddRegionStore(3, 20) @@ -1094,8 +1070,6 @@ func TestHotWriteRegionScheduleWithLeader(t *testing.T) { func TestHotWriteRegionScheduleWithPendingInfluence(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 checkHotWriteRegionScheduleWithPendingInfluence(re, 0) // 0: byte rate checkHotWriteRegionScheduleWithPendingInfluence(re, 1) // 1: key rate } @@ -1114,7 +1088,6 @@ func checkHotWriteRegionScheduleWithPendingInfluence(re *require.Assertions, dim pendingAmpFactor = old }() - tc.SetHotRegionCacheHitsThreshold(0) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) @@ -1189,8 +1162,6 @@ func checkHotWriteRegionScheduleWithPendingInfluence(re *require.Assertions, dim func TestHotWriteRegionScheduleWithRuleEnabled(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() tc.SetEnablePlacementRules(true) @@ -1199,7 +1170,6 @@ func TestHotWriteRegionScheduleWithRuleEnabled(t *testing.T) { hb.(*hotScheduler).conf.WriteLeaderPriorities = []string{utils.KeyPriority, utils.BytePriority} hb.(*hotScheduler).conf.SetHistorySampleDuration(0) - tc.SetHotRegionCacheHitsThreshold(0) key, err := hex.DecodeString("") re.NoError(err) // skip stddev check @@ -1282,7 +1252,6 @@ func TestHotReadRegionScheduleByteRateOnly(t *testing.T) { hb := scheduler.(*hotScheduler) hb.conf.ReadPriorities = []string{utils.BytePriority, utils.KeyPriority} hb.conf.SetHistorySampleDuration(0) - tc.SetHotRegionCacheHitsThreshold(0) // Add stores 1, 2, 3, 4, 5 with region counts 3, 2, 2, 2, 0. tc.AddRegionStore(1, 3) @@ -1396,8 +1365,6 @@ func TestHotReadRegionScheduleByteRateOnly(t *testing.T) { func TestHotReadRegionScheduleWithQuery(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() @@ -1408,7 +1375,6 @@ func TestHotReadRegionScheduleWithQuery(t *testing.T) { hb.(*hotScheduler).conf.RankFormulaVersion = "v1" hb.(*hotScheduler).conf.SetHistorySampleDuration(0) - tc.SetHotRegionCacheHitsThreshold(0) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) tc.AddRegionStore(3, 20) @@ -1432,8 +1398,6 @@ func TestHotReadRegionScheduleWithQuery(t *testing.T) { func TestHotReadRegionScheduleWithKeyRate(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() @@ -1445,7 +1409,6 @@ func TestHotReadRegionScheduleWithKeyRate(t *testing.T) { hb.(*hotScheduler).conf.ReadPriorities = []string{utils.BytePriority, utils.KeyPriority} hb.(*hotScheduler).conf.SetHistorySampleDuration(0) - tc.SetHotRegionCacheHitsThreshold(0) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) tc.AddRegionStore(3, 20) @@ -1491,8 +1454,6 @@ func TestHotReadRegionScheduleWithKeyRate(t *testing.T) { func TestHotReadRegionScheduleWithPendingInfluence(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 checkHotReadRegionScheduleWithPendingInfluence(re, 0) // 0: byte rate checkHotReadRegionScheduleWithPendingInfluence(re, 1) // 1: key rate } @@ -1515,7 +1476,6 @@ func checkHotReadRegionScheduleWithPendingInfluence(re *require.Assertions, dim pendingAmpFactor = old }() - tc.SetHotRegionCacheHitsThreshold(0) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) @@ -1614,8 +1574,6 @@ func checkHotReadRegionScheduleWithPendingInfluence(re *require.Assertions, dim func TestHotReadWithEvictLeaderScheduler(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() @@ -1625,7 +1583,6 @@ func TestHotReadWithEvictLeaderScheduler(t *testing.T) { hb.(*hotScheduler).conf.SetDstToleranceRatio(1) hb.(*hotScheduler).conf.SetStrictPickingStore(false) hb.(*hotScheduler).conf.ReadPriorities = []string{utils.BytePriority, utils.KeyPriority} - tc.SetHotRegionCacheHitsThreshold(0) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) @@ -1657,7 +1614,6 @@ func TestHotCacheUpdateCache(t *testing.T) { re := require.New(t) cancel, _, tc, _ := prepareSchedulersTest() defer cancel() - tc.SetHotRegionCacheHitsThreshold(0) // For read flow addRegionInfo(tc, utils.Read, []testRegionInfo{ @@ -1724,7 +1680,6 @@ func TestHotCacheKeyThresholds(t *testing.T) { { // only a few regions cancel, _, tc, _ := prepareSchedulersTest() defer cancel() - tc.SetHotRegionCacheHitsThreshold(0) addRegionInfo(tc, utils.Read, []testRegionInfo{ {1, []uint64{1, 2, 3}, 0, 1, 0}, {2, []uint64{1, 2, 3}, 0, 1 * units.KiB, 0}, @@ -1796,7 +1751,6 @@ func TestHotCacheByteAndKey(t *testing.T) { re := require.New(t) cancel, _, tc, _ := prepareSchedulersTest() defer cancel() - tc.SetHotRegionCacheHitsThreshold(0) statistics.ThresholdsUpdateInterval = 0 defer func() { statistics.ThresholdsUpdateInterval = 8 * time.Second @@ -2090,8 +2044,6 @@ func TestInfluenceByRWType(t *testing.T) { defer func() { schedulePeerPr = originValue }() - statistics.Denoising = false - statisticsInterval = 0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() @@ -2100,7 +2052,6 @@ func TestInfluenceByRWType(t *testing.T) { hb.(*hotScheduler).conf.SetDstToleranceRatio(1) hb.(*hotScheduler).conf.SetSrcToleranceRatio(1) hb.(*hotScheduler).conf.SetHistorySampleDuration(0) - tc.SetHotRegionCacheHitsThreshold(0) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) @@ -2214,8 +2165,6 @@ func checkHotReadPeerSchedule(re *require.Assertions, enablePlacementRules bool) func TestHotScheduleWithPriority(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() @@ -2231,7 +2180,6 @@ func TestHotScheduleWithPriority(t *testing.T) { stddevThreshold = origin }() - tc.SetHotRegionCacheHitsThreshold(0) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) @@ -2323,9 +2271,6 @@ func TestHotScheduleWithPriority(t *testing.T) { func TestHotScheduleWithStddev(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 - cancel, _, tc, oc := prepareSchedulersTest() defer cancel() hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) @@ -2334,7 +2279,6 @@ func TestHotScheduleWithStddev(t *testing.T) { hb.(*hotScheduler).conf.SetSrcToleranceRatio(1.0) hb.(*hotScheduler).conf.RankFormulaVersion = "v1" tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - tc.SetHotRegionCacheHitsThreshold(0) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) tc.AddRegionStore(3, 20) @@ -2384,9 +2328,6 @@ func TestHotScheduleWithStddev(t *testing.T) { func TestHotWriteLeaderScheduleWithPriority(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 - cancel, _, tc, oc := prepareSchedulersTest() defer cancel() hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) @@ -2396,7 +2337,6 @@ func TestHotWriteLeaderScheduleWithPriority(t *testing.T) { hb.(*hotScheduler).conf.SetHistorySampleDuration(0) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - tc.SetHotRegionCacheHitsThreshold(0) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) tc.AddRegionStore(3, 20) @@ -2428,9 +2368,6 @@ func TestHotWriteLeaderScheduleWithPriority(t *testing.T) { func TestCompatibility(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 - cancel, _, tc, oc := prepareSchedulersTest() defer cancel() hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) diff --git a/pkg/schedule/schedulers/hot_region_v2_test.go b/pkg/schedule/schedulers/hot_region_v2_test.go index 78a30cebaca1..25d6d94f7b16 100644 --- a/pkg/schedule/schedulers/hot_region_v2_test.go +++ b/pkg/schedule/schedulers/hot_region_v2_test.go @@ -21,7 +21,6 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/mock/mockcluster" "github.com/tikv/pd/pkg/schedule/operator" - "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/utils" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" @@ -33,8 +32,6 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimSecond(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - statistics.Denoising = false - statisticsInterval = 0 sche, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) re.NoError(err) hb := sche.(*hotScheduler) @@ -43,7 +40,6 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimSecond(t *testing.T) { hb.conf.SetRankFormulaVersion("v1") hb.conf.SetHistorySampleDuration(0) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - tc.SetHotRegionCacheHitsThreshold(0) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) tc.AddRegionStore(3, 20) @@ -96,8 +92,6 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimFirst(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - statistics.Denoising = false - sche, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) re.NoError(err) hb := sche.(*hotScheduler) @@ -106,7 +100,6 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimFirst(t *testing.T) { hb.conf.SetRankFormulaVersion("v1") hb.conf.SetHistorySampleDuration(0) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - tc.SetHotRegionCacheHitsThreshold(0) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) tc.AddRegionStore(3, 20) @@ -148,9 +141,6 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimFirst(t *testing.T) { func TestHotWriteRegionScheduleWithRevertRegionsDimFirstOnly(t *testing.T) { // This is a test that searchRevertRegions finds a solution of rank -2. re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 - cancel, _, tc, oc := prepareSchedulersTest() defer cancel() sche, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) @@ -161,7 +151,6 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimFirstOnly(t *testing.T) { hb.conf.SetRankFormulaVersion("v1") hb.conf.SetHistorySampleDuration(0) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - tc.SetHotRegionCacheHitsThreshold(0) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) tc.AddRegionStore(3, 20) @@ -212,9 +201,6 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimFirstOnly(t *testing.T) { func TestHotReadRegionScheduleWithRevertRegionsDimSecond(t *testing.T) { // This is a test that searchRevertRegions finds a solution of rank -1. re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 - cancel, _, tc, oc := prepareSchedulersTest() defer cancel() sche, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) @@ -225,7 +211,6 @@ func TestHotReadRegionScheduleWithRevertRegionsDimSecond(t *testing.T) { hb.conf.SetRankFormulaVersion("v1") hb.conf.SetHistorySampleDuration(0) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - tc.SetHotRegionCacheHitsThreshold(0) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) tc.AddRegionStore(3, 20) @@ -275,9 +260,6 @@ func TestHotReadRegionScheduleWithRevertRegionsDimSecond(t *testing.T) { func TestSkipUniformStore(t *testing.T) { re := require.New(t) - statistics.Denoising = false - statisticsInterval = 0 - cancel, _, tc, oc := prepareSchedulersTest() defer cancel() hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) @@ -287,7 +269,6 @@ func TestSkipUniformStore(t *testing.T) { hb.(*hotScheduler).conf.SetRankFormulaVersion("v2") hb.(*hotScheduler).conf.ReadPriorities = []string{utils.BytePriority, utils.KeyPriority} hb.(*hotScheduler).conf.SetHistorySampleDuration(0) - tc.SetHotRegionCacheHitsThreshold(0) tc.AddRegionStore(1, 20) tc.AddRegionStore(2, 20) tc.AddRegionStore(3, 20) @@ -439,7 +420,6 @@ func checkHotReadRegionScheduleWithSmallHotRegion(re *require.Assertions, highLo addOtherRegions func(*mockcluster.Cluster, *hotScheduler)) []*operator.Operator { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - statistics.Denoising = false sche, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) re.NoError(err) hb := sche.(*hotScheduler) @@ -447,7 +427,6 @@ func checkHotReadRegionScheduleWithSmallHotRegion(re *require.Assertions, highLo hb.conf.SetDstToleranceRatio(1) hb.conf.SetRankFormulaVersion("v2") hb.conf.ReadPriorities = []string{utils.QueryPriority, utils.BytePriority} - tc.SetHotRegionCacheHitsThreshold(0) tc.AddRegionStore(1, 40) tc.AddRegionStore(2, 10) tc.AddRegionStore(3, 10) @@ -470,7 +449,6 @@ func checkHotReadRegionScheduleWithSmallHotRegion(re *require.Assertions, highLo } } addRegionInfo(tc, utils.Read, regions) - tc.SetHotRegionCacheHitsThreshold(1) addOtherRegions(tc, hb) ops, _ := hb.Schedule(tc, false) return ops diff --git a/pkg/schedule/schedulers/scheduler_test.go b/pkg/schedule/schedulers/scheduler_test.go index 77c190ad943c..d30ef3ad0aa0 100644 --- a/pkg/schedule/schedulers/scheduler_test.go +++ b/pkg/schedule/schedulers/scheduler_test.go @@ -46,6 +46,7 @@ func prepareSchedulersTest(needToRunStream ...bool) (context.CancelFunc, config. stream = hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, needToRunStream[0]) } oc := operator.NewController(ctx, tc.GetBasicCluster(), tc.GetSchedulerConfig(), stream) + tc.SetHotRegionCacheHitsThreshold(1) return cancel, opt, tc, oc } @@ -183,7 +184,6 @@ func checkBalance(re *require.Assertions, enablePlacementRules bool) { tc.AddLeaderRegionWithWriteInfo(1, 1, 512*units.KiB*utils.RegionHeartBeatReportInterval, 0, 0, utils.RegionHeartBeatReportInterval, []uint64{2, 3}) tc.AddLeaderRegionWithWriteInfo(2, 1, 512*units.KiB*utils.RegionHeartBeatReportInterval, 0, 0, utils.RegionHeartBeatReportInterval, []uint64{3, 4}) tc.AddLeaderRegionWithWriteInfo(3, 1, 512*units.KiB*utils.RegionHeartBeatReportInterval, 0, 0, utils.RegionHeartBeatReportInterval, []uint64{2, 4}) - tc.SetHotRegionCacheHitsThreshold(0) // try to get an operator var ops []*operator.Operator @@ -218,7 +218,6 @@ func TestHotRegionScheduleAbnormalReplica(t *testing.T) { tc.AddRegionWithReadInfo(1, 1, 512*units.KiB*utils.StoreHeartBeatReportInterval, 0, 0, utils.StoreHeartBeatReportInterval, []uint64{2}) tc.AddRegionWithReadInfo(2, 2, 512*units.KiB*utils.StoreHeartBeatReportInterval, 0, 0, utils.StoreHeartBeatReportInterval, []uint64{1, 3}) tc.AddRegionWithReadInfo(3, 1, 512*units.KiB*utils.StoreHeartBeatReportInterval, 0, 0, utils.StoreHeartBeatReportInterval, []uint64{2, 3}) - tc.SetHotRegionCacheHitsThreshold(0) re.True(tc.IsRegionHot(tc.GetRegion(1))) re.False(hb.IsScheduleAllowed(tc)) } @@ -318,7 +317,6 @@ func TestSpecialUseHotRegion(t *testing.T) { hs, err := CreateScheduler(utils.Write.String(), oc, storage, cd) re.NoError(err) - tc.SetHotRegionCacheHitsThreshold(0) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.AddRegionStore(1, 10) tc.AddRegionStore(2, 4) @@ -368,7 +366,6 @@ func TestSpecialUseReserved(t *testing.T) { bs, err := CreateScheduler(BalanceRegionType, oc, storage, cd) re.NoError(err) - tc.SetHotRegionCacheHitsThreshold(0) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.AddRegionStore(1, 10) tc.AddRegionStore(2, 4) From db88a43f7783e8768730f8ab0ef6056e9d2eff04 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Wed, 17 Apr 2024 22:04:37 +0800 Subject: [PATCH 24/50] client/tso: use the TSO request pool at the tsoClient level to avoid data race (#8077) close tikv/pd#8055, ref tikv/pd#8076 Use the TSO request pool at the `tsoClient` level to avoid data race. Signed-off-by: JmPotato Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/client.go | 33 ++++++-------- client/tso_client.go | 61 ++++++++++++------------- client/tso_dispatcher.go | 32 -------------- client/tso_request.go | 96 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 83 deletions(-) create mode 100644 client/tso_request.go diff --git a/client/client.go b/client/client.go index 1852b77e4c6e..eaebef7e10c1 100644 --- a/client/client.go +++ b/client/client.go @@ -798,23 +798,7 @@ func (c *client) GetLocalTSAsync(ctx context.Context, dcLocation string) TSFutur defer span.Finish() } - req := c.getTSORequest(ctx, dcLocation) - if err := c.dispatchTSORequestWithRetry(req); err != nil { - req.tryDone(err) - } - return req -} - -func (c *client) getTSORequest(ctx context.Context, dcLocation string) *tsoRequest { - req := tsoReqPool.Get().(*tsoRequest) - // Set needed fields in the request before using it. - req.start = time.Now() - req.clientCtx = c.ctx - req.requestCtx = ctx - req.physical = 0 - req.logical = 0 - req.dcLocation = dcLocation - return req + return c.dispatchTSORequestWithRetry(ctx, dcLocation) } const ( @@ -822,10 +806,11 @@ const ( dispatchRetryCount = 2 ) -func (c *client) dispatchTSORequestWithRetry(req *tsoRequest) error { +func (c *client) dispatchTSORequestWithRetry(ctx context.Context, dcLocation string) TSFuture { var ( retryable bool err error + req *tsoRequest ) for i := 0; i < dispatchRetryCount; i++ { // Do not delay for the first time. @@ -838,12 +823,22 @@ func (c *client) dispatchTSORequestWithRetry(req *tsoRequest) error { err = errs.ErrClientGetTSO.FastGenByArgs("tso client is nil") continue } + // Get a new request from the pool if it's nil or not from the current pool. + if req == nil || req.pool != tsoClient.tsoReqPool { + req = tsoClient.getTSORequest(ctx, dcLocation) + } retryable, err = tsoClient.dispatchRequest(req) if !retryable { break } } - return err + if err != nil { + if req == nil { + return newTSORequestFastFail(err) + } + req.tryDone(err) + } + return req } func (c *client) GetTS(ctx context.Context) (physical int64, logical int64, err error) { diff --git a/client/tso_client.go b/client/tso_client.go index 5f8b12df36f6..8185b99d1d0d 100644 --- a/client/tso_client.go +++ b/client/tso_client.go @@ -43,33 +43,6 @@ type TSOClient interface { GetMinTS(ctx context.Context) (int64, int64, error) } -type tsoRequest struct { - start time.Time - clientCtx context.Context - requestCtx context.Context - done chan error - physical int64 - logical int64 - dcLocation string -} - -var tsoReqPool = sync.Pool{ - New: func() any { - return &tsoRequest{ - done: make(chan error, 1), - physical: 0, - logical: 0, - } - }, -} - -func (req *tsoRequest) tryDone(err error) { - select { - case req.done <- err: - default: - } -} - type tsoClient struct { ctx context.Context cancel context.CancelFunc @@ -84,6 +57,8 @@ type tsoClient struct { // tso allocator leader is switched. tsoAllocServingURLSwitchedCallback []func() + // tsoReqPool is the pool to recycle `*tsoRequest`. + tsoReqPool *sync.Pool // tsoDispatcher is used to dispatch different TSO requests to // the corresponding dc-location TSO channel. tsoDispatcher sync.Map // Same as map[string]*tsoDispatcher @@ -104,11 +79,20 @@ func newTSOClient( ) *tsoClient { ctx, cancel := context.WithCancel(ctx) c := &tsoClient{ - ctx: ctx, - cancel: cancel, - option: option, - svcDiscovery: svcDiscovery, - tsoStreamBuilderFactory: factory, + ctx: ctx, + cancel: cancel, + option: option, + svcDiscovery: svcDiscovery, + tsoStreamBuilderFactory: factory, + tsoReqPool: &sync.Pool{ + New: func() any { + return &tsoRequest{ + done: make(chan error, 1), + physical: 0, + logical: 0, + } + }, + }, checkTSDeadlineCh: make(chan struct{}), checkTSODispatcherCh: make(chan struct{}, 1), updateTSOConnectionCtxsCh: make(chan struct{}, 1), @@ -155,6 +139,19 @@ func (c *tsoClient) Close() { log.Info("tso client is closed") } +func (c *tsoClient) getTSORequest(ctx context.Context, dcLocation string) *tsoRequest { + req := c.tsoReqPool.Get().(*tsoRequest) + // Set needed fields in the request before using it. + req.start = time.Now() + req.pool = c.tsoReqPool + req.requestCtx = ctx + req.clientCtx = c.ctx + req.physical = 0 + req.logical = 0 + req.dcLocation = dcLocation + return req +} + // GetTSOAllocators returns {dc-location -> TSO allocator leader URL} connection map func (c *tsoClient) GetTSOAllocators() *sync.Map { return &c.tsoAllocators diff --git a/client/tso_dispatcher.go b/client/tso_dispatcher.go index ad3aa1c5d74a..d02fdd52af8d 100644 --- a/client/tso_dispatcher.go +++ b/client/tso_dispatcher.go @@ -115,38 +115,6 @@ func (c *tsoClient) dispatchRequest(request *tsoRequest) (bool, error) { return false, nil } -// TSFuture is a future which promises to return a TSO. -type TSFuture interface { - // Wait gets the physical and logical time, it would block caller if data is not available yet. - Wait() (int64, int64, error) -} - -func (req *tsoRequest) Wait() (physical int64, logical int64, err error) { - // If tso command duration is observed very high, the reason could be it - // takes too long for Wait() be called. - start := time.Now() - cmdDurationTSOAsyncWait.Observe(start.Sub(req.start).Seconds()) - select { - case err = <-req.done: - defer trace.StartRegion(req.requestCtx, "pdclient.tsoReqDone").End() - err = errors.WithStack(err) - defer tsoReqPool.Put(req) - if err != nil { - cmdFailDurationTSO.Observe(time.Since(req.start).Seconds()) - return 0, 0, err - } - physical, logical = req.physical, req.logical - now := time.Now() - cmdDurationWait.Observe(now.Sub(start).Seconds()) - cmdDurationTSO.Observe(now.Sub(req.start).Seconds()) - return - case <-req.requestCtx.Done(): - return 0, 0, errors.WithStack(req.requestCtx.Err()) - case <-req.clientCtx.Done(): - return 0, 0, errors.WithStack(req.clientCtx.Err()) - } -} - func (c *tsoClient) updateTSODispatcher() { // Set up the new TSO dispatcher and batch controller. c.GetTSOAllocators().Range(func(dcLocationKey, _ any) bool { diff --git a/client/tso_request.go b/client/tso_request.go new file mode 100644 index 000000000000..f30ceb5268a6 --- /dev/null +++ b/client/tso_request.go @@ -0,0 +1,96 @@ +// Copyright 2024 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pd + +import ( + "context" + "runtime/trace" + "sync" + "time" + + "github.com/pingcap/errors" +) + +// TSFuture is a future which promises to return a TSO. +type TSFuture interface { + // Wait gets the physical and logical time, it would block caller if data is not available yet. + Wait() (int64, int64, error) +} + +var ( + _ TSFuture = (*tsoRequest)(nil) + _ TSFuture = (*tsoRequestFastFail)(nil) +) + +type tsoRequest struct { + requestCtx context.Context + clientCtx context.Context + done chan error + physical int64 + logical int64 + dcLocation string + + // Runtime fields. + start time.Time + pool *sync.Pool +} + +// tryDone tries to send the result to the channel, it will not block. +func (req *tsoRequest) tryDone(err error) { + select { + case req.done <- err: + default: + } +} + +// Wait will block until the TSO result is ready. +func (req *tsoRequest) Wait() (physical int64, logical int64, err error) { + // If tso command duration is observed very high, the reason could be it + // takes too long for Wait() be called. + start := time.Now() + cmdDurationTSOAsyncWait.Observe(start.Sub(req.start).Seconds()) + select { + case err = <-req.done: + defer trace.StartRegion(req.requestCtx, "pdclient.tsoReqDone").End() + defer req.pool.Put(req) + err = errors.WithStack(err) + if err != nil { + cmdFailDurationTSO.Observe(time.Since(req.start).Seconds()) + return 0, 0, err + } + physical, logical = req.physical, req.logical + now := time.Now() + cmdDurationWait.Observe(now.Sub(start).Seconds()) + cmdDurationTSO.Observe(now.Sub(req.start).Seconds()) + return + case <-req.requestCtx.Done(): + return 0, 0, errors.WithStack(req.requestCtx.Err()) + case <-req.clientCtx.Done(): + return 0, 0, errors.WithStack(req.clientCtx.Err()) + } +} + +type tsoRequestFastFail struct { + err error +} + +func newTSORequestFastFail(err error) *tsoRequestFastFail { + return &tsoRequestFastFail{err} +} + +// Wait returns the error directly. +func (req *tsoRequestFastFail) Wait() (physical int64, logical int64, err error) { + return 0, 0, req.err +} From afdd48f3026776eaa7cbf8172e824b05a0a332fa Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 17 Apr 2024 22:21:07 +0800 Subject: [PATCH 25/50] *: make `TestEtcdClientSync` stable (#8084) close tikv/pd#8085 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/utils/etcdutil/etcdutil_test.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pkg/utils/etcdutil/etcdutil_test.go b/pkg/utils/etcdutil/etcdutil_test.go index 4c1e20fa73c3..c402081fa2ff 100644 --- a/pkg/utils/etcdutil/etcdutil_test.go +++ b/pkg/utils/etcdutil/etcdutil_test.go @@ -172,6 +172,7 @@ func TestEtcdClientSync(t *testing.T) { servers, client1, clean := NewTestEtcdCluster(t, 1) defer clean() etcd1, cfg1 := servers[0], servers[0].Config() + defer etcd1.Close() // Add a new member. etcd2 := MustAddEtcdMember(t, &cfg1, client1) @@ -180,10 +181,22 @@ func TestEtcdClientSync(t *testing.T) { // wait for etcd client sync endpoints checkEtcdEndpointNum(re, client1, 2) - // Remove the first member and close the etcd1. - _, err := RemoveEtcdMember(client1, uint64(etcd1.Server.ID())) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + // remove one member that is not the one we connected to. + resp, err := ListEtcdMembers(ctx, client1) + re.NoError(err) + + var memIDToRemove uint64 + for _, m := range resp.Members { + if m.ID != resp.Header.MemberId { + memIDToRemove = m.ID + break + } + } + + _, err = RemoveEtcdMember(client1, memIDToRemove) re.NoError(err) - etcd1.Close() // Check the client can get the new member with the new endpoints. checkEtcdEndpointNum(re, client1, 1) From 956684a0015e63b904b2ff6c1c6186f161f6de4b Mon Sep 17 00:00:00 2001 From: Shirly Date: Thu, 18 Apr 2024 10:12:06 +0800 Subject: [PATCH 26/50] cluster: print down peer when the related store is not disconnected (#8046) ref tikv/pd#4399 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/statistics/region_collection.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pkg/statistics/region_collection.go b/pkg/statistics/region_collection.go index 488763142e1c..cb0de6f601b8 100644 --- a/pkg/statistics/region_collection.go +++ b/pkg/statistics/region_collection.go @@ -22,6 +22,7 @@ import ( sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/utils/syncutil" + "go.uber.org/zap" ) // RegionInfoProvider is an interface to provide the region information. @@ -250,6 +251,7 @@ func (r *RegionStatistics) Observe(region *core.RegionInfo, stores []*core.Store regionDownPeerDuration.Observe(float64(time.Now().Unix() - info.startDownPeerTS)) } else { info.startDownPeerTS = time.Now().Unix() + logDownPeerWithNoDisconnectedStore(region, stores) } } else if typ == MissPeer && len(region.GetVoters()) < desiredVoters { if info.startMissVoterPeerTS != 0 { @@ -440,3 +442,24 @@ func notIsolatedStoresWithLabel(stores []*core.StoreInfo, label string) [][]*cor } return res } + +// logDownPeerWithNoDisconnectedStore logs down peers on connected stores. +// It won't log down peer when any store of the replica is disconnected which is +// used to avoid too many logs when a store is disconnected. +// TODO: it's not a good way to log down peer during process region heartbeat, we should handle it in another way. +// region: the region which has down peer +// stores: all stores that the region has peer on them +func logDownPeerWithNoDisconnectedStore(region *core.RegionInfo, stores []*core.StoreInfo) { + for _, store := range stores { + if store.IsDisconnected() { + return + } + } + for _, p := range region.GetDownPeers() { + log.Warn("region has down peer on connected store", + zap.Uint64("region-id", region.GetID()), + zap.Uint64("down-peer", p.GetPeer().GetId()), + zap.Uint64("down-seconds", p.GetDownSeconds()), + zap.Uint64("store-id", p.GetPeer().GetStoreId())) + } +} From ad5ba18205eb0ef2d3b4c609ed5a8a2d786dc0c4 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 18 Apr 2024 11:45:07 +0800 Subject: [PATCH 27/50] *: make `TestConcurrentlyReset` stable (#8082) close tikv/pd#6898 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/api/admin_test.go | 2 +- tests/integrations/tso/server_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/api/admin_test.go b/server/api/admin_test.go index 3a628a1de61a..f3b3dd64bd34 100644 --- a/server/api/admin_test.go +++ b/server/api/admin_test.go @@ -181,7 +181,7 @@ func (suite *adminTestSuite) TestPersistFile() { func makeTS(offset time.Duration) uint64 { physical := time.Now().Add(offset).UnixNano() / int64(time.Millisecond) - return uint64(physical << 18) + return uint64(physical) << 18 } func (suite *adminTestSuite) TestResetTS() { diff --git a/tests/integrations/tso/server_test.go b/tests/integrations/tso/server_test.go index ac3d914aa80b..5590ba68d371 100644 --- a/tests/integrations/tso/server_test.go +++ b/tests/integrations/tso/server_test.go @@ -152,12 +152,12 @@ func (suite *tsoServerTestSuite) TestConcurrentlyReset() { for i := 0; i < 2; i++ { go func() { defer wg.Done() - for j := 0; j <= 100; j++ { + for j := 0; j <= 50; j++ { // Get a copy of now then call base.add, because now is shared by all goroutines // and now.add() will add to itself which isn't atomic and multi-goroutine safe. base := now - physical := base.Add(time.Duration(2*j)*time.Minute).UnixNano() / int64(time.Millisecond) - ts := uint64(physical << 18) + physical := base.Add(time.Duration(j)*time.Minute).UnixNano() / int64(time.Millisecond) + ts := uint64(physical) << 18 suite.resetTS(ts, false, false) } }() From d64e6981eda4c93e38807cc77a1af5e7e203f132 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 18 Apr 2024 14:06:07 +0800 Subject: [PATCH 28/50] *: make `TestSplitKeyspaceGroup` stable (#8088) close tikv/pd#7380 Signed-off-by: Ryan Leung --- tests/server/apiv2/handlers/testutil.go | 42 ++++++++++++++++++------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/tests/server/apiv2/handlers/testutil.go b/tests/server/apiv2/handlers/testutil.go index d26ce7327143..c5682aafbce1 100644 --- a/tests/server/apiv2/handlers/testutil.go +++ b/tests/server/apiv2/handlers/testutil.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server/apiv2/handlers" "github.com/tikv/pd/tests" ) @@ -168,8 +169,14 @@ func tryCreateKeyspaceGroup(re *require.Assertions, server *tests.TestServer, re // MustLoadKeyspaceGroupByID loads the keyspace group by ID with HTTP API. func MustLoadKeyspaceGroupByID(re *require.Assertions, server *tests.TestServer, id uint32) *endpoint.KeyspaceGroup { - kg, code := TryLoadKeyspaceGroupByID(re, server, id) - re.Equal(http.StatusOK, code) + var ( + kg *endpoint.KeyspaceGroup + code int + ) + testutil.Eventually(re, func() bool { + kg, code = TryLoadKeyspaceGroupByID(re, server, id) + return code == http.StatusOK + }) return kg } @@ -232,15 +239,28 @@ func MustSplitKeyspaceGroup(re *require.Assertions, server *tests.TestServer, id // MustFinishSplitKeyspaceGroup finishes a keyspace group split with HTTP API. func MustFinishSplitKeyspaceGroup(re *require.Assertions, server *tests.TestServer, id uint32) { - httpReq, err := http.NewRequest(http.MethodDelete, server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d/split", id), http.NoBody) - re.NoError(err) - // Send request. - resp, err := dialClient.Do(httpReq) - re.NoError(err) - defer resp.Body.Close() - data, err := io.ReadAll(resp.Body) - re.NoError(err) - re.Equal(http.StatusOK, resp.StatusCode, string(data)) + testutil.Eventually(re, func() bool { + httpReq, err := http.NewRequest(http.MethodDelete, server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d/split", id), http.NoBody) + if err != nil { + return false + } + // Send request. + resp, err := dialClient.Do(httpReq) + if err != nil { + return false + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return false + } + if resp.StatusCode == http.StatusServiceUnavailable || + resp.StatusCode == http.StatusInternalServerError { + return false + } + re.Equal(http.StatusOK, resp.StatusCode, string(data)) + return true + }) } // MustMergeKeyspaceGroup merges keyspace groups with HTTP API. From 882d2e55951f7eba3f888a91872b08fea395e3e0 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 18 Apr 2024 15:58:37 +0800 Subject: [PATCH 29/50] *: make `TestRateLimitConfigReload` stable (#8092) close tikv/pd#8083 Signed-off-by: Ryan Leung --- tests/server/config/config_test.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/server/config/config_test.go b/tests/server/config/config_test.go index b6fcecbd47b1..57e4272f7ead 100644 --- a/tests/server/config/config_test.go +++ b/tests/server/config/config_test.go @@ -32,7 +32,6 @@ import ( tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/pkg/versioninfo" - "github.com/tikv/pd/server" "github.com/tikv/pd/server/config" "github.com/tikv/pd/tests" ) @@ -74,11 +73,7 @@ func TestRateLimitConfigReload(t *testing.T) { oldLeaderName := leader.GetServer().Name() leader.GetServer().GetMember().ResignEtcdLeader(leader.GetServer().Context(), oldLeaderName, "") - var servers []*server.Server - for _, s := range cluster.GetServers() { - servers = append(servers, s.GetServer()) - } - server.MustWaitLeader(re, servers) + re.NotEmpty(cluster.WaitLeader()) leader = cluster.GetLeaderServer() re.NotNil(leader) re.True(leader.GetServer().GetServiceMiddlewarePersistOptions().IsRateLimitEnabled()) From 5d3c8c105626ee86956f1c24d3b8750243a50961 Mon Sep 17 00:00:00 2001 From: ShuNing Date: Fri, 19 Apr 2024 12:12:06 +0800 Subject: [PATCH 30/50] schedule: fix the filter metrics flush (#8097) close tikv/pd#8098 schedule: fix the filter metrics flush Signed-off-by: nolouch --- pkg/schedule/schedulers/balance_leader.go | 2 +- pkg/schedule/schedulers/balance_region.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/schedule/schedulers/balance_leader.go b/pkg/schedule/schedulers/balance_leader.go index b4e6feb332cc..6976bc4d8ea8 100644 --- a/pkg/schedule/schedulers/balance_leader.go +++ b/pkg/schedule/schedulers/balance_leader.go @@ -354,6 +354,7 @@ func (l *balanceLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun if dryRun { collector = plan.NewCollector(basePlan) } + defer l.filterCounter.Flush() batch := l.conf.getBatch() balanceLeaderScheduleCounter.Inc() @@ -395,7 +396,6 @@ func (l *balanceLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun } } } - l.filterCounter.Flush() l.retryQuota.GC(append(sourceCandidate.stores, targetCandidate.stores...)) return result, collector.GetPlans() } diff --git a/pkg/schedule/schedulers/balance_region.go b/pkg/schedule/schedulers/balance_region.go index 98e3be6e08a8..e4202c133ad8 100644 --- a/pkg/schedule/schedulers/balance_region.go +++ b/pkg/schedule/schedulers/balance_region.go @@ -114,6 +114,7 @@ func (s *balanceRegionScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster func (s *balanceRegionScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { basePlan := plan.NewBalanceSchedulerPlan() + defer s.filterCounter.Flush() var collector *plan.Collector if dryRun { collector = plan.NewCollector(basePlan) @@ -217,7 +218,6 @@ func (s *balanceRegionScheduler) Schedule(cluster sche.SchedulerCluster, dryRun } s.retryQuota.Attenuate(solver.Source) } - s.filterCounter.Flush() s.retryQuota.GC(stores) return nil, collector.GetPlans() } From 32bf572de6b55cbcf6556021f8ad0d4485247240 Mon Sep 17 00:00:00 2001 From: Hu# Date: Fri, 19 Apr 2024 16:58:37 +0800 Subject: [PATCH 31/50] tests/etcd: remove mod to make test stable (#8102) ref tikv/pd#7969, close tikv/pd#8091 Signed-off-by: husharp --- pkg/utils/etcdutil/testutil.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/utils/etcdutil/testutil.go b/pkg/utils/etcdutil/testutil.go index d9464eeceeb9..13c10260a409 100644 --- a/pkg/utils/etcdutil/testutil.go +++ b/pkg/utils/etcdutil/testutil.go @@ -51,7 +51,7 @@ func NewTestSingleConfig() *embed.Config { } func genRandName() string { - return "test_etcd_" + strconv.FormatInt(time.Now().UnixNano()%10000, 10) + return "test_etcd_" + strconv.FormatInt(time.Now().UnixNano(), 10) } // NewTestEtcdCluster is used to create a etcd cluster for the unit test purpose. From 4a7bffcb3baf7a8e2515e70834d8567bc17439da Mon Sep 17 00:00:00 2001 From: Hu# Date: Fri, 19 Apr 2024 19:25:37 +0800 Subject: [PATCH 32/50] operator: make additional information thread safe (#8104) close tikv/pd#8079 Signed-off-by: husharp --- pkg/schedule/operator/create_operator.go | 4 +- pkg/schedule/operator/operator.go | 46 ++++++++----------- pkg/schedule/operator/operator_controller.go | 14 +++--- pkg/schedule/operator/status_tracker.go | 33 +++++++++++++ pkg/schedule/operator/status_tracker_test.go | 25 ++++++++++ pkg/schedule/scatter/region_scatterer.go | 4 +- pkg/schedule/scatter/region_scatterer_test.go | 2 +- pkg/schedule/schedulers/balance_leader.go | 4 +- pkg/schedule/schedulers/balance_region.go | 4 +- pkg/schedule/schedulers/balance_witness.go | 4 +- pkg/schedule/schedulers/hot_region.go | 4 +- pkg/schedule/schedulers/split_bucket.go | 2 +- 12 files changed, 98 insertions(+), 48 deletions(-) diff --git a/pkg/schedule/operator/create_operator.go b/pkg/schedule/operator/create_operator.go index 638230e30975..646805209335 100644 --- a/pkg/schedule/operator/create_operator.go +++ b/pkg/schedule/operator/create_operator.go @@ -170,8 +170,8 @@ func CreateSplitRegionOperator(desc string, region *core.RegionInfo, kind OpKind brief += fmt.Sprintf(" and keys %v", hexKeys) } op := NewOperator(desc, brief, region.GetID(), region.GetRegionEpoch(), kind|OpSplit, region.GetApproximateSize(), step) - op.AdditionalInfos["region-start-key"] = core.HexRegionKeyStr(logutil.RedactBytes(region.GetStartKey())) - op.AdditionalInfos["region-end-key"] = core.HexRegionKeyStr(logutil.RedactBytes(region.GetEndKey())) + op.SetAdditionalInfo("region-start-key", core.HexRegionKeyStr(logutil.RedactBytes(region.GetStartKey()))) + op.SetAdditionalInfo("region-end-key", core.HexRegionKeyStr(logutil.RedactBytes(region.GetEndKey()))) return op, nil } diff --git a/pkg/schedule/operator/operator.go b/pkg/schedule/operator/operator.go index b87a050969f3..de197c4fba40 100644 --- a/pkg/schedule/operator/operator.go +++ b/pkg/schedule/operator/operator.go @@ -15,7 +15,6 @@ package operator import ( - "encoding/json" "fmt" "reflect" "strconv" @@ -83,7 +82,7 @@ type Operator struct { level constant.PriorityLevel Counters []prometheus.Counter FinishedCounters []prometheus.Counter - AdditionalInfos map[string]string + additionalInfos opAdditionalInfo ApproximateSize int64 timeout time.Duration influence *OpInfluence @@ -100,16 +99,18 @@ func NewOperator(desc, brief string, regionID uint64, regionEpoch *metapb.Region maxDuration += v.Timeout(approximateSize).Seconds() } return &Operator{ - desc: desc, - brief: brief, - regionID: regionID, - regionEpoch: regionEpoch, - kind: kind, - steps: steps, - stepsTime: make([]int64, len(steps)), - status: NewOpStatusTracker(), - level: level, - AdditionalInfos: make(map[string]string), + desc: desc, + brief: brief, + regionID: regionID, + regionEpoch: regionEpoch, + kind: kind, + steps: steps, + stepsTime: make([]int64, len(steps)), + status: NewOpStatusTracker(), + level: level, + additionalInfos: opAdditionalInfo{ + value: make(map[string]string), + }, ApproximateSize: approximateSize, timeout: time.Duration(maxDuration) * time.Second, } @@ -118,8 +119,8 @@ func NewOperator(desc, brief string, regionID uint64, regionEpoch *metapb.Region // Sync some attribute with the given timeout. func (o *Operator) Sync(other *Operator) { o.timeout = other.timeout - o.AdditionalInfos[string(RelatedMergeRegion)] = strconv.FormatUint(other.RegionID(), 10) - other.AdditionalInfos[string(RelatedMergeRegion)] = strconv.FormatUint(o.RegionID(), 10) + o.SetAdditionalInfo(string(RelatedMergeRegion), strconv.FormatUint(other.RegionID(), 10)) + other.SetAdditionalInfo(string(RelatedMergeRegion), strconv.FormatUint(o.RegionID(), 10)) } func (o *Operator) String() string { @@ -297,8 +298,10 @@ func (o *Operator) CheckSuccess() bool { // Cancel marks the operator canceled. func (o *Operator) Cancel(reason ...CancelReasonType) bool { - if _, ok := o.AdditionalInfos[cancelReason]; !ok && len(reason) != 0 { - o.AdditionalInfos[cancelReason] = string(reason[0]) + o.additionalInfos.Lock() + defer o.additionalInfos.Unlock() + if _, ok := o.additionalInfos.value[cancelReason]; !ok && len(reason) != 0 { + o.additionalInfos.value[cancelReason] = string(reason[0]) } return o.status.To(CANCELED) } @@ -507,17 +510,6 @@ func (o *Operator) Record(finishTime time.Time) *OpRecord { return record } -// GetAdditionalInfo returns additional info with string -func (o *Operator) GetAdditionalInfo() string { - if len(o.AdditionalInfos) != 0 { - additionalInfo, err := json.Marshal(o.AdditionalInfos) - if err == nil { - return string(additionalInfo) - } - } - return "" -} - // IsLeaveJointStateOperator returns true if the desc is OpDescLeaveJointState. func (o *Operator) IsLeaveJointStateOperator() bool { return strings.EqualFold(o.desc, OpDescLeaveJointState) diff --git a/pkg/schedule/operator/operator_controller.go b/pkg/schedule/operator/operator_controller.go index 86e51fe70d63..d63e843f52a1 100644 --- a/pkg/schedule/operator/operator_controller.go +++ b/pkg/schedule/operator/operator_controller.go @@ -510,7 +510,7 @@ func (oc *Controller) addOperatorInner(op *Operator) bool { log.Info("add operator", zap.Uint64("region-id", regionID), zap.Reflect("operator", op), - zap.String("additional-info", op.GetAdditionalInfo())) + zap.String("additional-info", op.LogAdditionalInfo())) // If there is an old operator, replace it. The priority should be checked // already. @@ -657,7 +657,7 @@ func (oc *Controller) removeOperatorInner(op *Operator) bool { } func (oc *Controller) removeRelatedMergeOperator(op *Operator) { - relatedID, _ := strconv.ParseUint(op.AdditionalInfos[string(RelatedMergeRegion)], 10, 64) + relatedID, _ := strconv.ParseUint(op.GetAdditionalInfo(string(RelatedMergeRegion)), 10, 64) relatedOpi, ok := oc.operators.Load(relatedID) if !ok { return @@ -666,7 +666,7 @@ func (oc *Controller) removeRelatedMergeOperator(op *Operator) { if relatedOp != nil && relatedOp.Status() != CANCELED { log.Info("operator canceled related merge region", zap.Uint64("region-id", relatedOp.RegionID()), - zap.String("additional-info", relatedOp.GetAdditionalInfo()), + zap.String("additional-info", relatedOp.LogAdditionalInfo()), zap.Duration("takes", relatedOp.RunningTime())) oc.removeOperatorInner(relatedOp) relatedOp.Cancel(RelatedMergeRegion) @@ -695,7 +695,7 @@ func (oc *Controller) buryOperator(op *Operator) { zap.Uint64("region-id", op.RegionID()), zap.Duration("takes", op.RunningTime()), zap.Reflect("operator", op), - zap.String("additional-info", op.GetAdditionalInfo())) + zap.String("additional-info", op.LogAdditionalInfo())) operatorCounter.WithLabelValues(op.Desc(), "finish").Inc() operatorDuration.WithLabelValues(op.Desc()).Observe(op.RunningTime().Seconds()) for _, counter := range op.FinishedCounters { @@ -706,7 +706,7 @@ func (oc *Controller) buryOperator(op *Operator) { zap.Uint64("region-id", op.RegionID()), zap.Duration("takes", op.RunningTime()), zap.Reflect("operator", op), - zap.String("additional-info", op.GetAdditionalInfo())) + zap.String("additional-info", op.LogAdditionalInfo())) operatorCounter.WithLabelValues(op.Desc(), "replace").Inc() case EXPIRED: log.Info("operator expired", @@ -719,14 +719,14 @@ func (oc *Controller) buryOperator(op *Operator) { zap.Uint64("region-id", op.RegionID()), zap.Duration("takes", op.RunningTime()), zap.Reflect("operator", op), - zap.String("additional-info", op.GetAdditionalInfo())) + zap.String("additional-info", op.LogAdditionalInfo())) operatorCounter.WithLabelValues(op.Desc(), "timeout").Inc() case CANCELED: log.Info("operator canceled", zap.Uint64("region-id", op.RegionID()), zap.Duration("takes", op.RunningTime()), zap.Reflect("operator", op), - zap.String("additional-info", op.GetAdditionalInfo()), + zap.String("additional-info", op.LogAdditionalInfo()), ) operatorCounter.WithLabelValues(op.Desc(), "cancel").Inc() } diff --git a/pkg/schedule/operator/status_tracker.go b/pkg/schedule/operator/status_tracker.go index 0ba8135750cd..a74d94b18a47 100644 --- a/pkg/schedule/operator/status_tracker.go +++ b/pkg/schedule/operator/status_tracker.go @@ -15,6 +15,7 @@ package operator import ( + "encoding/json" "time" "github.com/tikv/pd/pkg/utils/syncutil" @@ -135,3 +136,35 @@ func (trk *OpStatusTracker) String() string { defer trk.rw.RUnlock() return OpStatusToString(trk.current) } + +type opAdditionalInfo struct { + syncutil.RWMutex + value map[string]string +} + +// SetAdditionalInfo sets additional info with key and value. +func (o *Operator) SetAdditionalInfo(key string, value string) { + o.additionalInfos.Lock() + defer o.additionalInfos.Unlock() + o.additionalInfos.value[key] = value +} + +// GetAdditionalInfo returns additional info with key. +func (o *Operator) GetAdditionalInfo(key string) string { + o.additionalInfos.RLock() + defer o.additionalInfos.RUnlock() + return o.additionalInfos.value[key] +} + +// LogAdditionalInfo returns additional info with string +func (o *Operator) LogAdditionalInfo() string { + o.additionalInfos.RLock() + defer o.additionalInfos.RUnlock() + if len(o.additionalInfos.value) != 0 { + additionalInfo, err := json.Marshal(o.additionalInfos.value) + if err == nil { + return string(additionalInfo) + } + } + return "" +} diff --git a/pkg/schedule/operator/status_tracker_test.go b/pkg/schedule/operator/status_tracker_test.go index e53b017229ab..8c897d1e545d 100644 --- a/pkg/schedule/operator/status_tracker_test.go +++ b/pkg/schedule/operator/status_tracker_test.go @@ -15,6 +15,8 @@ package operator import ( + "fmt" + "sync" "testing" "time" @@ -178,3 +180,26 @@ func checkReachTime(re *require.Assertions, trk *OpStatusTracker, reached ...OpS re.True(trk.ReachTimeOf(st).IsZero()) } } + +func TestAdditionalInfoConcurrent(t *testing.T) { + op := NewOperator("test", "test", 0, nil, OpAdmin, 0) + + var wg sync.WaitGroup + for i := 0; i < 1000; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + key := fmt.Sprintf("key%d", i) + value := fmt.Sprintf("value%d", i) + op.SetAdditionalInfo(key, value) + if op.GetAdditionalInfo(key) != value { + t.Errorf("unexpected value for key %s", key) + } + }(i) + } + wg.Wait() + + if logInfo := op.LogAdditionalInfo(); logInfo == "" { + t.Error("LogAdditionalInfo returned an empty string") + } +} diff --git a/pkg/schedule/scatter/region_scatterer.go b/pkg/schedule/scatter/region_scatterer.go index 898c4d052a71..bdec5c98c9c1 100644 --- a/pkg/schedule/scatter/region_scatterer.go +++ b/pkg/schedule/scatter/region_scatterer.go @@ -399,8 +399,8 @@ func (r *RegionScatterer) scatterRegion(region *core.RegionInfo, group string, s if op != nil { scatterSuccessCounter.Inc() r.Put(targetPeers, targetLeader, group) - op.AdditionalInfos["group"] = group - op.AdditionalInfos["leader-picked-count"] = strconv.FormatUint(leaderStorePickedCount, 10) + op.SetAdditionalInfo("group", group) + op.SetAdditionalInfo("leader-picked-count", strconv.FormatUint(leaderStorePickedCount, 10)) op.SetPriorityLevel(constant.High) } return op, nil diff --git a/pkg/schedule/scatter/region_scatterer_test.go b/pkg/schedule/scatter/region_scatterer_test.go index af41ed04b761..b0027e0e4155 100644 --- a/pkg/schedule/scatter/region_scatterer_test.go +++ b/pkg/schedule/scatter/region_scatterer_test.go @@ -679,7 +679,7 @@ func TestSelectedStoresTooFewPeers(t *testing.T) { re.NoError(err) re.False(isPeerCountChanged(op)) if op != nil { - re.Equal(group, op.AdditionalInfos["group"]) + re.Equal(group, op.GetAdditionalInfo("group")) } } } diff --git a/pkg/schedule/schedulers/balance_leader.go b/pkg/schedule/schedulers/balance_leader.go index 6976bc4d8ea8..6114d2a8f890 100644 --- a/pkg/schedule/schedulers/balance_leader.go +++ b/pkg/schedule/schedulers/balance_leader.go @@ -567,7 +567,7 @@ func (l *balanceLeaderScheduler) createOperator(solver *solver, collector *plan. op.FinishedCounters = append(op.FinishedCounters, balanceDirectionCounter.WithLabelValues(l.GetName(), solver.SourceMetricLabel(), solver.TargetMetricLabel()), ) - op.AdditionalInfos["sourceScore"] = strconv.FormatFloat(solver.sourceScore, 'f', 2, 64) - op.AdditionalInfos["targetScore"] = strconv.FormatFloat(solver.targetScore, 'f', 2, 64) + op.SetAdditionalInfo("sourceScore", strconv.FormatFloat(solver.sourceScore, 'f', 2, 64)) + op.SetAdditionalInfo("targetScore", strconv.FormatFloat(solver.targetScore, 'f', 2, 64)) return op } diff --git a/pkg/schedule/schedulers/balance_region.go b/pkg/schedule/schedulers/balance_region.go index e4202c133ad8..608d008a99ec 100644 --- a/pkg/schedule/schedulers/balance_region.go +++ b/pkg/schedule/schedulers/balance_region.go @@ -278,8 +278,8 @@ func (s *balanceRegionScheduler) transferPeer(solver *solver, collector *plan.Co op.FinishedCounters = append(op.FinishedCounters, balanceDirectionCounter.WithLabelValues(s.GetName(), sourceLabel, targetLabel), ) - op.AdditionalInfos["sourceScore"] = strconv.FormatFloat(solver.sourceScore, 'f', 2, 64) - op.AdditionalInfos["targetScore"] = strconv.FormatFloat(solver.targetScore, 'f', 2, 64) + op.SetAdditionalInfo("sourceScore", strconv.FormatFloat(solver.sourceScore, 'f', 2, 64)) + op.SetAdditionalInfo("targetScore", strconv.FormatFloat(solver.targetScore, 'f', 2, 64)) return op } diff --git a/pkg/schedule/schedulers/balance_witness.go b/pkg/schedule/schedulers/balance_witness.go index 3c4776c4666d..aa97874409ab 100644 --- a/pkg/schedule/schedulers/balance_witness.go +++ b/pkg/schedule/schedulers/balance_witness.go @@ -378,7 +378,7 @@ func (b *balanceWitnessScheduler) createOperator(solver *solver, collector *plan b.counter.WithLabelValues("move-witness", solver.SourceMetricLabel()+"-out"), b.counter.WithLabelValues("move-witness", solver.TargetMetricLabel()+"-in"), ) - op.AdditionalInfos["sourceScore"] = strconv.FormatFloat(solver.sourceScore, 'f', 2, 64) - op.AdditionalInfos["targetScore"] = strconv.FormatFloat(solver.targetScore, 'f', 2, 64) + op.SetAdditionalInfo("sourceScore", strconv.FormatFloat(solver.sourceScore, 'f', 2, 64)) + op.SetAdditionalInfo("targetScore", strconv.FormatFloat(solver.targetScore, 'f', 2, 64)) return op } diff --git a/pkg/schedule/schedulers/hot_region.go b/pkg/schedule/schedulers/hot_region.go index b4e904c14812..5e5e254596a7 100644 --- a/pkg/schedule/schedulers/hot_region.go +++ b/pkg/schedule/schedulers/hot_region.go @@ -1653,8 +1653,8 @@ func (bs *balanceSolver) splitBucketsByLoad(region *core.RegionInfo, bucketStats } op := bs.splitBucketsOperator(region, [][]byte{splitKey}) if op != nil { - op.AdditionalInfos["accLoads"] = strconv.FormatUint(acc-stats[splitIdx-1].Loads[dim], 10) - op.AdditionalInfos["totalLoads"] = strconv.FormatUint(totalLoads, 10) + op.SetAdditionalInfo("accLoads", strconv.FormatUint(acc-stats[splitIdx-1].Loads[dim], 10)) + op.SetAdditionalInfo("totalLoads", strconv.FormatUint(totalLoads, 10)) } return op } diff --git a/pkg/schedule/schedulers/split_bucket.go b/pkg/schedule/schedulers/split_bucket.go index 609510446c75..32e57ec9b3dd 100644 --- a/pkg/schedule/schedulers/split_bucket.go +++ b/pkg/schedule/schedulers/split_bucket.go @@ -297,7 +297,7 @@ func (s *splitBucketScheduler) splitBucket(plan *splitBucketPlan) []*operator.Op return nil } splitBucketNewOperatorCounter.Inc() - op.AdditionalInfos["hot-degree"] = strconv.FormatInt(int64(splitBucket.HotDegree), 10) + op.SetAdditionalInfo("hot-degree", strconv.FormatInt(int64(splitBucket.HotDegree), 10)) return []*operator.Operator{op} } return nil From 2f85f1fd30061d81a704a830d15fa0defc9103b5 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 22 Apr 2024 14:31:39 +0800 Subject: [PATCH 33/50] *: make `TestTSOKeyspaceGroupSplitElection` stable (#8110) close tikv/pd#8099 Signed-off-by: Ryan Leung --- pkg/tso/keyspace_group_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tso/keyspace_group_manager.go b/pkg/tso/keyspace_group_manager.go index d1e94d445cc7..2930357e2b4f 100644 --- a/pkg/tso/keyspace_group_manager.go +++ b/pkg/tso/keyspace_group_manager.go @@ -1439,7 +1439,7 @@ func (kgm *KeyspaceGroupManager) groupSplitPatroller() { defer kgm.wg.Done() patrolInterval := groupPatrolInterval failpoint.Inject("fastGroupSplitPatroller", func() { - patrolInterval = 200 * time.Millisecond + patrolInterval = time.Second }) ticker := time.NewTicker(patrolInterval) defer ticker.Stop() From 5cdf2526e703f89c541f068813bee5467361c3e2 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 22 Apr 2024 16:48:09 +0800 Subject: [PATCH 34/50] *: update dependencies (#8112) ref tikv/pd#4399 Signed-off-by: Ryan Leung --- client/go.mod | 10 +++++----- client/go.sum | 24 ++++++++++++------------ go.mod | 2 +- go.sum | 4 ++-- tests/integrations/go.mod | 2 +- tests/integrations/go.sum | 4 ++-- tools/go.mod | 2 +- tools/go.sum | 4 ++-- 8 files changed, 26 insertions(+), 26 deletions(-) diff --git a/client/go.mod b/client/go.mod index 9b2cb87f75e4..89799796521b 100644 --- a/client/go.mod +++ b/client/go.mod @@ -19,7 +19,7 @@ require ( go.uber.org/multierr v1.11.0 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 - google.golang.org/grpc v1.59.0 + google.golang.org/grpc v1.62.1 google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9 ) @@ -34,11 +34,11 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.46.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - golang.org/x/net v0.20.0 // indirect - golang.org/x/sys v0.16.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/client/go.sum b/client/go.sum index a58d351ebcfe..54942bb0bb84 100644 --- a/client/go.sum +++ b/client/go.sum @@ -22,8 +22,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -110,8 +110,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -122,8 +122,8 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -144,16 +144,16 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9 h1:ATnmU8nL2NfIyTSiBvJVDIDIr3qBmeW+c7z7XU21eWs= google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9/go.mod h1:j5uROIAAgi3YmtiETMt1LW0d/lHqQ7wwrIY4uGRXLQ4= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/go.mod b/go.mod index c76242f3753c..ff0cb20069a0 100644 --- a/go.mod +++ b/go.mod @@ -179,7 +179,7 @@ require ( golang.org/x/crypto v0.21.0 // indirect golang.org/x/image v0.10.0 // indirect golang.org/x/mod v0.13.0 // indirect - golang.org/x/net v0.22.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect diff --git a/go.sum b/go.sum index d11fad07aa6c..8c77a4b84da9 100644 --- a/go.sum +++ b/go.sum @@ -605,8 +605,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= diff --git a/tests/integrations/go.mod b/tests/integrations/go.mod index cdd244cafc19..bb231f747b75 100644 --- a/tests/integrations/go.mod +++ b/tests/integrations/go.mod @@ -168,7 +168,7 @@ require ( golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/image v0.10.0 // indirect - golang.org/x/net v0.22.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect diff --git a/tests/integrations/go.sum b/tests/integrations/go.sum index 79f7dddd130d..eeb2d73ba7fa 100644 --- a/tests/integrations/go.sum +++ b/tests/integrations/go.sum @@ -598,8 +598,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= diff --git a/tools/go.mod b/tools/go.mod index 9d8728f70347..8d0f0d4ec35e 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -167,7 +167,7 @@ require ( golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/image v0.10.0 // indirect - golang.org/x/net v0.22.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect diff --git a/tools/go.sum b/tools/go.sum index d7c7a4801b11..a3c41c16420a 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -599,8 +599,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= From 1d386f6f189feb4a9b83799efa593474b62e5aa3 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Mon, 22 Apr 2024 21:25:39 +0800 Subject: [PATCH 35/50] tests: avoid panic in cluster test (#8114) close tikv/pd#8113 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/cluster/cluster_test.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index ecd579d88817..945e354bb6cb 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -3655,7 +3655,8 @@ func TestInterval(t *testing.T) { func waitAddLearner(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { var res *pdpb.RegionHeartbeatResponse testutil.Eventually(re, func() bool { - if res = stream.Recv().(*pdpb.RegionHeartbeatResponse); res != nil { + if r := stream.Recv(); r != nil { + res = r.(*pdpb.RegionHeartbeatResponse) return res.GetRegionId() == region.GetID() && res.GetChangePeer().GetChangeType() == eraftpb.ConfChangeType_AddLearnerNode && res.GetChangePeer().GetPeer().GetStoreId() == storeID @@ -3671,7 +3672,8 @@ func waitAddLearner(re *require.Assertions, stream mockhbstream.HeartbeatStream, func waitPromoteLearner(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { var res *pdpb.RegionHeartbeatResponse testutil.Eventually(re, func() bool { - if res = stream.Recv().(*pdpb.RegionHeartbeatResponse); res != nil { + if r := stream.Recv(); r != nil { + res = r.(*pdpb.RegionHeartbeatResponse) return res.GetRegionId() == region.GetID() && res.GetChangePeer().GetChangeType() == eraftpb.ConfChangeType_AddNode && res.GetChangePeer().GetPeer().GetStoreId() == storeID @@ -3688,7 +3690,8 @@ func waitPromoteLearner(re *require.Assertions, stream mockhbstream.HeartbeatStr func waitRemovePeer(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { var res *pdpb.RegionHeartbeatResponse testutil.Eventually(re, func() bool { - if res = stream.Recv().(*pdpb.RegionHeartbeatResponse); res != nil { + if r := stream.Recv(); r != nil { + res = r.(*pdpb.RegionHeartbeatResponse) return res.GetRegionId() == region.GetID() && res.GetChangePeer().GetChangeType() == eraftpb.ConfChangeType_RemoveNode && res.GetChangePeer().GetPeer().GetStoreId() == storeID @@ -3704,7 +3707,8 @@ func waitRemovePeer(re *require.Assertions, stream mockhbstream.HeartbeatStream, func waitTransferLeader(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { var res *pdpb.RegionHeartbeatResponse testutil.Eventually(re, func() bool { - if res = stream.Recv().(*pdpb.RegionHeartbeatResponse); res != nil { + if r := stream.Recv(); r != nil { + res = r.(*pdpb.RegionHeartbeatResponse) if res.GetRegionId() == region.GetID() { for _, peer := range append(res.GetTransferLeader().GetPeers(), res.GetTransferLeader().GetPeer()) { if peer.GetStoreId() == storeID { From 1e65f9dda7d9b93e68178139ec3b4c31539a223d Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 24 Apr 2024 14:18:10 +0800 Subject: [PATCH 36/50] *: fix follower cannot handle member request (#8122) ref tikv/pd#7519 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/apiv2/handlers/micro_service.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server/apiv2/handlers/micro_service.go b/server/apiv2/handlers/micro_service.go index 209a4c954456..fd44665530f0 100644 --- a/server/apiv2/handlers/micro_service.go +++ b/server/apiv2/handlers/micro_service.go @@ -26,7 +26,6 @@ import ( // RegisterMicroService registers microservice handler to the router. func RegisterMicroService(r *gin.RouterGroup) { router := r.Group("ms") - router.Use(middlewares.BootstrapChecker()) router.GET("members/:service", GetMembers) router.GET("primary/:service", GetPrimary) } From 141186e958dcd3b88e3dd289c9294f6623c3eba3 Mon Sep 17 00:00:00 2001 From: ShuNing Date: Wed, 24 Apr 2024 14:54:11 +0800 Subject: [PATCH 37/50] *: optimize heartbeat process with concurrent runner - part 2 (#8052) ref tikv/pd#7897 Optimize heartbeat process - Split the statics updates on the subtree Signed-off-by: nolouch Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/core/region.go | 173 +++++++++++++++++++++++++++ pkg/core/region_test.go | 86 ++++++++++++- pkg/mcs/scheduling/server/cluster.go | 28 ++--- pkg/ratelimit/metrics.go | 52 ++++++++ pkg/ratelimit/runner.go | 18 +++ pkg/schedule/config/config.go | 2 +- server/cluster/cluster.go | 36 +++--- 7 files changed, 354 insertions(+), 41 deletions(-) create mode 100644 pkg/ratelimit/metrics.go diff --git a/pkg/core/region.go b/pkg/core/region.go index 41bfb4d31add..713e82cc36d8 100644 --- a/pkg/core/region.go +++ b/pkg/core/region.go @@ -856,6 +856,24 @@ func GenerateRegionGuideFunc(enableLog bool) RegionGuideFunc { } } +// RegionHeartbeatStageName is the name of the stage of the region heartbeat. +const ( + HandleStatsAsync = "HandleStatsAsync" + ObserveRegionStatsAsync = "ObserveRegionStatsAsync" + UpdateSubTree = "UpdateSubTree" + HandleOverlaps = "HandleOverlaps" + CollectRegionStatsAsync = "CollectRegionStatsAsync" + SaveRegionToKV = "SaveRegionToKV" +) + +// ExtraTaskOpts returns the task options for the task. +func ExtraTaskOpts(ctx *MetaProcessContext, name string) ratelimit.TaskOpts { + return ratelimit.TaskOpts{ + TaskName: name, + Limit: ctx.Limiter, + } +} + // RWLockStats is a read-write lock with statistics. type RWLockStats struct { syncutil.RWMutex @@ -1004,6 +1022,161 @@ func (r *RegionsInfo) AtomicCheckAndPutRegion(ctx *MetaProcessContext, region *R return overlaps, nil } +// CheckAndPutRootTree checks if the region is valid to put to the root, if valid then return error. +// Usually used with CheckAndPutSubTree together. +func (r *RegionsInfo) CheckAndPutRootTree(ctx *MetaProcessContext, region *RegionInfo) ([]*RegionInfo, error) { + tracer := ctx.Tracer + r.t.Lock() + var ols []*regionItem + origin := r.getRegionLocked(region.GetID()) + if origin == nil || !bytes.Equal(origin.GetStartKey(), region.GetStartKey()) || !bytes.Equal(origin.GetEndKey(), region.GetEndKey()) { + ols = r.tree.overlaps(®ionItem{RegionInfo: region}) + } + tracer.OnCheckOverlapsFinished() + err := check(region, origin, convertItemsToRegions(ols)) + if err != nil { + r.t.Unlock() + tracer.OnValidateRegionFinished() + return nil, err + } + tracer.OnValidateRegionFinished() + _, overlaps, _ := r.setRegionLocked(region, true, ols...) + r.t.Unlock() + tracer.OnSetRegionFinished() + return overlaps, nil +} + +// CheckAndPutSubTree checks if the region is valid to put to the sub tree, if valid then return error. +// Usually used with CheckAndPutRootTree together. +func (r *RegionsInfo) CheckAndPutSubTree(region *RegionInfo) { + // new region get from root tree again + var newRegion *RegionInfo + newRegion = r.GetRegion(region.GetID()) + if newRegion == nil { + newRegion = region + } + r.UpdateSubTreeOrderInsensitive(newRegion) +} + +// UpdateSubTreeOrderInsensitive updates the subtree. +// It's can used to update the subtree concurrently. +// because it can use concurrently, check region version to make sure the order. +// 1. if the version is stale, drop this update. +// 2. if the version is same, then only some statistic info need to be updated. +// in this situation, the order of update is not important. +// +// in another hand, the overlap regions need re-check, because the region tree and the subtree update is not atomic. +func (r *RegionsInfo) UpdateSubTreeOrderInsensitive(region *RegionInfo) { + var origin *RegionInfo + r.st.Lock() + defer r.st.Unlock() + originItem, ok := r.subRegions[region.GetID()] + if ok { + origin = originItem.RegionInfo + } + rangeChanged := true + + if origin != nil { + re := region.GetRegionEpoch() + oe := origin.GetRegionEpoch() + isTermBehind := region.GetTerm() > 0 && region.GetTerm() < origin.GetTerm() + if (isTermBehind || re.GetVersion() < oe.GetVersion() || re.GetConfVer() < oe.GetConfVer()) && !region.isRegionRecreated() { + // Region meta is stale, skip. + return + } + rangeChanged = !origin.rangeEqualsTo(region) + + if rangeChanged || !origin.peersEqualTo(region) { + // If the range or peers have changed, the sub regionTree needs to be cleaned up. + // TODO: Improve performance by deleting only the different peers. + r.removeRegionFromSubTreeLocked(origin) + } else { + // The region tree and the subtree update is not atomic and the region tree is updated first. + // If there are two thread needs to update region tree, + // t1: thread-A update region tree + // t2: thread-B: update region tree again + // t3: thread-B: update subtree + // t4: thread-A: update region subtree + // to keep region tree consistent with subtree, we need to drop this update. + if tree, ok := r.subRegions[region.GetID()]; ok { + r.updateSubTreeStat(origin, region) + tree.RegionInfo = region + } + return + } + } + + if rangeChanged { + overlaps := r.getOverlapRegionFromSubTreeLocked(region) + for _, re := range overlaps { + r.removeRegionFromSubTreeLocked(re) + } + } + + item := ®ionItem{region} + r.subRegions[region.GetID()] = item + // It has been removed and all information needs to be updated again. + // Set peers then. + setPeer := func(peersMap map[uint64]*regionTree, storeID uint64, item *regionItem) { + store, ok := peersMap[storeID] + if !ok { + store = newRegionTree() + peersMap[storeID] = store + } + store.update(item, false) + } + + // Add to leaders and followers. + for _, peer := range region.GetVoters() { + storeID := peer.GetStoreId() + if peer.GetId() == region.leader.GetId() { + // Add leader peer to leaders. + setPeer(r.leaders, storeID, item) + } else { + // Add follower peer to followers. + setPeer(r.followers, storeID, item) + } + } + + setPeers := func(peersMap map[uint64]*regionTree, peers []*metapb.Peer) { + for _, peer := range peers { + storeID := peer.GetStoreId() + setPeer(peersMap, storeID, item) + } + } + // Add to learners. + setPeers(r.learners, region.GetLearners()) + // Add to witnesses. + setPeers(r.witnesses, region.GetWitnesses()) + // Add to PendingPeers + setPeers(r.pendingPeers, region.GetPendingPeers()) +} + +func (r *RegionsInfo) getOverlapRegionFromSubTreeLocked(region *RegionInfo) []*RegionInfo { + it := ®ionItem{RegionInfo: region} + overlaps := make([]*RegionInfo, 0) + overlapsMap := make(map[uint64]struct{}) + collectFromItemSlice := func(peersMap map[uint64]*regionTree, storeID uint64) { + if tree, ok := peersMap[storeID]; ok { + items := tree.overlaps(it) + for _, item := range items { + if _, ok := overlapsMap[item.GetID()]; !ok { + overlapsMap[item.GetID()] = struct{}{} + overlaps = append(overlaps, item.RegionInfo) + } + } + } + } + for _, peer := range region.GetMeta().GetPeers() { + storeID := peer.GetStoreId() + collectFromItemSlice(r.leaders, storeID) + collectFromItemSlice(r.followers, storeID) + collectFromItemSlice(r.learners, storeID) + collectFromItemSlice(r.witnesses, storeID) + } + return overlaps +} + // GetRelevantRegions returns the relevant regions for a given region. func (r *RegionsInfo) GetRelevantRegions(region *RegionInfo) (origin *RegionInfo, overlaps []*RegionInfo) { r.t.RLock() diff --git a/pkg/core/region_test.go b/pkg/core/region_test.go index ae91886369f9..88683968f3fd 100644 --- a/pkg/core/region_test.go +++ b/pkg/core/region_test.go @@ -789,14 +789,15 @@ func randomBytes(n int) []byte { return bytes } -func newRegionInfoID(idAllocator id.Allocator) *RegionInfo { +func newRegionInfoIDRandom(idAllocator id.Allocator) *RegionInfo { var ( peers []*metapb.Peer leader *metapb.Peer ) + storeNum := 10 for i := 0; i < 3; i++ { id, _ := idAllocator.Alloc() - p := &metapb.Peer{Id: id, StoreId: id} + p := &metapb.Peer{Id: id, StoreId: uint64(i%storeNum + 1)} if i == 0 { leader = p } @@ -811,6 +812,8 @@ func newRegionInfoID(idAllocator id.Allocator) *RegionInfo { Peers: peers, }, leader, + SetApproximateSize(10), + SetApproximateKeys(10), ) } @@ -819,7 +822,7 @@ func BenchmarkAddRegion(b *testing.B) { idAllocator := mockid.NewIDAllocator() var items []*RegionInfo for i := 0; i < 10000000; i++ { - items = append(items, newRegionInfoID(idAllocator)) + items = append(items, newRegionInfoIDRandom(idAllocator)) } b.ResetTimer() for i := 0; i < b.N; i++ { @@ -858,3 +861,80 @@ func BenchmarkRegionFromHeartbeat(b *testing.B) { RegionFromHeartbeat(regionReq) } } + +func TestUpdateRegionEquivalence(t *testing.T) { + re := require.New(t) + regionsOld := NewRegionsInfo() + regionsNew := NewRegionsInfo() + storeNums := 5 + items := generateTestRegions(1000, storeNums) + + updateRegion := func(item *RegionInfo) { + // old way + ctx := ContextTODO() + regionsOld.AtomicCheckAndPutRegion(ctx, item) + // new way + ctx = ContextTODO() + regionsNew.CheckAndPutRootTree(ctx, item) + regionsNew.CheckAndPutSubTree(item) + } + checksEquivalence := func() { + re.Equal(regionsOld.GetRegionCount([]byte(""), []byte("")), regionsNew.GetRegionCount([]byte(""), []byte(""))) + re.Equal(regionsOld.GetRegionSizeByRange([]byte(""), []byte("")), regionsNew.GetRegionSizeByRange([]byte(""), []byte(""))) + checkRegions(re, regionsOld) + checkRegions(re, regionsNew) + + for i := 1; i <= storeNums; i++ { + re.Equal(regionsOld.GetStoreRegionCount(uint64(i)), regionsNew.GetStoreRegionCount(uint64(i))) + re.Equal(regionsOld.GetStoreLeaderCount(uint64(i)), regionsNew.GetStoreLeaderCount(uint64(i))) + re.Equal(regionsOld.GetStorePendingPeerCount(uint64(i)), regionsNew.GetStorePendingPeerCount(uint64(i))) + re.Equal(regionsOld.GetStoreLearnerRegionSize(uint64(i)), regionsNew.GetStoreLearnerRegionSize(uint64(i))) + re.Equal(regionsOld.GetStoreRegionSize(uint64(i)), regionsNew.GetStoreRegionSize(uint64(i))) + re.Equal(regionsOld.GetStoreLeaderRegionSize(uint64(i)), regionsNew.GetStoreLeaderRegionSize(uint64(i))) + re.Equal(regionsOld.GetStoreFollowerRegionSize(uint64(i)), regionsNew.GetStoreFollowerRegionSize(uint64(i))) + } + } + + // Add a region. + for _, item := range items { + updateRegion(item) + } + checksEquivalence() + + // Merge regions. + itemA, itemB := items[10], items[11] + itemMergedAB := itemA.Clone(WithEndKey(itemB.GetEndKey()), WithIncVersion()) + updateRegion(itemMergedAB) + checksEquivalence() + + // Split + itemA = itemA.Clone(WithIncVersion(), WithIncVersion()) + itemB = itemB.Clone(WithIncVersion(), WithIncVersion()) + updateRegion(itemA) + updateRegion(itemB) + checksEquivalence() +} + +func generateTestRegions(count int, storeNum int) []*RegionInfo { + var items []*RegionInfo + for i := 0; i < count; i++ { + peer1 := &metapb.Peer{StoreId: uint64(i%storeNum + 1), Id: uint64(i*storeNum + 1)} + peer2 := &metapb.Peer{StoreId: uint64((i+1)%storeNum + 1), Id: uint64(i*storeNum + 2)} + peer3 := &metapb.Peer{StoreId: uint64((i+2)%storeNum + 1), Id: uint64(i*storeNum + 3)} + if i%3 == 0 { + peer2.IsWitness = true + } + region := NewRegionInfo(&metapb.Region{ + Id: uint64(i + 1), + Peers: []*metapb.Peer{peer1, peer2, peer3}, + StartKey: []byte(fmt.Sprintf("%20d", i*10)), + EndKey: []byte(fmt.Sprintf("%20d", (i+1)*10)), + RegionEpoch: &metapb.RegionEpoch{ConfVer: 100, Version: 100}, + }, + peer1, + SetApproximateKeys(10), + SetApproximateSize(10)) + items = append(items, region) + } + return items +} diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index 42e8c3a35cb7..94b24f4ca16e 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -590,10 +590,7 @@ func (c *Cluster) processRegionHeartbeat(ctx *core.MetaProcessContext, region *c ctx.TaskRunner.RunTask( ctx, - ratelimit.TaskOpts{ - TaskName: "HandleStatsAsync", - Limit: ctx.Limiter, - }, + core.ExtraTaskOpts(ctx, core.HandleStatsAsync), func(_ context.Context) { cluster.HandleStatsAsync(c, region) }, @@ -610,10 +607,7 @@ func (c *Cluster) processRegionHeartbeat(ctx *core.MetaProcessContext, region *c if hasRegionStats && c.regionStats.RegionStatsNeedUpdate(region) { ctx.TaskRunner.RunTask( ctx, - ratelimit.TaskOpts{ - TaskName: "ObserveRegionStatsAsync", - Limit: ctx.Limiter, - }, + core.ExtraTaskOpts(ctx, core.ObserveRegionStatsAsync), func(_ context.Context) { if c.regionStats.RegionStatsNeedUpdate(region) { cluster.Collect(c, region, hasRegionStats) @@ -632,16 +626,21 @@ func (c *Cluster) processRegionHeartbeat(ctx *core.MetaProcessContext, region *c // However, it can't solve the race condition of concurrent heartbeats from the same region. // Async task in next PR. - if overlaps, err = c.AtomicCheckAndPutRegion(ctx, region); err != nil { + if overlaps, err = c.CheckAndPutRootTree(ctx, region); err != nil { tracer.OnSaveCacheFinished() return err } ctx.TaskRunner.RunTask( ctx, - ratelimit.TaskOpts{ - TaskName: "HandleOverlaps", - Limit: ctx.Limiter, + core.ExtraTaskOpts(ctx, core.UpdateSubTree), + func(_ context.Context) { + c.CheckAndPutSubTree(region) }, + ) + tracer.OnUpdateSubTreeFinished() + ctx.TaskRunner.RunTask( + ctx, + core.ExtraTaskOpts(ctx, core.HandleOverlaps), func(_ context.Context) { cluster.HandleOverlaps(c, overlaps) }, @@ -651,10 +650,7 @@ func (c *Cluster) processRegionHeartbeat(ctx *core.MetaProcessContext, region *c // handle region stats ctx.TaskRunner.RunTask( ctx, - ratelimit.TaskOpts{ - TaskName: "CollectRegionStatsAsync", - Limit: ctx.Limiter, - }, + core.ExtraTaskOpts(ctx, core.CollectRegionStatsAsync), func(_ context.Context) { cluster.Collect(c, region, hasRegionStats) }, diff --git a/pkg/ratelimit/metrics.go b/pkg/ratelimit/metrics.go new file mode 100644 index 000000000000..3c5020554a87 --- /dev/null +++ b/pkg/ratelimit/metrics.go @@ -0,0 +1,52 @@ +// Copyright 2024 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ratelimit + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const nameStr = "runner_name" + +var ( + RunnerTaskMaxWaitingDuration = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "ratelimit", + Name: "runner_task_max_waiting_duration_seconds", + Help: "The duration of tasks waiting in the runner.", + }, []string{nameStr}) + + RunnerTaskPendingTasks = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "ratelimit", + Name: "runner_task_pending_tasks", + Help: "The number of pending tasks in the runner.", + }, []string{nameStr}) + RunnerTaskFailedTasks = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "ratelimit", + Name: "runner_task_failed_tasks_total", + Help: "The number of failed tasks in the runner.", + }, []string{nameStr}) +) + +func init() { + prometheus.MustRegister(RunnerTaskMaxWaitingDuration) + prometheus.MustRegister(RunnerTaskPendingTasks) + prometheus.MustRegister(RunnerTaskFailedTasks) +} diff --git a/pkg/ratelimit/runner.go b/pkg/ratelimit/runner.go index 7f0ef21f7916..c4f2d5bc5ac6 100644 --- a/pkg/ratelimit/runner.go +++ b/pkg/ratelimit/runner.go @@ -21,6 +21,7 @@ import ( "time" "github.com/pingcap/log" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" ) @@ -53,6 +54,8 @@ type ConcurrentRunner struct { pendingMu sync.Mutex stopChan chan struct{} wg sync.WaitGroup + failedTaskCount prometheus.Counter + maxWaitingDuration prometheus.Gauge } // NewConcurrentRunner creates a new ConcurrentRunner. @@ -62,6 +65,8 @@ func NewConcurrentRunner(name string, maxPendingDuration time.Duration) *Concurr maxPendingDuration: maxPendingDuration, taskChan: make(chan *Task), pendingTasks: make([]*Task, 0, initialCapacity), + failedTaskCount: RunnerTaskFailedTasks.WithLabelValues(name), + maxWaitingDuration: RunnerTaskMaxWaitingDuration.WithLabelValues(name), } return s } @@ -77,6 +82,7 @@ type TaskOpts struct { func (s *ConcurrentRunner) Start() { s.stopChan = make(chan struct{}) s.wg.Add(1) + ticker := time.NewTicker(5 * time.Second) go func() { defer s.wg.Done() for { @@ -92,8 +98,19 @@ func (s *ConcurrentRunner) Start() { go s.run(task.Ctx, task.f, nil) } case <-s.stopChan: + s.pendingMu.Lock() + s.pendingTasks = make([]*Task, 0, initialCapacity) + s.pendingMu.Unlock() log.Info("stopping async task runner", zap.String("name", s.name)) return + case <-ticker.C: + maxDuration := time.Duration(0) + s.pendingMu.Lock() + if len(s.pendingTasks) > 0 { + maxDuration = time.Since(s.pendingTasks[0].submittedAt) + } + s.pendingMu.Unlock() + s.maxWaitingDuration.Set(maxDuration.Seconds()) } } }() @@ -144,6 +161,7 @@ func (s *ConcurrentRunner) RunTask(ctx context.Context, opt TaskOpts, f func(con if len(s.pendingTasks) > 0 { maxWait := time.Since(s.pendingTasks[0].submittedAt) if maxWait > s.maxPendingDuration { + s.failedTaskCount.Inc() return ErrMaxWaitingTasksExceeded } } diff --git a/pkg/schedule/config/config.go b/pkg/schedule/config/config.go index 1f3701763837..5a67a5474834 100644 --- a/pkg/schedule/config/config.go +++ b/pkg/schedule/config/config.go @@ -52,7 +52,7 @@ const ( defaultEnableJointConsensus = true defaultEnableTiKVSplitRegion = true defaultEnableHeartbeatBreakdownMetrics = true - defaultEnableHeartbeatConcurrentRunner = false + defaultEnableHeartbeatConcurrentRunner = true defaultEnableCrossTableMerge = true defaultEnableDiagnostic = true defaultStrictlyMatchLabel = false diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index dd59b63240ff..dbc6a6cadf33 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -1015,10 +1015,7 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { ctx.TaskRunner.RunTask( ctx.Context, - ratelimit.TaskOpts{ - TaskName: "HandleStatsAsync", - Limit: ctx.Limiter, - }, + core.ExtraTaskOpts(ctx, core.HandleStatsAsync), func(_ context.Context) { cluster.HandleStatsAsync(c, region) }, @@ -1039,10 +1036,7 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio if hasRegionStats && c.regionStats.RegionStatsNeedUpdate(region) { ctx.TaskRunner.RunTask( ctx.Context, - ratelimit.TaskOpts{ - TaskName: "ObserveRegionStatsAsync", - Limit: ctx.Limiter, - }, + core.ExtraTaskOpts(ctx, core.ObserveRegionStatsAsync), func(_ context.Context) { if c.regionStats.RegionStatsNeedUpdate(region) { cluster.Collect(c, region, hasRegionStats) @@ -1065,17 +1059,23 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio // check its validation again here. // // However, it can't solve the race condition of concurrent heartbeats from the same region. - if overlaps, err = c.core.AtomicCheckAndPutRegion(ctx, region); err != nil { + if overlaps, err = c.core.CheckAndPutRootTree(ctx, region); err != nil { tracer.OnSaveCacheFinished() return err } + ctx.TaskRunner.RunTask( + ctx, + core.ExtraTaskOpts(ctx, core.UpdateSubTree), + func(_ context.Context) { + c.CheckAndPutSubTree(region) + }, + ) + tracer.OnUpdateSubTreeFinished() + if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { ctx.TaskRunner.RunTask( ctx.Context, - ratelimit.TaskOpts{ - TaskName: "HandleOverlaps", - Limit: ctx.Limiter, - }, + core.ExtraTaskOpts(ctx, core.HandleOverlaps), func(_ context.Context) { cluster.HandleOverlaps(c, overlaps) }, @@ -1088,10 +1088,7 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio // handle region stats ctx.TaskRunner.RunTask( ctx.Context, - ratelimit.TaskOpts{ - TaskName: "CollectRegionStatsAsync", - Limit: ctx.Limiter, - }, + core.ExtraTaskOpts(ctx, core.CollectRegionStatsAsync), func(_ context.Context) { // TODO: Due to the accuracy requirements of the API "/regions/check/xxx", // region stats needs to be collected in API mode. @@ -1105,10 +1102,7 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio if saveKV { ctx.TaskRunner.RunTask( ctx.Context, - ratelimit.TaskOpts{ - TaskName: "SaveRegionToKV", - Limit: ctx.Limiter, - }, + core.ExtraTaskOpts(ctx, core.SaveRegionToKV), func(_ context.Context) { // If there are concurrent heartbeats from the same region, the last write will win even if // writes to storage in the critical area. So don't use mutex to protect it. From fa7841b212b42508467dd9d256fb8f55b745d2c5 Mon Sep 17 00:00:00 2001 From: Wallace Wu Date: Wed, 24 Apr 2024 15:42:11 +0800 Subject: [PATCH 38/50] Fix typo in keyspace config (#8109) ref tikv/pd#4399 Co-authored-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> Co-authored-by: ShuNing --- conf/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/config.toml b/conf/config.toml index 8b80a5044f1f..20f664a4c855 100644 --- a/conf/config.toml +++ b/conf/config.toml @@ -201,7 +201,7 @@ ## When enabled, usage data will be sent to PingCAP for improving user experience. # enable-telemetry = false -[keyspaces] +[keyspace] ## pre-alloc is used to pre-allocate keyspaces during pd bootstrap. ## Its value should be a list of strings, denotting the name of the keyspaces. ## Example: From b4f292d746b2bde1c6a7a1c7b5708a11c9a81536 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Wed, 24 Apr 2024 16:18:11 +0800 Subject: [PATCH 39/50] client/tso: organize the methods of TSO dispatcher (#8121) ref tikv/pd#8047 Organize the methods of the TSO dispatcher. Signed-off-by: JmPotato Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/tso_client.go | 334 +++++++++++++++++++++++++++++++++++++- client/tso_dispatcher.go | 343 ++------------------------------------- 2 files changed, 341 insertions(+), 336 deletions(-) diff --git a/client/tso_client.go b/client/tso_client.go index 8185b99d1d0d..347d1f6ec0ad 100644 --- a/client/tso_client.go +++ b/client/tso_client.go @@ -18,14 +18,22 @@ import ( "context" "fmt" "math/rand" + "runtime/trace" "sync" "time" + "github.com/opentracing/opentracing-go" + "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/tikv/pd/client/errs" + "github.com/tikv/pd/client/grpcutil" + "github.com/tikv/pd/client/tsoutil" "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/grpc/codes" healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" ) // TSOClient is the client used to get timestamps. @@ -127,18 +135,36 @@ func (c *tsoClient) Close() { c.wg.Wait() log.Info("close tso client") - c.tsoDispatcher.Range(func(_, dispatcherInterface any) bool { - if dispatcherInterface != nil { - dispatcher := dispatcherInterface.(*tsoDispatcher) - dispatcher.dispatcherCancel() - dispatcher.tsoBatchController.clear() - } - return true - }) - + c.closeTSODispatcher() log.Info("tso client is closed") } +func (c *tsoClient) scheduleCheckTSDeadline() { + select { + case c.checkTSDeadlineCh <- struct{}{}: + default: + } +} + +func (c *tsoClient) scheduleCheckTSODispatcher() { + select { + case c.checkTSODispatcherCh <- struct{}{}: + default: + } +} + +func (c *tsoClient) scheduleUpdateTSOConnectionCtxs() { + select { + case c.updateTSOConnectionCtxsCh <- struct{}{}: + default: + } +} + +// TSO Follower Proxy only supports the Global TSO proxy now. +func (c *tsoClient) allowTSOFollowerProxy(dc string) bool { + return dc == globalDCLocation && c.option.getEnableTSOFollowerProxy() +} + func (c *tsoClient) getTSORequest(ctx context.Context, dcLocation string) *tsoRequest { req := c.tsoReqPool.Get().(*tsoRequest) // Set needed fields in the request before using it. @@ -279,3 +305,293 @@ func (c *tsoClient) backupClientConn() (*grpc.ClientConn, string) { } return nil, "" } + +type tsoConnectionContext struct { + streamURL string + // Current stream to send gRPC requests, pdpb.PD_TsoClient for a leader/follower in the PD cluster, + // or tsopb.TSO_TsoClient for a primary/secondary in the TSO cluster + stream tsoStream + ctx context.Context + cancel context.CancelFunc +} + +func (c *tsoClient) updateTSOConnectionCtxs(updaterCtx context.Context, dc string, connectionCtxs *sync.Map) bool { + // Normal connection creating, it will be affected by the `enableForwarding`. + createTSOConnection := c.tryConnectToTSO + if c.allowTSOFollowerProxy(dc) { + createTSOConnection = c.tryConnectToTSOWithProxy + } + if err := createTSOConnection(updaterCtx, dc, connectionCtxs); err != nil { + log.Error("[tso] update connection contexts failed", zap.String("dc", dc), errs.ZapError(err)) + return false + } + return true +} + +// tryConnectToTSO will try to connect to the TSO allocator leader. If the connection becomes unreachable +// and enableForwarding is true, it will create a new connection to a follower to do the forwarding, +// while a new daemon will be created also to switch back to a normal leader connection ASAP the +// connection comes back to normal. +func (c *tsoClient) tryConnectToTSO( + dispatcherCtx context.Context, + dc string, + connectionCtxs *sync.Map, +) error { + var ( + networkErrNum uint64 + err error + stream tsoStream + url string + cc *grpc.ClientConn + ) + updateAndClear := func(newURL string, connectionCtx *tsoConnectionContext) { + if cc, loaded := connectionCtxs.LoadOrStore(newURL, connectionCtx); loaded { + // If the previous connection still exists, we should close it first. + cc.(*tsoConnectionContext).cancel() + connectionCtxs.Store(newURL, connectionCtx) + } + connectionCtxs.Range(func(url, cc any) bool { + if url.(string) != newURL { + cc.(*tsoConnectionContext).cancel() + connectionCtxs.Delete(url) + } + return true + }) + } + // retry several times before falling back to the follower when the network problem happens + + ticker := time.NewTicker(retryInterval) + defer ticker.Stop() + for i := 0; i < maxRetryTimes; i++ { + c.svcDiscovery.ScheduleCheckMemberChanged() + cc, url = c.GetTSOAllocatorClientConnByDCLocation(dc) + if cc != nil { + cctx, cancel := context.WithCancel(dispatcherCtx) + stream, err = c.tsoStreamBuilderFactory.makeBuilder(cc).build(cctx, cancel, c.option.timeout) + failpoint.Inject("unreachableNetwork", func() { + stream = nil + err = status.New(codes.Unavailable, "unavailable").Err() + }) + if stream != nil && err == nil { + updateAndClear(url, &tsoConnectionContext{url, stream, cctx, cancel}) + return nil + } + + if err != nil && c.option.enableForwarding { + // The reason we need to judge if the error code is equal to "Canceled" here is that + // when we create a stream we use a goroutine to manually control the timeout of the connection. + // There is no need to wait for the transport layer timeout which can reduce the time of unavailability. + // But it conflicts with the retry mechanism since we use the error code to decide if it is caused by network error. + // And actually the `Canceled` error can be regarded as a kind of network error in some way. + if rpcErr, ok := status.FromError(err); ok && (isNetworkError(rpcErr.Code()) || rpcErr.Code() == codes.Canceled) { + networkErrNum++ + } + } + cancel() + } else { + networkErrNum++ + } + select { + case <-dispatcherCtx.Done(): + return err + case <-ticker.C: + } + } + + if networkErrNum == maxRetryTimes { + // encounter the network error + backupClientConn, backupURL := c.backupClientConn() + if backupClientConn != nil { + log.Info("[tso] fall back to use follower to forward tso stream", zap.String("dc", dc), zap.String("follower-url", backupURL)) + forwardedHost, ok := c.GetTSOAllocatorServingURLByDCLocation(dc) + if !ok { + return errors.Errorf("cannot find the allocator leader in %s", dc) + } + + // create the follower stream + cctx, cancel := context.WithCancel(dispatcherCtx) + cctx = grpcutil.BuildForwardContext(cctx, forwardedHost) + stream, err = c.tsoStreamBuilderFactory.makeBuilder(backupClientConn).build(cctx, cancel, c.option.timeout) + if err == nil { + forwardedHostTrim := trimHTTPPrefix(forwardedHost) + addr := trimHTTPPrefix(backupURL) + // the goroutine is used to check the network and change back to the original stream + go c.checkAllocator(dispatcherCtx, cancel, dc, forwardedHostTrim, addr, url, updateAndClear) + requestForwarded.WithLabelValues(forwardedHostTrim, addr).Set(1) + updateAndClear(backupURL, &tsoConnectionContext{backupURL, stream, cctx, cancel}) + return nil + } + cancel() + } + } + return err +} + +// tryConnectToTSOWithProxy will create multiple streams to all the service endpoints to work as +// a TSO proxy to reduce the pressure of the main serving service endpoint. +func (c *tsoClient) tryConnectToTSOWithProxy(dispatcherCtx context.Context, dc string, connectionCtxs *sync.Map) error { + tsoStreamBuilders := c.getAllTSOStreamBuilders() + leaderAddr := c.svcDiscovery.GetServingURL() + forwardedHost, ok := c.GetTSOAllocatorServingURLByDCLocation(dc) + if !ok { + return errors.Errorf("cannot find the allocator leader in %s", dc) + } + // GC the stale one. + connectionCtxs.Range(func(addr, cc any) bool { + addrStr := addr.(string) + if _, ok := tsoStreamBuilders[addrStr]; !ok { + log.Info("[tso] remove the stale tso stream", + zap.String("dc", dc), + zap.String("addr", addrStr)) + cc.(*tsoConnectionContext).cancel() + connectionCtxs.Delete(addr) + } + return true + }) + // Update the missing one. + for addr, tsoStreamBuilder := range tsoStreamBuilders { + if _, ok = connectionCtxs.Load(addr); ok { + continue + } + log.Info("[tso] try to create tso stream", + zap.String("dc", dc), zap.String("addr", addr)) + cctx, cancel := context.WithCancel(dispatcherCtx) + // Do not proxy the leader client. + if addr != leaderAddr { + log.Info("[tso] use follower to forward tso stream to do the proxy", + zap.String("dc", dc), zap.String("addr", addr)) + cctx = grpcutil.BuildForwardContext(cctx, forwardedHost) + } + // Create the TSO stream. + stream, err := tsoStreamBuilder.build(cctx, cancel, c.option.timeout) + if err == nil { + if addr != leaderAddr { + forwardedHostTrim := trimHTTPPrefix(forwardedHost) + addrTrim := trimHTTPPrefix(addr) + requestForwarded.WithLabelValues(forwardedHostTrim, addrTrim).Set(1) + } + connectionCtxs.Store(addr, &tsoConnectionContext{addr, stream, cctx, cancel}) + continue + } + log.Error("[tso] create the tso stream failed", + zap.String("dc", dc), zap.String("addr", addr), errs.ZapError(err)) + cancel() + } + return nil +} + +// getAllTSOStreamBuilders returns a TSO stream builder for every service endpoint of TSO leader/followers +// or of keyspace group primary/secondaries. +func (c *tsoClient) getAllTSOStreamBuilders() map[string]tsoStreamBuilder { + var ( + addrs = c.svcDiscovery.GetServiceURLs() + streamBuilders = make(map[string]tsoStreamBuilder, len(addrs)) + cc *grpc.ClientConn + err error + ) + for _, addr := range addrs { + if len(addrs) == 0 { + continue + } + if cc, err = c.svcDiscovery.GetOrCreateGRPCConn(addr); err != nil { + continue + } + healthCtx, healthCancel := context.WithTimeout(c.ctx, c.option.timeout) + resp, err := healthpb.NewHealthClient(cc).Check(healthCtx, &healthpb.HealthCheckRequest{Service: ""}) + healthCancel() + if err == nil && resp.GetStatus() == healthpb.HealthCheckResponse_SERVING { + streamBuilders[addr] = c.tsoStreamBuilderFactory.makeBuilder(cc) + } + } + return streamBuilders +} + +type tsoInfo struct { + tsoServer string + reqKeyspaceGroupID uint32 + respKeyspaceGroupID uint32 + respReceivedAt time.Time + physical int64 + logical int64 +} + +func (c *tsoClient) processRequests( + stream tsoStream, dcLocation string, tbc *tsoBatchController, +) error { + requests := tbc.getCollectedRequests() + // nolint + for _, req := range requests { + defer trace.StartRegion(req.requestCtx, "pdclient.tsoReqSend").End() + if span := opentracing.SpanFromContext(req.requestCtx); span != nil && span.Tracer() != nil { + span = span.Tracer().StartSpan("pdclient.processRequests", opentracing.ChildOf(span.Context())) + defer span.Finish() + } + } + + count := int64(len(requests)) + reqKeyspaceGroupID := c.svcDiscovery.GetKeyspaceGroupID() + respKeyspaceGroupID, physical, logical, suffixBits, err := stream.processRequests( + c.svcDiscovery.GetClusterID(), c.svcDiscovery.GetKeyspaceID(), reqKeyspaceGroupID, + dcLocation, count, tbc.batchStartTime) + if err != nil { + tbc.finishCollectedRequests(0, 0, 0, err) + return err + } + // `logical` is the largest ts's logical part here, we need to do the subtracting before we finish each TSO request. + firstLogical := tsoutil.AddLogical(logical, -count+1, suffixBits) + curTSOInfo := &tsoInfo{ + tsoServer: stream.getServerURL(), + reqKeyspaceGroupID: reqKeyspaceGroupID, + respKeyspaceGroupID: respKeyspaceGroupID, + respReceivedAt: time.Now(), + physical: physical, + logical: tsoutil.AddLogical(firstLogical, count-1, suffixBits), + } + c.compareAndSwapTS(dcLocation, curTSOInfo, physical, firstLogical) + tbc.finishCollectedRequests(physical, firstLogical, suffixBits, nil) + return nil +} + +func (c *tsoClient) compareAndSwapTS( + dcLocation string, + curTSOInfo *tsoInfo, + physical, firstLogical int64, +) { + val, loaded := c.lastTSOInfoMap.LoadOrStore(dcLocation, curTSOInfo) + if !loaded { + return + } + lastTSOInfo := val.(*tsoInfo) + if lastTSOInfo.respKeyspaceGroupID != curTSOInfo.respKeyspaceGroupID { + log.Info("[tso] keyspace group changed", + zap.String("dc-location", dcLocation), + zap.Uint32("old-group-id", lastTSOInfo.respKeyspaceGroupID), + zap.Uint32("new-group-id", curTSOInfo.respKeyspaceGroupID)) + } + + // The TSO we get is a range like [largestLogical-count+1, largestLogical], so we save the last TSO's largest logical + // to compare with the new TSO's first logical. For example, if we have a TSO resp with logical 10, count 5, then + // all TSOs we get will be [6, 7, 8, 9, 10]. lastTSOInfo.logical stores the logical part of the largest ts returned + // last time. + if tsoutil.TSLessEqual(physical, firstLogical, lastTSOInfo.physical, lastTSOInfo.logical) { + log.Panic("[tso] timestamp fallback", + zap.String("dc-location", dcLocation), + zap.Uint32("keyspace", c.svcDiscovery.GetKeyspaceID()), + zap.String("last-ts", fmt.Sprintf("(%d, %d)", lastTSOInfo.physical, lastTSOInfo.logical)), + zap.String("cur-ts", fmt.Sprintf("(%d, %d)", physical, firstLogical)), + zap.String("last-tso-server", lastTSOInfo.tsoServer), + zap.String("cur-tso-server", curTSOInfo.tsoServer), + zap.Uint32("last-keyspace-group-in-request", lastTSOInfo.reqKeyspaceGroupID), + zap.Uint32("cur-keyspace-group-in-request", curTSOInfo.reqKeyspaceGroupID), + zap.Uint32("last-keyspace-group-in-response", lastTSOInfo.respKeyspaceGroupID), + zap.Uint32("cur-keyspace-group-in-response", curTSOInfo.respKeyspaceGroupID), + zap.Time("last-response-received-at", lastTSOInfo.respReceivedAt), + zap.Time("cur-response-received-at", curTSOInfo.respReceivedAt)) + } + lastTSOInfo.tsoServer = curTSOInfo.tsoServer + lastTSOInfo.reqKeyspaceGroupID = curTSOInfo.reqKeyspaceGroupID + lastTSOInfo.respKeyspaceGroupID = curTSOInfo.respKeyspaceGroupID + lastTSOInfo.respReceivedAt = curTSOInfo.respReceivedAt + lastTSOInfo.physical = curTSOInfo.physical + lastTSOInfo.logical = curTSOInfo.logical +} diff --git a/client/tso_dispatcher.go b/client/tso_dispatcher.go index d02fdd52af8d..7528293a7338 100644 --- a/client/tso_dispatcher.go +++ b/client/tso_dispatcher.go @@ -22,36 +22,16 @@ import ( "sync" "time" - "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/tikv/pd/client/errs" - "github.com/tikv/pd/client/grpcutil" "github.com/tikv/pd/client/retry" "github.com/tikv/pd/client/timerpool" - "github.com/tikv/pd/client/tsoutil" "go.uber.org/zap" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" healthpb "google.golang.org/grpc/health/grpc_health_v1" - "google.golang.org/grpc/status" ) -type tsoDispatcher struct { - dispatcherCancel context.CancelFunc - tsoBatchController *tsoBatchController -} - -type tsoInfo struct { - tsoServer string - reqKeyspaceGroupID uint32 - respKeyspaceGroupID uint32 - respReceivedAt time.Time - physical int64 - logical int64 -} - const ( tsLoopDCCheckInterval = time.Minute defaultMaxTSOBatchSize = 10000 // should be higher if client is sending requests in burst @@ -59,18 +39,9 @@ const ( maxRetryTimes = 6 ) -func (c *tsoClient) scheduleCheckTSODispatcher() { - select { - case c.checkTSODispatcherCh <- struct{}{}: - default: - } -} - -func (c *tsoClient) scheduleUpdateTSOConnectionCtxs() { - select { - case c.updateTSOConnectionCtxsCh <- struct{}{}: - default: - } +type tsoDispatcher struct { + dispatcherCancel context.CancelFunc + tsoBatchController *tsoBatchController } func (c *tsoClient) dispatchRequest(request *tsoRequest) (bool, error) { @@ -115,6 +86,17 @@ func (c *tsoClient) dispatchRequest(request *tsoRequest) (bool, error) { return false, nil } +func (c *tsoClient) closeTSODispatcher() { + c.tsoDispatcher.Range(func(_, dispatcherInterface any) bool { + if dispatcherInterface != nil { + dispatcher := dispatcherInterface.(*tsoDispatcher) + dispatcher.dispatcherCancel() + dispatcher.tsoBatchController.clear() + } + return true + }) +} + func (c *tsoClient) updateTSODispatcher() { // Set up the new TSO dispatcher and batch controller. c.GetTSOAllocators().Range(func(dcLocationKey, _ any) bool { @@ -212,13 +194,6 @@ func (c *tsoClient) watchTSDeadline(ctx context.Context, dcLocation string) { } } -func (c *tsoClient) scheduleCheckTSDeadline() { - select { - case c.checkTSDeadlineCh <- struct{}{}: - default: - } -} - func (c *tsoClient) tsoDispatcherCheckLoop() { defer c.wg.Done() @@ -445,7 +420,7 @@ tsoBatchLoop: // Choose a stream to send the TSO gRPC request. streamChoosingLoop: for { - connectionCtx := c.chooseStream(&connectionCtxs) + connectionCtx := chooseStream(&connectionCtxs) if connectionCtx != nil { streamURL, stream, streamCtx, cancel = connectionCtx.streamURL, connectionCtx.stream, connectionCtx.ctx, connectionCtx.cancel } @@ -541,14 +516,9 @@ tsoBatchLoop: } } -// TSO Follower Proxy only supports the Global TSO proxy now. -func (c *tsoClient) allowTSOFollowerProxy(dc string) bool { - return dc == globalDCLocation && c.option.getEnableTSOFollowerProxy() -} - // chooseStream uses the reservoir sampling algorithm to randomly choose a connection. // connectionCtxs will only have only one stream to choose when the TSO Follower Proxy is off. -func (*tsoClient) chooseStream(connectionCtxs *sync.Map) (connectionCtx *tsoConnectionContext) { +func chooseStream(connectionCtxs *sync.Map) (connectionCtx *tsoConnectionContext) { idx := 0 connectionCtxs.Range(func(_, cc any) bool { j := rand.Intn(idx + 1) @@ -560,284 +530,3 @@ func (*tsoClient) chooseStream(connectionCtxs *sync.Map) (connectionCtx *tsoConn }) return connectionCtx } - -type tsoConnectionContext struct { - streamURL string - // Current stream to send gRPC requests, pdpb.PD_TsoClient for a leader/follower in the PD cluster, - // or tsopb.TSO_TsoClient for a primary/secondary in the TSO cluster - stream tsoStream - ctx context.Context - cancel context.CancelFunc -} - -func (c *tsoClient) updateTSOConnectionCtxs(updaterCtx context.Context, dc string, connectionCtxs *sync.Map) bool { - // Normal connection creating, it will be affected by the `enableForwarding`. - createTSOConnection := c.tryConnectToTSO - if c.allowTSOFollowerProxy(dc) { - createTSOConnection = c.tryConnectToTSOWithProxy - } - if err := createTSOConnection(updaterCtx, dc, connectionCtxs); err != nil { - log.Error("[tso] update connection contexts failed", zap.String("dc", dc), errs.ZapError(err)) - return false - } - return true -} - -// tryConnectToTSO will try to connect to the TSO allocator leader. If the connection becomes unreachable -// and enableForwarding is true, it will create a new connection to a follower to do the forwarding, -// while a new daemon will be created also to switch back to a normal leader connection ASAP the -// connection comes back to normal. -func (c *tsoClient) tryConnectToTSO( - dispatcherCtx context.Context, - dc string, - connectionCtxs *sync.Map, -) error { - var ( - networkErrNum uint64 - err error - stream tsoStream - url string - cc *grpc.ClientConn - ) - updateAndClear := func(newURL string, connectionCtx *tsoConnectionContext) { - if cc, loaded := connectionCtxs.LoadOrStore(newURL, connectionCtx); loaded { - // If the previous connection still exists, we should close it first. - cc.(*tsoConnectionContext).cancel() - connectionCtxs.Store(newURL, connectionCtx) - } - connectionCtxs.Range(func(url, cc any) bool { - if url.(string) != newURL { - cc.(*tsoConnectionContext).cancel() - connectionCtxs.Delete(url) - } - return true - }) - } - // retry several times before falling back to the follower when the network problem happens - - ticker := time.NewTicker(retryInterval) - defer ticker.Stop() - for i := 0; i < maxRetryTimes; i++ { - c.svcDiscovery.ScheduleCheckMemberChanged() - cc, url = c.GetTSOAllocatorClientConnByDCLocation(dc) - if cc != nil { - cctx, cancel := context.WithCancel(dispatcherCtx) - stream, err = c.tsoStreamBuilderFactory.makeBuilder(cc).build(cctx, cancel, c.option.timeout) - failpoint.Inject("unreachableNetwork", func() { - stream = nil - err = status.New(codes.Unavailable, "unavailable").Err() - }) - if stream != nil && err == nil { - updateAndClear(url, &tsoConnectionContext{url, stream, cctx, cancel}) - return nil - } - - if err != nil && c.option.enableForwarding { - // The reason we need to judge if the error code is equal to "Canceled" here is that - // when we create a stream we use a goroutine to manually control the timeout of the connection. - // There is no need to wait for the transport layer timeout which can reduce the time of unavailability. - // But it conflicts with the retry mechanism since we use the error code to decide if it is caused by network error. - // And actually the `Canceled` error can be regarded as a kind of network error in some way. - if rpcErr, ok := status.FromError(err); ok && (isNetworkError(rpcErr.Code()) || rpcErr.Code() == codes.Canceled) { - networkErrNum++ - } - } - cancel() - } else { - networkErrNum++ - } - select { - case <-dispatcherCtx.Done(): - return err - case <-ticker.C: - } - } - - if networkErrNum == maxRetryTimes { - // encounter the network error - backupClientConn, backupURL := c.backupClientConn() - if backupClientConn != nil { - log.Info("[tso] fall back to use follower to forward tso stream", zap.String("dc", dc), zap.String("follower-url", backupURL)) - forwardedHost, ok := c.GetTSOAllocatorServingURLByDCLocation(dc) - if !ok { - return errors.Errorf("cannot find the allocator leader in %s", dc) - } - - // create the follower stream - cctx, cancel := context.WithCancel(dispatcherCtx) - cctx = grpcutil.BuildForwardContext(cctx, forwardedHost) - stream, err = c.tsoStreamBuilderFactory.makeBuilder(backupClientConn).build(cctx, cancel, c.option.timeout) - if err == nil { - forwardedHostTrim := trimHTTPPrefix(forwardedHost) - addr := trimHTTPPrefix(backupURL) - // the goroutine is used to check the network and change back to the original stream - go c.checkAllocator(dispatcherCtx, cancel, dc, forwardedHostTrim, addr, url, updateAndClear) - requestForwarded.WithLabelValues(forwardedHostTrim, addr).Set(1) - updateAndClear(backupURL, &tsoConnectionContext{backupURL, stream, cctx, cancel}) - return nil - } - cancel() - } - } - return err -} - -// getAllTSOStreamBuilders returns a TSO stream builder for every service endpoint of TSO leader/followers -// or of keyspace group primary/secondaries. -func (c *tsoClient) getAllTSOStreamBuilders() map[string]tsoStreamBuilder { - var ( - addrs = c.svcDiscovery.GetServiceURLs() - streamBuilders = make(map[string]tsoStreamBuilder, len(addrs)) - cc *grpc.ClientConn - err error - ) - for _, addr := range addrs { - if len(addrs) == 0 { - continue - } - if cc, err = c.svcDiscovery.GetOrCreateGRPCConn(addr); err != nil { - continue - } - healthCtx, healthCancel := context.WithTimeout(c.ctx, c.option.timeout) - resp, err := healthpb.NewHealthClient(cc).Check(healthCtx, &healthpb.HealthCheckRequest{Service: ""}) - healthCancel() - if err == nil && resp.GetStatus() == healthpb.HealthCheckResponse_SERVING { - streamBuilders[addr] = c.tsoStreamBuilderFactory.makeBuilder(cc) - } - } - return streamBuilders -} - -// tryConnectToTSOWithProxy will create multiple streams to all the service endpoints to work as -// a TSO proxy to reduce the pressure of the main serving service endpoint. -func (c *tsoClient) tryConnectToTSOWithProxy(dispatcherCtx context.Context, dc string, connectionCtxs *sync.Map) error { - tsoStreamBuilders := c.getAllTSOStreamBuilders() - leaderAddr := c.svcDiscovery.GetServingURL() - forwardedHost, ok := c.GetTSOAllocatorServingURLByDCLocation(dc) - if !ok { - return errors.Errorf("cannot find the allocator leader in %s", dc) - } - // GC the stale one. - connectionCtxs.Range(func(addr, cc any) bool { - addrStr := addr.(string) - if _, ok := tsoStreamBuilders[addrStr]; !ok { - log.Info("[tso] remove the stale tso stream", - zap.String("dc", dc), - zap.String("addr", addrStr)) - cc.(*tsoConnectionContext).cancel() - connectionCtxs.Delete(addr) - } - return true - }) - // Update the missing one. - for addr, tsoStreamBuilder := range tsoStreamBuilders { - if _, ok = connectionCtxs.Load(addr); ok { - continue - } - log.Info("[tso] try to create tso stream", - zap.String("dc", dc), zap.String("addr", addr)) - cctx, cancel := context.WithCancel(dispatcherCtx) - // Do not proxy the leader client. - if addr != leaderAddr { - log.Info("[tso] use follower to forward tso stream to do the proxy", - zap.String("dc", dc), zap.String("addr", addr)) - cctx = grpcutil.BuildForwardContext(cctx, forwardedHost) - } - // Create the TSO stream. - stream, err := tsoStreamBuilder.build(cctx, cancel, c.option.timeout) - if err == nil { - if addr != leaderAddr { - forwardedHostTrim := trimHTTPPrefix(forwardedHost) - addrTrim := trimHTTPPrefix(addr) - requestForwarded.WithLabelValues(forwardedHostTrim, addrTrim).Set(1) - } - connectionCtxs.Store(addr, &tsoConnectionContext{addr, stream, cctx, cancel}) - continue - } - log.Error("[tso] create the tso stream failed", - zap.String("dc", dc), zap.String("addr", addr), errs.ZapError(err)) - cancel() - } - return nil -} - -func (c *tsoClient) processRequests( - stream tsoStream, dcLocation string, tbc *tsoBatchController, -) error { - requests := tbc.getCollectedRequests() - // nolint - for _, req := range requests { - defer trace.StartRegion(req.requestCtx, "pdclient.tsoReqSend").End() - if span := opentracing.SpanFromContext(req.requestCtx); span != nil && span.Tracer() != nil { - span = span.Tracer().StartSpan("pdclient.processRequests", opentracing.ChildOf(span.Context())) - defer span.Finish() - } - } - - count := int64(len(requests)) - reqKeyspaceGroupID := c.svcDiscovery.GetKeyspaceGroupID() - respKeyspaceGroupID, physical, logical, suffixBits, err := stream.processRequests( - c.svcDiscovery.GetClusterID(), c.svcDiscovery.GetKeyspaceID(), reqKeyspaceGroupID, - dcLocation, count, tbc.batchStartTime) - if err != nil { - tbc.finishCollectedRequests(0, 0, 0, err) - return err - } - // `logical` is the largest ts's logical part here, we need to do the subtracting before we finish each TSO request. - firstLogical := tsoutil.AddLogical(logical, -count+1, suffixBits) - curTSOInfo := &tsoInfo{ - tsoServer: stream.getServerURL(), - reqKeyspaceGroupID: reqKeyspaceGroupID, - respKeyspaceGroupID: respKeyspaceGroupID, - respReceivedAt: time.Now(), - physical: physical, - logical: tsoutil.AddLogical(firstLogical, count-1, suffixBits), - } - c.compareAndSwapTS(dcLocation, curTSOInfo, physical, firstLogical) - tbc.finishCollectedRequests(physical, firstLogical, suffixBits, nil) - return nil -} - -func (c *tsoClient) compareAndSwapTS( - dcLocation string, - curTSOInfo *tsoInfo, - physical, firstLogical int64, -) { - val, loaded := c.lastTSOInfoMap.LoadOrStore(dcLocation, curTSOInfo) - if !loaded { - return - } - lastTSOInfo := val.(*tsoInfo) - if lastTSOInfo.respKeyspaceGroupID != curTSOInfo.respKeyspaceGroupID { - log.Info("[tso] keyspace group changed", - zap.String("dc-location", dcLocation), - zap.Uint32("old-group-id", lastTSOInfo.respKeyspaceGroupID), - zap.Uint32("new-group-id", curTSOInfo.respKeyspaceGroupID)) - } - - // The TSO we get is a range like [largestLogical-count+1, largestLogical], so we save the last TSO's largest logical - // to compare with the new TSO's first logical. For example, if we have a TSO resp with logical 10, count 5, then - // all TSOs we get will be [6, 7, 8, 9, 10]. lastTSOInfo.logical stores the logical part of the largest ts returned - // last time. - if tsoutil.TSLessEqual(physical, firstLogical, lastTSOInfo.physical, lastTSOInfo.logical) { - log.Panic("[tso] timestamp fallback", - zap.String("dc-location", dcLocation), - zap.Uint32("keyspace", c.svcDiscovery.GetKeyspaceID()), - zap.String("last-ts", fmt.Sprintf("(%d, %d)", lastTSOInfo.physical, lastTSOInfo.logical)), - zap.String("cur-ts", fmt.Sprintf("(%d, %d)", physical, firstLogical)), - zap.String("last-tso-server", lastTSOInfo.tsoServer), - zap.String("cur-tso-server", curTSOInfo.tsoServer), - zap.Uint32("last-keyspace-group-in-request", lastTSOInfo.reqKeyspaceGroupID), - zap.Uint32("cur-keyspace-group-in-request", curTSOInfo.reqKeyspaceGroupID), - zap.Uint32("last-keyspace-group-in-response", lastTSOInfo.respKeyspaceGroupID), - zap.Uint32("cur-keyspace-group-in-response", curTSOInfo.respKeyspaceGroupID), - zap.Time("last-response-received-at", lastTSOInfo.respReceivedAt), - zap.Time("cur-response-received-at", curTSOInfo.respReceivedAt)) - } - lastTSOInfo.tsoServer = curTSOInfo.tsoServer - lastTSOInfo.reqKeyspaceGroupID = curTSOInfo.reqKeyspaceGroupID - lastTSOInfo.respKeyspaceGroupID = curTSOInfo.respKeyspaceGroupID - lastTSOInfo.respReceivedAt = curTSOInfo.respReceivedAt - lastTSOInfo.physical = curTSOInfo.physical - lastTSOInfo.logical = curTSOInfo.logical -} From de5bad370d6f07d97b7641fce0fe93ffe203e4fc Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 25 Apr 2024 11:43:41 +0800 Subject: [PATCH 40/50] *: fix log level (#8118) close tikv/pd#8117 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- cmd/pd-server/main.go | 8 ++++---- pkg/mcs/resourcemanager/server/config.go | 6 ++---- pkg/mcs/scheduling/server/config/config.go | 6 ++---- pkg/mcs/tso/server/config.go | 6 ++---- pkg/mcs/utils/constant.go | 2 ++ server/config/config.go | 7 +++---- server/config/config_test.go | 16 ++++++++++++++++ 7 files changed, 31 insertions(+), 20 deletions(-) diff --git a/cmd/pd-server/main.go b/cmd/pd-server/main.go index 0bf28ccfdea5..bd75309ed8ad 100644 --- a/cmd/pd-server/main.go +++ b/cmd/pd-server/main.go @@ -102,7 +102,7 @@ func NewTSOServiceCommand() *cobra.Command { cmd.Flags().StringP("cacert", "", "", "path of file that contains list of trusted TLS CAs") cmd.Flags().StringP("cert", "", "", "path of file that contains X509 certificate in PEM format") cmd.Flags().StringP("key", "", "", "path of file that contains X509 key in PEM format") - cmd.Flags().StringP("log-level", "L", "info", "log level: debug, info, warn, error, fatal (default 'info')") + cmd.Flags().StringP("log-level", "L", "", "log level: debug, info, warn, error, fatal (default 'info')") cmd.Flags().StringP("log-file", "", "", "log file path") return cmd } @@ -122,7 +122,7 @@ func NewSchedulingServiceCommand() *cobra.Command { cmd.Flags().StringP("cacert", "", "", "path of file that contains list of trusted TLS CAs") cmd.Flags().StringP("cert", "", "", "path of file that contains X509 certificate in PEM format") cmd.Flags().StringP("key", "", "", "path of file that contains X509 key in PEM format") - cmd.Flags().StringP("log-level", "L", "info", "log level: debug, info, warn, error, fatal (default 'info')") + cmd.Flags().StringP("log-level", "L", "", "log level: debug, info, warn, error, fatal (default 'info')") cmd.Flags().StringP("log-file", "", "", "log file path") return cmd } @@ -142,7 +142,7 @@ func NewResourceManagerServiceCommand() *cobra.Command { cmd.Flags().StringP("cacert", "", "", "path of file that contains list of trusted TLS CAs") cmd.Flags().StringP("cert", "", "", "path of file that contains X509 certificate in PEM format") cmd.Flags().StringP("key", "", "", "path of file that contains X509 key in PEM format") - cmd.Flags().StringP("log-level", "L", "info", "log level: debug, info, warn, error, fatal (default 'info')") + cmd.Flags().StringP("log-level", "L", "", "log level: debug, info, warn, error, fatal (default 'info')") cmd.Flags().StringP("log-file", "", "", "log file path") return cmd } @@ -171,7 +171,7 @@ func addFlags(cmd *cobra.Command) { cmd.Flags().StringP("initial-cluster", "", "", "initial cluster configuration for bootstrapping, e,g. pd=http://127.0.0.1:2380") cmd.Flags().StringP("join", "", "", "join to an existing cluster (usage: cluster's '${advertise-client-urls}'") cmd.Flags().StringP("metrics-addr", "", "", "prometheus pushgateway address, leaves it empty will disable prometheus push") - cmd.Flags().StringP("log-level", "L", "info", "log level: debug, info, warn, error, fatal (default 'info')") + cmd.Flags().StringP("log-level", "L", "", "log level: debug, info, warn, error, fatal (default 'info')") cmd.Flags().StringP("log-file", "", "", "log file path") cmd.Flags().StringP("cacert", "", "", "path of file that contains list of trusted TLS CAs") cmd.Flags().StringP("cert", "", "", "path of file that contains X509 certificate in PEM format") diff --git a/pkg/mcs/resourcemanager/server/config.go b/pkg/mcs/resourcemanager/server/config.go index dfb2babe676e..70862ffb89c3 100644 --- a/pkg/mcs/resourcemanager/server/config.go +++ b/pkg/mcs/resourcemanager/server/config.go @@ -239,10 +239,6 @@ func (c *Config) Adjust(meta *toml.MetaData) error { c.adjustLog(configMetaData.Child("log")) c.Security.Encryption.Adjust() - if len(c.Log.Format) == 0 { - c.Log.Format = utils.DefaultLogFormat - } - c.Controller.Adjust(configMetaData.Child("controller")) configutil.AdjustInt64(&c.LeaderLease, utils.DefaultLeaderLease) @@ -253,6 +249,8 @@ func (c *Config) adjustLog(meta *configutil.ConfigMetaData) { if !meta.IsDefined("disable-error-verbose") { c.Log.DisableErrorVerbose = utils.DefaultDisableErrorVerbose } + configutil.AdjustString(&c.Log.Format, utils.DefaultLogFormat) + configutil.AdjustString(&c.Log.Level, utils.DefaultLogLevel) } // GetName returns the Name diff --git a/pkg/mcs/scheduling/server/config/config.go b/pkg/mcs/scheduling/server/config/config.go index 148a7015d116..091771bc38c3 100644 --- a/pkg/mcs/scheduling/server/config/config.go +++ b/pkg/mcs/scheduling/server/config/config.go @@ -148,10 +148,6 @@ func (c *Config) adjust(meta *toml.MetaData) error { c.adjustLog(configMetaData.Child("log")) c.Security.Encryption.Adjust() - if len(c.Log.Format) == 0 { - c.Log.Format = utils.DefaultLogFormat - } - configutil.AdjustInt64(&c.LeaderLease, utils.DefaultLeaderLease) if err := c.Schedule.Adjust(configMetaData.Child("schedule"), false); err != nil { @@ -164,6 +160,8 @@ func (c *Config) adjustLog(meta *configutil.ConfigMetaData) { if !meta.IsDefined("disable-error-verbose") { c.Log.DisableErrorVerbose = utils.DefaultDisableErrorVerbose } + configutil.AdjustString(&c.Log.Format, utils.DefaultLogFormat) + configutil.AdjustString(&c.Log.Level, utils.DefaultLogLevel) } // GetName returns the Name diff --git a/pkg/mcs/tso/server/config.go b/pkg/mcs/tso/server/config.go index c117dd72e384..06e9054e1174 100644 --- a/pkg/mcs/tso/server/config.go +++ b/pkg/mcs/tso/server/config.go @@ -226,10 +226,6 @@ func (c *Config) Adjust(meta *toml.MetaData) error { c.adjustLog(configMetaData.Child("log")) c.Security.Encryption.Adjust() - if len(c.Log.Format) == 0 { - c.Log.Format = utils.DefaultLogFormat - } - return nil } @@ -237,6 +233,8 @@ func (c *Config) adjustLog(meta *configutil.ConfigMetaData) { if !meta.IsDefined("disable-error-verbose") { c.Log.DisableErrorVerbose = utils.DefaultDisableErrorVerbose } + configutil.AdjustString(&c.Log.Format, utils.DefaultLogFormat) + configutil.AdjustString(&c.Log.Level, utils.DefaultLogLevel) } // Validate is used to validate if some configurations are right. diff --git a/pkg/mcs/utils/constant.go b/pkg/mcs/utils/constant.go index 21b57a3c5c55..c6c882f5179b 100644 --- a/pkg/mcs/utils/constant.go +++ b/pkg/mcs/utils/constant.go @@ -32,6 +32,8 @@ const ( DefaultHTTPGracefulShutdownTimeout = 5 * time.Second // DefaultLogFormat is the default log format DefaultLogFormat = "text" + // DefaultLogLevel is the default log level + DefaultLogLevel = "info" // DefaultDisableErrorVerbose is the default value of DisableErrorVerbose DefaultDisableErrorVerbose = true // DefaultLeaderLease is the default value of LeaderLease diff --git a/server/config/config.go b/server/config/config.go index 93aa3bb87d38..95d5dcb3257c 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -234,6 +234,7 @@ const ( minTSOUpdatePhysicalInterval = 1 * time.Millisecond defaultLogFormat = "text" + defaultLogLevel = "info" defaultServerMemoryLimit = 0 minServerMemoryLimit = 0 @@ -468,10 +469,6 @@ func (c *Config) Adjust(meta *toml.MetaData, reloading bool) error { c.Security.Encryption.Adjust() - if len(c.Log.Format) == 0 { - c.Log.Format = defaultLogFormat - } - c.Controller.Adjust(configMetaData.Child("controller")) return nil @@ -481,6 +478,8 @@ func (c *Config) adjustLog(meta *configutil.ConfigMetaData) { if !meta.IsDefined("disable-error-verbose") { c.Log.DisableErrorVerbose = defaultDisableErrorVerbose } + configutil.AdjustString(&c.Log.Format, defaultLogFormat) + configutil.AdjustString(&c.Log.Level, defaultLogLevel) } // Clone returns a cloned configuration. diff --git a/server/config/config_test.go b/server/config/config_test.go index 69cfafd8d361..d7abfe0746ae 100644 --- a/server/config/config_test.go +++ b/server/config/config_test.go @@ -246,6 +246,22 @@ tso-update-physical-interval = "15s" re.NoError(err) re.Equal(maxTSOUpdatePhysicalInterval, cfg.TSOUpdatePhysicalInterval.Duration) + + cfgData = ` +[log] +level = "debug" +` + flagSet = pflag.NewFlagSet("testlog", pflag.ContinueOnError) + flagSet.StringP("log-level", "L", "info", "log level: debug, info, warn, error, fatal (default 'info')") + flagSet.Parse(nil) + cfg = NewConfig() + err = cfg.Parse(flagSet) + re.NoError(err) + meta, err = toml.Decode(cfgData, &cfg) + re.NoError(err) + err = cfg.Adjust(&meta, false) + re.NoError(err) + re.Equal("debug", cfg.Log.Level) } func TestMigrateFlags(t *testing.T) { From 50c80407cd57e96a85452dd1601fcb41c1f263cf Mon Sep 17 00:00:00 2001 From: Hu# Date: Fri, 26 Apr 2024 10:12:11 +0800 Subject: [PATCH 41/50] scheduler: fix scatter counter name (#8124) close tikv/pd#8125 Signed-off-by: husharp --- pkg/schedule/filter/counter.go | 4 ++++ pkg/schedule/schedulers/balance_leader.go | 7 +++++++ pkg/schedule/schedulers/balance_region.go | 7 +++++++ pkg/schedule/schedulers/scatter_range.go | 2 ++ 4 files changed, 20 insertions(+) diff --git a/pkg/schedule/filter/counter.go b/pkg/schedule/filter/counter.go index 0120ef5b6663..29c75bbe41dc 100644 --- a/pkg/schedule/filter/counter.go +++ b/pkg/schedule/filter/counter.go @@ -188,6 +188,10 @@ func NewCounter(scope string) *Counter { return &Counter{counter: counter, scope: scope} } +func (c *Counter) SetScope(scope string) { + c.scope = scope +} + // Add adds the filter counter. func (c *Counter) inc(action action, filterType filterType, sourceID uint64, targetID uint64) { if _, ok := c.counter[action][filterType][sourceID]; !ok { diff --git a/pkg/schedule/schedulers/balance_leader.go b/pkg/schedule/schedulers/balance_leader.go index 6114d2a8f890..910ed86c7523 100644 --- a/pkg/schedule/schedulers/balance_leader.go +++ b/pkg/schedule/schedulers/balance_leader.go @@ -208,6 +208,13 @@ func (l *balanceLeaderScheduler) ServeHTTP(w http.ResponseWriter, r *http.Reques // BalanceLeaderCreateOption is used to create a scheduler with an option. type BalanceLeaderCreateOption func(s *balanceLeaderScheduler) +// WithBalanceLeaderFilterCounterName sets the filter counter name for the scheduler. +func WithBalanceLeaderFilterCounterName(name string) BalanceLeaderCreateOption { + return func(s *balanceLeaderScheduler) { + s.filterCounter.SetScope(name) + } +} + // WithBalanceLeaderName sets the name for the scheduler. func WithBalanceLeaderName(name string) BalanceLeaderCreateOption { return func(s *balanceLeaderScheduler) { diff --git a/pkg/schedule/schedulers/balance_region.go b/pkg/schedule/schedulers/balance_region.go index 608d008a99ec..bfc1a2364819 100644 --- a/pkg/schedule/schedulers/balance_region.go +++ b/pkg/schedule/schedulers/balance_region.go @@ -92,6 +92,13 @@ func WithBalanceRegionName(name string) BalanceRegionCreateOption { } } +// WithBalanceRegionFilterCounterName sets the filter counter name for the scheduler. +func WithBalanceRegionFilterCounterName(name string) BalanceRegionCreateOption { + return func(s *balanceRegionScheduler) { + s.filterCounter.SetScope(name) + } +} + func (s *balanceRegionScheduler) GetName() string { return s.conf.Name } diff --git a/pkg/schedule/schedulers/scatter_range.go b/pkg/schedule/schedulers/scatter_range.go index 8a2f0a5398bc..daa3c5cc5c13 100644 --- a/pkg/schedule/schedulers/scatter_range.go +++ b/pkg/schedule/schedulers/scatter_range.go @@ -138,11 +138,13 @@ func newScatterRangeScheduler(opController *operator.Controller, config *scatter opController, &balanceLeaderSchedulerConfig{Ranges: []core.KeyRange{core.NewKeyRange("", "")}}, WithBalanceLeaderName("scatter-range-leader"), + WithBalanceLeaderFilterCounterName("scatter-range-leader"), ), balanceRegion: newBalanceRegionScheduler( opController, &balanceRegionSchedulerConfig{Ranges: []core.KeyRange{core.NewKeyRange("", "")}}, WithBalanceRegionName("scatter-range-region"), + WithBalanceRegionFilterCounterName("scatter-range-region"), ), } return scheduler From aae410f16f8a16d873a8aad87acee3559db5e321 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Sun, 28 Apr 2024 15:44:57 +0800 Subject: [PATCH 42/50] client/tso: improve the switching of TSO stream (#8123) close tikv/pd#7997, ref tikv/pd#8047 Previously, without enabling the TSO Follower Proxy, we only passively update its stream when a TSO request fails. This means that we cannot automatically and gradually complete the TSO stream update after a service switch. This PR strengthens this logic, which can improve the success rate of TSO requests during service switching. Signed-off-by: JmPotato --- client/tso_client.go | 68 +++++---- client/tso_dispatcher.go | 181 ++++++++++++----------- tests/integrations/client/client_test.go | 35 +++++ 3 files changed, 169 insertions(+), 115 deletions(-) diff --git a/client/tso_client.go b/client/tso_client.go index 347d1f6ec0ad..e3bdb8359016 100644 --- a/client/tso_client.go +++ b/client/tso_client.go @@ -178,6 +178,14 @@ func (c *tsoClient) getTSORequest(ctx context.Context, dcLocation string) *tsoRe return req } +func (c *tsoClient) getTSODispatcher(dcLocation string) (*tsoDispatcher, bool) { + dispatcher, ok := c.tsoDispatcher.Load(dcLocation) + if !ok || dispatcher == nil { + return nil, false + } + return dispatcher.(*tsoDispatcher), true +} + // GetTSOAllocators returns {dc-location -> TSO allocator leader URL} connection map func (c *tsoClient) GetTSOAllocators() *sync.Map { return &c.tsoAllocators @@ -259,6 +267,7 @@ func (c *tsoClient) updateTSOGlobalServURL(url string) error { log.Info("[tso] switch dc tso global allocator serving url", zap.String("dc-location", globalDCLocation), zap.String("new-url", url)) + c.scheduleUpdateTSOConnectionCtxs() c.scheduleCheckTSODispatcher() return nil } @@ -333,40 +342,41 @@ func (c *tsoClient) updateTSOConnectionCtxs(updaterCtx context.Context, dc strin // while a new daemon will be created also to switch back to a normal leader connection ASAP the // connection comes back to normal. func (c *tsoClient) tryConnectToTSO( - dispatcherCtx context.Context, + ctx context.Context, dc string, connectionCtxs *sync.Map, ) error { var ( - networkErrNum uint64 - err error - stream tsoStream - url string - cc *grpc.ClientConn - ) - updateAndClear := func(newURL string, connectionCtx *tsoConnectionContext) { - if cc, loaded := connectionCtxs.LoadOrStore(newURL, connectionCtx); loaded { - // If the previous connection still exists, we should close it first. - cc.(*tsoConnectionContext).cancel() - connectionCtxs.Store(newURL, connectionCtx) + networkErrNum uint64 + err error + stream tsoStream + url string + cc *grpc.ClientConn + updateAndClear = func(newURL string, connectionCtx *tsoConnectionContext) { + // Only store the `connectionCtx` if it does not exist before. + connectionCtxs.LoadOrStore(newURL, connectionCtx) + // Remove all other `connectionCtx`s. + connectionCtxs.Range(func(url, cc any) bool { + if url.(string) != newURL { + cc.(*tsoConnectionContext).cancel() + connectionCtxs.Delete(url) + } + return true + }) } - connectionCtxs.Range(func(url, cc any) bool { - if url.(string) != newURL { - cc.(*tsoConnectionContext).cancel() - connectionCtxs.Delete(url) - } - return true - }) - } - // retry several times before falling back to the follower when the network problem happens + ) ticker := time.NewTicker(retryInterval) defer ticker.Stop() + // Retry several times before falling back to the follower when the network problem happens for i := 0; i < maxRetryTimes; i++ { c.svcDiscovery.ScheduleCheckMemberChanged() cc, url = c.GetTSOAllocatorClientConnByDCLocation(dc) + if _, ok := connectionCtxs.Load(url); ok { + return nil + } if cc != nil { - cctx, cancel := context.WithCancel(dispatcherCtx) + cctx, cancel := context.WithCancel(ctx) stream, err = c.tsoStreamBuilderFactory.makeBuilder(cc).build(cctx, cancel, c.option.timeout) failpoint.Inject("unreachableNetwork", func() { stream = nil @@ -392,7 +402,7 @@ func (c *tsoClient) tryConnectToTSO( networkErrNum++ } select { - case <-dispatcherCtx.Done(): + case <-ctx.Done(): return err case <-ticker.C: } @@ -409,14 +419,14 @@ func (c *tsoClient) tryConnectToTSO( } // create the follower stream - cctx, cancel := context.WithCancel(dispatcherCtx) + cctx, cancel := context.WithCancel(ctx) cctx = grpcutil.BuildForwardContext(cctx, forwardedHost) stream, err = c.tsoStreamBuilderFactory.makeBuilder(backupClientConn).build(cctx, cancel, c.option.timeout) if err == nil { forwardedHostTrim := trimHTTPPrefix(forwardedHost) addr := trimHTTPPrefix(backupURL) // the goroutine is used to check the network and change back to the original stream - go c.checkAllocator(dispatcherCtx, cancel, dc, forwardedHostTrim, addr, url, updateAndClear) + go c.checkAllocator(ctx, cancel, dc, forwardedHostTrim, addr, url, updateAndClear) requestForwarded.WithLabelValues(forwardedHostTrim, addr).Set(1) updateAndClear(backupURL, &tsoConnectionContext{backupURL, stream, cctx, cancel}) return nil @@ -429,7 +439,11 @@ func (c *tsoClient) tryConnectToTSO( // tryConnectToTSOWithProxy will create multiple streams to all the service endpoints to work as // a TSO proxy to reduce the pressure of the main serving service endpoint. -func (c *tsoClient) tryConnectToTSOWithProxy(dispatcherCtx context.Context, dc string, connectionCtxs *sync.Map) error { +func (c *tsoClient) tryConnectToTSOWithProxy( + ctx context.Context, + dc string, + connectionCtxs *sync.Map, +) error { tsoStreamBuilders := c.getAllTSOStreamBuilders() leaderAddr := c.svcDiscovery.GetServingURL() forwardedHost, ok := c.GetTSOAllocatorServingURLByDCLocation(dc) @@ -455,7 +469,7 @@ func (c *tsoClient) tryConnectToTSOWithProxy(dispatcherCtx context.Context, dc s } log.Info("[tso] try to create tso stream", zap.String("dc", dc), zap.String("addr", addr)) - cctx, cancel := context.WithCancel(dispatcherCtx) + cctx, cancel := context.WithCancel(ctx) // Do not proxy the leader client. if addr != leaderAddr { log.Info("[tso] use follower to forward tso stream to do the proxy", diff --git a/client/tso_dispatcher.go b/client/tso_dispatcher.go index 7528293a7338..c82ec777eca9 100644 --- a/client/tso_dispatcher.go +++ b/client/tso_dispatcher.go @@ -44,8 +44,17 @@ type tsoDispatcher struct { tsoBatchController *tsoBatchController } +func (td *tsoDispatcher) close() { + td.dispatcherCancel() + td.tsoBatchController.clear() +} + +func (td *tsoDispatcher) push(request *tsoRequest) { + td.tsoBatchController.tsoRequestCh <- request +} + func (c *tsoClient) dispatchRequest(request *tsoRequest) (bool, error) { - dispatcher, ok := c.tsoDispatcher.Load(request.dcLocation) + dispatcher, ok := c.getTSODispatcher(request.dcLocation) if !ok { err := errs.ErrClientGetTSO.FastGenByArgs(fmt.Sprintf("unknown dc-location %s to the client", request.dcLocation)) log.Error("[tso] dispatch tso request error", zap.String("dc-location", request.dcLocation), errs.ZapError(err)) @@ -70,7 +79,7 @@ func (c *tsoClient) dispatchRequest(request *tsoRequest) (bool, error) { failpoint.Inject("delayDispatchTSORequest", func() { time.Sleep(time.Second) }) - dispatcher.(*tsoDispatcher).tsoBatchController.tsoRequestCh <- request + dispatcher.push(request) } // Check the contexts again to make sure the request is not been sent to a closed dispatcher. // Never retry on these conditions to prevent unexpected data race. @@ -89,9 +98,7 @@ func (c *tsoClient) dispatchRequest(request *tsoRequest) (bool, error) { func (c *tsoClient) closeTSODispatcher() { c.tsoDispatcher.Range(func(_, dispatcherInterface any) bool { if dispatcherInterface != nil { - dispatcher := dispatcherInterface.(*tsoDispatcher) - dispatcher.dispatcherCancel() - dispatcher.tsoBatchController.clear() + dispatcherInterface.(*tsoDispatcher).close() } return true }) @@ -101,7 +108,7 @@ func (c *tsoClient) updateTSODispatcher() { // Set up the new TSO dispatcher and batch controller. c.GetTSOAllocators().Range(func(dcLocationKey, _ any) bool { dcLocation := dcLocationKey.(string) - if !c.checkTSODispatcher(dcLocation) { + if _, ok := c.getTSODispatcher(dcLocation); !ok { c.createTSODispatcher(dcLocation) } return true @@ -115,8 +122,8 @@ func (c *tsoClient) updateTSODispatcher() { } if _, exist := c.GetTSOAllocators().Load(dcLocation); !exist { log.Info("[tso] delete unused tso dispatcher", zap.String("dc-location", dcLocation)) - dispatcher.(*tsoDispatcher).dispatcherCancel() c.tsoDispatcher.Delete(dcLocation) + dispatcher.(*tsoDispatcher).close() } return true }) @@ -215,7 +222,7 @@ func (c *tsoClient) tsoDispatcherCheckLoop() { } func (c *tsoClient) checkAllocator( - dispatcherCtx context.Context, + ctx context.Context, forwardCancel context.CancelFunc, dc, forwardedHostTrim, addr, url string, updateAndClear func(newAddr string, connectionCtx *tsoConnectionContext)) { @@ -238,7 +245,7 @@ func (c *tsoClient) checkAllocator( healthCli = healthpb.NewHealthClient(cc) } if healthCli != nil { - healthCtx, healthCancel := context.WithTimeout(dispatcherCtx, c.option.timeout) + healthCtx, healthCancel := context.WithTimeout(ctx, c.option.timeout) resp, err := healthCli.Check(healthCtx, &healthpb.HealthCheckRequest{Service: ""}) failpoint.Inject("unreachableNetwork", func() { resp.Status = healthpb.HealthCheckResponse_UNKNOWN @@ -246,7 +253,7 @@ func (c *tsoClient) checkAllocator( healthCancel() if err == nil && resp.GetStatus() == healthpb.HealthCheckResponse_SERVING { // create a stream of the original allocator - cctx, cancel := context.WithCancel(dispatcherCtx) + cctx, cancel := context.WithCancel(ctx) stream, err := c.tsoStreamBuilderFactory.makeBuilder(cc).build(cctx, cancel, c.option.timeout) if err == nil && stream != nil { log.Info("[tso] recover the original tso stream since the network has become normal", zap.String("dc", dc), zap.String("url", url)) @@ -256,7 +263,7 @@ func (c *tsoClient) checkAllocator( } } select { - case <-dispatcherCtx.Done(): + case <-ctx.Done(): return case <-ticker.C: // To ensure we can get the latest allocator leader @@ -266,30 +273,19 @@ func (c *tsoClient) checkAllocator( } } -func (c *tsoClient) checkTSODispatcher(dcLocation string) bool { - dispatcher, ok := c.tsoDispatcher.Load(dcLocation) - if !ok || dispatcher == nil { - return false - } - return true -} - func (c *tsoClient) createTSODispatcher(dcLocation string) { dispatcherCtx, dispatcherCancel := context.WithCancel(c.ctx) - dispatcher := &tsoDispatcher{ - dispatcherCancel: dispatcherCancel, - tsoBatchController: newTSOBatchController( - make(chan *tsoRequest, defaultMaxTSOBatchSize*2), - defaultMaxTSOBatchSize), - } + tsoBatchController := newTSOBatchController( + make(chan *tsoRequest, defaultMaxTSOBatchSize*2), + defaultMaxTSOBatchSize, + ) failpoint.Inject("shortDispatcherChannel", func() { - dispatcher = &tsoDispatcher{ - dispatcherCancel: dispatcherCancel, - tsoBatchController: newTSOBatchController( - make(chan *tsoRequest, 1), - defaultMaxTSOBatchSize), - } + tsoBatchController = newTSOBatchController( + make(chan *tsoRequest, 1), + defaultMaxTSOBatchSize, + ) }) + dispatcher := &tsoDispatcher{dispatcherCancel, tsoBatchController} if _, ok := c.tsoDispatcher.LoadOrStore(dcLocation, dispatcher); !ok { // Successfully stored the value. Start the following goroutine. @@ -306,7 +302,7 @@ func (c *tsoClient) createTSODispatcher(dcLocation string) { } func (c *tsoClient) handleDispatcher( - dispatcherCtx context.Context, + ctx context.Context, dc string, tbc *tsoBatchController, ) { @@ -319,6 +315,7 @@ func (c *tsoClient) handleDispatcher( // url -> connectionContext connectionCtxs sync.Map ) + // Clean up the connectionCtxs when the dispatcher exits. defer func() { log.Info("[tso] exit tso dispatcher", zap.String("dc-location", dc)) // Cancel all connections. @@ -330,51 +327,8 @@ func (c *tsoClient) handleDispatcher( tbc.clear() c.wg.Done() }() - // Call updateTSOConnectionCtxs once to init the connectionCtxs first. - c.updateTSOConnectionCtxs(dispatcherCtx, dc, &connectionCtxs) - // Only the Global TSO needs to watch the updateTSOConnectionCtxsCh to sense the - // change of the cluster when TSO Follower Proxy is enabled. - // TODO: support TSO Follower Proxy for the Local TSO. - if dc == globalDCLocation { - go func() { - var updateTicker = &time.Ticker{} - setNewUpdateTicker := func(ticker *time.Ticker) { - if updateTicker.C != nil { - updateTicker.Stop() - } - updateTicker = ticker - } - // Set to nil before returning to ensure that the existing ticker can be GC. - defer setNewUpdateTicker(nil) - - for { - select { - case <-dispatcherCtx.Done(): - return - case <-c.option.enableTSOFollowerProxyCh: - enableTSOFollowerProxy := c.option.getEnableTSOFollowerProxy() - log.Info("[tso] tso follower proxy status changed", - zap.String("dc-location", dc), - zap.Bool("enable", enableTSOFollowerProxy)) - if enableTSOFollowerProxy && updateTicker.C == nil { - // Because the TSO Follower Proxy is enabled, - // the periodic check needs to be performed. - setNewUpdateTicker(time.NewTicker(memberUpdateInterval)) - } else if !enableTSOFollowerProxy && updateTicker.C != nil { - // Because the TSO Follower Proxy is disabled, - // the periodic check needs to be turned off. - setNewUpdateTicker(&time.Ticker{}) - } else { - // The status of TSO Follower Proxy does not change, and updateTSOConnectionCtxs is not triggered - continue - } - case <-updateTicker.C: - case <-c.updateTSOConnectionCtxsCh: - } - c.updateTSOConnectionCtxs(dispatcherCtx, dc, &connectionCtxs) - } - }() - } + // Daemon goroutine to update the connectionCtxs periodically and handle the `connectionCtxs` update event. + go c.connectionCtxsUpdater(ctx, dc, &connectionCtxs) // Loop through each batch of TSO requests and send them for processing. streamLoopTimer := time.NewTimer(c.option.timeout) @@ -383,7 +337,7 @@ func (c *tsoClient) handleDispatcher( tsoBatchLoop: for { select { - case <-dispatcherCtx.Done(): + case <-ctx.Done(): return default: } @@ -391,7 +345,7 @@ tsoBatchLoop: maxBatchWaitInterval := c.option.getMaxTSOBatchWaitInterval() // Once the TSO requests are collected, must make sure they could be finished or revoked eventually, // otherwise the upper caller may get blocked on waiting for the results. - if err = tbc.fetchPendingRequests(dispatcherCtx, maxBatchWaitInterval); err != nil { + if err = tbc.fetchPendingRequests(ctx, maxBatchWaitInterval); err != nil { // Finish the collected requests if the fetch failed. tbc.finishCollectedRequests(0, 0, 0, errors.WithStack(err)) if err == context.Canceled { @@ -427,14 +381,14 @@ tsoBatchLoop: // Check stream and retry if necessary. if stream == nil { log.Info("[tso] tso stream is not ready", zap.String("dc", dc)) - if c.updateTSOConnectionCtxs(dispatcherCtx, dc, &connectionCtxs) { + if c.updateTSOConnectionCtxs(ctx, dc, &connectionCtxs) { continue streamChoosingLoop } timer := time.NewTimer(retryInterval) select { - case <-dispatcherCtx.Done(): + case <-ctx.Done(): // Finish the collected requests if the context is canceled. - tbc.finishCollectedRequests(0, 0, 0, errors.WithStack(dispatcherCtx.Err())) + tbc.finishCollectedRequests(0, 0, 0, errors.WithStack(ctx.Err())) timer.Stop() return case <-streamLoopTimer.C: @@ -471,9 +425,9 @@ tsoBatchLoop: tsDeadlineCh, ok = c.tsDeadline.Load(dc) } select { - case <-dispatcherCtx.Done(): + case <-ctx.Done(): // Finish the collected requests if the context is canceled. - tbc.finishCollectedRequests(0, 0, 0, errors.WithStack(dispatcherCtx.Err())) + tbc.finishCollectedRequests(0, 0, 0, errors.WithStack(ctx.Err())) return case tsDeadlineCh.(chan *deadline) <- dl: } @@ -483,7 +437,7 @@ tsoBatchLoop: // If error happens during tso stream handling, reset stream and run the next trial. if err != nil { select { - case <-dispatcherCtx.Done(): + case <-ctx.Done(): return default: } @@ -498,9 +452,9 @@ tsoBatchLoop: stream = nil // Because ScheduleCheckMemberChanged is asynchronous, if the leader changes, we better call `updateMember` ASAP. if IsLeaderChange(err) { - if err := bo.Exec(dispatcherCtx, c.svcDiscovery.CheckMemberChanged); err != nil { + if err := bo.Exec(ctx, c.svcDiscovery.CheckMemberChanged); err != nil { select { - case <-dispatcherCtx.Done(): + case <-ctx.Done(): return default: } @@ -510,8 +464,59 @@ tsoBatchLoop: // will cancel the current stream, then the EOF error caused by cancel() // should not trigger the updateTSOConnectionCtxs here. // So we should only call it when the leader changes. - c.updateTSOConnectionCtxs(dispatcherCtx, dc, &connectionCtxs) + c.updateTSOConnectionCtxs(ctx, dc, &connectionCtxs) + } + } + } +} + +// updateTSOConnectionCtxs updates the `connectionCtxs` for the specified DC location regularly. +// TODO: implement support for the Local TSO. +func (c *tsoClient) connectionCtxsUpdater( + ctx context.Context, + dc string, + connectionCtxs *sync.Map, +) { + if dc != globalDCLocation { + return + } + log.Info("[tso] start tso connection contexts updater", zap.String("dc-location", dc)) + var updateTicker = &time.Ticker{} + setNewUpdateTicker := func(ticker *time.Ticker) { + if updateTicker.C != nil { + updateTicker.Stop() + } + updateTicker = ticker + } + // Set to nil before returning to ensure that the existing ticker can be GC. + defer setNewUpdateTicker(nil) + + for { + c.updateTSOConnectionCtxs(ctx, dc, connectionCtxs) + select { + case <-ctx.Done(): + log.Info("[tso] exit tso connection contexts updater", zap.String("dc-location", dc)) + return + case <-c.option.enableTSOFollowerProxyCh: + enableTSOFollowerProxy := c.option.getEnableTSOFollowerProxy() + log.Info("[tso] tso follower proxy status changed", + zap.String("dc-location", dc), + zap.Bool("enable", enableTSOFollowerProxy)) + if enableTSOFollowerProxy && updateTicker.C == nil { + // Because the TSO Follower Proxy is enabled, + // the periodic check needs to be performed. + setNewUpdateTicker(time.NewTicker(memberUpdateInterval)) + } else if !enableTSOFollowerProxy && updateTicker.C != nil { + // Because the TSO Follower Proxy is disabled, + // the periodic check needs to be turned off. + setNewUpdateTicker(&time.Ticker{}) + } else { + continue } + case <-updateTicker.C: + // Triggered periodically when the TSO Follower Proxy is enabled. + case <-c.updateTSOConnectionCtxsCh: + // Triggered by the leader/follower change. } } } diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 10be418c0294..dfe7a6980c78 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -26,6 +26,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -248,6 +249,40 @@ func TestLeaderTransferAndMoveCluster(t *testing.T) { wg.Wait() } +func TestGetTSAfterTransferLeader(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cluster, err := tests.NewTestCluster(ctx, 2) + re.NoError(err) + endpoints := runServer(re, cluster) + leader := cluster.WaitLeader() + re.NotEmpty(leader) + defer cluster.Destroy() + + cli := setupCli(ctx, re, endpoints, pd.WithCustomTimeoutOption(10*time.Second)) + defer cli.Close() + + var leaderSwitched atomic.Bool + cli.GetServiceDiscovery().AddServingURLSwitchedCallback(func() { + leaderSwitched.Store(true) + }) + err = cluster.GetServer(leader).ResignLeader() + re.NoError(err) + newLeader := cluster.WaitLeader() + re.NotEmpty(newLeader) + re.NotEqual(leader, newLeader) + leader = cluster.WaitLeader() + re.NotEmpty(leader) + err = cli.GetServiceDiscovery().CheckMemberChanged() + re.NoError(err) + + testutil.Eventually(re, leaderSwitched.Load) + // The leader stream must be updated after the leader switch is sensed by the client. + _, _, err = cli.GetTS(context.TODO()) + re.NoError(err) +} + func TestTSOAllocatorLeader(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) From 6e3cac7f95279f08ff31fe15e948afd0237881b8 Mon Sep 17 00:00:00 2001 From: ShuNing Date: Tue, 30 Apr 2024 10:45:27 +0800 Subject: [PATCH 43/50] server/api: add the api to show the regions in subtree by type (#8101) close tikv/pd#8100 server/api: add the api to show the regions in subtree by type Signed-off-by: nolouch Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/core/region.go | 54 ++++++++++++++++++++ pkg/response/store.go | 2 + server/api/region.go | 12 ++++- server/cluster/cluster.go | 5 ++ tools/pd-ctl/pdctl/command/region_command.go | 3 ++ tools/pd-ctl/tests/region/region_test.go | 12 ++++- 6 files changed, 86 insertions(+), 2 deletions(-) diff --git a/pkg/core/region.go b/pkg/core/region.go index 713e82cc36d8..8e3d4e5dec85 100644 --- a/pkg/core/region.go +++ b/pkg/core/region.go @@ -1541,6 +1541,60 @@ func (r *RegionsInfo) GetStoreRegions(storeID uint64) []*RegionInfo { return regions } +// SubTreeRegionType is the type of sub tree region. +type SubTreeRegionType string + +const ( + // AllInSubTree is all sub trees. + AllInSubTree SubTreeRegionType = "all" + // LeaderInSubTree is the leader sub tree. + LeaderInSubTree SubTreeRegionType = "leader" + // FollowerInSubTree is the follower sub tree. + FollowerInSubTree SubTreeRegionType = "follower" + // LearnerInSubTree is the learner sub tree. + LearnerInSubTree SubTreeRegionType = "learner" + // WitnessInSubTree is the witness sub tree. + WitnessInSubTree SubTreeRegionType = "witness" + // PendingPeerInSubTree is the pending peer sub tree. + PendingPeerInSubTree SubTreeRegionType = "pending" +) + +// GetStoreRegions gets all RegionInfo with a given storeID +func (r *RegionsInfo) GetStoreRegionsByTypeInSubTree(storeID uint64, typ SubTreeRegionType) ([]*RegionInfo, error) { + r.st.RLock() + var regions []*RegionInfo + switch typ { + case LeaderInSubTree: + if leaders, ok := r.leaders[storeID]; ok { + regions = leaders.scanRanges() + } + case FollowerInSubTree: + if followers, ok := r.followers[storeID]; ok { + regions = followers.scanRanges() + } + case LearnerInSubTree: + if learners, ok := r.learners[storeID]; ok { + regions = learners.scanRanges() + } + case WitnessInSubTree: + if witnesses, ok := r.witnesses[storeID]; ok { + regions = witnesses.scanRanges() + } + case PendingPeerInSubTree: + if pendingPeers, ok := r.pendingPeers[storeID]; ok { + regions = pendingPeers.scanRanges() + } + case AllInSubTree: + r.st.RUnlock() + return r.GetStoreRegions(storeID), nil + default: + return nil, errors.Errorf("unknown sub tree region type %v", typ) + } + + r.st.RUnlock() + return regions, nil +} + // GetStoreLeaderRegionSize get total size of store's leader regions func (r *RegionsInfo) GetStoreLeaderRegionSize(storeID uint64) int64 { r.st.RLock() diff --git a/pkg/response/store.go b/pkg/response/store.go index 1efe11bfb39c..8bff1e75e42d 100644 --- a/pkg/response/store.go +++ b/pkg/response/store.go @@ -64,6 +64,7 @@ type StoreStatus struct { RegionSize int64 `json:"region_size"` LearnerCount int `json:"learner_count,omitempty"` WitnessCount int `json:"witness_count,omitempty"` + PendingPeerCount int `json:"pending_peer_count,omitempty"` SlowScore uint64 `json:"slow_score,omitempty"` SlowTrend *SlowTrend `json:"slow_trend,omitempty"` SendingSnapCount uint32 `json:"sending_snap_count,omitempty"` @@ -117,6 +118,7 @@ func BuildStoreInfo(opt *sc.ScheduleConfig, store *core.StoreInfo) *StoreInfo { SlowTrend: slowTrend, SendingSnapCount: store.GetSendingSnapCount(), ReceivingSnapCount: store.GetReceivingSnapCount(), + PendingPeerCount: store.GetPendingPeerCount(), IsBusy: store.IsBusy(), }, } diff --git a/server/api/region.go b/server/api/region.go index dac92f247ca9..974b5e4fa120 100644 --- a/server/api/region.go +++ b/server/api/region.go @@ -218,7 +218,17 @@ func (h *regionsHandler) GetStoreRegions(w http.ResponseWriter, r *http.Request) h.rd.JSON(w, http.StatusBadRequest, err.Error()) return } - regions := rc.GetStoreRegions(uint64(id)) + // get type from query + typ := r.URL.Query().Get("type") + if len(typ) == 0 { + typ = string(core.AllInSubTree) + } + + regions, err := rc.GetStoreRegionsByTypeInSubTree(uint64(id), core.SubTreeRegionType(typ)) + if err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return + } b, err := response.MarshalRegionsInfoJSON(r.Context(), regions) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index dbc6a6cadf33..75301664b509 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -1198,6 +1198,11 @@ func (c *RaftCluster) GetStoreRegions(storeID uint64) []*core.RegionInfo { return c.core.GetStoreRegions(storeID) } +// GetStoreRegions returns all regions' information with a given storeID. +func (c *RaftCluster) GetStoreRegionsByType(storeID uint64) []*core.RegionInfo { + return c.core.GetStoreRegions(storeID) +} + // RandLeaderRegions returns some random regions that has leader on the store. func (c *RaftCluster) RandLeaderRegions(storeID uint64, ranges []core.KeyRange) []*core.RegionInfo { return c.core.RandLeaderRegions(storeID, ranges) diff --git a/tools/pd-ctl/pdctl/command/region_command.go b/tools/pd-ctl/pdctl/command/region_command.go index d7e19967c7a0..3536b01a606a 100644 --- a/tools/pd-ctl/pdctl/command/region_command.go +++ b/tools/pd-ctl/pdctl/command/region_command.go @@ -486,6 +486,7 @@ func NewRegionWithStoreCommand() *cobra.Command { Short: "show the regions of a specific store", Run: showRegionWithStoreCommandFunc, } + r.Flags().String("type", "all", "the type of the regions, could be 'all', 'leader', 'learner' or 'pending'") return r } @@ -496,6 +497,8 @@ func showRegionWithStoreCommandFunc(cmd *cobra.Command, args []string) { } storeID := args[0] prefix := regionsStorePrefix + "/" + storeID + flagType := cmd.Flag("type") + prefix += "?type=" + flagType.Value.String() r, err := doRequest(cmd, prefix, http.MethodGet, http.Header{}) if err != nil { cmd.Printf("Failed to get regions with the given storeID: %s\n", err) diff --git a/tools/pd-ctl/tests/region/region_test.go b/tools/pd-ctl/tests/region/region_test.go index b328fd882864..2952e137f3b4 100644 --- a/tools/pd-ctl/tests/region/region_test.go +++ b/tools/pd-ctl/tests/region/region_test.go @@ -108,6 +108,11 @@ func TestRegion(t *testing.T) { ) defer cluster.Destroy() + getRegionsByType := func(storeID uint64, regionType core.SubTreeRegionType) []*core.RegionInfo { + regions, _ := leaderServer.GetRaftCluster().GetStoreRegionsByTypeInSubTree(storeID, regionType) + return regions + } + var testRegionsCases = []struct { args []string expect []*core.RegionInfo @@ -118,7 +123,12 @@ func TestRegion(t *testing.T) { {[]string{"region", "sibling", "2"}, leaderServer.GetAdjacentRegions(leaderServer.GetRegionInfoByID(2))}, // region store command {[]string{"region", "store", "1"}, leaderServer.GetStoreRegions(1)}, - {[]string{"region", "store", "1"}, []*core.RegionInfo{r1, r2, r3, r4}}, + {[]string{"region", "store", "1", "--type=leader"}, getRegionsByType(1, core.LeaderInSubTree)}, + {[]string{"region", "store", "1", "--type=follower"}, getRegionsByType(1, core.FollowerInSubTree)}, + {[]string{"region", "store", "1", "--type=learner"}, getRegionsByType(1, core.LearnerInSubTree)}, + {[]string{"region", "store", "1", "--type=witness"}, getRegionsByType(1, core.WitnessInSubTree)}, + {[]string{"region", "store", "1", "--type=pending"}, getRegionsByType(1, core.PendingPeerInSubTree)}, + {[]string{"region", "store", "1", "--type=all"}, []*core.RegionInfo{r1, r2, r3, r4}}, // region check extra-peer command {[]string{"region", "check", "extra-peer"}, []*core.RegionInfo{r1}}, // region check miss-peer command From 8ee18a4b498669d0a23067e643e5fcfbfc521803 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 30 Apr 2024 11:46:57 +0800 Subject: [PATCH 44/50] pkg: fix `RegionStatsNeedUpdate` condition (#8133) ref tikv/pd#7897 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/statistics/region_collection.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/statistics/region_collection.go b/pkg/statistics/region_collection.go index cb0de6f601b8..565597b4efb3 100644 --- a/pkg/statistics/region_collection.go +++ b/pkg/statistics/region_collection.go @@ -158,14 +158,14 @@ func (r *RegionStatistics) RegionStatsNeedUpdate(region *core.RegionInfo) bool { region.IsOversized(int64(r.conf.GetRegionMaxSize()), int64(r.conf.GetRegionMaxKeys())) { return true } - // expected to be zero for below type - if r.IsRegionStatsType(regionID, PendingPeer) && len(region.GetPendingPeers()) == 0 { + + if r.IsRegionStatsType(regionID, PendingPeer) != (len(region.GetPendingPeers()) != 0) { return true } - if r.IsRegionStatsType(regionID, DownPeer) && len(region.GetDownPeers()) == 0 { + if r.IsRegionStatsType(regionID, DownPeer) != (len(region.GetDownPeers()) != 0) { return true } - if r.IsRegionStatsType(regionID, LearnerPeer) && len(region.GetLearners()) == 0 { + if r.IsRegionStatsType(regionID, LearnerPeer) != (len(region.GetLearners()) != 0) { return true } From 1679dbca25b3483d1375c7e747da27e99ad77360 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Tue, 30 Apr 2024 16:04:03 +0800 Subject: [PATCH 45/50] client/tso: decouple tsoClient and tsoDispatcher (#8134) ref tikv/pd#8047 - Decouple `tsoClient` and `tsoDispatcher` to appropriately categorize different methods. - Update `connectionCtxsUpdater` to support the Local TSO. Signed-off-by: JmPotato --- client/client.go | 6 +- client/tso_batch_controller.go | 4 +- client/tso_client.go | 334 +++++++++++++-------- client/tso_dispatcher.go | 523 +++++++++++++++------------------ client/tso_request.go | 2 +- 5 files changed, 450 insertions(+), 419 deletions(-) diff --git a/client/client.go b/client/client.go index eaebef7e10c1..1865fd0866ed 100644 --- a/client/client.go +++ b/client/client.go @@ -301,7 +301,7 @@ func (k *serviceModeKeeper) close() { fallthrough case pdpb.ServiceMode_PD_SVC_MODE: if k.tsoClient != nil { - k.tsoClient.Close() + k.tsoClient.close() } case pdpb.ServiceMode_UNKNOWN_SVC_MODE: } @@ -651,11 +651,11 @@ func (c *client) resetTSOClientLocked(mode pdpb.ServiceMode) { log.Warn("[pd] intend to switch to unknown service mode, just return") return } - newTSOCli.Setup() + newTSOCli.setup() // Replace the old TSO client. oldTSOClient := c.tsoClient c.tsoClient = newTSOCli - oldTSOClient.Close() + oldTSOClient.close() // Replace the old TSO service discovery if needed. oldTSOSvcDiscovery := c.tsoSvcDiscovery // If newTSOSvcDiscovery is nil, that's expected, as it means we are switching to PD service mode and diff --git a/client/tso_batch_controller.go b/client/tso_batch_controller.go index d7ba5d7e74bb..a713b7a187d8 100644 --- a/client/tso_batch_controller.go +++ b/client/tso_batch_controller.go @@ -139,9 +139,11 @@ func (tbc *tsoBatchController) adjustBestBatchSize() { func (tbc *tsoBatchController) finishCollectedRequests(physical, firstLogical int64, suffixBits uint32, err error) { for i := 0; i < tbc.collectedRequestCount; i++ { tsoReq := tbc.collectedRequests[i] + // Retrieve the request context before the request is done to trace without race. + requestCtx := tsoReq.requestCtx tsoReq.physical, tsoReq.logical = physical, tsoutil.AddLogical(firstLogical, int64(i), suffixBits) - defer trace.StartRegion(tsoReq.requestCtx, "pdclient.tsoReqDequeue").End() // nolint tsoReq.tryDone(err) + trace.StartRegion(requestCtx, "pdclient.tsoReqDequeue").End() } // Prevent the finished requests from being processed again. tbc.collectedRequestCount = 0 diff --git a/client/tso_client.go b/client/tso_client.go index e3bdb8359016..72b09d8054df 100644 --- a/client/tso_client.go +++ b/client/tso_client.go @@ -22,13 +22,11 @@ import ( "sync" "time" - "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/tikv/pd/client/errs" "github.com/tikv/pd/client/grpcutil" - "github.com/tikv/pd/client/tsoutil" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -36,6 +34,15 @@ import ( "google.golang.org/grpc/status" ) +const ( + tsoDispatcherCheckInterval = time.Minute + // defaultMaxTSOBatchSize is the default max size of the TSO request batch. + defaultMaxTSOBatchSize = 10000 + // retryInterval and maxRetryTimes are used to control the retry interval and max retry times. + retryInterval = 500 * time.Millisecond + maxRetryTimes = 6 +) + // TSOClient is the client used to get timestamps. type TSOClient interface { // GetTS gets a timestamp from PD or TSO microservice. @@ -70,14 +77,8 @@ type tsoClient struct { // tsoDispatcher is used to dispatch different TSO requests to // the corresponding dc-location TSO channel. tsoDispatcher sync.Map // Same as map[string]*tsoDispatcher - // dc-location -> deadline - tsDeadline sync.Map // Same as map[string]chan deadline - // dc-location -> *tsoInfo while the tsoInfo is the last TSO info - lastTSOInfoMap sync.Map // Same as map[string]*tsoInfo - - checkTSDeadlineCh chan struct{} - checkTSODispatcherCh chan struct{} - updateTSOConnectionCtxsCh chan struct{} + + checkTSODispatcherCh chan struct{} } // newTSOClient returns a new TSO client. @@ -101,49 +102,64 @@ func newTSOClient( } }, }, - checkTSDeadlineCh: make(chan struct{}), - checkTSODispatcherCh: make(chan struct{}, 1), - updateTSOConnectionCtxsCh: make(chan struct{}, 1), + checkTSODispatcherCh: make(chan struct{}, 1), } eventSrc := svcDiscovery.(tsoAllocatorEventSource) eventSrc.SetTSOLocalServURLsUpdatedCallback(c.updateTSOLocalServURLs) eventSrc.SetTSOGlobalServURLUpdatedCallback(c.updateTSOGlobalServURL) - c.svcDiscovery.AddServiceURLsSwitchedCallback(c.scheduleUpdateTSOConnectionCtxs) + c.svcDiscovery.AddServiceURLsSwitchedCallback(c.scheduleUpdateAllTSOConnectionCtxs) return c } -func (c *tsoClient) Setup() { +func (c *tsoClient) getOption() *option { return c.option } + +func (c *tsoClient) getServiceDiscovery() ServiceDiscovery { return c.svcDiscovery } + +func (c *tsoClient) setup() { c.svcDiscovery.CheckMemberChanged() c.updateTSODispatcher() // Start the daemons. - c.wg.Add(2) + c.wg.Add(1) go c.tsoDispatcherCheckLoop() - go c.tsCancelLoop() } -// Close closes the TSO client -func (c *tsoClient) Close() { +func (c *tsoClient) tsoDispatcherCheckLoop() { + log.Info("[tso] start tso dispatcher check loop") + defer log.Info("[tso] exit tso dispatcher check loop") + defer c.wg.Done() + + loopCtx, loopCancel := context.WithCancel(c.ctx) + defer loopCancel() + + ticker := time.NewTicker(tsoDispatcherCheckInterval) + defer ticker.Stop() + for { + c.updateTSODispatcher() + select { + case <-ticker.C: + case <-c.checkTSODispatcherCh: + case <-loopCtx.Done(): + return + } + } +} + +// close closes the TSO client +func (c *tsoClient) close() { if c == nil { return } - log.Info("closing tso client") + log.Info("[tso] closing tso client") c.cancel() c.wg.Wait() - log.Info("close tso client") + log.Info("[tso] close tso client") c.closeTSODispatcher() - log.Info("tso client is closed") -} - -func (c *tsoClient) scheduleCheckTSDeadline() { - select { - case c.checkTSDeadlineCh <- struct{}{}: - default: - } + log.Info("[tso] tso client is closed") } func (c *tsoClient) scheduleCheckTSODispatcher() { @@ -153,11 +169,21 @@ func (c *tsoClient) scheduleCheckTSODispatcher() { } } -func (c *tsoClient) scheduleUpdateTSOConnectionCtxs() { - select { - case c.updateTSOConnectionCtxsCh <- struct{}{}: - default: +// scheduleUpdateAllTSOConnectionCtxs update the TSO connection contexts for all dc-locations. +func (c *tsoClient) scheduleUpdateAllTSOConnectionCtxs() { + c.tsoDispatcher.Range(func(_, dispatcher any) bool { + dispatcher.(*tsoDispatcher).scheduleUpdateConnectionCtxs() + return true + }) +} + +// scheduleUpdateTSOConnectionCtxs update the TSO connection contexts for the given dc-location. +func (c *tsoClient) scheduleUpdateTSOConnectionCtxs(dcLocation string) { + dispatcher, ok := c.getTSODispatcher(dcLocation) + if !ok { + return } + dispatcher.scheduleUpdateConnectionCtxs() } // TSO Follower Proxy only supports the Global TSO proxy now. @@ -200,14 +226,12 @@ func (c *tsoClient) GetTSOAllocatorServingURLByDCLocation(dcLocation string) (st return url.(string), true } -// GetTSOAllocatorClientConnByDCLocation returns the tso allocator grpc client connection -// of the given dcLocation +// GetTSOAllocatorClientConnByDCLocation returns the TSO allocator gRPC client connection of the given dcLocation. func (c *tsoClient) GetTSOAllocatorClientConnByDCLocation(dcLocation string) (*grpc.ClientConn, string) { url, ok := c.tsoAllocators.Load(dcLocation) if !ok { - panic(fmt.Sprintf("the allocator leader in %s should exist", dcLocation)) + log.Fatal("[tso] the allocator leader should exist", zap.String("dc-location", dcLocation)) } - // todo: if we support local tso forward, we should get or create client conns. cc, ok := c.svcDiscovery.GetClientConns().Load(url) if !ok { return nil, url.(string) @@ -250,6 +274,8 @@ func (c *tsoClient) updateTSOLocalServURLs(allocatorMap map[string]string) error zap.String("dc-location", dcLocation), zap.String("new-url", url), zap.String("old-url", oldURL)) + // Should trigger the update of the connection contexts once the allocator leader is switched. + c.scheduleUpdateTSOConnectionCtxs(dcLocation) } // Garbage collection of the old TSO allocator primaries @@ -267,7 +293,7 @@ func (c *tsoClient) updateTSOGlobalServURL(url string) error { log.Info("[tso] switch dc tso global allocator serving url", zap.String("dc-location", globalDCLocation), zap.String("new-url", url)) - c.scheduleUpdateTSOConnectionCtxs() + c.scheduleUpdateTSOConnectionCtxs(globalDCLocation) c.scheduleCheckTSODispatcher() return nil } @@ -315,22 +341,27 @@ func (c *tsoClient) backupClientConn() (*grpc.ClientConn, string) { return nil, "" } +// tsoConnectionContext is used to store the context of a TSO stream connection. type tsoConnectionContext struct { - streamURL string - // Current stream to send gRPC requests, pdpb.PD_TsoClient for a leader/follower in the PD cluster, - // or tsopb.TSO_TsoClient for a primary/secondary in the TSO cluster - stream tsoStream ctx context.Context cancel context.CancelFunc + // Current URL of the stream connection. + streamURL string + // Current stream to send gRPC requests. + // - `pdpb.PD_TsoClient` for a leader/follower in the PD cluster. + // - `tsopb.TSO_TsoClient` for a primary/secondary in the TSO cluster. + stream tsoStream } -func (c *tsoClient) updateTSOConnectionCtxs(updaterCtx context.Context, dc string, connectionCtxs *sync.Map) bool { +// updateConnectionCtxs will choose the proper way to update the connections for the given dc-location. +// It will return a bool to indicate whether the update is successful. +func (c *tsoClient) updateConnectionCtxs(ctx context.Context, dc string, connectionCtxs *sync.Map) bool { // Normal connection creating, it will be affected by the `enableForwarding`. createTSOConnection := c.tryConnectToTSO if c.allowTSOFollowerProxy(dc) { createTSOConnection = c.tryConnectToTSOWithProxy } - if err := createTSOConnection(updaterCtx, dc, connectionCtxs); err != nil { + if err := createTSOConnection(ctx, dc, connectionCtxs); err != nil { log.Error("[tso] update connection contexts failed", zap.String("dc", dc), errs.ZapError(err)) return false } @@ -383,7 +414,7 @@ func (c *tsoClient) tryConnectToTSO( err = status.New(codes.Unavailable, "unavailable").Err() }) if stream != nil && err == nil { - updateAndClear(url, &tsoConnectionContext{url, stream, cctx, cancel}) + updateAndClear(url, &tsoConnectionContext{cctx, cancel, url, stream}) return nil } @@ -428,7 +459,7 @@ func (c *tsoClient) tryConnectToTSO( // the goroutine is used to check the network and change back to the original stream go c.checkAllocator(ctx, cancel, dc, forwardedHostTrim, addr, url, updateAndClear) requestForwarded.WithLabelValues(forwardedHostTrim, addr).Set(1) - updateAndClear(backupURL, &tsoConnectionContext{backupURL, stream, cctx, cancel}) + updateAndClear(backupURL, &tsoConnectionContext{cctx, cancel, backupURL, stream}) return nil } cancel() @@ -437,6 +468,59 @@ func (c *tsoClient) tryConnectToTSO( return err } +func (c *tsoClient) checkAllocator( + ctx context.Context, + forwardCancel context.CancelFunc, + dc, forwardedHostTrim, addr, url string, + updateAndClear func(newAddr string, connectionCtx *tsoConnectionContext), +) { + defer func() { + // cancel the forward stream + forwardCancel() + requestForwarded.WithLabelValues(forwardedHostTrim, addr).Set(0) + }() + cc, u := c.GetTSOAllocatorClientConnByDCLocation(dc) + var healthCli healthpb.HealthClient + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + // the pd/allocator leader change, we need to re-establish the stream + if u != url { + log.Info("[tso] the leader of the allocator leader is changed", zap.String("dc", dc), zap.String("origin", url), zap.String("new", u)) + return + } + if healthCli == nil && cc != nil { + healthCli = healthpb.NewHealthClient(cc) + } + if healthCli != nil { + healthCtx, healthCancel := context.WithTimeout(ctx, c.option.timeout) + resp, err := healthCli.Check(healthCtx, &healthpb.HealthCheckRequest{Service: ""}) + failpoint.Inject("unreachableNetwork", func() { + resp.Status = healthpb.HealthCheckResponse_UNKNOWN + }) + healthCancel() + if err == nil && resp.GetStatus() == healthpb.HealthCheckResponse_SERVING { + // create a stream of the original allocator + cctx, cancel := context.WithCancel(ctx) + stream, err := c.tsoStreamBuilderFactory.makeBuilder(cc).build(cctx, cancel, c.option.timeout) + if err == nil && stream != nil { + log.Info("[tso] recover the original tso stream since the network has become normal", zap.String("dc", dc), zap.String("url", url)) + updateAndClear(url, &tsoConnectionContext{cctx, cancel, url, stream}) + return + } + } + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + // To ensure we can get the latest allocator leader + // and once the leader is changed, we can exit this function. + cc, u = c.GetTSOAllocatorClientConnByDCLocation(dc) + } + } +} + // tryConnectToTSOWithProxy will create multiple streams to all the service endpoints to work as // a TSO proxy to reduce the pressure of the main serving service endpoint. func (c *tsoClient) tryConnectToTSOWithProxy( @@ -484,7 +568,7 @@ func (c *tsoClient) tryConnectToTSOWithProxy( addrTrim := trimHTTPPrefix(addr) requestForwarded.WithLabelValues(forwardedHostTrim, addrTrim).Set(1) } - connectionCtxs.Store(addr, &tsoConnectionContext{addr, stream, cctx, cancel}) + connectionCtxs.Store(addr, &tsoConnectionContext{cctx, cancel, addr, stream}) continue } log.Error("[tso] create the tso stream failed", @@ -520,92 +604,90 @@ func (c *tsoClient) getAllTSOStreamBuilders() map[string]tsoStreamBuilder { return streamBuilders } -type tsoInfo struct { - tsoServer string - reqKeyspaceGroupID uint32 - respKeyspaceGroupID uint32 - respReceivedAt time.Time - physical int64 - logical int64 +func (c *tsoClient) createTSODispatcher(dcLocation string) { + dispatcher := newTSODispatcher(c.ctx, dcLocation, defaultMaxTSOBatchSize, c) + if _, ok := c.tsoDispatcher.LoadOrStore(dcLocation, dispatcher); !ok { + // Create a new dispatcher for the dc-location to handle the TSO requests. + c.wg.Add(1) + go dispatcher.handleDispatcher(&c.wg) + } else { + dispatcher.close() + } } -func (c *tsoClient) processRequests( - stream tsoStream, dcLocation string, tbc *tsoBatchController, -) error { - requests := tbc.getCollectedRequests() - // nolint - for _, req := range requests { - defer trace.StartRegion(req.requestCtx, "pdclient.tsoReqSend").End() - if span := opentracing.SpanFromContext(req.requestCtx); span != nil && span.Tracer() != nil { - span = span.Tracer().StartSpan("pdclient.processRequests", opentracing.ChildOf(span.Context())) - defer span.Finish() +func (c *tsoClient) closeTSODispatcher() { + c.tsoDispatcher.Range(func(_, dispatcherInterface any) bool { + if dispatcherInterface != nil { + dispatcherInterface.(*tsoDispatcher).close() } - } + return true + }) +} - count := int64(len(requests)) - reqKeyspaceGroupID := c.svcDiscovery.GetKeyspaceGroupID() - respKeyspaceGroupID, physical, logical, suffixBits, err := stream.processRequests( - c.svcDiscovery.GetClusterID(), c.svcDiscovery.GetKeyspaceID(), reqKeyspaceGroupID, - dcLocation, count, tbc.batchStartTime) - if err != nil { - tbc.finishCollectedRequests(0, 0, 0, err) - return err - } - // `logical` is the largest ts's logical part here, we need to do the subtracting before we finish each TSO request. - firstLogical := tsoutil.AddLogical(logical, -count+1, suffixBits) - curTSOInfo := &tsoInfo{ - tsoServer: stream.getServerURL(), - reqKeyspaceGroupID: reqKeyspaceGroupID, - respKeyspaceGroupID: respKeyspaceGroupID, - respReceivedAt: time.Now(), - physical: physical, - logical: tsoutil.AddLogical(firstLogical, count-1, suffixBits), - } - c.compareAndSwapTS(dcLocation, curTSOInfo, physical, firstLogical) - tbc.finishCollectedRequests(physical, firstLogical, suffixBits, nil) - return nil +func (c *tsoClient) updateTSODispatcher() { + // Set up the new TSO dispatcher and batch controller. + c.GetTSOAllocators().Range(func(dcLocationKey, _ any) bool { + dcLocation := dcLocationKey.(string) + if _, ok := c.getTSODispatcher(dcLocation); !ok { + c.createTSODispatcher(dcLocation) + } + return true + }) + // Clean up the unused TSO dispatcher + c.tsoDispatcher.Range(func(dcLocationKey, dispatcher any) bool { + dcLocation := dcLocationKey.(string) + // Skip the Global TSO Allocator + if dcLocation == globalDCLocation { + return true + } + if _, exist := c.GetTSOAllocators().Load(dcLocation); !exist { + log.Info("[tso] delete unused tso dispatcher", zap.String("dc-location", dcLocation)) + c.tsoDispatcher.Delete(dcLocation) + dispatcher.(*tsoDispatcher).close() + } + return true + }) } -func (c *tsoClient) compareAndSwapTS( - dcLocation string, - curTSOInfo *tsoInfo, - physical, firstLogical int64, -) { - val, loaded := c.lastTSOInfoMap.LoadOrStore(dcLocation, curTSOInfo) - if !loaded { - return - } - lastTSOInfo := val.(*tsoInfo) - if lastTSOInfo.respKeyspaceGroupID != curTSOInfo.respKeyspaceGroupID { - log.Info("[tso] keyspace group changed", - zap.String("dc-location", dcLocation), - zap.Uint32("old-group-id", lastTSOInfo.respKeyspaceGroupID), - zap.Uint32("new-group-id", curTSOInfo.respKeyspaceGroupID)) +// dispatchRequest will send the TSO request to the corresponding TSO dispatcher. +func (c *tsoClient) dispatchRequest(request *tsoRequest) (bool, error) { + dispatcher, ok := c.getTSODispatcher(request.dcLocation) + if !ok { + err := errs.ErrClientGetTSO.FastGenByArgs(fmt.Sprintf("unknown dc-location %s to the client", request.dcLocation)) + log.Error("[tso] dispatch tso request error", zap.String("dc-location", request.dcLocation), errs.ZapError(err)) + c.svcDiscovery.ScheduleCheckMemberChanged() + // New dispatcher could be created in the meantime, which is retryable. + return true, err } - // The TSO we get is a range like [largestLogical-count+1, largestLogical], so we save the last TSO's largest logical - // to compare with the new TSO's first logical. For example, if we have a TSO resp with logical 10, count 5, then - // all TSOs we get will be [6, 7, 8, 9, 10]. lastTSOInfo.logical stores the logical part of the largest ts returned - // last time. - if tsoutil.TSLessEqual(physical, firstLogical, lastTSOInfo.physical, lastTSOInfo.logical) { - log.Panic("[tso] timestamp fallback", - zap.String("dc-location", dcLocation), - zap.Uint32("keyspace", c.svcDiscovery.GetKeyspaceID()), - zap.String("last-ts", fmt.Sprintf("(%d, %d)", lastTSOInfo.physical, lastTSOInfo.logical)), - zap.String("cur-ts", fmt.Sprintf("(%d, %d)", physical, firstLogical)), - zap.String("last-tso-server", lastTSOInfo.tsoServer), - zap.String("cur-tso-server", curTSOInfo.tsoServer), - zap.Uint32("last-keyspace-group-in-request", lastTSOInfo.reqKeyspaceGroupID), - zap.Uint32("cur-keyspace-group-in-request", curTSOInfo.reqKeyspaceGroupID), - zap.Uint32("last-keyspace-group-in-response", lastTSOInfo.respKeyspaceGroupID), - zap.Uint32("cur-keyspace-group-in-response", curTSOInfo.respKeyspaceGroupID), - zap.Time("last-response-received-at", lastTSOInfo.respReceivedAt), - zap.Time("cur-response-received-at", curTSOInfo.respReceivedAt)) + defer trace.StartRegion(request.requestCtx, "pdclient.tsoReqEnqueue").End() + select { + case <-request.requestCtx.Done(): + // Caller cancelled the request, no need to retry. + return false, request.requestCtx.Err() + case <-request.clientCtx.Done(): + // Client is closed, no need to retry. + return false, request.clientCtx.Err() + case <-c.ctx.Done(): + // tsoClient is closed due to the PD service mode switch, which is retryable. + return true, c.ctx.Err() + default: + // This failpoint will increase the possibility that the request is sent to a closed dispatcher. + failpoint.Inject("delayDispatchTSORequest", func() { + time.Sleep(time.Second) + }) + dispatcher.push(request) + } + // Check the contexts again to make sure the request is not been sent to a closed dispatcher. + // Never retry on these conditions to prevent unexpected data race. + select { + case <-request.requestCtx.Done(): + return false, request.requestCtx.Err() + case <-request.clientCtx.Done(): + return false, request.clientCtx.Err() + case <-c.ctx.Done(): + return false, c.ctx.Err() + default: } - lastTSOInfo.tsoServer = curTSOInfo.tsoServer - lastTSOInfo.reqKeyspaceGroupID = curTSOInfo.reqKeyspaceGroupID - lastTSOInfo.respKeyspaceGroupID = curTSOInfo.respKeyspaceGroupID - lastTSOInfo.respReceivedAt = curTSOInfo.respReceivedAt - lastTSOInfo.physical = curTSOInfo.physical - lastTSOInfo.logical = curTSOInfo.logical + return false, nil } diff --git a/client/tso_dispatcher.go b/client/tso_dispatcher.go index c82ec777eca9..d5b52ad60390 100644 --- a/client/tso_dispatcher.go +++ b/client/tso_dispatcher.go @@ -22,113 +22,19 @@ import ( "sync" "time" + "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/tikv/pd/client/errs" "github.com/tikv/pd/client/retry" "github.com/tikv/pd/client/timerpool" + "github.com/tikv/pd/client/tsoutil" "go.uber.org/zap" - healthpb "google.golang.org/grpc/health/grpc_health_v1" ) -const ( - tsLoopDCCheckInterval = time.Minute - defaultMaxTSOBatchSize = 10000 // should be higher if client is sending requests in burst - retryInterval = 500 * time.Millisecond - maxRetryTimes = 6 -) - -type tsoDispatcher struct { - dispatcherCancel context.CancelFunc - tsoBatchController *tsoBatchController -} - -func (td *tsoDispatcher) close() { - td.dispatcherCancel() - td.tsoBatchController.clear() -} - -func (td *tsoDispatcher) push(request *tsoRequest) { - td.tsoBatchController.tsoRequestCh <- request -} - -func (c *tsoClient) dispatchRequest(request *tsoRequest) (bool, error) { - dispatcher, ok := c.getTSODispatcher(request.dcLocation) - if !ok { - err := errs.ErrClientGetTSO.FastGenByArgs(fmt.Sprintf("unknown dc-location %s to the client", request.dcLocation)) - log.Error("[tso] dispatch tso request error", zap.String("dc-location", request.dcLocation), errs.ZapError(err)) - c.svcDiscovery.ScheduleCheckMemberChanged() - // New dispatcher could be created in the meantime, which is retryable. - return true, err - } - - defer trace.StartRegion(request.requestCtx, "pdclient.tsoReqEnqueue").End() - select { - case <-request.requestCtx.Done(): - // Caller cancelled the request, no need to retry. - return false, request.requestCtx.Err() - case <-request.clientCtx.Done(): - // Client is closed, no need to retry. - return false, request.clientCtx.Err() - case <-c.ctx.Done(): - // tsoClient is closed due to the PD service mode switch, which is retryable. - return true, c.ctx.Err() - default: - // This failpoint will increase the possibility that the request is sent to a closed dispatcher. - failpoint.Inject("delayDispatchTSORequest", func() { - time.Sleep(time.Second) - }) - dispatcher.push(request) - } - // Check the contexts again to make sure the request is not been sent to a closed dispatcher. - // Never retry on these conditions to prevent unexpected data race. - select { - case <-request.requestCtx.Done(): - return false, request.requestCtx.Err() - case <-request.clientCtx.Done(): - return false, request.clientCtx.Err() - case <-c.ctx.Done(): - return false, c.ctx.Err() - default: - } - return false, nil -} - -func (c *tsoClient) closeTSODispatcher() { - c.tsoDispatcher.Range(func(_, dispatcherInterface any) bool { - if dispatcherInterface != nil { - dispatcherInterface.(*tsoDispatcher).close() - } - return true - }) -} - -func (c *tsoClient) updateTSODispatcher() { - // Set up the new TSO dispatcher and batch controller. - c.GetTSOAllocators().Range(func(dcLocationKey, _ any) bool { - dcLocation := dcLocationKey.(string) - if _, ok := c.getTSODispatcher(dcLocation); !ok { - c.createTSODispatcher(dcLocation) - } - return true - }) - // Clean up the unused TSO dispatcher - c.tsoDispatcher.Range(func(dcLocationKey, dispatcher any) bool { - dcLocation := dcLocationKey.(string) - // Skip the Global TSO Allocator - if dcLocation == globalDCLocation { - return true - } - if _, exist := c.GetTSOAllocators().Load(dcLocation); !exist { - log.Info("[tso] delete unused tso dispatcher", zap.String("dc-location", dcLocation)) - c.tsoDispatcher.Delete(dcLocation) - dispatcher.(*tsoDispatcher).close() - } - return true - }) -} - +// deadline is used to control the TS request timeout manually, +// it will be sent to the `tsDeadlineCh` to be handled by the `watchTSDeadline` goroutine. type deadline struct { timer *time.Timer done chan struct{} @@ -148,173 +54,118 @@ func newTSDeadline( } } -func (c *tsoClient) tsCancelLoop() { - defer c.wg.Done() - - tsCancelLoopCtx, tsCancelLoopCancel := context.WithCancel(c.ctx) - defer tsCancelLoopCancel() - - ticker := time.NewTicker(tsLoopDCCheckInterval) - defer ticker.Stop() - for { - // Watch every dc-location's tsDeadlineCh - c.GetTSOAllocators().Range(func(dcLocation, _ any) bool { - c.watchTSDeadline(tsCancelLoopCtx, dcLocation.(string)) - return true - }) - select { - case <-c.checkTSDeadlineCh: - continue - case <-ticker.C: - continue - case <-tsCancelLoopCtx.Done(): - log.Info("exit tso requests cancel loop") - return - } - } +type tsoInfo struct { + tsoServer string + reqKeyspaceGroupID uint32 + respKeyspaceGroupID uint32 + respReceivedAt time.Time + physical int64 + logical int64 } -func (c *tsoClient) watchTSDeadline(ctx context.Context, dcLocation string) { - if _, exist := c.tsDeadline.Load(dcLocation); !exist { - tsDeadlineCh := make(chan *deadline, 1) - c.tsDeadline.Store(dcLocation, tsDeadlineCh) - go func(dc string, tsDeadlineCh <-chan *deadline) { - for { - select { - case d := <-tsDeadlineCh: - select { - case <-d.timer.C: - log.Error("[tso] tso request is canceled due to timeout", zap.String("dc-location", dc), errs.ZapError(errs.ErrClientGetTSOTimeout)) - d.cancel() - timerpool.GlobalTimerPool.Put(d.timer) - case <-d.done: - timerpool.GlobalTimerPool.Put(d.timer) - case <-ctx.Done(): - timerpool.GlobalTimerPool.Put(d.timer) - return - } - case <-ctx.Done(): - return - } - } - }(dcLocation, tsDeadlineCh) - } +type tsoServiceProvider interface { + getOption() *option + getServiceDiscovery() ServiceDiscovery + updateConnectionCtxs(ctx context.Context, dc string, connectionCtxs *sync.Map) bool } -func (c *tsoClient) tsoDispatcherCheckLoop() { - defer c.wg.Done() +type tsoDispatcher struct { + ctx context.Context + cancel context.CancelFunc + dc string - loopCtx, loopCancel := context.WithCancel(c.ctx) - defer loopCancel() + provider tsoServiceProvider + // URL -> *connectionContext + connectionCtxs *sync.Map + batchController *tsoBatchController + tsDeadlineCh chan *deadline + lastTSOInfo *tsoInfo - ticker := time.NewTicker(tsLoopDCCheckInterval) - defer ticker.Stop() - for { - c.updateTSODispatcher() - select { - case <-ticker.C: - case <-c.checkTSODispatcherCh: - case <-loopCtx.Done(): - log.Info("exit tso dispatcher loop") - return - } - } + updateConnectionCtxsCh chan struct{} } -func (c *tsoClient) checkAllocator( +func newTSODispatcher( ctx context.Context, - forwardCancel context.CancelFunc, - dc, forwardedHostTrim, addr, url string, - updateAndClear func(newAddr string, connectionCtx *tsoConnectionContext)) { - defer func() { - // cancel the forward stream - forwardCancel() - requestForwarded.WithLabelValues(forwardedHostTrim, addr).Set(0) - }() - cc, u := c.GetTSOAllocatorClientConnByDCLocation(dc) - var healthCli healthpb.HealthClient - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - for { - // the pd/allocator leader change, we need to re-establish the stream - if u != url { - log.Info("[tso] the leader of the allocator leader is changed", zap.String("dc", dc), zap.String("origin", url), zap.String("new", u)) - return - } - if healthCli == nil && cc != nil { - healthCli = healthpb.NewHealthClient(cc) - } - if healthCli != nil { - healthCtx, healthCancel := context.WithTimeout(ctx, c.option.timeout) - resp, err := healthCli.Check(healthCtx, &healthpb.HealthCheckRequest{Service: ""}) - failpoint.Inject("unreachableNetwork", func() { - resp.Status = healthpb.HealthCheckResponse_UNKNOWN - }) - healthCancel() - if err == nil && resp.GetStatus() == healthpb.HealthCheckResponse_SERVING { - // create a stream of the original allocator - cctx, cancel := context.WithCancel(ctx) - stream, err := c.tsoStreamBuilderFactory.makeBuilder(cc).build(cctx, cancel, c.option.timeout) - if err == nil && stream != nil { - log.Info("[tso] recover the original tso stream since the network has become normal", zap.String("dc", dc), zap.String("url", url)) - updateAndClear(url, &tsoConnectionContext{url, stream, cctx, cancel}) - return - } - } - } - select { - case <-ctx.Done(): - return - case <-ticker.C: - // To ensure we can get the latest allocator leader - // and once the leader is changed, we can exit this function. - cc, u = c.GetTSOAllocatorClientConnByDCLocation(dc) - } - } -} - -func (c *tsoClient) createTSODispatcher(dcLocation string) { - dispatcherCtx, dispatcherCancel := context.WithCancel(c.ctx) + dc string, + maxBatchSize int, + provider tsoServiceProvider, +) *tsoDispatcher { + dispatcherCtx, dispatcherCancel := context.WithCancel(ctx) tsoBatchController := newTSOBatchController( - make(chan *tsoRequest, defaultMaxTSOBatchSize*2), - defaultMaxTSOBatchSize, + make(chan *tsoRequest, maxBatchSize*2), + maxBatchSize, ) failpoint.Inject("shortDispatcherChannel", func() { tsoBatchController = newTSOBatchController( make(chan *tsoRequest, 1), - defaultMaxTSOBatchSize, + maxBatchSize, ) }) - dispatcher := &tsoDispatcher{dispatcherCancel, tsoBatchController} + td := &tsoDispatcher{ + ctx: dispatcherCtx, + cancel: dispatcherCancel, + dc: dc, + provider: provider, + connectionCtxs: &sync.Map{}, + batchController: tsoBatchController, + tsDeadlineCh: make(chan *deadline, 1), + updateConnectionCtxsCh: make(chan struct{}, 1), + } + go td.watchTSDeadline() + return td +} - if _, ok := c.tsoDispatcher.LoadOrStore(dcLocation, dispatcher); !ok { - // Successfully stored the value. Start the following goroutine. - // Each goroutine is responsible for handling the tso stream request for its dc-location. - // The only case that will make the dispatcher goroutine exit - // is that the loopCtx is done, otherwise there is no circumstance - // this goroutine should exit. - c.wg.Add(1) - go c.handleDispatcher(dispatcherCtx, dcLocation, dispatcher.tsoBatchController) - log.Info("[tso] tso dispatcher created", zap.String("dc-location", dcLocation)) - } else { - dispatcherCancel() +func (td *tsoDispatcher) watchTSDeadline() { + log.Info("[tso] start tso deadline watcher", zap.String("dc-location", td.dc)) + defer log.Info("[tso] exit tso deadline watcher", zap.String("dc-location", td.dc)) + for { + select { + case d := <-td.tsDeadlineCh: + select { + case <-d.timer.C: + log.Error("[tso] tso request is canceled due to timeout", + zap.String("dc-location", td.dc), errs.ZapError(errs.ErrClientGetTSOTimeout)) + d.cancel() + timerpool.GlobalTimerPool.Put(d.timer) + case <-d.done: + timerpool.GlobalTimerPool.Put(d.timer) + case <-td.ctx.Done(): + timerpool.GlobalTimerPool.Put(d.timer) + return + } + case <-td.ctx.Done(): + return + } } } -func (c *tsoClient) handleDispatcher( - ctx context.Context, - dc string, - tbc *tsoBatchController, -) { +func (td *tsoDispatcher) scheduleUpdateConnectionCtxs() { + select { + case td.updateConnectionCtxsCh <- struct{}{}: + default: + } +} + +func (td *tsoDispatcher) close() { + td.cancel() + td.batchController.clear() +} + +func (td *tsoDispatcher) push(request *tsoRequest) { + td.batchController.tsoRequestCh <- request +} + +func (td *tsoDispatcher) handleDispatcher(wg *sync.WaitGroup) { var ( - err error - streamURL string - stream tsoStream - streamCtx context.Context - cancel context.CancelFunc - // url -> connectionContext - connectionCtxs sync.Map + ctx = td.ctx + dc = td.dc + provider = td.provider + svcDiscovery = provider.getServiceDiscovery() + option = provider.getOption() + connectionCtxs = td.connectionCtxs + batchController = td.batchController ) + log.Info("[tso] tso dispatcher created", zap.String("dc-location", dc)) // Clean up the connectionCtxs when the dispatcher exits. defer func() { log.Info("[tso] exit tso dispatcher", zap.String("dc-location", dc)) @@ -324,14 +175,21 @@ func (c *tsoClient) handleDispatcher( return true }) // Clear the tso batch controller. - tbc.clear() - c.wg.Done() + batchController.clear() + wg.Done() }() // Daemon goroutine to update the connectionCtxs periodically and handle the `connectionCtxs` update event. - go c.connectionCtxsUpdater(ctx, dc, &connectionCtxs) + go td.connectionCtxsUpdater() + var ( + err error + streamCtx context.Context + cancel context.CancelFunc + streamURL string + stream tsoStream + ) // Loop through each batch of TSO requests and send them for processing. - streamLoopTimer := time.NewTimer(c.option.timeout) + streamLoopTimer := time.NewTimer(option.timeout) defer streamLoopTimer.Stop() bo := retry.InitialBackoffer(updateMemberBackOffBaseTime, updateMemberTimeout, updateMemberBackOffBaseTime) tsoBatchLoop: @@ -342,12 +200,12 @@ tsoBatchLoop: default: } // Start to collect the TSO requests. - maxBatchWaitInterval := c.option.getMaxTSOBatchWaitInterval() + maxBatchWaitInterval := option.getMaxTSOBatchWaitInterval() // Once the TSO requests are collected, must make sure they could be finished or revoked eventually, // otherwise the upper caller may get blocked on waiting for the results. - if err = tbc.fetchPendingRequests(ctx, maxBatchWaitInterval); err != nil { + if err = batchController.fetchPendingRequests(ctx, maxBatchWaitInterval); err != nil { // Finish the collected requests if the fetch failed. - tbc.finishCollectedRequests(0, 0, 0, errors.WithStack(err)) + batchController.finishCollectedRequests(0, 0, 0, errors.WithStack(err)) if err == context.Canceled { log.Info("[tso] stop fetching the pending tso requests due to context canceled", zap.String("dc-location", dc)) @@ -359,7 +217,7 @@ tsoBatchLoop: return } if maxBatchWaitInterval >= 0 { - tbc.adjustBestBatchSize() + batchController.adjustBestBatchSize() } // Stop the timer if it's not stopped. if !streamLoopTimer.Stop() { @@ -370,33 +228,33 @@ tsoBatchLoop: } // We need be careful here, see more details in the comments of Timer.Reset. // https://pkg.go.dev/time@master#Timer.Reset - streamLoopTimer.Reset(c.option.timeout) + streamLoopTimer.Reset(option.timeout) // Choose a stream to send the TSO gRPC request. streamChoosingLoop: for { - connectionCtx := chooseStream(&connectionCtxs) + connectionCtx := chooseStream(connectionCtxs) if connectionCtx != nil { - streamURL, stream, streamCtx, cancel = connectionCtx.streamURL, connectionCtx.stream, connectionCtx.ctx, connectionCtx.cancel + streamCtx, cancel, streamURL, stream = connectionCtx.ctx, connectionCtx.cancel, connectionCtx.streamURL, connectionCtx.stream } // Check stream and retry if necessary. if stream == nil { log.Info("[tso] tso stream is not ready", zap.String("dc", dc)) - if c.updateTSOConnectionCtxs(ctx, dc, &connectionCtxs) { + if provider.updateConnectionCtxs(ctx, dc, connectionCtxs) { continue streamChoosingLoop } timer := time.NewTimer(retryInterval) select { case <-ctx.Done(): // Finish the collected requests if the context is canceled. - tbc.finishCollectedRequests(0, 0, 0, errors.WithStack(ctx.Err())) + batchController.finishCollectedRequests(0, 0, 0, errors.WithStack(ctx.Err())) timer.Stop() return case <-streamLoopTimer.C: err = errs.ErrClientCreateTSOStream.FastGenByArgs(errs.RetryTimeoutErr) log.Error("[tso] create tso stream error", zap.String("dc-location", dc), errs.ZapError(err)) - c.svcDiscovery.ScheduleCheckMemberChanged() + svcDiscovery.ScheduleCheckMemberChanged() // Finish the collected requests if the stream is failed to be created. - tbc.finishCollectedRequests(0, 0, 0, errors.WithStack(err)) + batchController.finishCollectedRequests(0, 0, 0, errors.WithStack(err)) timer.Stop() continue tsoBatchLoop case <-timer.C: @@ -417,22 +275,16 @@ tsoBatchLoop: } } done := make(chan struct{}) - dl := newTSDeadline(c.option.timeout, done, cancel) - tsDeadlineCh, ok := c.tsDeadline.Load(dc) - for !ok || tsDeadlineCh == nil { - c.scheduleCheckTSDeadline() - time.Sleep(time.Millisecond * 100) - tsDeadlineCh, ok = c.tsDeadline.Load(dc) - } + dl := newTSDeadline(option.timeout, done, cancel) select { case <-ctx.Done(): // Finish the collected requests if the context is canceled. - tbc.finishCollectedRequests(0, 0, 0, errors.WithStack(ctx.Err())) + batchController.finishCollectedRequests(0, 0, 0, errors.WithStack(ctx.Err())) return - case tsDeadlineCh.(chan *deadline) <- dl: + case td.tsDeadlineCh <- dl: } // processRequests guarantees that the collected requests could be finished properly. - err = c.processRequests(stream, dc, tbc) + err = td.processRequests(stream, dc, td.batchController) close(done) // If error happens during tso stream handling, reset stream and run the next trial. if err != nil { @@ -441,7 +293,7 @@ tsoBatchLoop: return default: } - c.svcDiscovery.ScheduleCheckMemberChanged() + svcDiscovery.ScheduleCheckMemberChanged() log.Error("[tso] getTS error after processing requests", zap.String("dc-location", dc), zap.String("stream-url", streamURL), @@ -452,7 +304,7 @@ tsoBatchLoop: stream = nil // Because ScheduleCheckMemberChanged is asynchronous, if the leader changes, we better call `updateMember` ASAP. if IsLeaderChange(err) { - if err := bo.Exec(ctx, c.svcDiscovery.CheckMemberChanged); err != nil { + if err := bo.Exec(ctx, svcDiscovery.CheckMemberChanged); err != nil { select { case <-ctx.Done(): return @@ -460,28 +312,28 @@ tsoBatchLoop: } } // Because the TSO Follower Proxy could be configured online, - // If we change it from on -> off, background updateTSOConnectionCtxs + // If we change it from on -> off, background updateConnectionCtxs // will cancel the current stream, then the EOF error caused by cancel() - // should not trigger the updateTSOConnectionCtxs here. + // should not trigger the updateConnectionCtxs here. // So we should only call it when the leader changes. - c.updateTSOConnectionCtxs(ctx, dc, &connectionCtxs) + provider.updateConnectionCtxs(ctx, dc, connectionCtxs) } } } } -// updateTSOConnectionCtxs updates the `connectionCtxs` for the specified DC location regularly. -// TODO: implement support for the Local TSO. -func (c *tsoClient) connectionCtxsUpdater( - ctx context.Context, - dc string, - connectionCtxs *sync.Map, -) { - if dc != globalDCLocation { - return - } +// updateConnectionCtxs updates the `connectionCtxs` for the specified DC location regularly. +func (td *tsoDispatcher) connectionCtxsUpdater() { + var ( + ctx = td.ctx + dc = td.dc + connectionCtxs = td.connectionCtxs + provider = td.provider + option = td.provider.getOption() + updateTicker = &time.Ticker{} + ) + log.Info("[tso] start tso connection contexts updater", zap.String("dc-location", dc)) - var updateTicker = &time.Ticker{} setNewUpdateTicker := func(ticker *time.Ticker) { if updateTicker.C != nil { updateTicker.Stop() @@ -492,13 +344,17 @@ func (c *tsoClient) connectionCtxsUpdater( defer setNewUpdateTicker(nil) for { - c.updateTSOConnectionCtxs(ctx, dc, connectionCtxs) + provider.updateConnectionCtxs(ctx, dc, connectionCtxs) select { case <-ctx.Done(): log.Info("[tso] exit tso connection contexts updater", zap.String("dc-location", dc)) return - case <-c.option.enableTSOFollowerProxyCh: - enableTSOFollowerProxy := c.option.getEnableTSOFollowerProxy() + case <-option.enableTSOFollowerProxyCh: + // TODO: implement support of TSO Follower Proxy for the Local TSO. + if dc != globalDCLocation { + continue + } + enableTSOFollowerProxy := option.getEnableTSOFollowerProxy() log.Info("[tso] tso follower proxy status changed", zap.String("dc-location", dc), zap.Bool("enable", enableTSOFollowerProxy)) @@ -515,7 +371,7 @@ func (c *tsoClient) connectionCtxsUpdater( } case <-updateTicker.C: // Triggered periodically when the TSO Follower Proxy is enabled. - case <-c.updateTSOConnectionCtxsCh: + case <-td.updateConnectionCtxsCh: // Triggered by the leader/follower change. } } @@ -535,3 +391,94 @@ func chooseStream(connectionCtxs *sync.Map) (connectionCtx *tsoConnectionContext }) return connectionCtx } + +func (td *tsoDispatcher) processRequests( + stream tsoStream, dcLocation string, tbc *tsoBatchController, +) error { + var ( + requests = tbc.getCollectedRequests() + traceRegions = make([]*trace.Region, 0, len(requests)) + spans = make([]opentracing.Span, 0, len(requests)) + ) + for _, req := range requests { + traceRegions = append(traceRegions, trace.StartRegion(req.requestCtx, "pdclient.tsoReqSend")) + if span := opentracing.SpanFromContext(req.requestCtx); span != nil && span.Tracer() != nil { + spans = append(spans, span.Tracer().StartSpan("pdclient.processRequests", opentracing.ChildOf(span.Context()))) + } + } + defer func() { + for i := range spans { + spans[i].Finish() + } + for i := range traceRegions { + traceRegions[i].End() + } + }() + + var ( + count = int64(len(requests)) + svcDiscovery = td.provider.getServiceDiscovery() + clusterID = svcDiscovery.GetClusterID() + keyspaceID = svcDiscovery.GetKeyspaceID() + reqKeyspaceGroupID = svcDiscovery.GetKeyspaceGroupID() + ) + respKeyspaceGroupID, physical, logical, suffixBits, err := stream.processRequests( + clusterID, keyspaceID, reqKeyspaceGroupID, + dcLocation, count, tbc.batchStartTime) + if err != nil { + tbc.finishCollectedRequests(0, 0, 0, err) + return err + } + curTSOInfo := &tsoInfo{ + tsoServer: stream.getServerURL(), + reqKeyspaceGroupID: reqKeyspaceGroupID, + respKeyspaceGroupID: respKeyspaceGroupID, + respReceivedAt: time.Now(), + physical: physical, + logical: logical, + } + // `logical` is the largest ts's logical part here, we need to do the subtracting before we finish each TSO request. + firstLogical := tsoutil.AddLogical(logical, -count+1, suffixBits) + td.compareAndSwapTS(curTSOInfo, firstLogical) + tbc.finishCollectedRequests(physical, firstLogical, suffixBits, nil) + return nil +} + +func (td *tsoDispatcher) compareAndSwapTS( + curTSOInfo *tsoInfo, firstLogical int64, +) { + if td.lastTSOInfo != nil { + var ( + lastTSOInfo = td.lastTSOInfo + dc = td.dc + physical = curTSOInfo.physical + keyspaceID = td.provider.getServiceDiscovery().GetKeyspaceID() + ) + if td.lastTSOInfo.respKeyspaceGroupID != curTSOInfo.respKeyspaceGroupID { + log.Info("[tso] keyspace group changed", + zap.String("dc-location", dc), + zap.Uint32("old-group-id", lastTSOInfo.respKeyspaceGroupID), + zap.Uint32("new-group-id", curTSOInfo.respKeyspaceGroupID)) + } + // The TSO we get is a range like [largestLogical-count+1, largestLogical], so we save the last TSO's largest logical + // to compare with the new TSO's first logical. For example, if we have a TSO resp with logical 10, count 5, then + // all TSOs we get will be [6, 7, 8, 9, 10]. lastTSOInfo.logical stores the logical part of the largest ts returned + // last time. + if tsoutil.TSLessEqual(physical, firstLogical, lastTSOInfo.physical, lastTSOInfo.logical) { + log.Panic("[tso] timestamp fallback", + zap.String("dc-location", dc), + zap.Uint32("keyspace", keyspaceID), + zap.String("last-ts", fmt.Sprintf("(%d, %d)", lastTSOInfo.physical, lastTSOInfo.logical)), + zap.String("cur-ts", fmt.Sprintf("(%d, %d)", physical, firstLogical)), + zap.String("last-tso-server", lastTSOInfo.tsoServer), + zap.String("cur-tso-server", curTSOInfo.tsoServer), + zap.Uint32("last-keyspace-group-in-request", lastTSOInfo.reqKeyspaceGroupID), + zap.Uint32("cur-keyspace-group-in-request", curTSOInfo.reqKeyspaceGroupID), + zap.Uint32("last-keyspace-group-in-response", lastTSOInfo.respKeyspaceGroupID), + zap.Uint32("cur-keyspace-group-in-response", curTSOInfo.respKeyspaceGroupID), + zap.Time("last-response-received-at", lastTSOInfo.respReceivedAt), + zap.Time("cur-response-received-at", curTSOInfo.respReceivedAt)) + } + } + td.lastTSOInfo = curTSOInfo +} diff --git a/client/tso_request.go b/client/tso_request.go index f30ceb5268a6..b912fa354973 100644 --- a/client/tso_request.go +++ b/client/tso_request.go @@ -63,8 +63,8 @@ func (req *tsoRequest) Wait() (physical int64, logical int64, err error) { cmdDurationTSOAsyncWait.Observe(start.Sub(req.start).Seconds()) select { case err = <-req.done: - defer trace.StartRegion(req.requestCtx, "pdclient.tsoReqDone").End() defer req.pool.Put(req) + defer trace.StartRegion(req.requestCtx, "pdclient.tsoReqDone").End() err = errors.WithStack(err) if err != nil { cmdFailDurationTSO.Observe(time.Since(req.start).Seconds()) From f83febabecb98b95b098fd31a3664178f8a6b437 Mon Sep 17 00:00:00 2001 From: ShuNing Date: Mon, 6 May 2024 16:44:37 +0800 Subject: [PATCH 46/50] core: check stats healthy by reference (#8132) ref tikv/pd#7897, close tikv/pd#8131 Signed-off-by: nolouch Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/core/region.go | 47 +++++++++++---- pkg/core/region_test.go | 87 +++++++++++++++++++++++++++- pkg/core/region_tree.go | 26 +++++++++ pkg/core/region_tree_test.go | 1 + pkg/mcs/scheduling/server/cluster.go | 10 ++++ server/cluster/cluster.go | 10 ++++ 6 files changed, 168 insertions(+), 13 deletions(-) diff --git a/pkg/core/region.go b/pkg/core/region.go index 8e3d4e5dec85..b3a9e24428cb 100644 --- a/pkg/core/region.go +++ b/pkg/core/region.go @@ -81,6 +81,8 @@ type RegionInfo struct { buckets unsafe.Pointer // source is used to indicate region's source, such as Storage/Sync/Heartbeat. source RegionSource + // ref is used to indicate the reference count of the region in root-tree and sub-tree. + ref atomic.Int32 } // RegionSource is the source of region. @@ -106,6 +108,21 @@ func (r *RegionInfo) LoadedFromSync() bool { return r.source == Sync } +// IncRef increases the reference count. +func (r *RegionInfo) IncRef() { + r.ref.Add(1) +} + +// DecRef decreases the reference count. +func (r *RegionInfo) DecRef() { + r.ref.Add(-1) +} + +// GetRef returns the reference count. +func (r *RegionInfo) GetRef() int32 { + return r.ref.Load() +} + // NewRegionInfo creates RegionInfo with region's meta and leader peer. func NewRegionInfo(region *metapb.Region, leader *metapb.Peer, opts ...RegionCreateOption) *RegionInfo { regionInfo := &RegionInfo{ @@ -928,7 +945,7 @@ type RegionsInfo struct { // NewRegionsInfo creates RegionsInfo with tree, regions, leaders and followers func NewRegionsInfo() *RegionsInfo { return &RegionsInfo{ - tree: newRegionTree(), + tree: newRegionTreeWithCountRef(), regions: make(map[uint64]*regionItem), subRegions: make(map[uint64]*regionItem), leaders: make(map[uint64]*regionTree), @@ -1117,10 +1134,14 @@ func (r *RegionsInfo) UpdateSubTreeOrderInsensitive(region *RegionInfo) { r.subRegions[region.GetID()] = item // It has been removed and all information needs to be updated again. // Set peers then. - setPeer := func(peersMap map[uint64]*regionTree, storeID uint64, item *regionItem) { + setPeer := func(peersMap map[uint64]*regionTree, storeID uint64, item *regionItem, countRef bool) { store, ok := peersMap[storeID] if !ok { - store = newRegionTree() + if !countRef { + store = newRegionTree() + } else { + store = newRegionTreeWithCountRef() + } peersMap[storeID] = store } store.update(item, false) @@ -1131,17 +1152,17 @@ func (r *RegionsInfo) UpdateSubTreeOrderInsensitive(region *RegionInfo) { storeID := peer.GetStoreId() if peer.GetId() == region.leader.GetId() { // Add leader peer to leaders. - setPeer(r.leaders, storeID, item) + setPeer(r.leaders, storeID, item, true) } else { // Add follower peer to followers. - setPeer(r.followers, storeID, item) + setPeer(r.followers, storeID, item, false) } } setPeers := func(peersMap map[uint64]*regionTree, peers []*metapb.Peer) { for _, peer := range peers { storeID := peer.GetStoreId() - setPeer(peersMap, storeID, item) + setPeer(peersMap, storeID, item, false) } } // Add to learners. @@ -1309,10 +1330,14 @@ func (r *RegionsInfo) UpdateSubTree(region, origin *RegionInfo, overlaps []*Regi r.subRegions[region.GetID()] = item // It has been removed and all information needs to be updated again. // Set peers then. - setPeer := func(peersMap map[uint64]*regionTree, storeID uint64, item *regionItem) { + setPeer := func(peersMap map[uint64]*regionTree, storeID uint64, item *regionItem, countRef bool) { store, ok := peersMap[storeID] if !ok { - store = newRegionTree() + if !countRef { + store = newRegionTree() + } else { + store = newRegionTreeWithCountRef() + } peersMap[storeID] = store } store.update(item, false) @@ -1323,17 +1348,17 @@ func (r *RegionsInfo) UpdateSubTree(region, origin *RegionInfo, overlaps []*Regi storeID := peer.GetStoreId() if peer.GetId() == region.leader.GetId() { // Add leader peer to leaders. - setPeer(r.leaders, storeID, item) + setPeer(r.leaders, storeID, item, true) } else { // Add follower peer to followers. - setPeer(r.followers, storeID, item) + setPeer(r.followers, storeID, item, false) } } setPeers := func(peersMap map[uint64]*regionTree, peers []*metapb.Peer) { for _, peer := range peers { storeID := peer.GetStoreId() - setPeer(peersMap, storeID, item) + setPeer(peersMap, storeID, item, false) } } // Add to learners. diff --git a/pkg/core/region_test.go b/pkg/core/region_test.go index 88683968f3fd..43629fccda0e 100644 --- a/pkg/core/region_test.go +++ b/pkg/core/region_test.go @@ -874,9 +874,10 @@ func TestUpdateRegionEquivalence(t *testing.T) { ctx := ContextTODO() regionsOld.AtomicCheckAndPutRegion(ctx, item) // new way + newItem := item.Clone() ctx = ContextTODO() - regionsNew.CheckAndPutRootTree(ctx, item) - regionsNew.CheckAndPutSubTree(item) + regionsNew.CheckAndPutRootTree(ctx, newItem) + regionsNew.CheckAndPutSubTree(newItem) } checksEquivalence := func() { re.Equal(regionsOld.GetRegionCount([]byte(""), []byte("")), regionsNew.GetRegionCount([]byte(""), []byte(""))) @@ -884,6 +885,13 @@ func TestUpdateRegionEquivalence(t *testing.T) { checkRegions(re, regionsOld) checkRegions(re, regionsNew) + for _, r := range regionsOld.GetRegions() { + re.Equal(int32(2), r.GetRef(), fmt.Sprintf("inconsistent region %d", r.GetID())) + } + for _, r := range regionsNew.GetRegions() { + re.Equal(int32(2), r.GetRef(), fmt.Sprintf("inconsistent region %d", r.GetID())) + } + for i := 1; i <= storeNums; i++ { re.Equal(regionsOld.GetStoreRegionCount(uint64(i)), regionsNew.GetStoreRegionCount(uint64(i))) re.Equal(regionsOld.GetStoreLeaderCount(uint64(i)), regionsNew.GetStoreLeaderCount(uint64(i))) @@ -938,3 +946,78 @@ func generateTestRegions(count int, storeNum int) []*RegionInfo { } return items } + +func TestUpdateRegionEventualConsistency(t *testing.T) { + re := require.New(t) + regionsOld := NewRegionsInfo() + regionsNew := NewRegionsInfo() + i := 1 + storeNum := 5 + peer1 := &metapb.Peer{StoreId: uint64(i%storeNum + 1), Id: uint64(i*storeNum + 1)} + peer2 := &metapb.Peer{StoreId: uint64((i+1)%storeNum + 1), Id: uint64(i*storeNum + 2)} + peer3 := &metapb.Peer{StoreId: uint64((i+2)%storeNum + 1), Id: uint64(i*storeNum + 3)} + item := NewRegionInfo(&metapb.Region{ + Id: uint64(i + 1), + Peers: []*metapb.Peer{peer1, peer2, peer3}, + StartKey: []byte(fmt.Sprintf("%20d", i*10)), + EndKey: []byte(fmt.Sprintf("%20d", (i+1)*10)), + RegionEpoch: &metapb.RegionEpoch{ConfVer: 100, Version: 100}, + }, + peer1, + SetApproximateKeys(10), + SetApproximateSize(10), + ) + regionItemA := item + regionPendingItemA := regionItemA.Clone(WithPendingPeers([]*metapb.Peer{peer3})) + + regionItemB := regionItemA.Clone() + regionPendingItemB := regionItemB.Clone(WithPendingPeers([]*metapb.Peer{peer3})) + regionGuide := GenerateRegionGuideFunc(true) + + // Old way + { + ctx := ContextTODO() + regionsOld.AtomicCheckAndPutRegion(ctx, regionPendingItemA) + re.Equal(int32(2), regionPendingItemA.GetRef()) + // check new item + saveKV, saveCache, needSync := regionGuide(ctx, regionItemA, regionPendingItemA) + re.True(needSync) + re.True(saveCache) + re.False(saveKV) + // update cache + regionsOld.AtomicCheckAndPutRegion(ctx, regionItemA) + re.Equal(int32(2), regionItemA.GetRef()) + } + + // New way + { + // root tree part in order, and updated in order, updated regionPendingItemB first, then regionItemB + ctx := ContextTODO() + regionsNew.CheckAndPutRootTree(ctx, regionPendingItemB) + re.Equal(int32(1), regionPendingItemB.GetRef()) + ctx = ContextTODO() + regionsNew.CheckAndPutRootTree(ctx, regionItemB) + re.Equal(int32(1), regionItemB.GetRef()) + re.Equal(int32(0), regionPendingItemB.GetRef()) + + // subtree part missing order, updated regionItemB first, then regionPendingItemB + regionsNew.CheckAndPutSubTree(regionItemB) + re.Equal(int32(2), regionItemB.GetRef()) + re.Equal(int32(0), regionPendingItemB.GetRef()) + regionsNew.UpdateSubTreeOrderInsensitive(regionPendingItemB) + re.Equal(int32(1), regionItemB.GetRef()) + re.Equal(int32(1), regionPendingItemB.GetRef()) + + // heartbeat again, no need updates root tree + saveKV, saveCache, needSync := regionGuide(ctx, regionItemB, regionItemB) + re.False(needSync) + re.False(saveCache) + re.False(saveKV) + + // but need update sub tree again + item := regionsNew.GetRegion(regionItemB.GetID()) + re.Equal(int32(1), item.GetRef()) + regionsNew.CheckAndPutSubTree(item) + re.Equal(int32(2), item.GetRef()) + } +} diff --git a/pkg/core/region_tree.go b/pkg/core/region_tree.go index 8c928f391eb3..6c3c71c51588 100644 --- a/pkg/core/region_tree.go +++ b/pkg/core/region_tree.go @@ -69,6 +69,8 @@ type regionTree struct { totalWriteKeysRate float64 // count the number of regions that not loaded from storage. notFromStorageRegionsCnt int + // count reference of RegionInfo + countRef bool } func newRegionTree() *regionTree { @@ -81,6 +83,17 @@ func newRegionTree() *regionTree { } } +func newRegionTreeWithCountRef() *regionTree { + return ®ionTree{ + tree: btree.NewG[*regionItem](defaultBTreeDegree), + totalSize: 0, + totalWriteBytesRate: 0, + totalWriteKeysRate: 0, + notFromStorageRegionsCnt: 0, + countRef: true, + } +} + func (t *regionTree) length() int { if t == nil { return 0 @@ -140,6 +153,9 @@ func (t *regionTree) update(item *regionItem, withOverlaps bool, overlaps ...*re t.tree.Delete(old) } t.tree.ReplaceOrInsert(item) + if t.countRef { + item.RegionInfo.IncRef() + } result := make([]*RegionInfo, len(overlaps)) for i, overlap := range overlaps { old := overlap.RegionInfo @@ -155,6 +171,9 @@ func (t *regionTree) update(item *regionItem, withOverlaps bool, overlaps ...*re if !old.LoadedFromStorage() { t.notFromStorageRegionsCnt-- } + if t.countRef { + old.DecRef() + } } return result @@ -180,6 +199,10 @@ func (t *regionTree) updateStat(origin *RegionInfo, region *RegionInfo) { if !origin.LoadedFromStorage() && region.LoadedFromStorage() { t.notFromStorageRegionsCnt-- } + if t.countRef { + origin.DecRef() + region.IncRef() + } } // remove removes a region if the region is in the tree. @@ -199,6 +222,9 @@ func (t *regionTree) remove(region *RegionInfo) { regionWriteBytesRate, regionWriteKeysRate := result.GetWriteRate() t.totalWriteBytesRate -= regionWriteBytesRate t.totalWriteKeysRate -= regionWriteKeysRate + if t.countRef { + result.RegionInfo.DecRef() + } if !region.LoadedFromStorage() { t.notFromStorageRegionsCnt-- } diff --git a/pkg/core/region_tree_test.go b/pkg/core/region_tree_test.go index f4ef6cb67b3b..3f2ca0c1fb8f 100644 --- a/pkg/core/region_tree_test.go +++ b/pkg/core/region_tree_test.go @@ -159,6 +159,7 @@ func TestRegionTree(t *testing.T) { updateNewItem(tree, regionA) updateNewItem(tree, regionC) re.Nil(tree.overlaps(newRegionItem([]byte("b"), []byte("c")))) + re.Equal(regionC, tree.overlaps(newRegionItem([]byte("c"), []byte("d")))[0].RegionInfo) re.Equal(regionC, tree.overlaps(newRegionItem([]byte("a"), []byte("cc")))[1].RegionInfo) re.Nil(tree.search([]byte{})) re.Equal(regionA, tree.search([]byte("a"))) diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index 94b24f4ca16e..6e6df8e3775f 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -615,6 +615,16 @@ func (c *Cluster) processRegionHeartbeat(ctx *core.MetaProcessContext, region *c }, ) } + // region is not updated to the subtree. + if origin.GetRef() < 2 { + ctx.TaskRunner.RunTask( + ctx, + core.ExtraTaskOpts(ctx, core.UpdateSubTree), + func(_ context.Context) { + c.CheckAndPutSubTree(region) + }, + ) + } return nil } tracer.OnSaveCacheBegin() diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 75301664b509..ae3284e26945 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -1044,6 +1044,16 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio }, ) } + // region is not updated to the subtree. + if origin.GetRef() < 2 { + ctx.TaskRunner.RunTask( + ctx, + core.ExtraTaskOpts(ctx, core.UpdateSubTree), + func(_ context.Context) { + c.CheckAndPutSubTree(region) + }, + ) + } return nil } failpoint.Inject("concurrentRegionHeartbeat", func() { From b41c897f34a8c188389f6ae2f8e5043b10986bff Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 7 May 2024 10:56:36 +0800 Subject: [PATCH 47/50] *: make `TestScheduler` stable (#8093) close tikv/pd#7062 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- .../pd-ctl/tests/scheduler/scheduler_test.go | 485 +++++++++++------- 1 file changed, 289 insertions(+), 196 deletions(-) diff --git a/tools/pd-ctl/tests/scheduler/scheduler_test.go b/tools/pd-ctl/tests/scheduler/scheduler_test.go index 00fab12b99b0..96a1f5557f9f 100644 --- a/tools/pd-ctl/tests/scheduler/scheduler_test.go +++ b/tools/pd-ctl/tests/scheduler/scheduler_test.go @@ -136,26 +136,6 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { re.Contains(string(output), "Usage") } - checkSchedulerCommand := func(args []string, expected map[string]bool) { - if args != nil { - echo := mustExec(re, cmd, args, nil) - re.Contains(echo, "Success!") - } - testutil.Eventually(re, func() bool { - var schedulers []string - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, &schedulers) - if len(schedulers) != len(expected) { - return false - } - for _, scheduler := range schedulers { - if _, ok := expected[scheduler]; !ok { - return false - } - } - return true - }) - } - checkSchedulerConfigCommand := func(expectedConfig map[string]any, schedulerName string) { testutil.Eventually(re, func() bool { configInfo := make(map[string]any) @@ -179,7 +159,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { "balance-hot-region-scheduler": true, "evict-slow-store-scheduler": true, } - checkSchedulerCommand(nil, expected) + checkSchedulerCommand(re, cmd, pdAddr, nil, expected) // scheduler delete command args := []string{"-u", pdAddr, "scheduler", "remove", "balance-region-scheduler"} @@ -188,7 +168,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { "balance-hot-region-scheduler": true, "evict-slow-store-scheduler": true, } - checkSchedulerCommand(args, expected) + checkSchedulerCommand(re, cmd, pdAddr, args, expected) // avoid the influence of the scheduler order schedulers := []string{"evict-leader-scheduler", "grant-leader-scheduler", "evict-leader-scheduler", "grant-leader-scheduler"} @@ -232,7 +212,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { schedulers[idx]: true, "evict-slow-store-scheduler": true, } - checkSchedulerCommand(args, expected) + checkSchedulerCommand(re, cmd, pdAddr, args, expected) // scheduler config show command expectedConfig := make(map[string]any) @@ -250,7 +230,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { } // check update success - checkSchedulerCommand(args, expected) + checkSchedulerCommand(re, cmd, pdAddr, args, expected) expectedConfig["store-id-ranges"] = map[string]any{"2": []any{map[string]any{"end-key": "", "start-key": ""}}, "3": []any{map[string]any{"end-key": "", "start-key": ""}}} checkSchedulerConfigCommand(expectedConfig, schedulers[idx]) checkStorePause([]uint64{2, 3}, schedulers[idx]) @@ -262,7 +242,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { "balance-hot-region-scheduler": true, "evict-slow-store-scheduler": true, } - checkSchedulerCommand(args, expected) + checkSchedulerCommand(re, cmd, pdAddr, args, expected) checkStorePause([]uint64{}, schedulers[idx]) // scheduler add command @@ -273,7 +253,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { schedulers[idx]: true, "evict-slow-store-scheduler": true, } - checkSchedulerCommand(args, expected) + checkSchedulerCommand(re, cmd, pdAddr, args, expected) checkStorePause([]uint64{2}, schedulers[idx]) // scheduler add command twice @@ -284,7 +264,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { schedulers[idx]: true, "evict-slow-store-scheduler": true, } - checkSchedulerCommand(args, expected) + checkSchedulerCommand(re, cmd, pdAddr, args, expected) // check add success expectedConfig["store-id-ranges"] = map[string]any{"2": []any{map[string]any{"end-key": "", "start-key": ""}}, "4": []any{map[string]any{"end-key": "", "start-key": ""}}} @@ -299,7 +279,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { schedulers[idx]: true, "evict-slow-store-scheduler": true, } - checkSchedulerCommand(args, expected) + checkSchedulerCommand(re, cmd, pdAddr, args, expected) // check remove success expectedConfig["store-id-ranges"] = map[string]any{"2": []any{map[string]any{"end-key": "", "start-key": ""}}} @@ -313,16 +293,175 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { "balance-hot-region-scheduler": true, "evict-slow-store-scheduler": true, } - checkSchedulerCommand(args, expected) + checkSchedulerCommand(re, cmd, pdAddr, args, expected) checkStorePause([]uint64{}, schedulers[idx]) } + // test remove and add scheduler + echo := mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-region-scheduler"}, nil) + re.Contains(echo, "Success!") + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-region-scheduler"}, nil) + re.Contains(echo, "Success!") + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-region-scheduler"}, nil) + re.NotContains(echo, "Success!") + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-region-scheduler"}, nil) + re.Contains(echo, "Success!") + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "evict-leader-scheduler", "1"}, nil) + re.Contains(echo, "Success! The scheduler is created.") + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "evict-leader-scheduler", "2"}, nil) + re.Contains(echo, "Success! The scheduler has been applied to the store.") + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "evict-leader-scheduler-1"}, nil) + re.Contains(echo, "Success!") + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "evict-leader-scheduler-2"}, nil) + re.Contains(echo, "Success!") + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "evict-leader-scheduler-1"}, nil) + re.Contains(echo, "404") + testutil.Eventually(re, func() bool { // wait for removed scheduler to be synced to scheduling server. + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "evict-leader-scheduler"}, nil) + return strings.Contains(echo, "[404] scheduler not found") + }) + + // test remove and add + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-hot-region-scheduler"}, nil) + re.Contains(echo, "Success") + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-hot-region-scheduler"}, nil) + re.Contains(echo, "Success") + + // test show scheduler with paused and disabled status. + checkSchedulerWithStatusCommand := func(status string, expected []string) { + testutil.Eventually(re, func() bool { + var schedulers []string + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show", "--status", status}, &schedulers) + return reflect.DeepEqual(expected, schedulers) + }) + } + + // test scatter range scheduler + for _, name := range []string{ + "test", "test#", "?test", + /* TODO: to handle case like "tes&t", we need to modify the server's JSON render to unescape the HTML characters */ + } { + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "scatter-range", "--format=raw", "a", "b", name}, nil) + re.Contains(echo, "Success!") + schedulerName := fmt.Sprintf("scatter-range-%s", name) + // test show scheduler + testutil.Eventually(re, func() bool { + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) + return strings.Contains(echo, schedulerName) + }) + // test remove scheduler + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", schedulerName}, nil) + re.Contains(echo, "Success!") + testutil.Eventually(re, func() bool { + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) + return !strings.Contains(echo, schedulerName) + }) + } + + mustUsage([]string{"-u", pdAddr, "scheduler", "pause", "balance-leader-scheduler"}) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "pause", "balance-leader-scheduler", "60"}, nil) + re.Contains(echo, "Success!") + checkSchedulerWithStatusCommand("paused", []string{ + "balance-leader-scheduler", + }) + result := make(map[string]any) + testutil.Eventually(re, func() bool { + mightExec(re, cmd, []string{"-u", pdAddr, "scheduler", "describe", "balance-leader-scheduler"}, &result) + return len(result) != 0 && result["status"] == "paused" && result["summary"] == "" + }, testutil.WithWaitFor(30*time.Second)) + + mustUsage([]string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler", "60"}) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler"}, nil) + re.Contains(echo, "Success!") + checkSchedulerWithStatusCommand("paused", []string{}) + + // set label scheduler to disabled manually. + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "label-scheduler"}, nil) + re.Contains(echo, "Success!") + cfg := leaderServer.GetServer().GetScheduleConfig() + origin := cfg.Schedulers + cfg.Schedulers = sc.SchedulerConfigs{{Type: "label", Disable: true}} + err := leaderServer.GetServer().SetScheduleConfig(*cfg) + re.NoError(err) + checkSchedulerWithStatusCommand("disabled", []string{"label-scheduler"}) + // reset Schedulers in ScheduleConfig + cfg.Schedulers = origin + err = leaderServer.GetServer().SetScheduleConfig(*cfg) + re.NoError(err) + checkSchedulerWithStatusCommand("disabled", []string{}) +} + +func (suite *schedulerTestSuite) TestSchedulerConfig() { + suite.env.RunTestInTwoModes(suite.checkSchedulerConfig) +} + +func (suite *schedulerTestSuite) checkSchedulerConfig(cluster *pdTests.TestCluster) { + re := suite.Require() + pdAddr := cluster.GetConfig().GetClientURL() + cmd := ctl.GetRootCmd() + + stores := []*metapb.Store{ + { + Id: 1, + State: metapb.StoreState_Up, + LastHeartbeat: time.Now().UnixNano(), + }, + { + Id: 2, + State: metapb.StoreState_Up, + LastHeartbeat: time.Now().UnixNano(), + }, + { + Id: 3, + State: metapb.StoreState_Up, + LastHeartbeat: time.Now().UnixNano(), + }, + { + Id: 4, + State: metapb.StoreState_Up, + LastHeartbeat: time.Now().UnixNano(), + }, + } + for _, store := range stores { + pdTests.MustPutStore(re, cluster, store) + } + + // note: because pdqsort is an unstable sort algorithm, set ApproximateSize for this region. + pdTests.MustPutRegion(re, cluster, 1, 1, []byte("a"), []byte("b"), core.SetApproximateSize(10)) + + // test evict-slow-store && evict-slow-trend schedulers config + evictSlownessSchedulers := []string{"evict-slow-store-scheduler", "evict-slow-trend-scheduler"} + for _, schedulerName := range evictSlownessSchedulers { + echo := mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", schedulerName}, nil) + if strings.Contains(echo, "Success!") { + re.Contains(echo, "Success!") + } else { + re.Contains(echo, "scheduler existed") + } + testutil.Eventually(re, func() bool { + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) + return strings.Contains(echo, schedulerName) + }) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", schedulerName, "set", "recovery-duration", "100"}, nil) + re.Contains(echo, "Success! Config updated.") + conf := make(map[string]any) + testutil.Eventually(re, func() bool { + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", schedulerName, "show"}, &conf) + return conf["recovery-duration"] == 100. + }) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", schedulerName}, nil) + re.Contains(echo, "Success!") + testutil.Eventually(re, func() bool { + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) + return !strings.Contains(echo, schedulerName) + }) + } // test shuffle region config - checkSchedulerCommand([]string{"-u", pdAddr, "scheduler", "add", "shuffle-region-scheduler"}, map[string]bool{ + checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "add", "shuffle-region-scheduler"}, map[string]bool{ + "balance-region-scheduler": true, "balance-leader-scheduler": true, "balance-hot-region-scheduler": true, "shuffle-region-scheduler": true, - "evict-slow-store-scheduler": true, }) var roles []string mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "show-roles"}, &roles) @@ -336,13 +475,18 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler"}, &roles) re.Equal([]string{"learner"}, roles) + checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "remove", "shuffle-region-scheduler"}, map[string]bool{ + "balance-region-scheduler": true, + "balance-leader-scheduler": true, + "balance-hot-region-scheduler": true, + }) + // test grant hot region scheduler config - checkSchedulerCommand([]string{"-u", pdAddr, "scheduler", "add", "grant-hot-region-scheduler", "1", "1,2,3"}, map[string]bool{ + checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "add", "grant-hot-region-scheduler", "1", "1,2,3"}, map[string]bool{ + "balance-region-scheduler": true, "balance-leader-scheduler": true, "balance-hot-region-scheduler": true, - "shuffle-region-scheduler": true, "grant-hot-region-scheduler": true, - "evict-slow-store-scheduler": true, }) var conf3 map[string]any expected3 := map[string]any{ @@ -359,31 +503,97 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "grant-hot-region-scheduler"}, &conf3) return reflect.DeepEqual(expected3, conf3) }) + checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "remove", "grant-hot-region-scheduler"}, map[string]bool{ + "balance-region-scheduler": true, + "balance-leader-scheduler": true, + "balance-hot-region-scheduler": true, + }) - // test remove and add scheduler - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-region-scheduler"}, nil) + // test shuffle hot region scheduler + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "shuffle-hot-region-scheduler"}, nil) re.Contains(echo, "Success!") - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-region-scheduler"}, nil) + testutil.Eventually(re, func() bool { + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) + return strings.Contains(echo, "shuffle-hot-region-scheduler") + }) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-hot-region-scheduler", "set", "limit", "127"}, nil) re.Contains(echo, "Success!") - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-region-scheduler"}, nil) - re.NotContains(echo, "Success!") - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-region-scheduler"}, nil) + conf := make(map[string]any) + testutil.Eventually(re, func() bool { + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-hot-region-scheduler", "show"}, &conf) + return conf["limit"] == 127. + }) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "shuffle-hot-region-scheduler"}, nil) re.Contains(echo, "Success!") - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "evict-leader-scheduler", "1"}, nil) - re.Contains(echo, "Success! The scheduler is created.") - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "evict-leader-scheduler", "2"}, nil) - re.Contains(echo, "Success! The scheduler has been applied to the store.") - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "evict-leader-scheduler-1"}, nil) + testutil.Eventually(re, func() bool { + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) + return !strings.Contains(echo, "shuffle-hot-region-scheduler") + }) + + // test balance leader config + conf = make(map[string]any) + conf1 := make(map[string]any) + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler", "show"}, &conf) + re.Equal(4., conf["batch"]) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler", "set", "batch", "3"}, nil) re.Contains(echo, "Success!") - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "evict-leader-scheduler-2"}, nil) + testutil.Eventually(re, func() bool { + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler"}, &conf1) + return conf1["batch"] == 3. + }) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-leader-scheduler"}, nil) + re.NotContains(echo, "Success!") + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-leader-scheduler"}, nil) re.Contains(echo, "Success!") - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "evict-leader-scheduler-1"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-leader-scheduler"}, nil) re.Contains(echo, "404") - testutil.Eventually(re, func() bool { // wait for removed scheduler to be synced to scheduling server. - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "evict-leader-scheduler"}, nil) - return strings.Contains(echo, "[404] scheduler not found") + re.Contains(echo, "PD:scheduler:ErrSchedulerNotFound]scheduler not found") + // The scheduling service need time to sync from PD. + testutil.Eventually(re, func() bool { + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler"}, nil) + return strings.Contains(echo, "404") && strings.Contains(echo, "scheduler not found") }) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-leader-scheduler"}, nil) + re.Contains(echo, "Success!") +} + +func (suite *schedulerTestSuite) TestHotRegionSchedulerConfig() { + suite.env.RunTestInTwoModes(suite.checkHotRegionSchedulerConfig) +} +func (suite *schedulerTestSuite) checkHotRegionSchedulerConfig(cluster *pdTests.TestCluster) { + re := suite.Require() + pdAddr := cluster.GetConfig().GetClientURL() + cmd := ctl.GetRootCmd() + + stores := []*metapb.Store{ + { + Id: 1, + State: metapb.StoreState_Up, + LastHeartbeat: time.Now().UnixNano(), + }, + { + Id: 2, + State: metapb.StoreState_Up, + LastHeartbeat: time.Now().UnixNano(), + }, + { + Id: 3, + State: metapb.StoreState_Up, + LastHeartbeat: time.Now().UnixNano(), + }, + { + Id: 4, + State: metapb.StoreState_Up, + LastHeartbeat: time.Now().UnixNano(), + }, + } + for _, store := range stores { + pdTests.MustPutStore(re, cluster, store) + } + // note: because pdqsort is an unstable sort algorithm, set ApproximateSize for this region. + pdTests.MustPutRegion(re, cluster, 1, 1, []byte("a"), []byte("b"), core.SetApproximateSize(10)) + leaderServer := cluster.GetLeaderServer() // test hot region config expected1 := map[string]any{ "min-hot-byte-rate": float64(100), @@ -407,13 +617,20 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { return reflect.DeepEqual(expect, conf1) }) } - + // scheduler show command + expected := map[string]bool{ + "balance-region-scheduler": true, + "balance-leader-scheduler": true, + "balance-hot-region-scheduler": true, + "evict-slow-store-scheduler": true, + } + checkSchedulerCommand(re, cmd, pdAddr, nil, expected) var conf map[string]any mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "list"}, &conf) re.Equal(expected1, conf) mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "show"}, &conf) re.Equal(expected1, conf) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "src-tolerance-ratio", "1.02"}, nil) + echo := mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "src-tolerance-ratio", "1.02"}, nil) re.Contains(echo, "Success!") expected1["src-tolerance-ratio"] = 1.02 checkHotSchedulerConfig(expected1) @@ -505,150 +722,6 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "write-peer-priorities", "query,byte"}, nil) re.Contains(echo, "query is not allowed to be set in priorities for write-peer-priorities") checkHotSchedulerConfig(expected1) - - // test remove and add - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-hot-region-scheduler"}, nil) - re.Contains(echo, "Success") - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-hot-region-scheduler"}, nil) - re.Contains(echo, "Success") - - // test balance leader config - conf = make(map[string]any) - conf1 := make(map[string]any) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler", "show"}, &conf) - re.Equal(4., conf["batch"]) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler", "set", "batch", "3"}, nil) - re.Contains(echo, "Success!") - testutil.Eventually(re, func() bool { - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler"}, &conf1) - return conf1["batch"] == 3. - }) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-leader-scheduler"}, nil) - re.NotContains(echo, "Success!") - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-leader-scheduler"}, nil) - re.Contains(echo, "Success!") - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-leader-scheduler"}, nil) - re.Contains(echo, "404") - re.Contains(echo, "PD:scheduler:ErrSchedulerNotFound]scheduler not found") - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler"}, nil) - re.Contains(echo, "404") - re.Contains(echo, "scheduler not found") - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-leader-scheduler"}, nil) - re.Contains(echo, "Success!") - - // test evict-slow-store && evict-slow-trend schedulers config - evictSlownessSchedulers := []string{"evict-slow-store-scheduler", "evict-slow-trend-scheduler"} - for _, schedulerName := range evictSlownessSchedulers { - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", schedulerName}, nil) - if strings.Contains(echo, "Success!") { - re.Contains(echo, "Success!") - } else { - re.Contains(echo, "scheduler existed") - } - testutil.Eventually(re, func() bool { - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) - return strings.Contains(echo, schedulerName) - }) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", schedulerName, "set", "recovery-duration", "100"}, nil) - re.Contains(echo, "Success! Config updated.") - conf = make(map[string]any) - testutil.Eventually(re, func() bool { - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", schedulerName, "show"}, &conf) - return conf["recovery-duration"] == 100. - }) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", schedulerName}, nil) - re.Contains(echo, "Success!") - testutil.Eventually(re, func() bool { - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) - return !strings.Contains(echo, schedulerName) - }) - } - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "evict-slow-store-scheduler"}, nil) - re.Contains(echo, "Success!") - - // test shuffle hot region scheduler - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "shuffle-hot-region-scheduler"}, nil) - re.Contains(echo, "Success!") - testutil.Eventually(re, func() bool { - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) - return strings.Contains(echo, "shuffle-hot-region-scheduler") - }) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-hot-region-scheduler", "set", "limit", "127"}, nil) - re.Contains(echo, "Success!") - conf = make(map[string]any) - testutil.Eventually(re, func() bool { - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-hot-region-scheduler", "show"}, &conf) - return conf["limit"] == 127. - }) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "shuffle-hot-region-scheduler"}, nil) - re.Contains(echo, "Success!") - testutil.Eventually(re, func() bool { - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) - return !strings.Contains(echo, "shuffle-hot-region-scheduler") - }) - - // test show scheduler with paused and disabled status. - checkSchedulerWithStatusCommand := func(status string, expected []string) { - testutil.Eventually(re, func() bool { - var schedulers []string - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show", "--status", status}, &schedulers) - return reflect.DeepEqual(expected, schedulers) - }) - } - - // test scatter range scheduler - for _, name := range []string{ - "test", "test#", "?test", - /* TODO: to handle case like "tes&t", we need to modify the server's JSON render to unescape the HTML characters */ - } { - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "scatter-range", "--format=raw", "a", "b", name}, nil) - re.Contains(echo, "Success!") - schedulerName := fmt.Sprintf("scatter-range-%s", name) - // test show scheduler - testutil.Eventually(re, func() bool { - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) - return strings.Contains(echo, schedulerName) - }) - // test remove scheduler - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", schedulerName}, nil) - re.Contains(echo, "Success!") - testutil.Eventually(re, func() bool { - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) - return !strings.Contains(echo, schedulerName) - }) - } - - mustUsage([]string{"-u", pdAddr, "scheduler", "pause", "balance-leader-scheduler"}) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "pause", "balance-leader-scheduler", "60"}, nil) - re.Contains(echo, "Success!") - checkSchedulerWithStatusCommand("paused", []string{ - "balance-leader-scheduler", - }) - result := make(map[string]any) - testutil.Eventually(re, func() bool { - mightExec(re, cmd, []string{"-u", pdAddr, "scheduler", "describe", "balance-leader-scheduler"}, &result) - return len(result) != 0 && result["status"] == "paused" && result["summary"] == "" - }, testutil.WithWaitFor(30*time.Second)) - - mustUsage([]string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler", "60"}) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler"}, nil) - re.Contains(echo, "Success!") - checkSchedulerWithStatusCommand("paused", []string{}) - - // set label scheduler to disabled manually. - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "label-scheduler"}, nil) - re.Contains(echo, "Success!") - cfg := leaderServer.GetServer().GetScheduleConfig() - origin := cfg.Schedulers - cfg.Schedulers = sc.SchedulerConfigs{{Type: "label", Disable: true}} - err := leaderServer.GetServer().SetScheduleConfig(*cfg) - re.NoError(err) - checkSchedulerWithStatusCommand("disabled", []string{"label-scheduler"}) - // reset Schedulers in ScheduleConfig - cfg.Schedulers = origin - err = leaderServer.GetServer().SetScheduleConfig(*cfg) - re.NoError(err) - checkSchedulerWithStatusCommand("disabled", []string{}) } func (suite *schedulerTestSuite) TestSchedulerDiagnostic() { @@ -731,3 +804,23 @@ func mightExec(re *require.Assertions, cmd *cobra.Command, args []string, v any) } json.Unmarshal(output, v) } + +func checkSchedulerCommand(re *require.Assertions, cmd *cobra.Command, pdAddr string, args []string, expected map[string]bool) { + if args != nil { + echo := mustExec(re, cmd, args, nil) + re.Contains(echo, "Success!") + } + testutil.Eventually(re, func() bool { + var schedulers []string + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, &schedulers) + if len(schedulers) != len(expected) { + return false + } + for _, scheduler := range schedulers { + if _, ok := expected[scheduler]; !ok { + return false + } + } + return true + }) +} From a3c5950c30786a0c4d0d44bd30155117cf55f1cd Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 7 May 2024 11:32:37 +0800 Subject: [PATCH 48/50] *: split into multiple runner for heartbeat (#8130) ref tikv/pd#7897 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/core/context.go | 3 +- pkg/core/region.go | 37 ++------- pkg/mcs/scheduling/server/cluster.go | 40 +++++---- pkg/ratelimit/runner.go | 118 ++++++++++++++++----------- pkg/ratelimit/runner_test.go | 36 ++++---- server/cluster/cluster.go | 45 +++++----- server/cluster/cluster_worker.go | 13 +-- 7 files changed, 151 insertions(+), 141 deletions(-) diff --git a/pkg/core/context.go b/pkg/core/context.go index ab149378b1d1..a0f51e556806 100644 --- a/pkg/core/context.go +++ b/pkg/core/context.go @@ -25,7 +25,7 @@ type MetaProcessContext struct { context.Context Tracer RegionHeartbeatProcessTracer TaskRunner ratelimit.Runner - Limiter *ratelimit.ConcurrencyLimiter + LogRunner ratelimit.Runner } // NewMetaProcessContext creates a new MetaProcessContext. @@ -35,6 +35,7 @@ func ContextTODO() *MetaProcessContext { Context: context.TODO(), Tracer: NewNoopHeartbeatProcessTracer(), TaskRunner: ratelimit.NewSyncRunner(), + LogRunner: ratelimit.NewSyncRunner(), // Limit default is nil } } diff --git a/pkg/core/region.go b/pkg/core/region.go index b3a9e24428cb..be8f392f05e2 100644 --- a/pkg/core/region.go +++ b/pkg/core/region.go @@ -744,33 +744,26 @@ func GenerateRegionGuideFunc(enableLog bool) RegionGuideFunc { // Save to storage if meta is updated. // Save to cache if meta or leader is updated, or contains any down/pending peer. return func(ctx *MetaProcessContext, region, origin *RegionInfo) (saveKV, saveCache, needSync bool) { - taskRunner := ctx.TaskRunner - limiter := ctx.Limiter + logRunner := ctx.LogRunner // print log asynchronously debug, info := d, i - if taskRunner != nil { + if logRunner != nil { debug = func(msg string, fields ...zap.Field) { - taskRunner.RunTask( + logRunner.RunTask( ctx.Context, - ratelimit.TaskOpts{ - TaskName: "Log", - Limit: limiter, - }, func(_ context.Context) { d(msg, fields...) }, + ratelimit.WithTaskName("DebugLog"), ) } info = func(msg string, fields ...zap.Field) { - taskRunner.RunTask( + logRunner.RunTask( ctx.Context, - ratelimit.TaskOpts{ - TaskName: "Log", - Limit: limiter, - }, func(_ context.Context) { i(msg, fields...) }, + ratelimit.WithTaskName("InfoLog"), ) } } @@ -873,24 +866,6 @@ func GenerateRegionGuideFunc(enableLog bool) RegionGuideFunc { } } -// RegionHeartbeatStageName is the name of the stage of the region heartbeat. -const ( - HandleStatsAsync = "HandleStatsAsync" - ObserveRegionStatsAsync = "ObserveRegionStatsAsync" - UpdateSubTree = "UpdateSubTree" - HandleOverlaps = "HandleOverlaps" - CollectRegionStatsAsync = "CollectRegionStatsAsync" - SaveRegionToKV = "SaveRegionToKV" -) - -// ExtraTaskOpts returns the task options for the task. -func ExtraTaskOpts(ctx *MetaProcessContext, name string) ratelimit.TaskOpts { - return ratelimit.TaskOpts{ - TaskName: name, - Limit: ctx.Limiter, - } -} - // RWLockStats is a read-write lock with statistics. type RWLockStats struct { syncutil.RWMutex diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index 6e6df8e3775f..2a5302b34dcc 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -54,8 +54,8 @@ type Cluster struct { clusterID uint64 running atomic.Bool - taskRunner ratelimit.Runner - hbConcurrencyLimiter *ratelimit.ConcurrencyLimiter + heartbeatRunnner ratelimit.Runner + logRunner ratelimit.Runner } const ( @@ -64,7 +64,8 @@ const ( collectWaitTime = time.Minute // heartbeat relative const - hbConcurrentRunner = "heartbeat-concurrent-task-runner" + heartbeatTaskRunner = "heartbeat-task-runner" + logTaskRunner = "log-task-runner" ) var syncRunner = ratelimit.NewSyncRunner() @@ -92,8 +93,8 @@ func NewCluster(parentCtx context.Context, persistConfig *config.PersistConfig, clusterID: clusterID, checkMembershipCh: checkMembershipCh, - taskRunner: ratelimit.NewConcurrentRunner(hbConcurrentRunner, time.Minute), - hbConcurrencyLimiter: ratelimit.NewConcurrencyLimiter(uint64(runtime.NumCPU() * 2)), + heartbeatRunnner: ratelimit.NewConcurrentRunner(heartbeatTaskRunner, ratelimit.NewConcurrencyLimiter(uint64(runtime.NumCPU()*2)), time.Minute), + logRunner: ratelimit.NewConcurrentRunner(logTaskRunner, ratelimit.NewConcurrencyLimiter(uint64(runtime.NumCPU()*2)), time.Minute), } c.coordinator = schedule.NewCoordinator(ctx, c, hbStreams) err = c.ruleManager.Initialize(persistConfig.GetMaxReplicas(), persistConfig.GetLocationLabels(), persistConfig.GetIsolationLevel()) @@ -530,7 +531,8 @@ func (c *Cluster) StartBackgroundJobs() { go c.runUpdateStoreStats() go c.runCoordinator() go c.runMetricsCollectionJob() - c.taskRunner.Start() + c.heartbeatRunnner.Start() + c.logRunner.Start() c.running.Store(true) } @@ -541,7 +543,8 @@ func (c *Cluster) StopBackgroundJobs() { } c.running.Store(false) c.coordinator.Stop() - c.taskRunner.Stop() + c.heartbeatRunnner.Stop() + c.logRunner.Stop() c.cancel() c.wg.Wait() } @@ -557,16 +560,17 @@ func (c *Cluster) HandleRegionHeartbeat(region *core.RegionInfo) error { if c.persistConfig.GetScheduleConfig().EnableHeartbeatBreakdownMetrics { tracer = core.NewHeartbeatProcessTracer() } - var runner ratelimit.Runner - runner = syncRunner + var taskRunner, logRunner ratelimit.Runner + taskRunner, logRunner = syncRunner, syncRunner if c.persistConfig.GetScheduleConfig().EnableHeartbeatConcurrentRunner { - runner = c.taskRunner + taskRunner = c.heartbeatRunnner + logRunner = c.logRunner } ctx := &core.MetaProcessContext{ Context: c.ctx, - Limiter: c.hbConcurrencyLimiter, Tracer: tracer, - TaskRunner: runner, + TaskRunner: taskRunner, + LogRunner: logRunner, } tracer.Begin() if err := c.processRegionHeartbeat(ctx, region); err != nil { @@ -590,10 +594,10 @@ func (c *Cluster) processRegionHeartbeat(ctx *core.MetaProcessContext, region *c ctx.TaskRunner.RunTask( ctx, - core.ExtraTaskOpts(ctx, core.HandleStatsAsync), func(_ context.Context) { cluster.HandleStatsAsync(c, region) }, + ratelimit.WithTaskName(ratelimit.HandleStatsAsync), ) tracer.OnAsyncHotStatsFinished() hasRegionStats := c.regionStats != nil @@ -607,22 +611,22 @@ func (c *Cluster) processRegionHeartbeat(ctx *core.MetaProcessContext, region *c if hasRegionStats && c.regionStats.RegionStatsNeedUpdate(region) { ctx.TaskRunner.RunTask( ctx, - core.ExtraTaskOpts(ctx, core.ObserveRegionStatsAsync), func(_ context.Context) { if c.regionStats.RegionStatsNeedUpdate(region) { cluster.Collect(c, region, hasRegionStats) } }, + ratelimit.WithTaskName(ratelimit.ObserveRegionStatsAsync), ) } // region is not updated to the subtree. if origin.GetRef() < 2 { ctx.TaskRunner.RunTask( ctx, - core.ExtraTaskOpts(ctx, core.UpdateSubTree), func(_ context.Context) { c.CheckAndPutSubTree(region) }, + ratelimit.WithTaskName(ratelimit.UpdateSubTree), ) } return nil @@ -642,28 +646,28 @@ func (c *Cluster) processRegionHeartbeat(ctx *core.MetaProcessContext, region *c } ctx.TaskRunner.RunTask( ctx, - core.ExtraTaskOpts(ctx, core.UpdateSubTree), func(_ context.Context) { c.CheckAndPutSubTree(region) }, + ratelimit.WithTaskName(ratelimit.UpdateSubTree), ) tracer.OnUpdateSubTreeFinished() ctx.TaskRunner.RunTask( ctx, - core.ExtraTaskOpts(ctx, core.HandleOverlaps), func(_ context.Context) { cluster.HandleOverlaps(c, overlaps) }, + ratelimit.WithTaskName(ratelimit.HandleOverlaps), ) } tracer.OnSaveCacheFinished() // handle region stats ctx.TaskRunner.RunTask( ctx, - core.ExtraTaskOpts(ctx, core.CollectRegionStatsAsync), func(_ context.Context) { cluster.Collect(c, region, hasRegionStats) }, + ratelimit.WithTaskName(ratelimit.CollectRegionStatsAsync), ) tracer.OnCollectRegionStatsFinished() return nil diff --git a/pkg/ratelimit/runner.go b/pkg/ratelimit/runner.go index c4f2d5bc5ac6..44ee54971f53 100644 --- a/pkg/ratelimit/runner.go +++ b/pkg/ratelimit/runner.go @@ -25,11 +25,21 @@ import ( "go.uber.org/zap" ) +// RegionHeartbeatStageName is the name of the stage of the region heartbeat. +const ( + HandleStatsAsync = "HandleStatsAsync" + ObserveRegionStatsAsync = "ObserveRegionStatsAsync" + UpdateSubTree = "UpdateSubTree" + HandleOverlaps = "HandleOverlaps" + CollectRegionStatsAsync = "CollectRegionStatsAsync" + SaveRegionToKV = "SaveRegionToKV" +) + const initialCapacity = 100 // Runner is the interface for running tasks. type Runner interface { - RunTask(ctx context.Context, opt TaskOpts, f func(context.Context)) error + RunTask(ctx context.Context, f func(context.Context), opts ...TaskOption) error Start() Stop() } @@ -37,7 +47,7 @@ type Runner interface { // Task is a task to be run. type Task struct { Ctx context.Context - Opts TaskOpts + Opts *TaskOpts f func(context.Context) submittedAt time.Time } @@ -48,6 +58,7 @@ var ErrMaxWaitingTasksExceeded = errors.New("max waiting tasks exceeded") // ConcurrentRunner is a simple task runner that limits the number of concurrent tasks. type ConcurrentRunner struct { name string + limiter *ConcurrencyLimiter maxPendingDuration time.Duration taskChan chan *Task pendingTasks []*Task @@ -59,9 +70,10 @@ type ConcurrentRunner struct { } // NewConcurrentRunner creates a new ConcurrentRunner. -func NewConcurrentRunner(name string, maxPendingDuration time.Duration) *ConcurrentRunner { +func NewConcurrentRunner(name string, limiter *ConcurrencyLimiter, maxPendingDuration time.Duration) *ConcurrentRunner { s := &ConcurrentRunner{ name: name, + limiter: limiter, maxPendingDuration: maxPendingDuration, taskChan: make(chan *Task), pendingTasks: make([]*Task, 0, initialCapacity), @@ -75,63 +87,70 @@ func NewConcurrentRunner(name string, maxPendingDuration time.Duration) *Concurr type TaskOpts struct { // TaskName is a human-readable name for the operation. TODO: metrics by name. TaskName string - Limit *ConcurrencyLimiter +} + +// TaskOption configures TaskOp +type TaskOption func(opts *TaskOpts) + +// WithTaskName specify the task name. +func WithTaskName(name string) TaskOption { + return func(opts *TaskOpts) { opts.TaskName = name } } // Start starts the runner. -func (s *ConcurrentRunner) Start() { - s.stopChan = make(chan struct{}) - s.wg.Add(1) +func (cr *ConcurrentRunner) Start() { + cr.stopChan = make(chan struct{}) + cr.wg.Add(1) ticker := time.NewTicker(5 * time.Second) go func() { - defer s.wg.Done() + defer cr.wg.Done() for { select { - case task := <-s.taskChan: - if task.Opts.Limit != nil { - token, err := task.Opts.Limit.Acquire(context.Background()) + case task := <-cr.taskChan: + if cr.limiter != nil { + token, err := cr.limiter.Acquire(context.Background()) if err != nil { continue } - go s.run(task.Ctx, task.f, token) + go cr.run(task.Ctx, task.f, token) } else { - go s.run(task.Ctx, task.f, nil) + go cr.run(task.Ctx, task.f, nil) } - case <-s.stopChan: - s.pendingMu.Lock() - s.pendingTasks = make([]*Task, 0, initialCapacity) - s.pendingMu.Unlock() - log.Info("stopping async task runner", zap.String("name", s.name)) + case <-cr.stopChan: + cr.pendingMu.Lock() + cr.pendingTasks = make([]*Task, 0, initialCapacity) + cr.pendingMu.Unlock() + log.Info("stopping async task runner", zap.String("name", cr.name)) return case <-ticker.C: maxDuration := time.Duration(0) - s.pendingMu.Lock() - if len(s.pendingTasks) > 0 { - maxDuration = time.Since(s.pendingTasks[0].submittedAt) + cr.pendingMu.Lock() + if len(cr.pendingTasks) > 0 { + maxDuration = time.Since(cr.pendingTasks[0].submittedAt) } - s.pendingMu.Unlock() - s.maxWaitingDuration.Set(maxDuration.Seconds()) + cr.pendingMu.Unlock() + cr.maxWaitingDuration.Set(maxDuration.Seconds()) } } }() } -func (s *ConcurrentRunner) run(ctx context.Context, task func(context.Context), token *TaskToken) { +func (cr *ConcurrentRunner) run(ctx context.Context, task func(context.Context), token *TaskToken) { task(ctx) if token != nil { token.Release() - s.processPendingTasks() + cr.processPendingTasks() } } -func (s *ConcurrentRunner) processPendingTasks() { - s.pendingMu.Lock() - defer s.pendingMu.Unlock() - for len(s.pendingTasks) > 0 { - task := s.pendingTasks[0] +func (cr *ConcurrentRunner) processPendingTasks() { + cr.pendingMu.Lock() + defer cr.pendingMu.Unlock() + for len(cr.pendingTasks) > 0 { + task := cr.pendingTasks[0] select { - case s.taskChan <- task: - s.pendingTasks = s.pendingTasks[1:] + case cr.taskChan <- task: + cr.pendingTasks = cr.pendingTasks[1:] return default: return @@ -140,33 +159,38 @@ func (s *ConcurrentRunner) processPendingTasks() { } // Stop stops the runner. -func (s *ConcurrentRunner) Stop() { - close(s.stopChan) - s.wg.Wait() +func (cr *ConcurrentRunner) Stop() { + close(cr.stopChan) + cr.wg.Wait() } // RunTask runs the task asynchronously. -func (s *ConcurrentRunner) RunTask(ctx context.Context, opt TaskOpts, f func(context.Context)) error { +func (cr *ConcurrentRunner) RunTask(ctx context.Context, f func(context.Context), opts ...TaskOption) error { + taskOpts := &TaskOpts{} + for _, opt := range opts { + opt(taskOpts) + } task := &Task{ Ctx: ctx, - Opts: opt, f: f, + Opts: taskOpts, } - s.processPendingTasks() + + cr.processPendingTasks() select { - case s.taskChan <- task: + case cr.taskChan <- task: default: - s.pendingMu.Lock() - defer s.pendingMu.Unlock() - if len(s.pendingTasks) > 0 { - maxWait := time.Since(s.pendingTasks[0].submittedAt) - if maxWait > s.maxPendingDuration { - s.failedTaskCount.Inc() + cr.pendingMu.Lock() + defer cr.pendingMu.Unlock() + if len(cr.pendingTasks) > 0 { + maxWait := time.Since(cr.pendingTasks[0].submittedAt) + if maxWait > cr.maxPendingDuration { + cr.failedTaskCount.Inc() return ErrMaxWaitingTasksExceeded } } task.submittedAt = time.Now() - s.pendingTasks = append(s.pendingTasks, task) + cr.pendingTasks = append(cr.pendingTasks, task) } return nil } @@ -180,7 +204,7 @@ func NewSyncRunner() *SyncRunner { } // RunTask runs the task synchronously. -func (*SyncRunner) RunTask(ctx context.Context, _ TaskOpts, f func(context.Context)) error { +func (*SyncRunner) RunTask(ctx context.Context, f func(context.Context), _ ...TaskOption) error { f(ctx) return nil } diff --git a/pkg/ratelimit/runner_test.go b/pkg/ratelimit/runner_test.go index 507f8cf4ee8a..ccbf6ed59ed3 100644 --- a/pkg/ratelimit/runner_test.go +++ b/pkg/ratelimit/runner_test.go @@ -25,8 +25,7 @@ import ( func TestConcurrentRunner(t *testing.T) { t.Run("RunTask", func(t *testing.T) { - limiter := NewConcurrencyLimiter(1) - runner := NewConcurrentRunner("test", time.Second) + runner := NewConcurrentRunner("test", NewConcurrencyLimiter(1), time.Second) runner.Start() defer runner.Stop() @@ -34,33 +33,34 @@ func TestConcurrentRunner(t *testing.T) { for i := 0; i < 10; i++ { time.Sleep(50 * time.Millisecond) wg.Add(1) - err := runner.RunTask(context.Background(), TaskOpts{ - TaskName: "test1", - Limit: limiter, - }, func(context.Context) { - defer wg.Done() - time.Sleep(100 * time.Millisecond) - }) + err := runner.RunTask( + context.Background(), + func(context.Context) { + defer wg.Done() + time.Sleep(100 * time.Millisecond) + }, + WithTaskName("test1"), + ) require.NoError(t, err) } wg.Wait() }) t.Run("MaxPendingDuration", func(t *testing.T) { - limiter := NewConcurrencyLimiter(1) - runner := NewConcurrentRunner("test", 2*time.Millisecond) + runner := NewConcurrentRunner("test", NewConcurrencyLimiter(1), 2*time.Millisecond) runner.Start() defer runner.Stop() var wg sync.WaitGroup for i := 0; i < 10; i++ { wg.Add(1) - err := runner.RunTask(context.Background(), TaskOpts{ - TaskName: "test2", - Limit: limiter, - }, func(context.Context) { - defer wg.Done() - time.Sleep(100 * time.Millisecond) - }) + err := runner.RunTask( + context.Background(), + func(context.Context) { + defer wg.Done() + time.Sleep(100 * time.Millisecond) + }, + WithTaskName("test2"), + ) if err != nil { wg.Done() // task 0 running diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index ae3284e26945..8889fdf87b67 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -107,7 +107,8 @@ const ( minSnapshotDurationSec = 5 // heartbeat relative const - hbConcurrentRunner = "heartbeat-async-task-runner" + heartbeatTaskRunner = "heartbeat-async-task-runner" + logTaskRunner = "log-async-task-runner" ) // Server is the interface for cluster. @@ -172,8 +173,8 @@ type RaftCluster struct { independentServices sync.Map hbstreams *hbstream.HeartbeatStreams - taskRunner ratelimit.Runner - hbConcurrencyLimiter *ratelimit.ConcurrencyLimiter + heartbeatRunnner ratelimit.Runner + logRunner ratelimit.Runner } // Status saves some state information. @@ -190,15 +191,15 @@ type Status struct { func NewRaftCluster(ctx context.Context, clusterID uint64, basicCluster *core.BasicCluster, storage storage.Storage, regionSyncer *syncer.RegionSyncer, etcdClient *clientv3.Client, httpClient *http.Client) *RaftCluster { return &RaftCluster{ - serverCtx: ctx, - clusterID: clusterID, - regionSyncer: regionSyncer, - httpClient: httpClient, - etcdClient: etcdClient, - core: basicCluster, - storage: storage, - taskRunner: ratelimit.NewConcurrentRunner(hbConcurrentRunner, time.Minute), - hbConcurrencyLimiter: ratelimit.NewConcurrencyLimiter(uint64(runtime.NumCPU() * 2)), + serverCtx: ctx, + clusterID: clusterID, + regionSyncer: regionSyncer, + httpClient: httpClient, + etcdClient: etcdClient, + core: basicCluster, + storage: storage, + heartbeatRunnner: ratelimit.NewConcurrentRunner(heartbeatTaskRunner, ratelimit.NewConcurrencyLimiter(uint64(runtime.NumCPU()*2)), time.Minute), + logRunner: ratelimit.NewConcurrentRunner(logTaskRunner, ratelimit.NewConcurrencyLimiter(uint64(runtime.NumCPU()*2)), time.Minute), } } @@ -356,7 +357,8 @@ func (c *RaftCluster) Start(s Server) error { go c.startGCTuner() c.running = true - c.taskRunner.Start() + c.heartbeatRunnner.Start() + c.logRunner.Start() return nil } @@ -750,7 +752,8 @@ func (c *RaftCluster) Stop() { if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { c.stopSchedulingJobs() } - c.taskRunner.Stop() + c.heartbeatRunnner.Stop() + c.logRunner.Stop() c.Unlock() c.wg.Wait() @@ -1015,10 +1018,10 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { ctx.TaskRunner.RunTask( ctx.Context, - core.ExtraTaskOpts(ctx, core.HandleStatsAsync), func(_ context.Context) { cluster.HandleStatsAsync(c, region) }, + ratelimit.WithTaskName(ratelimit.HandleStatsAsync), ) } tracer.OnAsyncHotStatsFinished() @@ -1036,22 +1039,22 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio if hasRegionStats && c.regionStats.RegionStatsNeedUpdate(region) { ctx.TaskRunner.RunTask( ctx.Context, - core.ExtraTaskOpts(ctx, core.ObserveRegionStatsAsync), func(_ context.Context) { if c.regionStats.RegionStatsNeedUpdate(region) { cluster.Collect(c, region, hasRegionStats) } }, + ratelimit.WithTaskName(ratelimit.ObserveRegionStatsAsync), ) } // region is not updated to the subtree. if origin.GetRef() < 2 { ctx.TaskRunner.RunTask( ctx, - core.ExtraTaskOpts(ctx, core.UpdateSubTree), func(_ context.Context) { c.CheckAndPutSubTree(region) }, + ratelimit.WithTaskName(ratelimit.UpdateSubTree), ) } return nil @@ -1075,20 +1078,20 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio } ctx.TaskRunner.RunTask( ctx, - core.ExtraTaskOpts(ctx, core.UpdateSubTree), func(_ context.Context) { c.CheckAndPutSubTree(region) }, + ratelimit.WithTaskName(ratelimit.UpdateSubTree), ) tracer.OnUpdateSubTreeFinished() if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { ctx.TaskRunner.RunTask( ctx.Context, - core.ExtraTaskOpts(ctx, core.HandleOverlaps), func(_ context.Context) { cluster.HandleOverlaps(c, overlaps) }, + ratelimit.WithTaskName(ratelimit.HandleOverlaps), ) } regionUpdateCacheEventCounter.Inc() @@ -1098,13 +1101,13 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio // handle region stats ctx.TaskRunner.RunTask( ctx.Context, - core.ExtraTaskOpts(ctx, core.CollectRegionStatsAsync), func(_ context.Context) { // TODO: Due to the accuracy requirements of the API "/regions/check/xxx", // region stats needs to be collected in API mode. // We need to think of a better way to reduce this part of the cost in the future. cluster.Collect(c, region, hasRegionStats) }, + ratelimit.WithTaskName(ratelimit.CollectRegionStatsAsync), ) tracer.OnCollectRegionStatsFinished() @@ -1112,7 +1115,6 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio if saveKV { ctx.TaskRunner.RunTask( ctx.Context, - core.ExtraTaskOpts(ctx, core.SaveRegionToKV), func(_ context.Context) { // If there are concurrent heartbeats from the same region, the last write will win even if // writes to storage in the critical area. So don't use mutex to protect it. @@ -1134,6 +1136,7 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio } regionUpdateKVEventCounter.Inc() }, + ratelimit.WithTaskName(ratelimit.SaveRegionToKV), ) } } diff --git a/server/cluster/cluster_worker.go b/server/cluster/cluster_worker.go index 5c2bb9502970..14a4d0c71a11 100644 --- a/server/cluster/cluster_worker.go +++ b/server/cluster/cluster_worker.go @@ -39,16 +39,19 @@ func (c *RaftCluster) HandleRegionHeartbeat(region *core.RegionInfo) error { if c.GetScheduleConfig().EnableHeartbeatBreakdownMetrics { tracer = core.NewHeartbeatProcessTracer() } - var runner ratelimit.Runner - runner = syncRunner + + var taskRunner, logRunner ratelimit.Runner + taskRunner, logRunner = syncRunner, syncRunner if c.GetScheduleConfig().EnableHeartbeatConcurrentRunner { - runner = c.taskRunner + taskRunner = c.heartbeatRunnner + logRunner = c.logRunner } + ctx := &core.MetaProcessContext{ Context: c.ctx, - Limiter: c.hbConcurrencyLimiter, Tracer: tracer, - TaskRunner: runner, + TaskRunner: taskRunner, + LogRunner: logRunner, } tracer.Begin() if err := c.processRegionHeartbeat(ctx, region); err != nil { From 44d57b6ed1756cf9317cd5698562697538bf9b2d Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 8 May 2024 15:17:38 +0800 Subject: [PATCH 49/50] metrics: fix patrol panel (#8116) close tikv/pd#8115 Signed-off-by: lhy1024 --- metrics/grafana/pd.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index 9941004e0c23..018afe1d6a93 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -7909,7 +7909,7 @@ "tableColumn": "", "targets": [ { - "expr": "pd_checker_patrol_regions_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"} != 0", + "expr": "max(max(pd_checker_patrol_regions_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})by(instance))", "legendFormat": "{{instance}}", "format": "time_series", "intervalFactor": 1, From 740f15e6529ea7607be801fcc8a5747f96f6585d Mon Sep 17 00:00:00 2001 From: JmPotato Date: Wed, 8 May 2024 16:31:08 +0800 Subject: [PATCH 50/50] *: individually check the scheduling halt for online unsafe recovery (#8147) ref tikv/pd#6493, close tikv/pd#8095 Individually check the scheduling halt for online unsafe recovery to avoid unexpectedly persisting the halt option in the intermediate process. Signed-off-by: JmPotato Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/scheduling/server/cluster.go | 7 +++++++ pkg/mcs/scheduling/server/config/config.go | 4 ++++ pkg/mcs/scheduling/server/grpc_service.go | 2 +- pkg/schedule/config/config_provider.go | 2 +- pkg/schedule/coordinator.go | 6 +----- pkg/schedule/core/cluster_informer.go | 1 + pkg/schedule/schedulers/scheduler_controller.go | 12 ++---------- pkg/unsaferecovery/unsafe_recovery_controller.go | 9 ++++----- server/cluster/cluster.go | 8 ++++++++ server/cluster/cluster_worker.go | 8 ++------ server/config/persist_options.go | 15 ++++++++++----- server/forward.go | 2 +- server/server.go | 3 +++ 13 files changed, 45 insertions(+), 34 deletions(-) diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index 2a5302b34dcc..c36964a059c5 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -692,3 +692,10 @@ func (c *Cluster) DropCacheAllRegion() { func (c *Cluster) DropCacheRegion(id uint64) { c.RemoveRegionIfExist(id) } + +// IsSchedulingHalted returns whether the scheduling is halted. +// Currently, the microservice scheduling is halted when: +// - The `HaltScheduling` persist option is set to true. +func (c *Cluster) IsSchedulingHalted() bool { + return c.persistConfig.IsSchedulingHalted() +} diff --git a/pkg/mcs/scheduling/server/config/config.go b/pkg/mcs/scheduling/server/config/config.go index 091771bc38c3..9dc6590a0b4b 100644 --- a/pkg/mcs/scheduling/server/config/config.go +++ b/pkg/mcs/scheduling/server/config/config.go @@ -682,6 +682,10 @@ func (o *PersistConfig) SetSplitMergeInterval(splitMergeInterval time.Duration) o.SetScheduleConfig(v) } +// SetSchedulingAllowanceStatus sets the scheduling allowance status to help distinguish the source of the halt. +// TODO: support this metrics for the scheduling service in the future. +func (*PersistConfig) SetSchedulingAllowanceStatus(bool, string) {} + // SetHaltScheduling set HaltScheduling. func (o *PersistConfig) SetHaltScheduling(halt bool, _ string) { v := o.GetScheduleConfig().Clone() diff --git a/pkg/mcs/scheduling/server/grpc_service.go b/pkg/mcs/scheduling/server/grpc_service.go index 62ec1c1118fd..605ec73dad5e 100644 --- a/pkg/mcs/scheduling/server/grpc_service.go +++ b/pkg/mcs/scheduling/server/grpc_service.go @@ -275,7 +275,7 @@ func (s *Service) AskBatchSplit(_ context.Context, request *schedulingpb.AskBatc }, nil } - if c.persistConfig.IsSchedulingHalted() { + if c.IsSchedulingHalted() { return nil, errs.ErrSchedulingIsHalted.FastGenByArgs() } if !c.persistConfig.IsTikvRegionSplitEnabled() { diff --git a/pkg/schedule/config/config_provider.go b/pkg/schedule/config/config_provider.go index 20c7f0dc2cf1..90e489f86f3e 100644 --- a/pkg/schedule/config/config_provider.go +++ b/pkg/schedule/config/config_provider.go @@ -46,7 +46,7 @@ func IsSchedulerRegistered(name string) bool { type SchedulerConfigProvider interface { SharedConfigProvider - IsSchedulingHalted() bool + SetSchedulingAllowanceStatus(bool, string) GetStoresLimit() map[uint64]StoreLimitConfig IsSchedulerDisabled(string) bool diff --git a/pkg/schedule/coordinator.go b/pkg/schedule/coordinator.go index 35d9c2029a1b..5ab38aad81d1 100644 --- a/pkg/schedule/coordinator.go +++ b/pkg/schedule/coordinator.go @@ -178,7 +178,7 @@ func (c *Coordinator) PatrolRegions() { log.Info("patrol regions has been stopped") return } - if c.isSchedulingHalted() { + if c.cluster.IsSchedulingHalted() { continue } @@ -207,10 +207,6 @@ func (c *Coordinator) PatrolRegions() { } } -func (c *Coordinator) isSchedulingHalted() bool { - return c.cluster.GetSchedulerConfig().IsSchedulingHalted() -} - func (c *Coordinator) checkRegions(startKey []byte) (key []byte, regions []*core.RegionInfo) { regions = c.cluster.ScanRegions(startKey, nil, patrolScanRegionLimit) if len(regions) == 0 { diff --git a/pkg/schedule/core/cluster_informer.go b/pkg/schedule/core/cluster_informer.go index 63dacd0c30dd..b97459d26ea6 100644 --- a/pkg/schedule/core/cluster_informer.go +++ b/pkg/schedule/core/cluster_informer.go @@ -43,6 +43,7 @@ type SchedulerCluster interface { GetSchedulerConfig() sc.SchedulerConfigProvider GetRegionLabeler() *labeler.RegionLabeler GetStoreConfig() sc.StoreConfigProvider + IsSchedulingHalted() bool } // CheckerCluster is an aggregate interface that wraps multiple interfaces diff --git a/pkg/schedule/schedulers/scheduler_controller.go b/pkg/schedule/schedulers/scheduler_controller.go index 5953ecac5e3b..334a2f1199af 100644 --- a/pkg/schedule/schedulers/scheduler_controller.go +++ b/pkg/schedule/schedulers/scheduler_controller.go @@ -115,7 +115,7 @@ func (c *Controller) CollectSchedulerMetrics() { var allowScheduler float64 // If the scheduler is not allowed to schedule, it will disappear in Grafana panel. // See issue #1341. - if !s.IsPaused() && !c.isSchedulingHalted() { + if !s.IsPaused() && !c.cluster.IsSchedulingHalted() { allowScheduler = 1 } schedulerStatusGauge.WithLabelValues(s.Scheduler.GetName(), "allow").Set(allowScheduler) @@ -131,10 +131,6 @@ func (c *Controller) CollectSchedulerMetrics() { ruleStatusGauge.WithLabelValues("group_count").Set(float64(groupCnt)) } -func (c *Controller) isSchedulingHalted() bool { - return c.cluster.GetSchedulerConfig().IsSchedulingHalted() -} - // ResetSchedulerMetrics resets metrics of all schedulers. func ResetSchedulerMetrics() { schedulerStatusGauge.Reset() @@ -526,7 +522,7 @@ func (s *ScheduleController) AllowSchedule(diagnosable bool) bool { } return false } - if s.isSchedulingHalted() { + if s.cluster.IsSchedulingHalted() { if diagnosable { s.diagnosticRecorder.SetResultFromStatus(Halted) } @@ -541,10 +537,6 @@ func (s *ScheduleController) AllowSchedule(diagnosable bool) bool { return true } -func (s *ScheduleController) isSchedulingHalted() bool { - return s.cluster.GetSchedulerConfig().IsSchedulingHalted() -} - // IsPaused returns if a scheduler is paused. func (s *ScheduleController) IsPaused() bool { delayUntil := atomic.LoadInt64(&s.delayUntil) diff --git a/pkg/unsaferecovery/unsafe_recovery_controller.go b/pkg/unsaferecovery/unsafe_recovery_controller.go index 044dbd182e2a..d2f6125c3f3a 100644 --- a/pkg/unsaferecovery/unsafe_recovery_controller.go +++ b/pkg/unsaferecovery/unsafe_recovery_controller.go @@ -493,12 +493,11 @@ func (u *Controller) GetStage() stage { } func (u *Controller) changeStage(stage stage) { - u.stage = stage - // Halt and resume the scheduling once the running state changed. - running := isRunning(stage) - if opt := u.cluster.GetSchedulerConfig(); opt.IsSchedulingHalted() != running { - opt.SetHaltScheduling(running, "online-unsafe-recovery") + // If the running stage changes, update the scheduling allowance status to add or remove "online-unsafe-recovery" halt. + if running := isRunning(stage); running != isRunning(u.stage) { + u.cluster.GetSchedulerConfig().SetSchedulingAllowanceStatus(running, "online-unsafe-recovery") } + u.stage = stage var output StageOutput output.Time = time.Now().Format("2006-01-02 15:04:05.000") diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 8889fdf87b67..69c3f46d21ef 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -843,6 +843,14 @@ func (c *RaftCluster) SetPDServerConfig(cfg *config.PDServerConfig) { c.opt.SetPDServerConfig(cfg) } +// IsSchedulingHalted returns whether the scheduling is halted. +// Currently, the PD scheduling is halted when: +// - The `HaltScheduling` persist option is set to true. +// - Online unsafe recovery is running. +func (c *RaftCluster) IsSchedulingHalted() bool { + return c.opt.IsSchedulingHalted() || c.unsafeRecoveryController.IsRunning() +} + // GetUnsafeRecoveryController returns the unsafe recovery controller. func (c *RaftCluster) GetUnsafeRecoveryController() *unsaferecovery.Controller { return c.unsafeRecoveryController diff --git a/server/cluster/cluster_worker.go b/server/cluster/cluster_worker.go index 14a4d0c71a11..fcddea58b18b 100644 --- a/server/cluster/cluster_worker.go +++ b/server/cluster/cluster_worker.go @@ -69,7 +69,7 @@ func (c *RaftCluster) HandleRegionHeartbeat(region *core.RegionInfo) error { // HandleAskSplit handles the split request. func (c *RaftCluster) HandleAskSplit(request *pdpb.AskSplitRequest) (*pdpb.AskSplitResponse, error) { - if c.isSchedulingHalted() { + if c.IsSchedulingHalted() { return nil, errs.ErrSchedulingIsHalted.FastGenByArgs() } if !c.opt.IsTikvRegionSplitEnabled() { @@ -112,13 +112,9 @@ func (c *RaftCluster) HandleAskSplit(request *pdpb.AskSplitRequest) (*pdpb.AskSp return split, nil } -func (c *RaftCluster) isSchedulingHalted() bool { - return c.opt.IsSchedulingHalted() -} - // HandleAskBatchSplit handles the batch split request. func (c *RaftCluster) HandleAskBatchSplit(request *pdpb.AskBatchSplitRequest) (*pdpb.AskBatchSplitResponse, error) { - if c.isSchedulingHalted() { + if c.IsSchedulingHalted() { return nil, errs.ErrSchedulingIsHalted.FastGenByArgs() } if !c.opt.IsTikvRegionSplitEnabled() { diff --git a/server/config/persist_options.go b/server/config/persist_options.go index 62118dde593f..6f5dc50f2054 100644 --- a/server/config/persist_options.go +++ b/server/config/persist_options.go @@ -987,11 +987,8 @@ func (o *PersistOptions) SetAllStoresLimitTTL(ctx context.Context, client *clien var haltSchedulingStatus = schedulingAllowanceStatusGauge.WithLabelValues("halt-scheduling") -// SetHaltScheduling set HaltScheduling. -func (o *PersistOptions) SetHaltScheduling(halt bool, source string) { - v := o.GetScheduleConfig().Clone() - v.HaltScheduling = halt - o.SetScheduleConfig(v) +// SetSchedulingAllowanceStatus sets the scheduling allowance status to help distinguish the source of the halt. +func (*PersistOptions) SetSchedulingAllowanceStatus(halt bool, source string) { if halt { haltSchedulingStatus.Set(1) schedulingAllowanceStatusGauge.WithLabelValues(source).Set(1) @@ -1001,6 +998,14 @@ func (o *PersistOptions) SetHaltScheduling(halt bool, source string) { } } +// SetHaltScheduling set HaltScheduling. +func (o *PersistOptions) SetHaltScheduling(halt bool, source string) { + v := o.GetScheduleConfig().Clone() + v.HaltScheduling = halt + o.SetScheduleConfig(v) + o.SetSchedulingAllowanceStatus(halt, source) +} + // IsSchedulingHalted returns if PD scheduling is halted. func (o *PersistOptions) IsSchedulingHalted() bool { if o == nil { diff --git a/server/forward.go b/server/forward.go index 13bad4c76002..650833e1fc17 100644 --- a/server/forward.go +++ b/server/forward.go @@ -264,7 +264,7 @@ func forwardRegionHeartbeatToScheduling(rc *cluster.RaftCluster, forwardStream s return } // TODO: find a better way to halt scheduling immediately. - if rc.GetOpts().IsSchedulingHalted() { + if rc.IsSchedulingHalted() { continue } // The error types defined for schedulingpb and pdpb are different, so we need to convert them. diff --git a/server/server.go b/server/server.go index 8d7b83cfe4ae..af9f48f8c9b0 100644 --- a/server/server.go +++ b/server/server.go @@ -1042,6 +1042,7 @@ func (s *Server) GetScheduleConfig() *sc.ScheduleConfig { } // SetScheduleConfig sets the balance config information. +// This function is exported to be used by the API. func (s *Server) SetScheduleConfig(cfg sc.ScheduleConfig) error { if err := cfg.Validate(); err != nil { return err @@ -1060,6 +1061,8 @@ func (s *Server) SetScheduleConfig(cfg sc.ScheduleConfig) error { errs.ZapError(err)) return err } + // Update the scheduling halt status at the same time. + s.persistOptions.SetSchedulingAllowanceStatus(cfg.HaltScheduling, "manually") log.Info("schedule config is updated", zap.Reflect("new", cfg), zap.Reflect("old", old)) return nil }