Skip to content

Commit

Permalink
fixing tests
Browse files Browse the repository at this point in the history 
Signed-off-by: Liu Cong <[email protected]>
  • Loading branch information
@innerr
innerr committed Feb 8, 2023
1 parent 64bef69 commit 3fd765c
Show file tree
Hide file tree
Showing 4 changed files with 29 additions and 45 deletions.
1 change: 1 addition & 0 deletions server/schedulers/balance_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,7 @@ func (suite *balanceLeaderSchedulerTestSuite) SetupTest() {
suite.opt = config.NewTestOptions()
suite.tc = mockcluster.NewCluster(suite.ctx, suite.opt)
suite.oc = schedule.NewOperatorController(suite.ctx, suite.tc, nil)
Register()
lb, err := schedule.CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceLeaderType, []string{"", ""}))
suite.NoError(err)
suite.lb = lb
Expand Down
1 change: 1 addition & 0 deletions server/schedulers/evict_slow_store_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ func (suite *evictSlowStoreTestSuite) SetupTest() {

suite.oc = schedule.NewOperatorController(suite.ctx, nil, nil)
storage := storage.NewStorageWithMemoryBackend()
Register()
var err error
suite.es, err = schedule.CreateScheduler(EvictSlowStoreType, suite.oc, storage, schedule.ConfigSliceDecoder(EvictSlowStoreType, []string{}))
suite.NoError(err)
Expand Down
24 changes: 14 additions & 10 deletions server/schedulers/evict_slow_trend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"go.uber.org/zap"

"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/log"
"github.com/tikv/pd/pkg/core"
"github.com/tikv/pd/pkg/storage/endpoint"
Expand Down Expand Up @@ -285,7 +284,7 @@ func chooseEvictCandidate(cluster schedule.Cluster) (slowStore *core.StoreInfo)
if slowTrend != nil && slowTrend.CauseRate > alterEpsilon && slowTrend.ResultRate < -alterEpsilon {
candidates = append(candidates, store)
storeSlowTrendActionStatusGauge.WithLabelValues("cand.add").Inc()
log.Info("evict-slow-trend-scheduler canptured candidate",
log.Info("evict-slow-trend-scheduler pre-canptured candidate",
zap.Uint64("store-id", store.GetID()),
zap.Float64("cause-rate", slowTrend.CauseRate),
zap.Float64("result-rate", slowTrend.ResultRate),
Expand All @@ -301,23 +300,28 @@ func chooseEvictCandidate(cluster schedule.Cluster) (slowStore *core.StoreInfo)
return
}

affectedStoreThreshold := int(float64(len(stores)) * cluster.GetOpts().GetSlowStoreEvictingAffectedStoreRatioThreshold())
if affectedStoreCount < affectedStoreThreshold {
storeSlowTrendActionStatusGauge.WithLabelValues("cand.none:affect-a-few").Inc()
return
}

// TODO: Calculate to judge if one store is way slower than the others
if len(candidates) > 1 {
storeSlowTrendActionStatusGauge.WithLabelValues("cand.none:too-many").Inc()
return
}

store := candidates[0]

affectedStoreThreshold := int(float64(len(stores)) * cluster.GetOpts().GetSlowStoreEvictingAffectedStoreRatioThreshold())
if affectedStoreCount < affectedStoreThreshold {
log.Info("evict-slow-trend-scheduler failed to confirm candidate: it only affect a few stores", zap.Uint64("store-id", store.GetID()))
storeSlowTrendActionStatusGauge.WithLabelValues("cand.none:affect-a-few").Inc()
return
}

if !checkStoreSlowerThanOthers(cluster, store) {
log.Info("evict-slow-trend-scheduler failed to confirm candidate: it's not slower than others", zap.Uint64("store-id", store.GetID()))
storeSlowTrendActionStatusGauge.WithLabelValues("cand.none:not-slower").Inc()
return
}

storeSlowTrendActionStatusGauge.WithLabelValues("cand.add").Inc()
log.Info("evict-slow-trend-scheduler canptured candidate", zap.Uint64("store-id", store.GetID()))
return store
}

Expand Down Expand Up @@ -433,7 +437,7 @@ func checkStoreFasterThanOthers(cluster schedule.Cluster, target *core.StoreInfo
}
storeSlowTrendMiscGauge.WithLabelValues("store.check-faster:count").Set(float64(fasterThanStores))
storeSlowTrendMiscGauge.WithLabelValues("store.check-faster:expected").Set(float64(expected))
return fasterThanStores > expected
return fasterThanStores >= expected
}

const alterEpsilon = 1e-9
48 changes: 13 additions & 35 deletions server/schedulers/evict_slow_trend_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,15 +52,15 @@ func (suite *evictSlowTrendTestSuite) SetupTest() {
suite.tc = mockcluster.NewCluster(suite.ctx, opt)

suite.tc.AddLeaderStore(1, 10)
suite.tc.AddLeaderStore(2, 100)
suite.tc.AddLeaderStore(3, 10)
suite.tc.AddLeaderStore(2, 99)
suite.tc.AddLeaderStore(3, 100)
suite.tc.AddLeaderRegion(1, 1, 2, 3)
suite.tc.AddLeaderRegion(2, 2, 1, 3)
suite.tc.AddLeaderRegion(3, 3, 1, 2)

now := time.Now()
for i := 1; i <= 3; i++ {
storeInfo := suite.tc.GetStore(1)
storeInfo := suite.tc.GetStore(uint64(i))
newStoreInfo := storeInfo.Clone(func(store *core.StoreInfo) {
store.GetStoreStats().SlowTrend = &pdpb.SlowTrend{
CauseValue: 5.0e6,
Expand All @@ -74,6 +74,7 @@ func (suite *evictSlowTrendTestSuite) SetupTest() {

suite.oc = schedule.NewOperatorController(suite.ctx, nil, nil)
storage := storage.NewStorageWithMemoryBackend()
Register()
var err error
suite.es, err = schedule.CreateScheduler(EvictSlowTrendType, suite.oc, storage, schedule.ConfigSliceDecoder(EvictSlowTrendType, []string{}))
suite.NoError(err)
Expand All @@ -90,8 +91,8 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrend() {
suite.True(ok)

// Set store-1 to slow status, generate evict candidate
suite.Equal(es2.conf.evictedStore(), 0)
suite.Equal(es2.conf.candidate(), 0)
suite.Equal(es2.conf.evictedStore(), uint64(0))
suite.Equal(es2.conf.candidate(), uint64(0))
storeInfo := suite.tc.GetStore(1)
newStoreInfo := storeInfo.Clone(func(store *core.StoreInfo) {
store.GetStoreStats().SlowTrend = &pdpb.SlowTrend{
Expand All @@ -105,8 +106,8 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrend() {
suite.True(suite.es.IsScheduleAllowed(suite.tc))
ops, _ := suite.es.Schedule(suite.tc, false)
suite.Empty(ops)
suite.Equal(es2.conf.candidate(), 1)
suite.Equal(es2.conf.evictedStore(), 0)
suite.Equal(es2.conf.candidate(), uint64(1))
suite.Equal(es2.conf.evictedStore(), uint64(0))

// Update other stores' heartbeat-ts, do evicting
for storeID := uint64(2); storeID <= uint64(3); storeID++ {
Expand All @@ -117,10 +118,10 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrend() {
suite.tc.PutStore(newStoreInfo)
}
ops, _ = suite.es.Schedule(suite.tc, false)
testutil.CheckMultiTargetTransferLeader(suite.Require(), ops[0], operator.OpLeader, 1, []uint64{2})
testutil.CheckMultiTargetTransferLeader(suite.Require(), ops[0], operator.OpLeader, 1, []uint64{2, 3})
suite.Equal(EvictSlowTrendType, ops[0].Desc())
suite.Equal(es2.conf.candidate(), 0)
suite.Equal(es2.conf.evictedStore(), 1)
suite.Equal(es2.conf.candidate(), uint64(0))
suite.Equal(es2.conf.evictedStore(), uint64(1))
// Cannot balance leaders to store 1
ops, _ = suite.bs.Schedule(suite.tc, false)
suite.Empty(ops)
Expand All @@ -135,12 +136,12 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrend() {
}
})
suite.tc.PutStore(newStoreInfo)
// Evict leader scheduler of store 1 should be removed, then leader can be balanced from store-2 to store-1
// Evict leader scheduler of store 1 should be removed, then leaders should be balanced from store-3 to store-1
ops, _ = suite.es.Schedule(suite.tc, false)
suite.Empty(ops)
suite.Zero(es2.conf.evictedStore())
ops, _ = suite.bs.Schedule(suite.tc, false)
testutil.CheckTransferLeader(suite.Require(), ops[0], operator.OpLeader, 2, 1)
testutil.CheckTransferLeader(suite.Require(), ops[0], operator.OpLeader, 3, 1)

// no slow store need to evict.
ops, _ = suite.es.Schedule(suite.tc, false)
Expand Down Expand Up @@ -176,26 +177,3 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrendPrepare() {
// prepare with evict store.
suite.es.Prepare(suite.tc)
}

func (suite *evictSlowTrendTestSuite) TestEvictSlowTrendPersistFail() {
persisFail := "github.com/tikv/pd/server/schedulers/persistFail"
suite.NoError(failpoint.Enable(persisFail, "return(true)"))

storeInfo := suite.tc.GetStore(1)
newStoreInfo := storeInfo.Clone(func(store *core.StoreInfo) {
store.GetStoreStats().SlowTrend = &pdpb.SlowTrend{
CauseValue: 5.0e8,
CauseRate: 1e7,
ResultValue: 3.0e3,
ResultRate: -1e7,
}
})
suite.tc.PutStore(newStoreInfo)
suite.True(suite.es.IsScheduleAllowed(suite.tc))
// Add evict leader scheduler to store 1
ops, _ := suite.es.Schedule(suite.tc, false)
suite.Empty(ops)
suite.NoError(failpoint.Disable(persisFail))
ops, _ = suite.es.Schedule(suite.tc, false)
suite.NotEmpty(ops)
}

0 comments on commit 3fd765c

Please sign in to comment.