Log when health rate limiter value changes (#4648)

temporalio · Jul 21, 2023 · 268e274 · 268e274
1 parent 32b4219
commit 268e274
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 0 deletions.
diff --git a/common/log/tag/zap_tag.go b/common/log/tag/zap_tag.go
@@ -102,6 +102,12 @@ func NewInt32(key string, value int32) ZapTag {
 	}
 }
 
+func NewFloat64(key string, value float64) ZapTag {
+	return ZapTag{
+		field: zap.Float64(key, value),
+	}
+}
+
 func NewBoolTag(key string, value bool) ZapTag {
 	return ZapTag{
 		field: zap.Bool(key, value),

diff --git a/common/persistence/client/health_request_rate_limiter.go b/common/persistence/client/health_request_rate_limiter.go
@@ -152,11 +152,13 @@ func (rl *HealthRequestRateLimiterImpl) refreshRate() {
 		rl.curRateMultiplier = math.Max(rl.minRateMultiplier, rl.curRateMultiplier-rl.curOptions.RateBackoffStepSize)
 		rl.rateLimiter.SetRate(rl.curRateMultiplier * rl.rateFn())
 		rl.rateLimiter.SetBurst(int(rl.rateToBurstRatio * rl.rateFn()))
+		rl.logger.Info("Health threshold exceeded, reducing rate limit.", tag.NewFloat64("newMulti", rl.curRateMultiplier), tag.NewFloat64("newRate", rl.rateLimiter.Rate()), tag.NewFloat64("latencyAvg", rl.healthSignals.AverageLatency()), tag.NewFloat64("errorRatio", rl.healthSignals.ErrorRatio()))
 	} else if rl.curRateMultiplier < rl.maxRateMultiplier {
 		// already doing backoff and under thresholds, increase limit
 		rl.curRateMultiplier = math.Min(rl.maxRateMultiplier, rl.curRateMultiplier+rl.curOptions.RateIncreaseStepSize)
 		rl.rateLimiter.SetRate(rl.curRateMultiplier * rl.rateFn())
 		rl.rateLimiter.SetBurst(int(rl.rateToBurstRatio * rl.rateFn()))
+		rl.logger.Info("System healthy, increasing rate limit.", tag.NewFloat64("newMulti", rl.curRateMultiplier), tag.NewFloat64("newRate", rl.rateLimiter.Rate()), tag.NewFloat64("latencyAvg", rl.healthSignals.AverageLatency()), tag.NewFloat64("errorRatio", rl.healthSignals.ErrorRatio()))
 	}
 }