From 7a125d94e6d90f5e02095c74f04585dccd4fe1dc Mon Sep 17 00:00:00 2001 From: Yichao Yang Date: Tue, 18 Jul 2023 13:11:18 -0700 Subject: [PATCH] Allow configurating history cache non-user context lock timeout (#4645) --- common/dynamicconfig/constants.go | 6 ++++++ service/history/configs/config.go | 24 +++++++++++++---------- service/history/workflow/cache/cache.go | 26 +++++++++++++------------ 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index feb0cc5d278..e15409edbb4 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -475,6 +475,12 @@ const ( HistoryCacheMaxSize = "history.cacheMaxSize" // HistoryCacheTTL is TTL of history cache HistoryCacheTTL = "history.cacheTTL" + // HistoryCacheNonUserContextLockTimeout controls how long non-user call (callerType != API or Operator) + // will wait on workflow lock acquisition. Requires service restart to take effect. + HistoryCacheNonUserContextLockTimeout = "history.cacheNonUserContextLockTimeout" + // HistoryStartupMembershipJoinDelay is the duration a history instance waits + // before joining membership after starting. + HistoryStartupMembershipJoinDelay = "history.startupMembershipJoinDelay" // HistoryShutdownDrainDuration is the duration of traffic drain during shutdown HistoryShutdownDrainDuration = "history.shutdownDrainDuration" // EventsCacheInitialSize is initial size of events cache diff --git a/service/history/configs/config.go b/service/history/configs/config.go index b8e09246ab8..43bba3d129f 100644 --- a/service/history/configs/config.go +++ b/service/history/configs/config.go @@ -65,9 +65,10 @@ type Config struct { // HistoryCache settings // Change of these configs require shard restart - HistoryCacheInitialSize dynamicconfig.IntPropertyFn - HistoryCacheMaxSize dynamicconfig.IntPropertyFn - HistoryCacheTTL dynamicconfig.DurationPropertyFn + HistoryCacheInitialSize dynamicconfig.IntPropertyFn + HistoryCacheMaxSize dynamicconfig.IntPropertyFn + HistoryCacheTTL dynamicconfig.DurationPropertyFn + HistoryCacheNonUserContextLockTimeout dynamicconfig.DurationPropertyFn // EventsCache settings // Change of these configs require shard restart @@ -345,13 +346,16 @@ func NewConfig( VisibilityDisableOrderByClause: dc.GetBoolPropertyFnWithNamespaceFilter(dynamicconfig.VisibilityDisableOrderByClause, true), VisibilityEnableManualPagination: dc.GetBoolPropertyFnWithNamespaceFilter(dynamicconfig.VisibilityEnableManualPagination, true), - EmitShardLagLog: dc.GetBoolProperty(dynamicconfig.EmitShardLagLog, false), - HistoryCacheInitialSize: dc.GetIntProperty(dynamicconfig.HistoryCacheInitialSize, 128), - HistoryCacheMaxSize: dc.GetIntProperty(dynamicconfig.HistoryCacheMaxSize, 512), - HistoryCacheTTL: dc.GetDurationProperty(dynamicconfig.HistoryCacheTTL, time.Hour), - EventsCacheInitialSize: dc.GetIntProperty(dynamicconfig.EventsCacheInitialSize, 128*1024), // 128KB - EventsCacheMaxSize: dc.GetIntProperty(dynamicconfig.EventsCacheMaxSize, 512*1024), // 512KB - EventsCacheTTL: dc.GetDurationProperty(dynamicconfig.EventsCacheTTL, time.Hour), + EmitShardLagLog: dc.GetBoolProperty(dynamicconfig.EmitShardLagLog, false), + HistoryCacheInitialSize: dc.GetIntProperty(dynamicconfig.HistoryCacheInitialSize, 128), + HistoryCacheMaxSize: dc.GetIntProperty(dynamicconfig.HistoryCacheMaxSize, 512), + HistoryCacheTTL: dc.GetDurationProperty(dynamicconfig.HistoryCacheTTL, time.Hour), + HistoryCacheNonUserContextLockTimeout: dc.GetDurationProperty(dynamicconfig.HistoryCacheNonUserContextLockTimeout, 500*time.Millisecond), + + EventsCacheInitialSize: dc.GetIntProperty(dynamicconfig.EventsCacheInitialSize, 128*1024), // 128KB + EventsCacheMaxSize: dc.GetIntProperty(dynamicconfig.EventsCacheMaxSize, 512*1024), // 512KB + EventsCacheTTL: dc.GetDurationProperty(dynamicconfig.EventsCacheTTL, time.Hour), + RangeSizeBits: 20, // 20 bits for sequencer, 2^20 sequence number for any range AcquireShardInterval: dc.GetDurationProperty(dynamicconfig.AcquireShardInterval, time.Minute), AcquireShardConcurrency: dc.GetIntProperty(dynamicconfig.AcquireShardConcurrency, 10), diff --git a/service/history/workflow/cache/cache.go b/service/history/workflow/cache/cache.go index 3c62befb71b..b4dc87253dc 100644 --- a/service/history/workflow/cache/cache.go +++ b/service/history/workflow/cache/cache.go @@ -44,7 +44,6 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" - "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/shard" "go.temporal.io/server/service/history/workflow" @@ -78,7 +77,8 @@ type ( shard shard.Context logger log.Logger metricsHandler metrics.Handler - config *configs.Config + + nonUserContextLockTimeout time.Duration } NewCacheFn func(shard shard.Context) Cache @@ -87,10 +87,12 @@ type ( var NoopReleaseFn ReleaseCacheFunc = func(err error) {} const ( - cacheNotReleased int32 = 0 - cacheReleased int32 = 1 - workflowLockTimeoutTailTime = 500 * time.Millisecond - nonApiContextLockTimeout = 500 * time.Millisecond + cacheNotReleased int32 = 0 + cacheReleased int32 = 1 +) + +const ( + workflowLockTimeoutTailTime = 500 * time.Millisecond ) func NewCache(shard shard.Context) Cache { @@ -101,11 +103,11 @@ func NewCache(shard shard.Context) Cache { opts.Pin = true return &CacheImpl{ - Cache: cache.New(config.HistoryCacheMaxSize(), opts), - shard: shard, - logger: log.With(shard.GetLogger(), tag.ComponentHistoryCache), - metricsHandler: shard.GetMetricsHandler().WithTags(metrics.CacheTypeTag(metrics.MutableStateCacheTypeTagValue)), - config: config, + Cache: cache.New(config.HistoryCacheMaxSize(), opts), + shard: shard, + logger: log.With(shard.GetLogger(), tag.ComponentHistoryCache), + metricsHandler: shard.GetMetricsHandler().WithTags(metrics.CacheTypeTag(metrics.MutableStateCacheTypeTagValue)), + nonUserContextLockTimeout: config.HistoryCacheNonUserContextLockTimeout(), } } @@ -220,7 +222,7 @@ func (c *CacheImpl) lockWorkflowExecution(ctx context.Context, if deadline, ok := ctx.Deadline(); ok { var cancel context.CancelFunc if headers.GetCallerInfo(ctx).CallerType != headers.CallerTypeAPI { - newDeadline := time.Now().Add(nonApiContextLockTimeout) + newDeadline := time.Now().Add(c.nonUserContextLockTimeout) if newDeadline.Before(deadline) { ctx, cancel = context.WithDeadline(ctx, newDeadline) defer cancel()