From 4576a1046dc66e78562331c90a79a830450a9fa3 Mon Sep 17 00:00:00 2001 From: Alex Shtin Date: Thu, 19 Jan 2023 11:03:37 -0800 Subject: [PATCH] Add debug.TimeoutMultiplier to all timeouts (#3815) --- common/persistence/cassandra/test.go | 3 ++- .../persistence/nosql/nosqlplugin/cassandra/gocql/client.go | 5 +++-- .../persistence-tests/clusterMetadataManagerTest.go | 3 ++- .../persistence-tests/historyV2PersistenceTest.go | 3 ++- .../persistence-tests/metadataPersistenceV2Test.go | 3 ++- .../persistence/persistence-tests/queuePersistenceTest.go | 3 ++- common/persistence/sql/sqlplugin/tests/context.go | 6 ++++-- common/persistence/tests/execution_mutable_state.go | 3 ++- common/persistence/tests/execution_mutable_state_task.go | 3 ++- common/persistence/tests/history_store.go | 3 ++- common/persistence/tests/shard.go | 3 ++- common/persistence/tests/task_queue.go | 3 ++- common/persistence/tests/task_queue_task.go | 3 ++- .../persistence/tests/visibility_persistence_suite_test.go | 3 ++- .../store/elasticsearch/visibility_store_read_test.go | 3 ++- service/history/queues/queue_base.go | 3 ++- service/history/shard/context_impl.go | 3 ++- service/history/transferQueueTaskExecutorBase.go | 5 +++-- service/matching/taskQueueManager.go | 3 ++- 19 files changed, 42 insertions(+), 22 deletions(-) diff --git a/common/persistence/cassandra/test.go b/common/persistence/cassandra/test.go index f048e25ae03..91c3dffa878 100644 --- a/common/persistence/cassandra/test.go +++ b/common/persistence/cassandra/test.go @@ -31,6 +31,7 @@ import ( "go.temporal.io/server/common" "go.temporal.io/server/common/config" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" @@ -76,7 +77,7 @@ func NewTestCluster(keyspace, username, password, host string, port int, schemaD Hosts: host, Port: port, MaxConns: 2, - ConnectTimeout: 30 * time.Second, + ConnectTimeout: 30 * time.Second * debug.TimeoutMultiplier, Keyspace: keyspace, } result.faultInjection = faultInjection diff --git a/common/persistence/nosql/nosqlplugin/cassandra/gocql/client.go b/common/persistence/nosql/nosqlplugin/cassandra/gocql/client.go index 503dcdacb7e..33897639518 100644 --- a/common/persistence/nosql/nosqlplugin/cassandra/gocql/client.go +++ b/common/persistence/nosql/nosqlplugin/cassandra/gocql/client.go @@ -38,6 +38,7 @@ import ( "go.temporal.io/server/common/auth" "go.temporal.io/server/common/config" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/persistence/nosql/nosqlplugin/cassandra/translator" "go.temporal.io/server/common/resolver" ) @@ -143,8 +144,8 @@ func NewCassandraCluster( cluster.Timeout = cfg.ConnectTimeout cluster.ConnectTimeout = cfg.ConnectTimeout } else { - cluster.Timeout = 10 * time.Second - cluster.ConnectTimeout = 10 * time.Second + cluster.Timeout = 10 * time.Second * debug.TimeoutMultiplier + cluster.ConnectTimeout = 10 * time.Second * debug.TimeoutMultiplier } cluster.ProtoVersion = 4 diff --git a/common/persistence/persistence-tests/clusterMetadataManagerTest.go b/common/persistence/persistence-tests/clusterMetadataManagerTest.go index 99d43c1f11f..95a032e14e0 100644 --- a/common/persistence/persistence-tests/clusterMetadataManagerTest.go +++ b/common/persistence/persistence-tests/clusterMetadataManagerTest.go @@ -35,6 +35,7 @@ import ( versionpb "go.temporal.io/api/version/v1" persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common/debug" p "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/primitives" ) @@ -60,7 +61,7 @@ func (s *ClusterMetadataManagerSuite) SetupSuite() { func (s *ClusterMetadataManagerSuite) SetupTest() { // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil s.Assertions = require.New(s.T()) - s.ctx, s.cancel = context.WithTimeout(context.Background(), time.Second*30) + s.ctx, s.cancel = context.WithTimeout(context.Background(), 30*time.Second*debug.TimeoutMultiplier) } // TearDownTest implementation diff --git a/common/persistence/persistence-tests/historyV2PersistenceTest.go b/common/persistence/persistence-tests/historyV2PersistenceTest.go index f47be0888f3..a728ce3f17e 100644 --- a/common/persistence/persistence-tests/historyV2PersistenceTest.go +++ b/common/persistence/persistence-tests/historyV2PersistenceTest.go @@ -38,6 +38,7 @@ import ( historypb "go.temporal.io/api/history/v1" "go.temporal.io/api/serviceerror" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/persistence/serialization" persistencespb "go.temporal.io/server/api/persistence/v1" @@ -91,7 +92,7 @@ func (s *HistoryV2PersistenceSuite) SetupTest() { // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil s.Assertions = require.New(s.T()) - s.ctx, s.cancel = context.WithTimeout(context.Background(), time.Second*30) + s.ctx, s.cancel = context.WithTimeout(context.Background(), 30*time.Second*debug.TimeoutMultiplier) } // TearDownTest implementation diff --git a/common/persistence/persistence-tests/metadataPersistenceV2Test.go b/common/persistence/persistence-tests/metadataPersistenceV2Test.go index c6ece67b235..9ce11bcd0ce 100644 --- a/common/persistence/persistence-tests/metadataPersistenceV2Test.go +++ b/common/persistence/persistence-tests/metadataPersistenceV2Test.go @@ -42,6 +42,7 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common/cluster" + "go.temporal.io/server/common/debug" p "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/primitives/timestamp" ) @@ -67,7 +68,7 @@ func (m *MetadataPersistenceSuiteV2) SetupSuite() { func (m *MetadataPersistenceSuiteV2) SetupTest() { // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil m.Assertions = require.New(m.T()) - m.ctx, m.cancel = context.WithTimeout(context.Background(), time.Second*30) + m.ctx, m.cancel = context.WithTimeout(context.Background(), 30*time.Second*debug.TimeoutMultiplier) // cleanup the namespace created var token []byte diff --git a/common/persistence/persistence-tests/queuePersistenceTest.go b/common/persistence/persistence-tests/queuePersistenceTest.go index 528b9843b50..3ad63d47e74 100644 --- a/common/persistence/persistence-tests/queuePersistenceTest.go +++ b/common/persistence/persistence-tests/queuePersistenceTest.go @@ -34,6 +34,7 @@ import ( enumsspb "go.temporal.io/server/api/enums/v1" replicationspb "go.temporal.io/server/api/replication/v1" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/persistence" ) @@ -58,7 +59,7 @@ func (s *QueuePersistenceSuite) SetupSuite() { func (s *QueuePersistenceSuite) SetupTest() { // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil s.Assertions = require.New(s.T()) - s.ctx, s.cancel = context.WithTimeout(context.Background(), time.Second*30) + s.ctx, s.cancel = context.WithTimeout(context.Background(), 30*time.Second*debug.TimeoutMultiplier) } func (s *QueuePersistenceSuite) TearDownTest() { diff --git a/common/persistence/sql/sqlplugin/tests/context.go b/common/persistence/sql/sqlplugin/tests/context.go index ca1ccd2c1d0..aaed2f018e9 100644 --- a/common/persistence/sql/sqlplugin/tests/context.go +++ b/common/persistence/sql/sqlplugin/tests/context.go @@ -27,11 +27,13 @@ package tests import ( "context" "time" + + "go.temporal.io/server/common/debug" ) const ( - executionTimeout = 2 * time.Second - visibilityTimeout = 4 * time.Second + executionTimeout = 2 * time.Second * debug.TimeoutMultiplier + visibilityTimeout = 4 * time.Second * debug.TimeoutMultiplier ) func newExecutionContext() context.Context { diff --git a/common/persistence/tests/execution_mutable_state.go b/common/persistence/tests/execution_mutable_state.go index 17beeee7f30..74cf9652bef 100644 --- a/common/persistence/tests/execution_mutable_state.go +++ b/common/persistence/tests/execution_mutable_state.go @@ -41,6 +41,7 @@ import ( enumsspb "go.temporal.io/server/api/enums/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common/convert" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" p "go.temporal.io/server/common/persistence" @@ -100,7 +101,7 @@ func (s *ExecutionMutableStateSuite) TearDownSuite() { func (s *ExecutionMutableStateSuite) SetupTest() { s.Assertions = require.New(s.T()) - s.Ctx, s.Cancel = context.WithTimeout(context.Background(), time.Second*30) + s.Ctx, s.Cancel = context.WithTimeout(context.Background(), 30*time.Second*debug.TimeoutMultiplier) s.ShardID++ resp, err := s.ShardManager.GetOrCreateShard(s.Ctx, &p.GetOrCreateShardRequest{ diff --git a/common/persistence/tests/execution_mutable_state_task.go b/common/persistence/tests/execution_mutable_state_task.go index de20c9205c3..c99b18b9a4e 100644 --- a/common/persistence/tests/execution_mutable_state_task.go +++ b/common/persistence/tests/execution_mutable_state_task.go @@ -40,6 +40,7 @@ import ( "go.temporal.io/api/serviceerror" persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" @@ -102,7 +103,7 @@ func NewExecutionMutableStateTaskSuite( func (s *ExecutionMutableStateTaskSuite) SetupTest() { s.Assertions = require.New(s.T()) - s.Ctx, s.Cancel = context.WithTimeout(context.Background(), time.Second*30) + s.Ctx, s.Cancel = context.WithTimeout(context.Background(), 30*time.Second*debug.TimeoutMultiplier) s.ShardID++ resp, err := s.ShardManager.GetOrCreateShard(s.Ctx, &p.GetOrCreateShardRequest{ diff --git a/common/persistence/tests/history_store.go b/common/persistence/tests/history_store.go index c9fa28f4d55..d8b7f995b04 100644 --- a/common/persistence/tests/history_store.go +++ b/common/persistence/tests/history_store.go @@ -39,6 +39,7 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" p "go.temporal.io/server/common/persistence" @@ -101,7 +102,7 @@ func (s *HistoryEventsSuite) TearDownSuite() { func (s *HistoryEventsSuite) SetupTest() { s.Assertions = require.New(s.T()) - s.Ctx, s.Cancel = context.WithTimeout(context.Background(), time.Second*30) + s.Ctx, s.Cancel = context.WithTimeout(context.Background(), 30*time.Second*debug.TimeoutMultiplier) } func (s *HistoryEventsSuite) TearDownTest() { diff --git a/common/persistence/tests/shard.go b/common/persistence/tests/shard.go index e931068bfe9..0068cc3ea22 100644 --- a/common/persistence/tests/shard.go +++ b/common/persistence/tests/shard.go @@ -33,6 +33,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/log" p "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/persistence/serialization" @@ -79,7 +80,7 @@ func (s *ShardSuite) TearDownSuite() { func (s *ShardSuite) SetupTest() { s.Assertions = require.New(s.T()) - s.Ctx, s.Cancel = context.WithTimeout(context.Background(), time.Second*30) + s.Ctx, s.Cancel = context.WithTimeout(context.Background(), 30*time.Second*debug.TimeoutMultiplier) s.ShardID++ } diff --git a/common/persistence/tests/task_queue.go b/common/persistence/tests/task_queue.go index 053fa3740f2..7bbd819ea50 100644 --- a/common/persistence/tests/task_queue.go +++ b/common/persistence/tests/task_queue.go @@ -37,6 +37,7 @@ import ( "go.temporal.io/api/serviceerror" persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/log" p "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/persistence/serialization" @@ -86,7 +87,7 @@ func (s *TaskQueueSuite) TearDownSuite() { func (s *TaskQueueSuite) SetupTest() { s.Assertions = require.New(s.T()) - s.ctx, s.cancel = context.WithTimeout(context.Background(), time.Second*30) + s.ctx, s.cancel = context.WithTimeout(context.Background(), 30*time.Second*debug.TimeoutMultiplier) s.stickyTTL = time.Second * 10 s.namespaceID = uuid.New().String() diff --git a/common/persistence/tests/task_queue_task.go b/common/persistence/tests/task_queue_task.go index 11789164361..21f6cb3bfb9 100644 --- a/common/persistence/tests/task_queue_task.go +++ b/common/persistence/tests/task_queue_task.go @@ -37,6 +37,7 @@ import ( clockspb "go.temporal.io/server/api/clock/v1" persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/log" p "go.temporal.io/server/common/persistence" @@ -88,7 +89,7 @@ func (s *TaskQueueTaskSuite) TearDownSuite() { func (s *TaskQueueTaskSuite) SetupTest() { s.Assertions = require.New(s.T()) - s.ctx, s.cancel = context.WithTimeout(context.Background(), time.Second*30) + s.ctx, s.cancel = context.WithTimeout(context.Background(), 30*time.Second*debug.TimeoutMultiplier) s.stickyTTL = time.Second * 10 s.taskTTL = time.Second * 16 diff --git a/common/persistence/tests/visibility_persistence_suite_test.go b/common/persistence/tests/visibility_persistence_suite_test.go index 0f8c9b4afe0..ffea7043eab 100644 --- a/common/persistence/tests/visibility_persistence_suite_test.go +++ b/common/persistence/tests/visibility_persistence_suite_test.go @@ -35,6 +35,7 @@ import ( enumspb "go.temporal.io/api/enums/v1" workflowpb "go.temporal.io/api/workflow/v1" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" @@ -88,7 +89,7 @@ func (s *VisibilityPersistenceSuite) SetupSuite() { func (s *VisibilityPersistenceSuite) SetupTest() { // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil s.Assertions = require.New(s.T()) - s.ctx, s.cancel = context.WithTimeout(context.Background(), time.Second*30) + s.ctx, s.cancel = context.WithTimeout(context.Background(), 30*time.Second*debug.TimeoutMultiplier) } func (s *VisibilityPersistenceSuite) TearDownTest() { diff --git a/common/persistence/visibility/store/elasticsearch/visibility_store_read_test.go b/common/persistence/visibility/store/elasticsearch/visibility_store_read_test.go index 0b1cbc86797..05cfbb925a4 100644 --- a/common/persistence/visibility/store/elasticsearch/visibility_store_read_test.go +++ b/common/persistence/visibility/store/elasticsearch/visibility_store_read_test.go @@ -42,6 +42,7 @@ import ( enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/api/serviceerror" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" @@ -117,7 +118,7 @@ func (s *ESVisibilitySuite) SetupTest() { // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil s.Assertions = require.New(s.T()) - esProcessorAckTimeout := dynamicconfig.GetDurationPropertyFn(1 * time.Minute) + esProcessorAckTimeout := dynamicconfig.GetDurationPropertyFn(1 * time.Minute * debug.TimeoutMultiplier) visibilityDisableOrderByClause := dynamicconfig.GetBoolPropertyFn(false) s.controller = gomock.NewController(s.T()) diff --git a/service/history/queues/queue_base.go b/service/history/queues/queue_base.go index f01ebe6f771..70ccc666297 100644 --- a/service/history/queues/queue_base.go +++ b/service/history/queues/queue_base.go @@ -33,6 +33,7 @@ import ( "go.temporal.io/server/common" "go.temporal.io/server/common/backoff" "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/log" @@ -53,7 +54,7 @@ const ( // task alert & action nonDefaultReaderMaxPendingTaskCoefficient = 0.8 - queueIOTimeout = 5 * time.Second + queueIOTimeout = 5 * time.Second * debug.TimeoutMultiplier // Force creating new slice every forceNewSliceDuration // so that the last slice in the default reader won't grow diff --git a/service/history/shard/context_impl.go b/service/history/shard/context_impl.go index 8cd42f4bffe..d094477a4b6 100644 --- a/service/history/shard/context_impl.go +++ b/service/history/shard/context_impl.go @@ -50,6 +50,7 @@ import ( cclock "go.temporal.io/server/common/clock" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/convert" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/future" "go.temporal.io/server/common/headers" @@ -82,7 +83,7 @@ const ( ) const ( - shardIOTimeout = 5 * time.Second + shardIOTimeout = 5 * time.Second * debug.TimeoutMultiplier pendingMaxReplicationTaskID = math.MaxInt64 ) diff --git a/service/history/transferQueueTaskExecutorBase.go b/service/history/transferQueueTaskExecutorBase.go index 65b6339636f..c56ef7f58cc 100644 --- a/service/history/transferQueueTaskExecutorBase.go +++ b/service/history/transferQueueTaskExecutorBase.go @@ -36,6 +36,7 @@ import ( "go.temporal.io/server/api/historyservice/v1" "go.temporal.io/server/api/matchingservice/v1" "go.temporal.io/server/common" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" @@ -55,8 +56,8 @@ import ( ) const ( - taskTimeout = time.Second * 3 - taskGetExecutionTimeout = time.Second + taskTimeout = time.Second * 3 * debug.TimeoutMultiplier + taskGetExecutionTimeout = time.Second * debug.TimeoutMultiplier taskHistoryOpTimeout = 20 * time.Second ) diff --git a/service/matching/taskQueueManager.go b/service/matching/taskQueueManager.go index 3153f4358a6..c16f2891485 100644 --- a/service/matching/taskQueueManager.go +++ b/service/matching/taskQueueManager.go @@ -47,6 +47,7 @@ import ( "go.temporal.io/server/common/backoff" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/cluster" + "go.temporal.io/server/common/debug" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/future" "go.temporal.io/server/common/headers" @@ -66,7 +67,7 @@ const ( // Fake Task ID to wrap a task for syncmatch syncMatchTaskId = -137 - ioTimeout = 5 * time.Second + ioTimeout = 5 * time.Second * debug.TimeoutMultiplier // Threshold for counting a AddTask call as a no recent poller call noPollerThreshold = time.Minute * 2