Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Capture panic in replication task processing #3799

Merged
merged 4 commits into from
Jan 12, 2023
Merged

Capture panic in replication task processing #3799

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c681221
Capture panic in replication task processing
yycptt Jan 11, 2023
81f7e72
clean up
yycptt Jan 11, 2023
a5a36d3
lint???
yycptt Jan 11, 2023
9d87056
Merge branch 'master' into replication-task-panic-wrapper
yycptt Jan 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion service/history/replication/dlq_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ func (r *dlqHandlerImpl) MergeMessages(
}

for _, task := range replicationTasks {
if _, err := taskExecutor.Execute(
if err := taskExecutor.Execute(
ctx,
task,
true,
Expand Down
2 changes: 1 addition & 1 deletion service/history/replication/dlq_handler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ func (s *dlqHandlerSuite) TestMergeMessages() {
Return(&adminservice.GetDLQReplicationMessagesResponse{
ReplicationTasks: []*replicationspb.ReplicationTask{remoteTask},
}, nil)
s.taskExecutor.EXPECT().Execute(gomock.Any(), remoteTask, true).Return("", nil)
s.taskExecutor.EXPECT().Execute(gomock.Any(), remoteTask, true).Return(nil)
s.executionManager.EXPECT().RangeDeleteReplicationTaskFromDLQ(gomock.Any(), &persistence.RangeDeleteReplicationTaskFromDLQRequest{
RangeCompleteHistoryTasksRequest: persistence.RangeCompleteHistoryTasksRequest{
ShardID: s.mockShard.GetShardID(),
Expand Down
13 changes: 3 additions & 10 deletions service/history/replication/task_executor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ import (

type (
TaskExecutor interface {
Execute(ctx context.Context, replicationTask *replicationspb.ReplicationTask, forceApply bool) (string, error)
Execute(ctx context.Context, replicationTask *replicationspb.ReplicationTask, forceApply bool) error
}

TaskExecutorParams struct {
Expand Down Expand Up @@ -107,32 +107,25 @@ func (e *taskExecutorImpl) Execute(
ctx context.Context,
replicationTask *replicationspb.ReplicationTask,
forceApply bool,
) (string, error) {
) error {
var err error
var operation string
switch replicationTask.GetTaskType() {
case enumsspb.REPLICATION_TASK_TYPE_SYNC_SHARD_STATUS_TASK:
// Shard status will be sent as part of the Replication message without kafka
operation = metrics.SyncShardTaskScope
case enumsspb.REPLICATION_TASK_TYPE_SYNC_ACTIVITY_TASK:
operation = metrics.SyncActivityTaskScope
err = e.handleActivityTask(ctx, replicationTask, forceApply)
case enumsspb.REPLICATION_TASK_TYPE_HISTORY_METADATA_TASK:
// Without kafka we should not have size limits so we don't necessary need this in the new replication scheme.
operation = metrics.HistoryMetadataReplicationTaskScope
case enumsspb.REPLICATION_TASK_TYPE_HISTORY_V2_TASK:
operation = metrics.HistoryReplicationTaskScope
err = e.handleHistoryReplicationTask(ctx, replicationTask, forceApply)
case enumsspb.REPLICATION_TASK_TYPE_SYNC_WORKFLOW_STATE_TASK:
operation = metrics.SyncWorkflowStateTaskScope
err = e.handleSyncWorkflowStateTask(ctx, replicationTask, forceApply)
default:
e.logger.Error("Unknown task type.")
operation = metrics.ReplicatorScope
err = ErrUnknownReplicationTask
}

return operation, err
return err
}

func (e *taskExecutorImpl) handleActivityTask(
Expand Down
7 changes: 3 additions & 4 deletions service/history/replication/task_executor_mock.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 5 additions & 5 deletions service/history/replication/task_executor_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ func (s *taskExecutorSuite) TestProcessTaskOnce_SyncActivityReplicationTask() {
}

s.mockEngine.EXPECT().SyncActivity(gomock.Any(), request).Return(nil)
_, err := s.replicationTaskExecutor.Execute(context.Background(), task, true)
err := s.replicationTaskExecutor.Execute(context.Background(), task, true)
s.NoError(err)
}

Expand Down Expand Up @@ -297,7 +297,7 @@ func (s *taskExecutorSuite) TestProcessTaskOnce_SyncActivityReplicationTask_Rese
int64(456),
)
s.mockEngine.EXPECT().SyncActivity(gomock.Any(), request).Return(nil)
_, err := s.replicationTaskExecutor.Execute(context.Background(), task, true)
err := s.replicationTaskExecutor.Execute(context.Background(), task, true)
s.NoError(err)
}

Expand Down Expand Up @@ -330,7 +330,7 @@ func (s *taskExecutorSuite) TestProcess_HistoryReplicationTask() {
}

s.mockEngine.EXPECT().ReplicateEventsV2(gomock.Any(), request).Return(nil)
_, err := s.replicationTaskExecutor.Execute(context.Background(), task, true)
err := s.replicationTaskExecutor.Execute(context.Background(), task, true)
s.NoError(err)
}

Expand Down Expand Up @@ -385,7 +385,7 @@ func (s *taskExecutorSuite) TestProcess_HistoryReplicationTask_Resend() {
int64(456),
)
s.mockEngine.EXPECT().ReplicateEventsV2(gomock.Any(), request).Return(nil)
_, err := s.replicationTaskExecutor.Execute(context.Background(), task, true)
err := s.replicationTaskExecutor.Execute(context.Background(), task, true)
s.NoError(err)
}

Expand All @@ -406,6 +406,6 @@ func (s *taskExecutorSuite) TestProcessTaskOnce_SyncWorkflowStateTask() {

s.mockEngine.EXPECT().ReplicateWorkflowState(gomock.Any(), gomock.Any()).Return(nil)

_, err := s.replicationTaskExecutor.Execute(context.Background(), task, true)
err := s.replicationTaskExecutor.Execute(context.Background(), task, true)
s.NoError(err)
}
35 changes: 31 additions & 4 deletions service/history/replication/task_processor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,14 +324,25 @@ func (p *taskProcessorImpl) handleSyncShardStatus(
func (p *taskProcessorImpl) handleReplicationTask(
ctx context.Context,
replicationTask *replicationspb.ReplicationTask,
) error {
) (retErr error) {
_ = p.rateLimiter.Wait(ctx)

operation := func() error {
operation, err := p.replicationTaskExecutor.Execute(ctx, replicationTask, false)
p.emitTaskMetrics(operation, err)
err := p.replicationTaskExecutor.Execute(ctx, replicationTask, false)
p.emitTaskMetrics(replicationTask.GetTaskType(), err)
return err
}

var panicErr error
defer func() {
if panicErr != nil {
retErr = panicErr
p.emitTaskMetrics(replicationTask.GetTaskType(), panicErr)
}
}()

defer log.CapturePanic(p.logger, &panicErr)

return backoff.ThrottleRetry(operation, p.taskRetryPolicy, p.isRetryableError)
}

Expand Down Expand Up @@ -495,7 +506,23 @@ func (p *taskProcessorImpl) paginationFn(_ []byte) ([]interface{}, []byte, error
}
}

func (p *taskProcessorImpl) emitTaskMetrics(operation string, err error) {
func (p *taskProcessorImpl) emitTaskMetrics(taskType enumsspb.ReplicationTaskType, err error) {
var operation string
switch taskType {
case enumsspb.REPLICATION_TASK_TYPE_SYNC_SHARD_STATUS_TASK:
operation = metrics.SyncShardTaskScope
case enumsspb.REPLICATION_TASK_TYPE_SYNC_ACTIVITY_TASK:
operation = metrics.SyncActivityTaskScope
case enumsspb.REPLICATION_TASK_TYPE_HISTORY_METADATA_TASK:
operation = metrics.HistoryMetadataReplicationTaskScope
case enumsspb.REPLICATION_TASK_TYPE_HISTORY_V2_TASK:
operation = metrics.HistoryReplicationTaskScope
case enumsspb.REPLICATION_TASK_TYPE_SYNC_WORKFLOW_STATE_TASK:
operation = metrics.SyncWorkflowStateTaskScope
default:
operation = metrics.ReplicatorScope
}

metricsScope := p.metricsHandler.WithTags(metrics.OperationTag(operation))
if common.IsContextDeadlineExceededErr(err) || common.IsContextCanceledErr(err) {
metricsScope.Counter(metrics.ServiceErrContextTimeoutCounter.GetMetricName()).Record(1)
Expand Down
16 changes: 14 additions & 2 deletions service/history/replication/task_processor_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ func (s *taskProcessorSuite) TestHandleReplicationTask_SyncActivity() {
VisibilityTime: &now,
}

s.mockReplicationTaskExecutor.EXPECT().Execute(gomock.Any(), task, false).Return("", nil)
s.mockReplicationTaskExecutor.EXPECT().Execute(gomock.Any(), task, false).Return(nil)
err := s.replicationTaskProcessor.handleReplicationTask(context.Background(), task)
s.NoError(err)
}
Expand Down Expand Up @@ -243,11 +243,23 @@ func (s *taskProcessorSuite) TestHandleReplicationTask_History() {
VisibilityTime: &now,
}

s.mockReplicationTaskExecutor.EXPECT().Execute(gomock.Any(), task, false).Return("", nil)
s.mockReplicationTaskExecutor.EXPECT().Execute(gomock.Any(), task, false).Return(nil)
err = s.replicationTaskProcessor.handleReplicationTask(context.Background(), task)
s.NoError(err)
}

func (s *taskProcessorSuite) TestHandleReplicationTask_Panic() {
task := &replicationspb.ReplicationTask{}

s.mockReplicationTaskExecutor.EXPECT().Execute(gomock.Any(), task, false).DoAndReturn(
func(_ context.Context, _ *replicationspb.ReplicationTask, _ bool) error {
panic("test replication task panic")
},
)
err := s.replicationTaskProcessor.handleReplicationTask(context.Background(), task)
s.Error(err)
}

func (s *taskProcessorSuite) TestHandleReplicationDLQTask_SyncActivity() {
namespaceID := uuid.NewRandom().String()
workflowID := uuid.New()
Expand Down