Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

properly fix migration in mem ack aggregation #4571

Merged
merged 8 commits into from
Jul 7, 2023
Merged

properly fix migration in mem ack aggregation #4571

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
b4a90a6
Fix migration in mem ack aggregation
wxing1292 Jun 30, 2023
9e04466
Revert "Fix migration in mem ack aggregation"
wxing1292 Jun 30, 2023
8f0c72f
Fix migration in mem ack aggregation
wxing1292 Jun 30, 2023
39532e8
UT
wxing1292 Jun 30, 2023
a528d0d
Merge branch 'master' into fix-migration-2
wxing1292 Jul 5, 2023
604f36a
Merge branch 'master' into fix-migration-2
wxing1292 Jul 6, 2023
0a87e86
fix logic
wxing1292 Jul 7, 2023
c9383c7
Merge branch 'master' into fix-migration-2
wxing1292 Jul 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 32 additions & 8 deletions service/history/shard/context_impl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -383,16 +383,23 @@ func (s *ContextImpl) UpdateReplicationQueueReaderState(
// UpdateRemoteClusterInfo deprecated
// Deprecated use UpdateRemoteReaderInfo in the future instead
func (s *ContextImpl) UpdateRemoteClusterInfo(
cluster string,
clusterName string,
ackTaskID int64,
ackTimestamp time.Time,
) {
s.wLock()
defer s.wUnlock()

remoteClusterInfo := s.getOrUpdateRemoteClusterInfoLocked(cluster)
remoteClusterInfo.AckedReplicationTaskIDs[s.shardID] = ackTaskID
remoteClusterInfo.AckedReplicationTimestamps[s.shardID] = ackTimestamp
clusterInfo := s.clusterMetadata.GetAllClusterInfo()
remoteClusterInfo := s.getOrUpdateRemoteClusterInfoLocked(clusterName)
for _, remoteShardID := range common.MapShardID(
clusterInfo[s.clusterMetadata.GetCurrentClusterName()].ShardCount,
clusterInfo[clusterName].ShardCount,
s.shardID,
) {
remoteClusterInfo.AckedReplicationTaskIDs[remoteShardID] = ackTaskID
remoteClusterInfo.AckedReplicationTimestamps[remoteShardID] = ackTimestamp
Comment on lines +400 to +401
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One thing I am not following, why the acked task id can be used across different share ids?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

for polling mode:
targer cluster's 1 poller may poll & ack on behalf of >= 1 shards, here source cluster should update all relevant shard's ack task IDs

for streaming mode:
each target shard will has its own grpc stream & update ack using reader ID (cluster ID + shard ID)

}
}

// UpdateRemoteReaderInfo do not use streaming replication until remoteClusterInfo is updated to allow both
Expand Down Expand Up @@ -1716,10 +1723,27 @@ func (s *ContextImpl) GetReplicationStatus(clusterNames []string) (map[string]*h
continue
}

remoteShardID := s.shardID
remoteClusters[clusterName] = &historyservice.ShardReplicationStatusPerCluster{
AckedTaskId: v.AckedReplicationTaskIDs[remoteShardID],
AckedTaskVisibilityTime: timestamp.TimePtr(v.AckedReplicationTimestamps[remoteShardID]),
for _, remoteShardID := range common.MapShardID(
clusterInfo[s.clusterMetadata.GetCurrentClusterName()].ShardCount,
clusterInfo[clusterName].ShardCount,
s.shardID,
) {
ackTaskID := v.AckedReplicationTaskIDs[remoteShardID] // default to 0
ackTimestamp := v.AckedReplicationTimestamps[remoteShardID]
if ackTimestamp.IsZero() {
ackTimestamp = time.Unix(0, 0)
}
if record, ok := remoteClusters[clusterName]; !ok {
remoteClusters[clusterName] = &historyservice.ShardReplicationStatusPerCluster{
AckedTaskId: ackTaskID,
AckedTaskVisibilityTime: timestamp.TimePtr(ackTimestamp),
}
} else if record.AckedTaskId > ackTaskID {
remoteClusters[clusterName] = &historyservice.ShardReplicationStatusPerCluster{
AckedTaskId: ackTaskID,
AckedTaskVisibilityTime: timestamp.TimePtr(ackTimestamp),
}
}
}
}

Expand Down
181 changes: 180 additions & 1 deletion service/history/shard/context_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"context"
"errors"
"fmt"
"math/rand"
"testing"
"time"

Expand All @@ -36,6 +37,7 @@ import (
"github.com/stretchr/testify/suite"
"go.temporal.io/api/enums/v1"

"go.temporal.io/server/api/historyservice/v1"
persistencespb "go.temporal.io/server/api/persistence/v1"
"go.temporal.io/server/common/backoff"
"go.temporal.io/server/common/clock"
Expand All @@ -54,6 +56,7 @@ type (
*require.Assertions

controller *gomock.Controller
shardID int32
mockShard *ContextTest
mockClusterMetadata *cluster.MockMetadata
mockShardManager *persistence.MockShardManager
Expand All @@ -75,11 +78,12 @@ func (s *contextSuite) SetupTest() {

s.controller = gomock.NewController(s.T())

s.shardID = 1
s.timeSource = clock.NewEventTimeSource()
shardContext := NewTestContextWithTimeSource(
s.controller,
&persistencespb.ShardInfo{
ShardId: 0,
ShardId: s.shardID,
RangeId: 1,
},
tests.NewDynamicConfig(),
Expand Down Expand Up @@ -499,3 +503,178 @@ func (s *contextSuite) TestHandoverNamespace() {
_, ok = handoverNS[namespaceEntry.Name().String()]
s.False(ok)
}

func (s *contextSuite) TestUpdateGetRemoteClusterInfo_Legacy_8_4() {
clusterMetadata := cluster.NewMockMetadata(s.controller)
clusterMetadata.EXPECT().GetClusterID().Return(cluster.TestCurrentClusterInitialFailoverVersion).AnyTimes()
clusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestCurrentClusterName).AnyTimes()
clusterMetadata.EXPECT().GetAllClusterInfo().Return(map[string]cluster.ClusterInformation{
cluster.TestCurrentClusterName: {
Enabled: true,
InitialFailoverVersion: cluster.TestCurrentClusterInitialFailoverVersion,
RPCAddress: cluster.TestCurrentClusterFrontendAddress,
ShardCount: 8,
},
cluster.TestAlternativeClusterName: {
Enabled: true,
InitialFailoverVersion: cluster.TestAlternativeClusterInitialFailoverVersion,
RPCAddress: cluster.TestAlternativeClusterFrontendAddress,
ShardCount: 4,
},
}).AnyTimes()
s.mockShard.clusterMetadata = clusterMetadata

ackTaskID := rand.Int63()
ackTimestamp := time.Unix(0, rand.Int63())
s.mockShard.UpdateRemoteClusterInfo(
cluster.TestAlternativeClusterName,
ackTaskID,
ackTimestamp,
)
remoteAckStatus, _, err := s.mockShard.GetReplicationStatus([]string{cluster.TestAlternativeClusterName})
s.NoError(err)
s.Equal(map[string]*historyservice.ShardReplicationStatusPerCluster{
cluster.TestAlternativeClusterName: {
AckedTaskId: ackTaskID,
AckedTaskVisibilityTime: timestamp.TimePtr(ackTimestamp),
},
}, remoteAckStatus)
}

func (s *contextSuite) TestUpdateGetRemoteClusterInfo_Legacy_4_8() {
clusterMetadata := cluster.NewMockMetadata(s.controller)
clusterMetadata.EXPECT().GetClusterID().Return(cluster.TestCurrentClusterInitialFailoverVersion).AnyTimes()
clusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestCurrentClusterName).AnyTimes()
clusterMetadata.EXPECT().GetAllClusterInfo().Return(map[string]cluster.ClusterInformation{
cluster.TestCurrentClusterName: {
Enabled: true,
InitialFailoverVersion: cluster.TestCurrentClusterInitialFailoverVersion,
RPCAddress: cluster.TestCurrentClusterFrontendAddress,
ShardCount: 4,
},
cluster.TestAlternativeClusterName: {
Enabled: true,
InitialFailoverVersion: cluster.TestAlternativeClusterInitialFailoverVersion,
RPCAddress: cluster.TestAlternativeClusterFrontendAddress,
ShardCount: 8,
},
}).AnyTimes()
s.mockShard.clusterMetadata = clusterMetadata

ackTaskID := rand.Int63()
ackTimestamp := time.Unix(0, rand.Int63())
s.mockShard.UpdateRemoteClusterInfo(
cluster.TestAlternativeClusterName,
ackTaskID,
ackTimestamp,
)
remoteAckStatus, _, err := s.mockShard.GetReplicationStatus([]string{cluster.TestAlternativeClusterName})
s.NoError(err)
s.Equal(map[string]*historyservice.ShardReplicationStatusPerCluster{
cluster.TestAlternativeClusterName: {
AckedTaskId: ackTaskID,
AckedTaskVisibilityTime: timestamp.TimePtr(ackTimestamp),
},
}, remoteAckStatus)
}

func (s *contextSuite) TestUpdateGetRemoteReaderInfo_8_4() {
clusterMetadata := cluster.NewMockMetadata(s.controller)
clusterMetadata.EXPECT().GetClusterID().Return(cluster.TestCurrentClusterInitialFailoverVersion).AnyTimes()
clusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestCurrentClusterName).AnyTimes()
clusterMetadata.EXPECT().GetAllClusterInfo().Return(map[string]cluster.ClusterInformation{
cluster.TestCurrentClusterName: {
Enabled: true,
InitialFailoverVersion: cluster.TestCurrentClusterInitialFailoverVersion,
RPCAddress: cluster.TestCurrentClusterFrontendAddress,
ShardCount: 8,
},
cluster.TestAlternativeClusterName: {
Enabled: true,
InitialFailoverVersion: cluster.TestAlternativeClusterInitialFailoverVersion,
RPCAddress: cluster.TestAlternativeClusterFrontendAddress,
ShardCount: 4,
},
}).AnyTimes()
s.mockShard.clusterMetadata = clusterMetadata

ackTaskID := rand.Int63()
ackTimestamp := time.Unix(0, rand.Int63())
err := s.mockShard.UpdateRemoteReaderInfo(
ReplicationReaderIDFromClusterShardID(
cluster.TestAlternativeClusterInitialFailoverVersion,
1,
),
ackTaskID,
ackTimestamp,
)
s.NoError(err)
remoteAckStatus, _, err := s.mockShard.GetReplicationStatus([]string{cluster.TestAlternativeClusterName})
s.NoError(err)
s.Equal(map[string]*historyservice.ShardReplicationStatusPerCluster{
cluster.TestAlternativeClusterName: {
AckedTaskId: ackTaskID,
AckedTaskVisibilityTime: timestamp.TimePtr(ackTimestamp),
},
}, remoteAckStatus)
}

func (s *contextSuite) TestUpdateGetRemoteReaderInfo_4_8() {
clusterMetadata := cluster.NewMockMetadata(s.controller)
clusterMetadata.EXPECT().GetClusterID().Return(cluster.TestCurrentClusterInitialFailoverVersion).AnyTimes()
clusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestCurrentClusterName).AnyTimes()
clusterMetadata.EXPECT().GetAllClusterInfo().Return(map[string]cluster.ClusterInformation{
cluster.TestCurrentClusterName: {
Enabled: true,
InitialFailoverVersion: cluster.TestCurrentClusterInitialFailoverVersion,
RPCAddress: cluster.TestCurrentClusterFrontendAddress,
ShardCount: 4,
},
cluster.TestAlternativeClusterName: {
Enabled: true,
InitialFailoverVersion: cluster.TestAlternativeClusterInitialFailoverVersion,
RPCAddress: cluster.TestAlternativeClusterFrontendAddress,
ShardCount: 8,
},
}).AnyTimes()
s.mockShard.clusterMetadata = clusterMetadata

ack1TaskID := rand.Int63()
ack1Timestamp := time.Unix(0, rand.Int63())
err := s.mockShard.UpdateRemoteReaderInfo(
ReplicationReaderIDFromClusterShardID(
cluster.TestAlternativeClusterInitialFailoverVersion,
1, // maps to local shard 1
),
ack1TaskID,
ack1Timestamp,
)
s.NoError(err)
ack5TaskID := rand.Int63()
ack5Timestamp := time.Unix(0, rand.Int63())
err = s.mockShard.UpdateRemoteReaderInfo(
ReplicationReaderIDFromClusterShardID(
cluster.TestAlternativeClusterInitialFailoverVersion,
5, // maps to local shard 1
),
ack5TaskID,
ack5Timestamp,
)
s.NoError(err)

ackTaskID := ack1TaskID
ackTimestamp := ack1Timestamp
if ackTaskID > ack5TaskID {
ackTaskID = ack5TaskID
ackTimestamp = ack5Timestamp
}

remoteAckStatus, _, err := s.mockShard.GetReplicationStatus([]string{cluster.TestAlternativeClusterName})
s.NoError(err)
s.Equal(map[string]*historyservice.ShardReplicationStatusPerCluster{
cluster.TestAlternativeClusterName: {
AckedTaskId: ackTaskID,
AckedTaskVisibilityTime: timestamp.TimePtr(ackTimestamp),
},
}, remoteAckStatus)
}