From 95e5b69d038f796e71f69ab1513b86862ea1a771 Mon Sep 17 00:00:00 2001 From: irfan sharif Date: Fri, 29 Oct 2021 12:07:01 -0400 Subject: [PATCH] kvserver: resurrect using_applied_state_key in ReplicaState Fixes #72029. using_applied_state_key was previously used to check whether the Range had upgraded to begin using the RangeAppliedState key. When set to true, replicas receiving the state (through a ReplicatedEvalResult) knew to begin using the new key. In #58088 (in 21.1) we introduced a migration to iterate through all ranges in the system and have them start using the RangeAppliedState key. In 21.2, this field was always set to true -- 21.2 nodes were already using the RangeAppliedState key, or receiving messages from 21.1 nodes that were also using the RangeAppliedState key. In 21.2 (and in 21.1 for that matter) we didn't need to trigger the "if set to true in an incoming message, start using the RangeAppliedState key" code path. When looking to get rid of this field in 22.1 (#70464), we observed that there was an unintentional read of this field in 21.2 nodes (see #72029 and \#72222); the saga is as follows: - Removing this field in 22.1 meant it was set as false when received at 21.2 nodes. - This should've been fine! We weren't using this field to trigger any upgrade code paths (like it was originally intended for). - It turns out that in 21.2 we were using the ReplicaState from the incoming snapshot to update our in-memory replica state - Because the proto field was being phased out, there was now a divergence between the on-disk state (field set to true, from earlier 21.2 operations) and the in-memory state (field set to false, because sent from a version that attempted to get rid of this field). Removing proto fields from the replica state are not possible until we stop using the protobuf copy of the replica state when applying a snapshot (#72222). Once that's done, we should be able to stop sending the replica state as part of the snapshot in the subsequent release. Release note: None --- pkg/kv/kvserver/kvserverpb/state.pb.go | 209 +++++++++++++++++-------- pkg/kv/kvserver/kvserverpb/state.proto | 37 ++++- pkg/kv/kvserver/replica.go | 5 + pkg/kv/kvserver/replica_command.go | 4 + pkg/kv/kvserver/store_snapshot.go | 4 + 5 files changed, 190 insertions(+), 69 deletions(-) diff --git a/pkg/kv/kvserver/kvserverpb/state.pb.go b/pkg/kv/kvserver/kvserverpb/state.pb.go index 77b5e7a4a728..be7726953ab3 100644 --- a/pkg/kv/kvserver/kvserverpb/state.pb.go +++ b/pkg/kv/kvserver/kvserverpb/state.pb.go @@ -64,6 +64,41 @@ type ReplicaState struct { // not be served. GCThreshold *hlc.Timestamp `protobuf:"bytes,6,opt,name=gc_threshold,json=gcThreshold,proto3" json:"gc_threshold,omitempty"` Stats *enginepb.MVCCStats `protobuf:"bytes,7,opt,name=stats,proto3" json:"stats,omitempty"` + // DeprecatedUsingAppliedStateKey was previously used to check whether the + // Range had upgraded to begin using the RangeAppliedState key. When set to + // true, replicas receiving the state (through a ReplicatedEvalResult) knew to + // begin using the new key. In #58088 (21.1) we introduced a migration to + // iterate through all ranges in the system and have them start using the + // RangeAppliedState key. This field was always set to true in 21.2, nodes + // nodes were already using the RangeAppliedState key, or receiving messages + // from 21.1 nodes that were also already using the RangeAppliedState key. In + // 21.2 (and in 21.1 for that matter) we no longer needed to trigger the "if + // set to true in an incoming message, upgrade to start using the + // RangeAppliedState key" code path. + // + // When looking to get rid of this field in 22.1 (#70464), we observed that + // there was an unintentional read of this field in 21.2 nodes (#72029). The + // saga is as follows: + // - Removing this field in 22.1 meant it was set as false when received at + // 21.2 nodes. + // - This should've been fine! We weren't using this field to trigger any + // upgrade code paths (like it was originally intended for). + // - It turns out that in 21.2 we were using the ReplicaState from the + // incoming snapshot to update our in-memory replica state + // - Because the proto field was being phased out, there was now a divergence + // between the on-disk state (field set to true, from earlier 21.2 + // operations) and the in-memory state (field set to false, because sent + // from a version that attempted to get rid of this field). + // + // Removing proto fields from the replica state are not possible until we stop + // using the protobuf copy of the replica state when applying a snapshot + // (#72222). Once that's done, we should be able to stop sending the replica + // state as part of the snapshot in the subsequent release. + // + // TODO(irfansharif): Remove this field in the release after the one where we + // stop consulting the protobuf copy of the replica state when applying a + // snapshot. + DeprecatedUsingAppliedStateKey bool `protobuf:"varint,11,opt,name=deprecated_using_applied_state_key,json=deprecatedUsingAppliedStateKey,proto3" json:"deprecated_using_applied_state_key,omitempty"` // Version tells us which migrations can be assumed to have run against this // particular replica. When we introduce backwards incompatible changes to the // replica state (for example using the unreplicated truncated state instead @@ -251,74 +286,76 @@ func init() { } var fileDescriptor_cc107fbd3ff296cb = []byte{ - // 1065 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcf, 0x6e, 0xdb, 0x36, - 0x18, 0x8f, 0x66, 0xa5, 0x71, 0x68, 0x27, 0x71, 0x99, 0x25, 0x51, 0xd3, 0xd5, 0x36, 0x3c, 0x6c, - 0xf0, 0x80, 0x4e, 0xc2, 0xba, 0x61, 0xc5, 0xfe, 0x00, 0x43, 0x9c, 0x00, 0x83, 0x8d, 0x64, 0x48, - 0x18, 0xa3, 0x03, 0xb6, 0x83, 0x40, 0x4b, 0x8c, 0x4c, 0x44, 0x16, 0x55, 0x92, 0x36, 0xd2, 0x3e, - 0xc5, 0x1e, 0x61, 0x87, 0x3d, 0xc0, 0x1e, 0x23, 0xc7, 0x1c, 0x7b, 0x32, 0x56, 0xe7, 0xb2, 0xcb, - 0x5e, 0x60, 0xa7, 0x81, 0xa4, 0xe4, 0xd8, 0x4e, 0x80, 0x66, 0x40, 0x6f, 0xd4, 0xf7, 0xfd, 0x7e, - 0xdf, 0x3f, 0xfe, 0x3e, 0x49, 0xa0, 0x71, 0x3e, 0xf2, 0xce, 0x47, 0x82, 0xf0, 0x11, 0xe1, 0xd3, - 0x43, 0xda, 0xf3, 0x84, 0xc4, 0x92, 0xb8, 0x29, 0x67, 0x92, 0xc1, 0x5a, 0xc0, 0x82, 0x73, 0xce, - 0x70, 0xd0, 0x77, 0xcf, 0x47, 0x6e, 0x0e, 0x72, 0x85, 0x64, 0x1c, 0x47, 0x24, 0xed, 0xed, 0x3e, - 0xce, 0x8e, 0x1e, 0x49, 0x22, 0x9a, 0x90, 0xb4, 0xe7, 0x0d, 0x46, 0x41, 0x60, 0xd8, 0xbb, 0x8f, - 0x35, 0x33, 0xed, 0x79, 0x34, 0x91, 0x84, 0x27, 0x38, 0xf6, 0x39, 0x3e, 0x93, 0x99, 0x73, 0x3b, - 0x77, 0x0e, 0x88, 0xc4, 0x21, 0x96, 0x38, 0xb3, 0xc3, 0xdc, 0x3e, 0x63, 0x73, 0x86, 0x92, 0xc6, - 0x5e, 0x3f, 0x0e, 0x3c, 0x49, 0x07, 0x44, 0x48, 0x3c, 0x48, 0x33, 0xcf, 0x87, 0x11, 0x8b, 0x98, - 0x3e, 0x7a, 0xea, 0x64, 0xac, 0x8d, 0xb7, 0x36, 0x28, 0x23, 0x92, 0xc6, 0x34, 0xc0, 0xa7, 0xaa, - 0x1b, 0xf8, 0x14, 0x40, 0x95, 0xda, 0xc7, 0x69, 0x1a, 0x53, 0x12, 0xfa, 0x34, 0x09, 0xc9, 0x85, - 0x63, 0xd5, 0xad, 0xa6, 0x8d, 0x2a, 0xca, 0xb3, 0x67, 0x1c, 0x6d, 0x65, 0x87, 0x2e, 0xd8, 0x8c, - 0x09, 0x16, 0x64, 0x01, 0xfe, 0x81, 0x86, 0x3f, 0xd4, 0xae, 0x39, 0xfc, 0xd7, 0xc0, 0x0e, 0x89, - 0x08, 0x9c, 0x42, 0xdd, 0x6a, 0x96, 0x9e, 0x35, 0xdc, 0x9b, 0xa1, 0x65, 0xbd, 0xb8, 0x08, 0x27, - 0x11, 0x39, 0x20, 0x22, 0xe0, 0x34, 0x95, 0x8c, 0x23, 0x8d, 0x87, 0x2e, 0x58, 0xd6, 0xc1, 0x1c, - 0x5b, 0x13, 0x9d, 0x3b, 0x88, 0x87, 0xca, 0x8f, 0x0c, 0x0c, 0xfe, 0x04, 0x36, 0x24, 0x1f, 0x26, - 0x01, 0x96, 0x24, 0xf4, 0xf5, 0x35, 0x39, 0xcb, 0x9a, 0xf9, 0xc9, 0x9d, 0x29, 0xcf, 0x64, 0x37, - 0x47, 0xeb, 0x29, 0xa0, 0x75, 0x39, 0xf7, 0x0c, 0x4f, 0x40, 0x39, 0x0a, 0x7c, 0xd9, 0xe7, 0x44, - 0xf4, 0x59, 0x1c, 0x3a, 0x0f, 0x74, 0xb0, 0x27, 0x33, 0xc1, 0xd4, 0xdc, 0xdd, 0x7e, 0x1c, 0xb8, - 0xdd, 0x7c, 0xee, 0xad, 0x8d, 0xc9, 0xb8, 0x56, 0xfa, 0x71, 0xbf, 0x9b, 0xb3, 0x50, 0x29, 0x0a, - 0xa6, 0x0f, 0xf0, 0x3b, 0xb0, 0xac, 0x0a, 0x13, 0xce, 0xca, 0xad, 0xc2, 0x32, 0xa5, 0xb8, 0xb9, - 0x52, 0xdc, 0xa3, 0x17, 0xfb, 0xfb, 0xaa, 0x10, 0x81, 0x0c, 0x07, 0x7e, 0x05, 0x56, 0x46, 0x84, - 0x0b, 0xca, 0x12, 0xa7, 0xac, 0xe9, 0xbb, 0x77, 0xf4, 0xf5, 0xc2, 0x20, 0x50, 0x0e, 0x85, 0x3f, - 0x83, 0x2d, 0x7d, 0xb7, 0x41, 0xcc, 0x04, 0x09, 0xfd, 0xa9, 0x42, 0x9c, 0xb5, 0xfb, 0xb4, 0x63, - 0x5f, 0x8e, 0x6b, 0x4b, 0x68, 0x53, 0x45, 0xd8, 0xd7, 0x01, 0xa6, 0xae, 0x6f, 0xed, 0xbf, 0x7f, - 0xaf, 0x59, 0x1d, 0xbb, 0x58, 0xac, 0xac, 0x76, 0xec, 0xe2, 0x6a, 0x05, 0x74, 0xec, 0x22, 0xa8, - 0x94, 0x3a, 0x76, 0xb1, 0x54, 0x29, 0x37, 0xfe, 0x59, 0x01, 0xab, 0xfa, 0x5a, 0xdb, 0xc9, 0x19, - 0x83, 0x47, 0xa6, 0x6f, 0xa2, 0x35, 0x55, 0x7a, 0xf6, 0xb9, 0xfb, 0x8e, 0xc5, 0x71, 0x67, 0xe5, - 0xd9, 0x2a, 0xaa, 0x22, 0xae, 0xc6, 0x35, 0xcb, 0x4c, 0x82, 0xc0, 0x27, 0x00, 0xc4, 0x58, 0xc8, - 0x39, 0xe1, 0xad, 0x2a, 0x8b, 0x11, 0x5c, 0x0d, 0x94, 0x92, 0xe1, 0xc0, 0x4f, 0x49, 0x12, 0xd2, - 0x24, 0xd2, 0xba, 0xb3, 0x11, 0x48, 0x86, 0x83, 0x63, 0x63, 0xc9, 0x01, 0x21, 0x67, 0x69, 0x4a, - 0x42, 0xad, 0x12, 0x03, 0x38, 0x30, 0x16, 0xd8, 0x00, 0x6b, 0x7a, 0x68, 0x31, 0x8b, 0x7c, 0x41, - 0x5f, 0x13, 0x7d, 0xf7, 0x05, 0x54, 0x52, 0xc6, 0x43, 0x16, 0x9d, 0xd2, 0xd7, 0x04, 0x7e, 0x91, - 0x0d, 0x36, 0xc7, 0xf8, 0x92, 0x0f, 0x85, 0x24, 0xa1, 0x03, 0xea, 0x56, 0xb3, 0x88, 0xe0, 0x0c, - 0xb6, 0x6b, 0x3c, 0xf0, 0x7b, 0xb0, 0x8b, 0xd3, 0x94, 0xb3, 0x0b, 0x3a, 0xc0, 0x92, 0xf8, 0x29, - 0x67, 0x29, 0x13, 0x38, 0xf6, 0x5f, 0x0e, 0x99, 0xc4, 0x5a, 0x13, 0x05, 0xe4, 0xcc, 0x20, 0x8e, - 0x33, 0xc0, 0x89, 0xf2, 0xc3, 0x6f, 0xc0, 0xa3, 0x79, 0x86, 0xdf, 0x53, 0x5b, 0x68, 0x86, 0xb0, - 0xae, 0xc9, 0xdb, 0xe9, 0x2c, 0xa3, 0x85, 0x05, 0x31, 0x13, 0xf9, 0x01, 0x7c, 0xb4, 0x40, 0xe5, - 0xc4, 0xec, 0xf0, 0xcb, 0x21, 0x19, 0x12, 0x67, 0xa3, 0x5e, 0x68, 0x16, 0xd0, 0xa3, 0x39, 0x36, - 0x32, 0x88, 0x13, 0x05, 0x80, 0x9f, 0x82, 0x0d, 0xae, 0x6e, 0xd3, 0x1f, 0xe0, 0x0b, 0xbf, 0xf7, - 0x4a, 0x12, 0xe1, 0x14, 0x75, 0xc6, 0x35, 0x6d, 0x3e, 0xc2, 0x17, 0x2d, 0x65, 0x84, 0xbf, 0x82, - 0x1d, 0x1c, 0x48, 0x3a, 0x22, 0xb7, 0xf5, 0x56, 0xbe, 0xbf, 0xde, 0xb6, 0x4c, 0x8c, 0x05, 0xc5, - 0xc1, 0xe7, 0x60, 0x47, 0x67, 0x3b, 0x23, 0x24, 0xf4, 0x39, 0x89, 0xa8, 0x90, 0x1c, 0x4b, 0xca, - 0x12, 0xa1, 0xc5, 0x5c, 0x40, 0xdb, 0x53, 0x37, 0x9a, 0xf5, 0xc2, 0xcf, 0xc0, 0xaa, 0x24, 0x09, - 0x4e, 0xa4, 0x4f, 0x43, 0xa7, 0xa2, 0x6e, 0xbb, 0x55, 0x9e, 0x8c, 0x6b, 0xc5, 0xae, 0x36, 0xb6, - 0x0f, 0x50, 0xd1, 0xb8, 0xdb, 0x21, 0x24, 0x60, 0x67, 0xb1, 0x72, 0x3f, 0x65, 0x31, 0x0d, 0x5e, - 0x39, 0xb0, 0x6e, 0x35, 0xd7, 0xe7, 0xb4, 0x3b, 0xf7, 0xfe, 0x5a, 0xa8, 0xf6, 0x58, 0x93, 0xd0, - 0x56, 0x70, 0x97, 0x19, 0xfe, 0x69, 0x81, 0x8f, 0x6f, 0xe5, 0x11, 0x34, 0x24, 0x92, 0xe3, 0x44, - 0xa4, 0x8c, 0x2b, 0x71, 0x9f, 0x31, 0x67, 0x53, 0x0f, 0xed, 0xf9, 0xbb, 0xf7, 0x45, 0x55, 0x70, - 0x4a, 0x43, 0xd2, 0xcd, 0xf9, 0x6a, 0xef, 0x5a, 0x4d, 0x35, 0xce, 0xc9, 0xb8, 0x56, 0x5f, 0x28, - 0xee, 0x16, 0x12, 0xd5, 0x83, 0xdb, 0x08, 0x39, 0x8b, 0x98, 0xee, 0xbb, 0x5d, 0x59, 0x9e, 0xee, - 0x7b, 0xa9, 0x52, 0xee, 0xd8, 0xc5, 0x87, 0x15, 0xd8, 0xf8, 0xa3, 0x00, 0xb6, 0xef, 0x2e, 0x02, - 0x76, 0xc0, 0x3a, 0x37, 0xeb, 0x9c, 0x89, 0x22, 0x7b, 0x0b, 0xdc, 0x4b, 0x0a, 0x6b, 0x19, 0xd5, - 0x34, 0x00, 0x87, 0xa0, 0x94, 0xc7, 0x8a, 0x31, 0xd5, 0xab, 0x5f, 0x68, 0x75, 0x27, 0xe3, 0x1a, - 0xc8, 0xde, 0x18, 0x87, 0x7b, 0xed, 0x7f, 0xc7, 0xb5, 0x56, 0x44, 0x65, 0x7f, 0xd8, 0x73, 0x03, - 0x36, 0xf0, 0xa6, 0x49, 0xc2, 0xde, 0xcd, 0xd9, 0x4b, 0xcf, 0x23, 0x6f, 0xf6, 0x0b, 0x6f, 0x2a, - 0x93, 0xc2, 0x0b, 0xa4, 0xfa, 0xae, 0xec, 0xb5, 0x11, 0xc8, 0x12, 0x1d, 0x62, 0xaa, 0x5a, 0x08, - 0x48, 0x22, 0x39, 0x8e, 0xf3, 0x16, 0x0a, 0xff, 0xa3, 0x85, 0x8c, 0x7a, 0xd3, 0x42, 0x1e, 0x4b, - 0xb5, 0x60, 0xdf, 0xb4, 0xb0, 0x6f, 0xcc, 0xef, 0xb1, 0x85, 0x2c, 0xd1, 0x21, 0xa6, 0xe6, 0xfa, - 0x5a, 0x4f, 0x2f, 0xdf, 0x56, 0x97, 0x2e, 0x27, 0x55, 0xeb, 0x6a, 0x52, 0xb5, 0xde, 0x4c, 0xaa, - 0xd6, 0x5f, 0x93, 0xaa, 0xf5, 0xdb, 0x75, 0x75, 0xe9, 0xea, 0xba, 0xba, 0xf4, 0xe6, 0xba, 0xba, - 0xf4, 0x0b, 0xb8, 0xf9, 0xd7, 0xe9, 0x3d, 0xd0, 0xff, 0x0b, 0x5f, 0xfe, 0x17, 0x00, 0x00, 0xff, - 0xff, 0x26, 0xb9, 0x1b, 0x85, 0x0c, 0x09, 0x00, 0x00, + // 1098 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4d, 0x6f, 0xe3, 0xc4, + 0x1b, 0xaf, 0xff, 0x71, 0x77, 0xd3, 0x49, 0xda, 0x66, 0xa7, 0xff, 0xb6, 0xde, 0x2e, 0x9b, 0x44, + 0x41, 0xa0, 0x20, 0x2d, 0xb6, 0x58, 0x10, 0x2b, 0x5e, 0x24, 0xd4, 0xb4, 0x12, 0x4a, 0x68, 0x51, + 0xeb, 0x86, 0x45, 0x82, 0x83, 0x35, 0xb1, 0xa7, 0xce, 0x28, 0x8e, 0xc7, 0x3b, 0x33, 0x8e, 0xda, + 0xfd, 0x14, 0x7c, 0x04, 0x0e, 0x48, 0x5c, 0xf9, 0x18, 0x3d, 0xf6, 0xb8, 0xa7, 0x08, 0xd2, 0x0b, + 0x17, 0xbe, 0x00, 0x27, 0x34, 0x33, 0x76, 0xde, 0x5a, 0x69, 0x8b, 0xc4, 0x6d, 0xf2, 0x3c, 0xbf, + 0xdf, 0x3c, 0x6f, 0xbf, 0x67, 0x62, 0xd0, 0x18, 0x8c, 0x9c, 0xc1, 0x88, 0x63, 0x36, 0xc2, 0x6c, + 0x7a, 0x48, 0x7a, 0x0e, 0x17, 0x48, 0x60, 0x3b, 0x61, 0x54, 0x50, 0x58, 0xf3, 0xa9, 0x3f, 0x60, + 0x14, 0xf9, 0x7d, 0x7b, 0x30, 0xb2, 0x73, 0x90, 0xcd, 0x05, 0x65, 0x28, 0xc4, 0x49, 0x6f, 0xef, + 0x49, 0x76, 0x74, 0x70, 0x1c, 0x92, 0x18, 0x27, 0x3d, 0x67, 0x38, 0xf2, 0x7d, 0xcd, 0xde, 0x7b, + 0xa2, 0x98, 0x49, 0xcf, 0x21, 0xb1, 0xc0, 0x2c, 0x46, 0x91, 0xc7, 0xd0, 0xb9, 0xc8, 0x9c, 0x3b, + 0xb9, 0x73, 0x88, 0x05, 0x0a, 0x90, 0x40, 0x99, 0x1d, 0xe6, 0xf6, 0x39, 0x9b, 0x95, 0x0a, 0x12, + 0x39, 0xfd, 0xc8, 0x77, 0x04, 0x19, 0x62, 0x2e, 0xd0, 0x30, 0xc9, 0x3c, 0xff, 0x0f, 0x69, 0x48, + 0xd5, 0xd1, 0x91, 0x27, 0x6d, 0x6d, 0xfc, 0xba, 0x0a, 0xca, 0x2e, 0x4e, 0x22, 0xe2, 0xa3, 0x33, + 0x59, 0x0d, 0x7c, 0x06, 0xa0, 0x0c, 0xed, 0xa1, 0x24, 0x89, 0x08, 0x0e, 0x3c, 0x12, 0x07, 0xf8, + 0xc2, 0x32, 0xea, 0x46, 0xd3, 0x74, 0x2b, 0xd2, 0xb3, 0xaf, 0x1d, 0x6d, 0x69, 0x87, 0x36, 0xd8, + 0x8a, 0x30, 0xe2, 0x78, 0x09, 0xfe, 0x3f, 0x05, 0x7f, 0xa4, 0x5c, 0x0b, 0xf8, 0x4f, 0x81, 0x19, + 0x60, 0xee, 0x5b, 0x85, 0xba, 0xd1, 0x2c, 0x3d, 0x6f, 0xd8, 0xb3, 0xa6, 0x65, 0xb5, 0xd8, 0x2e, + 0x8a, 0x43, 0x7c, 0x88, 0xb9, 0xcf, 0x48, 0x22, 0x28, 0x73, 0x15, 0x1e, 0xda, 0x60, 0x55, 0x5d, + 0x66, 0x99, 0x8a, 0x68, 0xdd, 0x41, 0x3c, 0x92, 0x7e, 0x57, 0xc3, 0xe0, 0xb7, 0x60, 0x53, 0xb0, + 0x34, 0xf6, 0x91, 0xc0, 0x81, 0xa7, 0xc6, 0x64, 0xad, 0x2a, 0xe6, 0x7b, 0x77, 0x86, 0x3c, 0x17, + 0xdd, 0x1c, 0xad, 0xba, 0xe0, 0x6e, 0x88, 0x85, 0xdf, 0xf0, 0x14, 0x94, 0x43, 0xdf, 0x13, 0x7d, + 0x86, 0x79, 0x9f, 0x46, 0x81, 0xf5, 0x40, 0x5d, 0xf6, 0x74, 0xee, 0x32, 0xd9, 0x77, 0xbb, 0x1f, + 0xf9, 0x76, 0x37, 0xef, 0x7b, 0x6b, 0x73, 0x32, 0xae, 0x95, 0xbe, 0x3e, 0xe8, 0xe6, 0x2c, 0xb7, + 0x14, 0xfa, 0xd3, 0x1f, 0xf0, 0x0b, 0xb0, 0x2a, 0x13, 0xe3, 0xd6, 0xc3, 0x5b, 0x89, 0x65, 0x4a, + 0xb1, 0x73, 0xa5, 0xd8, 0xc7, 0x2f, 0x0f, 0x0e, 0x64, 0x22, 0xdc, 0xd5, 0x1c, 0xd8, 0x01, 0x8d, + 0x00, 0x27, 0x0c, 0xeb, 0x02, 0x53, 0x4e, 0xe2, 0x70, 0x3a, 0x02, 0x55, 0xae, 0x37, 0xc0, 0x97, + 0x56, 0xa9, 0x6e, 0x34, 0x8b, 0x6e, 0x75, 0x86, 0xfc, 0x4e, 0x02, 0xb3, 0x81, 0xa8, 0xc2, 0xbe, + 0xc1, 0x97, 0xf0, 0x13, 0xf0, 0x70, 0x84, 0x19, 0x27, 0x34, 0xb6, 0xca, 0x2a, 0x95, 0xbd, 0x3b, + 0x7a, 0xf4, 0x52, 0x23, 0xdc, 0x1c, 0x0a, 0xbf, 0x07, 0xdb, 0x4a, 0x27, 0x7e, 0x44, 0x39, 0x0e, + 0xbc, 0xa9, 0xda, 0xac, 0xf5, 0xfb, 0xb4, 0xc6, 0xbc, 0x1a, 0xd7, 0x56, 0xdc, 0x2d, 0x79, 0xc3, + 0x81, 0xba, 0x60, 0xea, 0xfa, 0xdc, 0xfc, 0xf3, 0xe7, 0x9a, 0xd1, 0x31, 0x8b, 0xc5, 0xca, 0x5a, + 0xc7, 0x2c, 0xae, 0x55, 0x40, 0xc7, 0x2c, 0x82, 0x4a, 0xa9, 0xf1, 0xd7, 0x43, 0xb0, 0xa6, 0xc4, + 0xd1, 0x8e, 0xcf, 0x29, 0x3c, 0xd6, 0xdd, 0xc3, 0x4a, 0x99, 0xa5, 0xe7, 0x1f, 0xda, 0x6f, 0x59, + 0x3f, 0x7b, 0x5e, 0xe4, 0xad, 0xa2, 0x0c, 0x7f, 0x3d, 0xae, 0x19, 0xba, 0x9f, 0x18, 0x3e, 0x05, + 0x20, 0x42, 0x5c, 0x2c, 0xc8, 0x77, 0x4d, 0x5a, 0xb4, 0x6c, 0x6b, 0xa0, 0x14, 0xa7, 0x43, 0x2f, + 0xc1, 0x71, 0x40, 0xe2, 0x50, 0xa9, 0xd7, 0x74, 0x41, 0x9c, 0x0e, 0x4f, 0xb4, 0x25, 0x07, 0x04, + 0x8c, 0x26, 0x09, 0x0e, 0x94, 0xd6, 0x34, 0xe0, 0x50, 0x5b, 0x60, 0x03, 0xac, 0xab, 0x76, 0x45, + 0x34, 0xf4, 0x38, 0x79, 0x8d, 0x95, 0x82, 0x0a, 0x6e, 0x49, 0x1a, 0x8f, 0x68, 0x78, 0x46, 0x5e, + 0x63, 0xf8, 0x51, 0xd6, 0xd2, 0x1c, 0xe3, 0x09, 0x96, 0x72, 0x81, 0x03, 0x0b, 0xa8, 0x39, 0xc2, + 0x39, 0x6c, 0x57, 0x7b, 0xe0, 0x97, 0x60, 0x0f, 0x25, 0x09, 0xa3, 0x17, 0x64, 0x28, 0x87, 0x9e, + 0x30, 0x9a, 0x50, 0x8e, 0x22, 0xef, 0x55, 0x4a, 0x05, 0x52, 0xca, 0x2a, 0xb8, 0xd6, 0x1c, 0xe2, + 0x24, 0x03, 0x9c, 0x4a, 0x3f, 0xfc, 0x0c, 0x3c, 0x5e, 0x64, 0x78, 0x3d, 0xb9, 0xcb, 0xba, 0x09, + 0x1b, 0x8a, 0xbc, 0x93, 0xcc, 0x33, 0x5a, 0x88, 0x63, 0xdd, 0x91, 0xaf, 0xc0, 0x3b, 0x4b, 0x54, + 0x86, 0xf5, 0x4b, 0xf0, 0x2a, 0xc5, 0x29, 0xb6, 0x36, 0xeb, 0x85, 0x66, 0xc1, 0x7d, 0xbc, 0xc0, + 0x76, 0x35, 0xe2, 0x54, 0x02, 0xe0, 0xfb, 0x60, 0x93, 0xc9, 0x69, 0x7a, 0x43, 0x74, 0xe1, 0xf5, + 0x2e, 0x05, 0xe6, 0x56, 0x51, 0x45, 0x5c, 0x57, 0xe6, 0x63, 0x74, 0xd1, 0x92, 0x46, 0xf8, 0x23, + 0xd8, 0x45, 0xbe, 0x20, 0x23, 0x7c, 0x5b, 0x69, 0xe5, 0xfb, 0x2b, 0x6d, 0x5b, 0xdf, 0xb1, 0xa4, + 0x35, 0xf8, 0x02, 0xec, 0xaa, 0x68, 0xe7, 0x18, 0x07, 0x1e, 0xc3, 0x21, 0xe1, 0x82, 0x21, 0x41, + 0x68, 0xcc, 0x95, 0x8c, 0x0b, 0xee, 0xce, 0xd4, 0xed, 0xce, 0x7b, 0xe1, 0x07, 0x60, 0x4d, 0xe0, + 0x18, 0xc5, 0xc2, 0x23, 0x81, 0x55, 0x91, 0xd3, 0x6e, 0x95, 0x27, 0xe3, 0x5a, 0xb1, 0xab, 0x8c, + 0xed, 0x43, 0xb7, 0xa8, 0xdd, 0xed, 0x00, 0x62, 0xb0, 0xbb, 0x9c, 0xb9, 0x97, 0xd0, 0x88, 0xf8, + 0x97, 0x16, 0xac, 0x1b, 0xcd, 0x8d, 0x05, 0xed, 0x2e, 0xbc, 0x82, 0x4b, 0xd9, 0x9e, 0x28, 0x92, + 0xbb, 0xed, 0xdf, 0x65, 0x86, 0xbf, 0x19, 0xe0, 0xdd, 0x5b, 0x71, 0x38, 0x09, 0xb0, 0x60, 0x28, + 0xe6, 0x09, 0x65, 0x52, 0xdc, 0xe7, 0xd4, 0xda, 0x52, 0x4d, 0x7b, 0xf1, 0xf6, 0x7d, 0x91, 0x19, + 0x9c, 0x91, 0x00, 0x77, 0x73, 0xbe, 0xdc, 0xbb, 0x56, 0x53, 0xb6, 0x73, 0x32, 0xae, 0xd5, 0x97, + 0x92, 0xbb, 0x85, 0x74, 0xeb, 0xfe, 0x6d, 0x84, 0x98, 0x47, 0x4c, 0x37, 0xdd, 0xac, 0xac, 0x4e, + 0x37, 0xbd, 0x54, 0x29, 0x77, 0xcc, 0xe2, 0xa3, 0x0a, 0x6c, 0xfc, 0x52, 0x00, 0x3b, 0x77, 0x27, + 0x01, 0x3b, 0x60, 0x83, 0xe9, 0x75, 0xce, 0x44, 0x91, 0xbd, 0x02, 0xf7, 0x92, 0xc2, 0x7a, 0x46, + 0xd5, 0x05, 0xc0, 0x14, 0x94, 0xf2, 0xbb, 0x22, 0x44, 0xd4, 0xea, 0x17, 0x5a, 0xdd, 0xc9, 0xb8, + 0x06, 0xb2, 0x17, 0xe3, 0x68, 0xbf, 0xfd, 0xf7, 0xb8, 0xd6, 0x0a, 0x89, 0xe8, 0xa7, 0x3d, 0xdb, + 0xa7, 0x43, 0x67, 0x1a, 0x24, 0xe8, 0xcd, 0xce, 0x4e, 0x32, 0x08, 0x9d, 0xf9, 0xef, 0x04, 0x9d, + 0x99, 0xe0, 0x8e, 0x2f, 0xe4, 0xbf, 0xd3, 0x7e, 0xdb, 0x05, 0x59, 0xa0, 0x23, 0x44, 0x64, 0x09, + 0x3e, 0x8e, 0x05, 0x43, 0x51, 0x5e, 0x42, 0xe1, 0x5f, 0x94, 0x90, 0x51, 0x67, 0x25, 0xe4, 0x77, + 0xc9, 0x12, 0xcc, 0x59, 0x09, 0x07, 0xda, 0xfc, 0x1f, 0x96, 0x90, 0x05, 0x3a, 0x42, 0x44, 0x8f, + 0xaf, 0xf5, 0xec, 0xea, 0x8f, 0xea, 0xca, 0xd5, 0xa4, 0x6a, 0x5c, 0x4f, 0xaa, 0xc6, 0x9b, 0x49, + 0xd5, 0xf8, 0x7d, 0x52, 0x35, 0x7e, 0xba, 0xa9, 0xae, 0x5c, 0xdf, 0x54, 0x57, 0xde, 0xdc, 0x54, + 0x57, 0x7e, 0x00, 0xb3, 0x2f, 0xa6, 0xde, 0x03, 0xf5, 0xd5, 0xf1, 0xf1, 0x3f, 0x01, 0x00, 0x00, + 0xff, 0xff, 0x6e, 0xa8, 0xa8, 0x9c, 0x52, 0x09, 0x00, 0x00, } func (this *ReplicaState) Equal(that interface{}) bool { @@ -361,6 +398,9 @@ func (this *ReplicaState) Equal(that interface{}) bool { if !this.Stats.Equal(that1.Stats) { return false } + if this.DeprecatedUsingAppliedStateKey != that1.DeprecatedUsingAppliedStateKey { + return false + } if !this.Version.Equal(that1.Version) { return false } @@ -515,6 +555,16 @@ func (m *ReplicaState) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x62 } + if m.DeprecatedUsingAppliedStateKey { + i-- + if m.DeprecatedUsingAppliedStateKey { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x58 + } if m.Stats != nil { { size, err := m.Stats.MarshalToSizedBuffer(dAtA[:i]) @@ -822,6 +872,9 @@ func (m *ReplicaState) Size() (n int) { l = m.Stats.Size() n += 1 + l + sovState(uint64(l)) } + if m.DeprecatedUsingAppliedStateKey { + n += 2 + } if m.Version != nil { l = m.Version.Size() n += 1 + l + sovState(uint64(l)) @@ -1158,6 +1211,26 @@ func (m *ReplicaState) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field DeprecatedUsingAppliedStateKey", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowState + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.DeprecatedUsingAppliedStateKey = bool(v != 0) case 12: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) diff --git a/pkg/kv/kvserver/kvserverpb/state.proto b/pkg/kv/kvserver/kvserverpb/state.proto index 857873a119ff..8f484529721f 100644 --- a/pkg/kv/kvserver/kvserverpb/state.proto +++ b/pkg/kv/kvserver/kvserverpb/state.proto @@ -59,6 +59,41 @@ message ReplicaState { // not be served. util.hlc.Timestamp gc_threshold = 6 [(gogoproto.customname) = "GCThreshold"]; storage.enginepb.MVCCStats stats = 7; + // DeprecatedUsingAppliedStateKey was previously used to check whether the + // Range had upgraded to begin using the RangeAppliedState key. When set to + // true, replicas receiving the state (through a ReplicatedEvalResult) knew to + // begin using the new key. In #58088 (21.1) we introduced a migration to + // iterate through all ranges in the system and have them start using the + // RangeAppliedState key. This field was always set to true in 21.2, nodes + // nodes were already using the RangeAppliedState key, or receiving messages + // from 21.1 nodes that were also already using the RangeAppliedState key. In + // 21.2 (and in 21.1 for that matter) we no longer needed to trigger the "if + // set to true in an incoming message, upgrade to start using the + // RangeAppliedState key" code path. + // + // When looking to get rid of this field in 22.1 (#70464), we observed that + // there was an unintentional read of this field in 21.2 nodes (#72029). The + // saga is as follows: + // - Removing this field in 22.1 meant it was set as false when received at + // 21.2 nodes. + // - This should've been fine! We weren't using this field to trigger any + // upgrade code paths (like it was originally intended for). + // - It turns out that in 21.2 we were using the ReplicaState from the + // incoming snapshot to update our in-memory replica state + // - Because the proto field was being phased out, there was now a divergence + // between the on-disk state (field set to true, from earlier 21.2 + // operations) and the in-memory state (field set to false, because sent + // from a version that attempted to get rid of this field). + // + // Removing proto fields from the replica state are not possible until we stop + // using the protobuf copy of the replica state when applying a snapshot + // (#72222). Once that's done, we should be able to stop sending the replica + // state as part of the snapshot in the subsequent release. + // + // TODO(irfansharif): Remove this field in the release after the one where we + // stop consulting the protobuf copy of the replica state when applying a + // snapshot. + bool deprecated_using_applied_state_key = 11; // Version tells us which migrations can be assumed to have run against this // particular replica. When we introduce backwards incompatible changes to the // replica state (for example using the unreplicated truncated state instead @@ -99,7 +134,7 @@ message ReplicaState { // "follower reads" at or below this timestamp. util.hlc.Timestamp raft_closed_timestamp = 13 [(gogoproto.nullable) = false]; - reserved 8, 9, 10, 11; + reserved 8, 9, 10; } // RangeInfo is used for reporting status information about a range out through diff --git a/pkg/kv/kvserver/replica.go b/pkg/kv/kvserver/replica.go index 798f5a9f59b0..8adb15ddad5f 100644 --- a/pkg/kv/kvserver/replica.go +++ b/pkg/kv/kvserver/replica.go @@ -1202,6 +1202,11 @@ func (r *Replica) assertStateRaftMuLockedReplicaMuRLocked( if err != nil { log.Fatalf(ctx, "%v", err) } + + // We don't care about this field; see comment on + // DeprecatedUsingAppliedStateKey for more details. This can be removed once + // we stop loading the replica state from snapshot protos. + diskState.DeprecatedUsingAppliedStateKey = r.mu.state.DeprecatedUsingAppliedStateKey if !diskState.Equal(r.mu.state) { // The roundabout way of printing here is to expose this information in sentry.io. // diff --git a/pkg/kv/kvserver/replica_command.go b/pkg/kv/kvserver/replica_command.go index cf7a9db9a649..d5abccc0198e 100644 --- a/pkg/kv/kvserver/replica_command.go +++ b/pkg/kv/kvserver/replica_command.go @@ -2451,6 +2451,10 @@ func (r *Replica) sendSnapshot( Term: snap.RaftSnap.Metadata.Term, } + // See comment on DeprecatedUsingAppliedStateKey for why we need to set this + // explicitly for snapshots going out to followers. + snap.State.DeprecatedUsingAppliedStateKey = true + req := SnapshotRequest_Header{ State: snap.State, DeprecatedUnreplicatedTruncatedState: true, diff --git a/pkg/kv/kvserver/store_snapshot.go b/pkg/kv/kvserver/store_snapshot.go index 68af0e53c8ef..c8f8fd4e0a6a 100644 --- a/pkg/kv/kvserver/store_snapshot.go +++ b/pkg/kv/kvserver/store_snapshot.go @@ -941,6 +941,10 @@ func SendEmptySnapshot( if err != nil { return err } + // See comment on DeprecatedUsingAppliedStateKey for why we need to set this + // explicitly for snapshots going out to followers. + state.DeprecatedUsingAppliedStateKey = true + hs, err := sl.LoadHardState(ctx, eng) if err != nil { return err