server: introduce a hook for short-running migrations

TODO: see if this migration is actually "short-running". That is,
in a sufficiently large cluster, does this cause significant load?

----

This is a baby version of #39182 that handles only short-running
migrations but is useful in itself because it allows us to migrate us
fully into the following two KV below-Raft migrations:

1. use the RangeAppliedState on all ranges
2. use the unreplicated TruncatedState on all ranges

These two migrations have been around for a while and it has been
getting in the way of things. While ideally we introduce #39182 in the
near future and use it to address a larger class of migration concerns,
this PR serves as a starting point and work done on #39182 should be
able to absorb this PR with relative ease.

With this PR, legacy code related to 1) and 2) above can be removed from
`master` once the 20.1 branch is cut, i.e. roughly in April 2020.

The main ingredients in this PR are

a) introduce a hook that is called during `SET CLUSTER SETTING version`,
   after the change has been validated but before it is made.
b) introduce a KV-level `Migrate` ranged write command that triggers
   the migrations for 1) and 2) above. It is called from the hook for
   all of the keyspace.
c) before returning to the client, `Migrate` waits for the command to
   be durably applied on all followers.

Trying to get this 100% correct has proven tricky, perhaps foreshadowing
similar issues that expect us once we try #39182 in earnest. For one,
the mechanism only reaches replicas that members of the raft group, that
is, it won't touch replicas which are gc'able. For the migrations at
hand this means that in 20.2 there may - in theory - still be replicas
that have a replicated truncated state. I believe that our recent
efforts around not re-using replicas across replicaIDs has made sure
that this isn't an issue for this particular migration, but in general
it will have to remain on the radar. Similarly, it seems hard to prove
conclusively that no snapshot is in-flight that would set up a new
follower with a state predating the explicit migration, though it would
be exceptionally rare in practice.

Release note (general change): version upgrades now perform maintenance
duties that may slightly delay the `SET CLUSTER SETTING version` command
and may cause a small amount of additional load on the cluster while an
upgrade is being finalized.

pkg/keys/constants.go

@@ -142,11 +142,6 @@ var (
 	LocalRangeMax    = LocalRangePrefix.PrefixEnd()
 	// LocalQueueLastProcessedSuffix is the suffix for replica queue state keys.
 	LocalQueueLastProcessedSuffix = roachpb.RKey("qlpt")
-	// LocalRangeDescriptorJointSuffix is the suffix for keys storing
-	// range descriptors. The value is a struct of type RangeDescriptor.
-	//
-	// TODO(tbg): decide what to actually store here. This is still unused.
-	LocalRangeDescriptorJointSuffix = roachpb.RKey("rdjt")
 	// LocalRangeDescriptorSuffix is the suffix for keys storing
 	// range descriptors. The value is a struct of type RangeDescriptor.
 	LocalRangeDescriptorSuffix = roachpb.RKey("rdsc")

pkg/keys/doc.go

@@ -195,10 +195,9 @@ var keymap = [...]interface{}{ // lint:ignore U1001
 	//   as a whole. They are replicated and addressable. Typical examples are
 	//   the range descriptor and transaction records. They all share
 	//   `LocalRangePrefix`.
-	QueueLastProcessedKey,   // "qlpt"
-	RangeDescriptorJointKey, // "rdjt"
-	RangeDescriptorKey,      // "rdsc"
-	TransactionKey,          // "txn-"
+	QueueLastProcessedKey, // "qlpt"
+	RangeDescriptorKey,    // "rdsc"
+	TransactionKey,        // "txn-"
 
 	//   4. Store local keys: These contain metadata about an individual store.
 	//   They are unreplicated and unaddressable. The typical example is the

pkg/keys/keys.go

@@ -401,16 +401,6 @@ func RangeDescriptorKey(key roachpb.RKey) roachpb.Key {
 	return MakeRangeKey(key, LocalRangeDescriptorSuffix, nil)
 }
 
-// RangeDescriptorJointKey returns a range-local key for the "joint descriptor"
-// for the range with specified key. This key is not versioned and it is set if
-// and only if the range is in a joint configuration that it yet has to transition
-// out of.
-func RangeDescriptorJointKey(key roachpb.RKey) roachpb.Key {
-	return MakeRangeKey(key, LocalRangeDescriptorJointSuffix, nil)
-}
-
-var _ = RangeDescriptorJointKey // silence unused check
-
 // TransactionKey returns a transaction key based on the provided
 // transaction key and ID. The base key is encoded in order to
 // guarantee that all transaction records for a range sort together.

pkg/roachpb/api.go

@@ -578,6 +578,8 @@ func (*ImportRequest) Method() Method { return Import }
 // Method implements the Request interface.
 func (*AdminScatterRequest) Method() Method { return AdminScatter }
 
+func (*MigrateRequest) Method() Method { return Migrate }
+
 // Method implements the Request interface.
 func (*AddSSTableRequest) Method() Method { return AddSSTable }
 
@@ -821,6 +823,11 @@ func (r *AdminScatterRequest) ShallowCopy() Request {
 	return &shallowCopy
 }
 
+func (r *MigrateRequest) ShallowCopy() Request {
+	shallowCopy := *r
+	return &shallowCopy
+}
+
 // ShallowCopy implements the Request interface.
 func (r *AddSSTableRequest) ShallowCopy() Request {
 	shallowCopy := *r
@@ -1120,6 +1127,7 @@ func (*ExportRequest) flags() int                        { return isRead | isRan
 func (*ImportRequest) flags() int                        { return isAdmin | isAlone }
 func (*AdminScatterRequest) flags() int                  { return isAdmin | isRange | isAlone }
 func (*AdminVerifyProtectedTimestampRequest) flags() int { return isAdmin | isRange | isAlone }
+func (*MigrateRequest) flags() int                       { return isWrite | isRange | isAlone }
 func (*AddSSTableRequest) flags() int {
 	return isWrite | isRange | isAlone | isUnsplittable | canBackpressure
 }

pkg/roachpb/api.proto

@@ -1467,6 +1467,23 @@ message AdminScatterResponse {
   repeated Range ranges = 2 [(gogoproto.nullable) = false];
 }
 
+// MigrateRequest forces all ranges overlapping it to proactively move out of
+// any legacy modes that they are currently in. When this command returns, the
+// ranges are ready to run with the most up to date cluster version supported
+// by this binary.
+message MigrateRequest {
+  option (gogoproto.equal) = true;
+
+  RequestHeader header = 1 [(gogoproto.nullable) = false, (gogoproto.embed) = true];
+  // The new version that will become active next.
+  Version NewVersion = 2 [(gogoproto.nullable) = false];
+}
+
+// MigrateResponse is the response to an Migrate operation.
+message MigrateResponse {
+  ResponseHeader header = 1 [(gogoproto.nullable) = false, (gogoproto.embed) = true];
+}
+
 // AdminVerifyProtectedTimestampRequest is the argument to the
 // AdminVerifyProtectedTimestamp method which ensures that the specified record
 // will be seen before data can be garbage collected at the timestamp.
@@ -1704,6 +1721,7 @@ message RequestUnion {
     SubsumeRequest subsume = 43;
     RangeStatsRequest range_stats = 44;
     AdminVerifyProtectedTimestampRequest admin_verify_protected_timestamp = 49;
+    MigrateRequest migrate = 50;
   }
   reserved 8, 15, 23, 25, 27;
 }
@@ -1755,6 +1773,7 @@ message ResponseUnion {
     SubsumeResponse subsume = 43;
     RangeStatsResponse range_stats = 44;
     AdminVerifyProtectedTimestampResponse admin_verify_protected_timestamp = 49;
+    MigrateResponse migrate = 50;
   }
   reserved 8, 15, 23, 25, 27, 28;
 }