diff --git a/pkg/internal/client/txn.go b/pkg/internal/client/txn.go index bea5d24f8eed..8635c1183c45 100644 --- a/pkg/internal/client/txn.go +++ b/pkg/internal/client/txn.go @@ -980,7 +980,7 @@ func firstWriteIndex(ba roachpb.BatchRequest) (int, *roachpb.Error) { // the txn. If the error is not a RetryableTxnError, then this is a no-op. For a // retryable error, the Transaction proto is either initialized with the updated // proto from the error, or a new Transaction proto is initialized. -// This needs to be called when a SQL hits a retryable error outside of the +// This needs to be called when a SQL statement hits a retryable error outside of the // TxnCoordSender, like in distSQL or retryable errors in SQL logic. func (txn *Txn) UpdateStateOnRetryableErr(ctx context.Context, pErr roachpb.Error) { txn.mu.Lock() diff --git a/pkg/kv/txn_coord_sender.go b/pkg/kv/txn_coord_sender.go index aaac029f07ee..8ca54c36ea52 100644 --- a/pkg/kv/txn_coord_sender.go +++ b/pkg/kv/txn_coord_sender.go @@ -886,6 +886,8 @@ func (tc *TxnCoordSender) updateState( tc.metrics.RestartsSerializable.Inc(1) case roachpb.RETRY_POSSIBLE_REPLAY: tc.metrics.RestartsPossibleReplay.Inc(1) + case roachpb.RETRY_TABLE_FROM_FUTURE, roachpb.RETRY_TABLE_VERSION_CHANGE: + panic("illegal transaction retry error") } } newTxn = roachpb.PrepareTransactionForRetry(ctx, pErr, ba.UserPriority) diff --git a/pkg/roachpb/errors.pb.go b/pkg/roachpb/errors.pb.go index c56c5813a858..35d8b88a726d 100644 --- a/pkg/roachpb/errors.pb.go +++ b/pkg/roachpb/errors.pb.go @@ -38,6 +38,11 @@ const ( // that the write timestamp on the table descriptor is after the // transaction timestamp. RETRY_TABLE_FROM_FUTURE TransactionRetryReason = 5 + // A transaction using a table descriptor with a specific version + // for some commands, sees a new table version that it cannot + // use for more commands. Retry until it can run the entire + // transaction with one table version. + RETRY_TABLE_VERSION_CHANGE TransactionRetryReason = 6 ) var TransactionRetryReason_name = map[int32]string{ @@ -47,14 +52,16 @@ var TransactionRetryReason_name = map[int32]string{ 3: "RETRY_SERIALIZABLE", 4: "RETRY_POSSIBLE_REPLAY", 5: "RETRY_TABLE_FROM_FUTURE", + 6: "RETRY_TABLE_VERSION_CHANGE", } var TransactionRetryReason_value = map[string]int32{ - "RETRY_REASON_UNKNOWN": 0, - "RETRY_WRITE_TOO_OLD": 1, - "RETRY_DELETE_RANGE": 2, - "RETRY_SERIALIZABLE": 3, - "RETRY_POSSIBLE_REPLAY": 4, - "RETRY_TABLE_FROM_FUTURE": 5, + "RETRY_REASON_UNKNOWN": 0, + "RETRY_WRITE_TOO_OLD": 1, + "RETRY_DELETE_RANGE": 2, + "RETRY_SERIALIZABLE": 3, + "RETRY_POSSIBLE_REPLAY": 4, + "RETRY_TABLE_FROM_FUTURE": 5, + "RETRY_TABLE_VERSION_CHANGE": 6, } func (x TransactionRetryReason) Enum() *TransactionRetryReason { @@ -5357,127 +5364,128 @@ var ( func init() { proto.RegisterFile("cockroach/pkg/roachpb/errors.proto", fileDescriptorErrors) } var fileDescriptorErrors = []byte{ - // 1950 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x58, 0xcb, 0x6f, 0x1b, 0xc7, - 0x19, 0xd7, 0x4a, 0x94, 0x29, 0x7e, 0x14, 0x45, 0x6a, 0x2c, 0xcb, 0x6b, 0x39, 0xa1, 0x94, 0x4d, - 0x9a, 0x3a, 0x2e, 0x20, 0xa6, 0x4a, 0x5d, 0x20, 0x6e, 0x0b, 0x84, 0x12, 0x57, 0x0a, 0x2d, 0x89, - 0x14, 0x86, 0x54, 0xd4, 0x24, 0x05, 0xb6, 0x2b, 0xee, 0x98, 0xdc, 0x78, 0xb9, 0x4b, 0xcf, 0xce, - 0x5a, 0xd2, 0xa5, 0xe7, 0x1e, 0x7b, 0x6c, 0x6f, 0x01, 0x8a, 0xde, 0x8a, 0xa2, 0xd7, 0xde, 0x7b, - 0xf0, 0xa5, 0x40, 0x8f, 0x45, 0x0f, 0x42, 0xab, 0x02, 0xfd, 0x23, 0x7c, 0x0a, 0xe6, 0xb1, 0xe4, - 0x52, 0x5a, 0xca, 0x44, 0x6e, 0xcb, 0xef, 0xf1, 0x9b, 0xc7, 0xf7, 0xf8, 0x7d, 0x43, 0x30, 0x3a, - 0x41, 0xe7, 0x05, 0x0d, 0xec, 0x4e, 0xaf, 0x32, 0x78, 0xd1, 0xad, 0x88, 0xaf, 0xc1, 0x69, 0x85, - 0x50, 0x1a, 0xd0, 0x70, 0x73, 0x40, 0x03, 0x16, 0xa0, 0xe5, 0xa1, 0xcd, 0xa6, 0xd2, 0xaf, 0x7d, - 0x90, 0xee, 0xd6, 0x27, 0xcc, 0x76, 0x6c, 0x66, 0x4b, 0xc7, 0xb5, 0x8d, 0x74, 0xab, 0x84, 0xc5, - 0x87, 0xe3, 0x16, 0x11, 0x73, 0xbd, 0x4a, 0xcf, 0xeb, 0x54, 0x98, 0xdb, 0x27, 0x21, 0xb3, 0xfb, - 0x03, 0x65, 0xb7, 0xd2, 0x0d, 0xba, 0x81, 0xf8, 0xac, 0xf0, 0x2f, 0x29, 0x35, 0xfe, 0x3a, 0x0b, - 0x77, 0x1b, 0x01, 0x3b, 0x20, 0x76, 0x48, 0x3e, 0x0f, 0x3c, 0x87, 0x50, 0x93, 0xef, 0x1b, 0xd5, - 0x20, 0x4b, 0xc9, 0xc0, 0x73, 0x3b, 0xb6, 0xae, 0x6d, 0x68, 0x8f, 0xf2, 0x5b, 0x1f, 0x6c, 0xde, - 0x38, 0xc2, 0x26, 0x96, 0x16, 0x35, 0x12, 0x76, 0xa8, 0x3b, 0x60, 0x01, 0xdd, 0xce, 0xbc, 0xbe, - 0x5c, 0x9f, 0xc1, 0xb1, 0x2b, 0xda, 0x83, 0x45, 0x8f, 0x23, 0x5b, 0x3d, 0x01, 0xad, 0xcf, 0x4e, - 0x0f, 0x85, 0xf3, 0xde, 0x68, 0x4f, 0xe8, 0x09, 0x2c, 0x50, 0xdb, 0xef, 0x12, 0xcb, 0x75, 0xf4, - 0xb9, 0x0d, 0xed, 0xd1, 0xdc, 0xf6, 0x1a, 0x5f, 0xe9, 0xea, 0x72, 0x3d, 0x8b, 0xb9, 0xbc, 0x5e, - 0x7b, 0x33, 0xfa, 0xc4, 0x59, 0x61, 0x5b, 0x77, 0xd0, 0x26, 0xcc, 0x0b, 0x14, 0x3d, 0x23, 0x16, - 0xd6, 0x53, 0x16, 0x16, 0x27, 0xc7, 0xd2, 0x0c, 0xbd, 0x0f, 0xd0, 0x89, 0x42, 0x16, 0xf4, 0xad, - 0x7e, 0xd8, 0xd5, 0xe7, 0x37, 0xb4, 0x47, 0x39, 0x75, 0xa4, 0x9c, 0x94, 0x1f, 0x86, 0x5d, 0x63, - 0x15, 0x56, 0x1a, 0x81, 0x43, 0x8e, 0x7d, 0xfb, 0x95, 0xed, 0x7a, 0xf6, 0xa9, 0x47, 0xc4, 0x95, - 0x19, 0xfb, 0x80, 0xc4, 0x06, 0x1a, 0x01, 0xdb, 0x0d, 0x22, 0xdf, 0x91, 0x17, 0x99, 0xdc, 0xb9, - 0x36, 0xf5, 0xce, 0x8d, 0x3f, 0xcc, 0xc2, 0x3d, 0x21, 0xdc, 0x27, 0x17, 0x87, 0x6e, 0xd8, 0xb7, - 0x59, 0xa7, 0x27, 0x01, 0x3f, 0x81, 0x65, 0x4a, 0x5e, 0x46, 0x24, 0x64, 0x56, 0xc8, 0x6c, 0xca, - 0xac, 0x17, 0xe4, 0x42, 0x20, 0x2f, 0x6e, 0x67, 0xdf, 0x5c, 0xae, 0xcf, 0xed, 0x93, 0x0b, 0x5c, - 0x54, 0x16, 0x2d, 0x6e, 0xb0, 0x4f, 0x2e, 0x50, 0x05, 0x62, 0x91, 0x45, 0x7c, 0x47, 0xb8, 0xcc, - 0x8e, 0xbb, 0x14, 0x94, 0xde, 0xf4, 0x1d, 0xee, 0x70, 0x08, 0xa5, 0xbe, 0x5a, 0x96, 0x38, 0x96, - 0xd8, 0x95, 0xb8, 0xf8, 0xfc, 0x96, 0x91, 0x16, 0x3d, 0xae, 0x4f, 0xc4, 0xae, 0x38, 0xf2, 0x15, - 0x2a, 0xb4, 0x0f, 0xc5, 0x30, 0xea, 0x76, 0x49, 0xc8, 0x86, 0x68, 0x99, 0xa9, 0xd1, 0x96, 0x86, - 0xae, 0x42, 0x63, 0xfc, 0x5d, 0x03, 0x03, 0x13, 0xdb, 0x39, 0x71, 0x59, 0xcf, 0xf5, 0x8f, 0xfd, - 0x0e, 0xa1, 0xcc, 0x76, 0x7d, 0x76, 0x51, 0xf7, 0x19, 0xa1, 0xaf, 0x6c, 0x4f, 0x5e, 0xd4, 0x33, - 0x58, 0xa2, 0xc4, 0x76, 0xac, 0x61, 0x21, 0xa8, 0x4c, 0x7e, 0x37, 0xb1, 0x24, 0xaf, 0x96, 0xcd, - 0x9e, 0xd7, 0xd9, 0x6c, 0xc7, 0x46, 0x2a, 0xde, 0x05, 0xee, 0x3a, 0x14, 0x22, 0x0c, 0x88, 0x9c, - 0xbb, 0x21, 0x73, 0xfd, 0x6e, 0x02, 0x6f, 0x76, 0x7a, 0xbc, 0xe5, 0xd8, 0x7d, 0xa8, 0x30, 0x1e, - 0xc0, 0xfd, 0x36, 0xb5, 0xfd, 0xd0, 0xee, 0x30, 0x37, 0xf0, 0xab, 0xa7, 0x01, 0x65, 0x44, 0x26, - 0x8d, 0xf1, 0x35, 0xac, 0x24, 0x54, 0x47, 0x51, 0xa8, 0x62, 0xbf, 0x03, 0x30, 0x88, 0xc2, 0x1e, - 0x21, 0x16, 0x3b, 0xf7, 0xd5, 0x71, 0xca, 0x29, 0x37, 0x98, 0x70, 0x8e, 0xf3, 0x57, 0xfa, 0xb5, - 0xcf, 0x7d, 0xe3, 0xd7, 0x70, 0x2f, 0xa1, 0xc7, 0x84, 0xd1, 0x0b, 0x89, 0xbe, 0x07, 0x77, 0x28, - 0xb1, 0xc3, 0x40, 0x22, 0x2f, 0x6d, 0x7d, 0x74, 0x3b, 0xb2, 0xf0, 0xc4, 0xc2, 0x41, 0x2d, 0xa2, - 0xdc, 0x0d, 0x1d, 0x56, 0xc7, 0xec, 0x06, 0x9e, 0x2d, 0x97, 0x30, 0x3e, 0x1e, 0xd3, 0xb4, 0x98, - 0xcd, 0xa2, 0x50, 0x2e, 0xbe, 0x0a, 0x73, 0xbc, 0xe6, 0xb4, 0x44, 0xcd, 0x71, 0x81, 0xd1, 0x82, - 0xd2, 0x09, 0x75, 0x19, 0xe1, 0xb1, 0xf5, 0x99, 0xb4, 0xfd, 0x14, 0xb2, 0xae, 0xf8, 0x19, 0xea, - 0xda, 0xc6, 0xdc, 0xa3, 0xfc, 0xd6, 0x83, 0x94, 0x9d, 0x4a, 0x87, 0xb8, 0x23, 0x29, 0xfb, 0x67, - 0x99, 0x85, 0xd9, 0xd2, 0x9c, 0xf1, 0x27, 0x4d, 0xa1, 0xb6, 0x83, 0xa0, 0xe9, 0xa9, 0x4a, 0xad, - 0x42, 0xee, 0x7b, 0xa5, 0xca, 0xc8, 0x0b, 0x35, 0xa0, 0x64, 0x77, 0x58, 0x64, 0x7b, 0xdf, 0x2f, - 0x49, 0x8a, 0xd2, 0x79, 0x94, 0x22, 0x2b, 0x80, 0x9a, 0x03, 0x4c, 0x5e, 0x46, 0x2e, 0x25, 0x61, - 0xfb, 0xdc, 0x97, 0x97, 0xd8, 0x82, 0x95, 0x9d, 0xc0, 0x77, 0x5c, 0x7e, 0x85, 0xbb, 0xb6, 0xeb, - 0xa9, 0xac, 0x41, 0x3f, 0x83, 0x45, 0xb5, 0xfa, 0x2b, 0xdb, 0x8b, 0x88, 0x3a, 0x43, 0x5a, 0xd3, - 0xfb, 0x82, 0xeb, 0x71, 0x5e, 0x5a, 0x8b, 0x1f, 0xc6, 0x5f, 0x34, 0x40, 0xb2, 0x17, 0x92, 0x6f, - 0x48, 0x27, 0xce, 0x44, 0x54, 0x86, 0x6c, 0x9f, 0x84, 0xa1, 0xdd, 0x25, 0x63, 0xa1, 0x89, 0x85, - 0xe8, 0xe7, 0x90, 0x53, 0x8d, 0x83, 0x38, 0xea, 0xa8, 0x13, 0xbb, 0x6c, 0x7c, 0x5f, 0x43, 0x07, - 0xf4, 0x14, 0x16, 0xe2, 0xba, 0x50, 0xdd, 0xe5, 0x6d, 0xce, 0x43, 0x7b, 0xe3, 0xc7, 0x90, 0x6b, - 0x11, 0x7f, 0xba, 0x6d, 0xaa, 0xb0, 0xbf, 0x84, 0x95, 0x6a, 0xff, 0xd4, 0xed, 0x46, 0x41, 0x14, - 0x62, 0x12, 0x46, 0x1e, 0x9b, 0xee, 0x90, 0x9f, 0x42, 0xfe, 0x8c, 0xda, 0x83, 0x01, 0x71, 0x2c, - 0x42, 0xe9, 0x2d, 0xc7, 0x14, 0x70, 0x18, 0x94, 0xb1, 0x49, 0xa9, 0x71, 0x9f, 0xb7, 0xf1, 0xe7, - 0x6c, 0x8f, 0x06, 0xd1, 0xa0, 0x46, 0x3c, 0x32, 0x2c, 0x71, 0x0b, 0x56, 0x15, 0xe7, 0xed, 0x04, - 0x94, 0x46, 0x03, 0x1e, 0x4c, 0xb9, 0x9b, 0xf7, 0x20, 0x27, 0x66, 0x07, 0xeb, 0x7a, 0x3d, 0x2c, - 0x08, 0xf1, 0x61, 0xd8, 0x45, 0x06, 0xe4, 0x06, 0x34, 0xe8, 0x90, 0x30, 0x54, 0xb7, 0xbe, 0x30, - 0x2c, 0xf3, 0x58, 0x6c, 0xb4, 0x00, 0xa9, 0x05, 0x92, 0x49, 0xfe, 0x0b, 0x00, 0x45, 0xce, 0x31, - 0x21, 0xcd, 0x6f, 0x97, 0x15, 0x21, 0xe5, 0x94, 0xbd, 0xa0, 0xa4, 0xd1, 0x0f, 0x1e, 0x30, 0xf9, - 0xe9, 0x70, 0x8e, 0x6b, 0xb1, 0x80, 0xde, 0xe4, 0xb8, 0x90, 0x4b, 0x53, 0x38, 0x4e, 0x58, 0x4b, - 0x8e, 0x53, 0x9f, 0x38, 0x2b, 0x6c, 0xeb, 0x8e, 0xf1, 0xff, 0x25, 0xc8, 0x0b, 0x80, 0x1a, 0x61, - 0xb6, 0xeb, 0xa1, 0x23, 0x28, 0xf9, 0x01, 0xb3, 0xc6, 0x26, 0x06, 0x99, 0xc3, 0x1f, 0xa6, 0xdc, - 0x75, 0xca, 0xd4, 0x82, 0x97, 0xfc, 0x31, 0x21, 0x3a, 0x84, 0xa2, 0x24, 0x5f, 0x8e, 0xfb, 0x9c, - 0x6f, 0x58, 0x05, 0xef, 0x07, 0x93, 0x68, 0x67, 0xec, 0x60, 0xb8, 0x40, 0x93, 0x32, 0xf4, 0x05, - 0x20, 0x09, 0xf7, 0x82, 0x5c, 0x58, 0x31, 0xc5, 0xa9, 0xc4, 0x7d, 0x34, 0x09, 0xf1, 0x3a, 0x81, - 0xe3, 0x12, 0xbd, 0x26, 0x46, 0xbf, 0x81, 0x0d, 0xc1, 0x54, 0x67, 0x82, 0xd0, 0xac, 0x68, 0xc4, - 0x68, 0x96, 0xab, 0x28, 0x4d, 0xd1, 0xe5, 0x93, 0xd4, 0xd1, 0xe9, 0x6d, 0x54, 0x88, 0xdf, 0xa5, - 0xb7, 0xd9, 0xa0, 0xaf, 0xe1, 0x2e, 0x1b, 0x75, 0x65, 0xcb, 0x96, 0x54, 0x24, 0xe6, 0x9f, 0xfc, - 0xd6, 0xe3, 0xdb, 0x59, 0x20, 0xc9, 0x5b, 0x18, 0xb1, 0x1b, 0x0a, 0x84, 0xa1, 0x94, 0x04, 0xe7, - 0x3c, 0xa4, 0xdf, 0x11, 0xc8, 0x3f, 0xbc, 0x1d, 0x79, 0x48, 0x7b, 0xb8, 0xc8, 0xc6, 0xa5, 0xe8, - 0x18, 0x96, 0x93, 0x98, 0x94, 0x33, 0x91, 0x9e, 0x9d, 0x18, 0x87, 0x54, 0xba, 0xc3, 0xc9, 0x6d, - 0x09, 0x31, 0xfa, 0x25, 0x24, 0x0f, 0xc0, 0xc7, 0x2b, 0x16, 0x85, 0xfa, 0x82, 0xc0, 0x7d, 0x0b, - 0x19, 0x26, 0xa8, 0x0c, 0x27, 0xf7, 0x26, 0xe5, 0x68, 0x17, 0x16, 0xcf, 0x38, 0xdf, 0x58, 0x92, - 0x87, 0xf4, 0x9c, 0xc0, 0x7c, 0x3f, 0x05, 0xf3, 0x3a, 0xd9, 0xe1, 0xfc, 0xd9, 0x48, 0x82, 0xf6, - 0xa0, 0x20, 0x71, 0x58, 0x10, 0x58, 0x81, 0xe7, 0xe8, 0x70, 0x3b, 0x50, 0xa2, 0xf4, 0x15, 0x90, - 0x94, 0xf0, 0xca, 0x08, 0x06, 0x16, 0x55, 0xd4, 0x22, 0xc6, 0x89, 0xfc, 0xc4, 0xca, 0xb8, 0xc9, - 0x41, 0xb8, 0x10, 0x24, 0x65, 0x3c, 0xc8, 0x9d, 0x98, 0x92, 0xac, 0xe7, 0x82, 0x93, 0xf4, 0xc5, - 0x89, 0x41, 0x4e, 0x63, 0x2f, 0x5c, 0xec, 0x8c, 0x4b, 0xd1, 0x01, 0x2c, 0xc9, 0x56, 0x40, 0x15, - 0x23, 0xe9, 0x85, 0x89, 0x3b, 0xbc, 0xc9, 0x5c, 0xb8, 0xe0, 0x25, 0x65, 0x7c, 0x87, 0x7e, 0xe0, - 0x10, 0x2b, 0x1a, 0x8d, 0xed, 0xfa, 0xd2, 0xc4, 0x1d, 0xa6, 0x0d, 0xf8, 0xb8, 0xe8, 0x8f, 0x4b, - 0xd1, 0xc7, 0x90, 0x09, 0x89, 0xef, 0xe8, 0x45, 0x81, 0xf3, 0x4e, 0x0a, 0xce, 0x90, 0xa1, 0xb0, - 0xb0, 0x94, 0x1d, 0xe4, 0x39, 0xb3, 0xba, 0x9c, 0x0f, 0x2c, 0x47, 0x12, 0x82, 0x5e, 0xba, 0xa5, - 0x83, 0xa4, 0x70, 0x07, 0xef, 0x20, 0xe3, 0x62, 0x9e, 0xb9, 0x71, 0x5b, 0xef, 0x0c, 0xe9, 0x44, - 0x5f, 0x9e, 0x98, 0xb9, 0xe9, 0xd4, 0x83, 0x97, 0xe9, 0x75, 0xb9, 0x68, 0xa1, 0x0a, 0x39, 0xce, - 0x39, 0x34, 0xb9, 0x85, 0xde, 0x20, 0x1c, 0x3e, 0x48, 0x27, 0x64, 0xd7, 0x4b, 0x8c, 0x8a, 0xd9, - 0x50, 0x5f, 0x9d, 0xa6, 0xc4, 0x12, 0x73, 0xe4, 0x58, 0x89, 0x49, 0x39, 0x0f, 0xb0, 0x1d, 0x93, - 0xbb, 0x45, 0x05, 0xbb, 0xeb, 0x6b, 0x13, 0x03, 0x9c, 0x36, 0x07, 0xe0, 0xa2, 0x3d, 0x2e, 0xe5, - 0x87, 0x97, 0xc4, 0x36, 0xe2, 0x8f, 0x87, 0x13, 0x0f, 0x7f, 0x93, 0x18, 0x71, 0x21, 0x4c, 0xca, - 0xd0, 0x37, 0xf0, 0xb0, 0x67, 0xfb, 0x8e, 0xc7, 0xdf, 0x40, 0xbc, 0xe1, 0xf0, 0x24, 0xe2, 0xa5, - 0x67, 0x09, 0x62, 0xd7, 0xdf, 0x11, 0xd0, 0x3f, 0x4a, 0x81, 0xfe, 0x5c, 0x7a, 0xe1, 0xd8, 0x69, - 0x58, 0x86, 0x7a, 0x6f, 0x82, 0xe6, 0x69, 0xe6, 0xf5, 0xb7, 0xeb, 0xda, 0xb3, 0xcc, 0xc2, 0x83, - 0xd2, 0x9a, 0xf1, 0x91, 0xe0, 0xd9, 0xa3, 0x20, 0x14, 0xe5, 0x85, 0xd6, 0x60, 0xde, 0xf5, 0x1d, - 0x72, 0xae, 0xe8, 0x5f, 0x4e, 0x0e, 0x52, 0x64, 0xfc, 0x79, 0x0e, 0xe6, 0xa7, 0x1b, 0x8a, 0x7e, - 0x35, 0x4e, 0x1a, 0x94, 0x88, 0xd7, 0xa8, 0x60, 0xc3, 0xa5, 0xd4, 0xfb, 0x19, 0x0b, 0xa5, 0x30, - 0x56, 0x90, 0x88, 0xdd, 0xd0, 0xa0, 0x1d, 0x28, 0x44, 0x3e, 0x39, 0x1f, 0x04, 0x21, 0x71, 0x44, - 0x77, 0xca, 0x4c, 0xf3, 0xd8, 0xc1, 0x8b, 0x43, 0x27, 0xde, 0x95, 0x2a, 0x90, 0x0f, 0xa8, 0xdb, - 0x75, 0x7d, 0x8b, 0x57, 0xae, 0xe0, 0xb3, 0xf9, 0xed, 0x25, 0xbe, 0xe6, 0x9b, 0xcb, 0xf5, 0x3b, - 0xbc, 0xc6, 0xeb, 0x35, 0x0c, 0xd2, 0x84, 0xff, 0x42, 0x3f, 0x85, 0x3b, 0x8e, 0x98, 0x45, 0x14, - 0x43, 0x95, 0x27, 0xcd, 0x78, 0x72, 0x62, 0xc1, 0xca, 0x1a, 0xfd, 0x24, 0xbe, 0xd1, 0xec, 0x6d, - 0x6e, 0x71, 0x00, 0xd4, 0x5d, 0xa3, 0x27, 0x30, 0xe7, 0x07, 0x67, 0x8a, 0x5f, 0xa6, 0x7a, 0x20, - 0x70, 0xfb, 0xa7, 0x99, 0xdf, 0x7f, 0xbb, 0x3e, 0xa3, 0x26, 0xda, 0x43, 0xb8, 0x7f, 0xec, 0x5f, - 0xcf, 0x01, 0x19, 0xbf, 0x2d, 0xc8, 0x0c, 0x4c, 0x4a, 0x6f, 0x79, 0x05, 0x08, 0x3b, 0x85, 0x2c, - 0x6c, 0x8d, 0x7f, 0x68, 0xa0, 0x4f, 0xca, 0xb5, 0x49, 0x2f, 0x34, 0x74, 0x02, 0x77, 0x78, 0x0e, - 0xbb, 0x8e, 0xfa, 0x4b, 0xe1, 0xb3, 0x7f, 0x5f, 0xae, 0x7f, 0xd2, 0x75, 0x59, 0x2f, 0x3a, 0xdd, - 0xec, 0x04, 0xfd, 0xca, 0x70, 0x61, 0xe7, 0xb4, 0x92, 0xf2, 0x5f, 0x55, 0x14, 0xb9, 0xce, 0xe6, - 0xf1, 0x71, 0xbd, 0x76, 0x75, 0xb9, 0x3e, 0xdf, 0x3e, 0xf7, 0xeb, 0x35, 0x3c, 0xcf, 0xce, 0xfd, - 0xba, 0x83, 0x3e, 0x83, 0x7c, 0x22, 0x33, 0xd4, 0x9c, 0xf5, 0xb6, 0x0c, 0x48, 0xba, 0x3c, 0xfe, - 0x9b, 0x76, 0xed, 0x25, 0x3a, 0x7c, 0xb1, 0x22, 0x1d, 0x56, 0xb0, 0xd9, 0xc6, 0x5f, 0x5a, 0xd8, - 0xac, 0xb6, 0x9a, 0x0d, 0xeb, 0xb8, 0xb1, 0xdf, 0x68, 0x9e, 0x34, 0x4a, 0x33, 0xe8, 0x3e, 0xdc, - 0x95, 0x9a, 0x13, 0x5c, 0x6f, 0x9b, 0x56, 0xbb, 0xd9, 0xb4, 0x9a, 0x07, 0xb5, 0x92, 0x86, 0x56, - 0x01, 0x49, 0x45, 0xcd, 0x3c, 0x30, 0xdb, 0xa6, 0x85, 0xab, 0x8d, 0x3d, 0xb3, 0x34, 0x3b, 0x92, - 0xb7, 0x4c, 0x5c, 0xaf, 0x1e, 0xd4, 0xbf, 0xaa, 0x6e, 0x1f, 0x98, 0xa5, 0x39, 0xf4, 0x00, 0xee, - 0x49, 0xf9, 0x51, 0xb3, 0xd5, 0xaa, 0x6f, 0x1f, 0x98, 0x16, 0x36, 0x8f, 0x0e, 0xaa, 0x5f, 0x96, - 0x32, 0xe8, 0x21, 0xdc, 0x97, 0xaa, 0x36, 0xb7, 0xb5, 0x76, 0x71, 0xf3, 0xd0, 0xda, 0x3d, 0x6e, - 0x1f, 0x63, 0xb3, 0x34, 0xbf, 0x96, 0xf9, 0xed, 0x1f, 0xcb, 0x33, 0x8f, 0x9f, 0x02, 0xba, 0x59, - 0x31, 0x68, 0x01, 0x32, 0x8d, 0x66, 0xc3, 0x2c, 0xcd, 0xa0, 0x3c, 0x64, 0xb7, 0xab, 0x3b, 0xfb, - 0xcd, 0xdd, 0xdd, 0x92, 0x86, 0x0a, 0x90, 0xab, 0x1f, 0x1e, 0x9a, 0xb5, 0x7a, 0xb5, 0x6d, 0x96, - 0x66, 0xb7, 0xdf, 0x7b, 0xfd, 0xdf, 0xf2, 0xcc, 0xeb, 0xab, 0xb2, 0xf6, 0xcf, 0xab, 0xb2, 0xf6, - 0xaf, 0xab, 0xb2, 0xf6, 0x9f, 0xab, 0xb2, 0xf6, 0xbb, 0xff, 0x95, 0x67, 0xbe, 0xca, 0xaa, 0x3b, - 0xfb, 0x2e, 0x00, 0x00, 0xff, 0xff, 0xd7, 0xa0, 0x51, 0xfb, 0xb6, 0x14, 0x00, 0x00, + // 1966 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x58, 0xcb, 0x4f, 0x23, 0xc9, + 0x19, 0xc7, 0x60, 0x30, 0xfe, 0x8c, 0xb1, 0xa9, 0x61, 0xa0, 0x87, 0xd9, 0x35, 0x6c, 0xef, 0x66, + 0xc3, 0x4e, 0x24, 0xbc, 0x61, 0x33, 0x91, 0x76, 0x92, 0x48, 0x6b, 0x70, 0xc3, 0x78, 0x00, 0x1b, + 0x95, 0xcd, 0x90, 0xdd, 0x8d, 0xd4, 0x69, 0xdc, 0x35, 0x76, 0xef, 0xb4, 0xbb, 0x3d, 0xd5, 0xd5, + 0x03, 0x5c, 0x72, 0xce, 0x31, 0xc7, 0xe4, 0xb6, 0x52, 0x94, 0x5b, 0x14, 0xe5, 0x8f, 0xc8, 0x61, + 0x2e, 0x91, 0x72, 0x4c, 0x72, 0x40, 0x09, 0x91, 0xf2, 0x47, 0xcc, 0x69, 0x55, 0x8f, 0xb6, 0xdb, + 0xd0, 0x66, 0xac, 0xbd, 0xb5, 0xbf, 0xc7, 0xaf, 0x1e, 0xdf, 0xe3, 0xf7, 0x95, 0x41, 0x6f, 0xfb, + 0xed, 0x97, 0xd4, 0xb7, 0xda, 0xdd, 0x72, 0xff, 0x65, 0xa7, 0x2c, 0xbe, 0xfa, 0x67, 0x65, 0x42, + 0xa9, 0x4f, 0x83, 0xad, 0x3e, 0xf5, 0x99, 0x8f, 0x96, 0x06, 0x36, 0x5b, 0x4a, 0xbf, 0xf6, 0x51, + 0xb2, 0x5b, 0x8f, 0x30, 0xcb, 0xb6, 0x98, 0x25, 0x1d, 0xd7, 0x36, 0x92, 0xad, 0x62, 0x16, 0x1f, + 0x8f, 0x5a, 0x84, 0xcc, 0x71, 0xcb, 0x5d, 0xb7, 0x5d, 0x66, 0x4e, 0x8f, 0x04, 0xcc, 0xea, 0xf5, + 0x95, 0xdd, 0x72, 0xc7, 0xef, 0xf8, 0xe2, 0xb3, 0xcc, 0xbf, 0xa4, 0x54, 0xff, 0xeb, 0x34, 0xdc, + 0xab, 0xfb, 0xec, 0x90, 0x58, 0x01, 0x79, 0xea, 0xbb, 0x36, 0xa1, 0x06, 0xdf, 0x37, 0xaa, 0x42, + 0x86, 0x92, 0xbe, 0xeb, 0xb4, 0x2d, 0x2d, 0xb5, 0x91, 0xda, 0xcc, 0x6d, 0x7f, 0xb4, 0x75, 0xeb, + 0x08, 0x5b, 0x58, 0x5a, 0x54, 0x49, 0xd0, 0xa6, 0x4e, 0x9f, 0xf9, 0x74, 0x27, 0xfd, 0xe6, 0x6a, + 0x7d, 0x0a, 0x47, 0xae, 0x68, 0x1f, 0x16, 0x5c, 0x8e, 0x6c, 0x76, 0x05, 0xb4, 0x36, 0x3d, 0x39, + 0x14, 0xce, 0xb9, 0xc3, 0x3d, 0xa1, 0xc7, 0x30, 0x4f, 0x2d, 0xaf, 0x43, 0x4c, 0xc7, 0xd6, 0x66, + 0x36, 0x52, 0x9b, 0x33, 0x3b, 0x6b, 0x7c, 0xa5, 0xeb, 0xab, 0xf5, 0x0c, 0xe6, 0xf2, 0x5a, 0xf5, + 0xed, 0xf0, 0x13, 0x67, 0x84, 0x6d, 0xcd, 0x46, 0x5b, 0x30, 0x2b, 0x50, 0xb4, 0xb4, 0x58, 0x58, + 0x4b, 0x58, 0x58, 0x9c, 0x1c, 0x4b, 0x33, 0xf4, 0x21, 0x40, 0x3b, 0x0c, 0x98, 0xdf, 0x33, 0x7b, + 0x41, 0x47, 0x9b, 0xdd, 0x48, 0x6d, 0x66, 0xd5, 0x91, 0xb2, 0x52, 0x7e, 0x14, 0x74, 0xf4, 0x15, + 0x58, 0xae, 0xfb, 0x36, 0x39, 0xf1, 0xac, 0xd7, 0x96, 0xe3, 0x5a, 0x67, 0x2e, 0x11, 0x57, 0xa6, + 0x1f, 0x00, 0x12, 0x1b, 0xa8, 0xfb, 0x6c, 0xcf, 0x0f, 0x3d, 0x5b, 0x5e, 0x64, 0x7c, 0xe7, 0xa9, + 0x89, 0x77, 0xae, 0xff, 0x61, 0x1a, 0xee, 0x0b, 0xe1, 0x01, 0xb9, 0x3c, 0x72, 0x82, 0x9e, 0xc5, + 0xda, 0x5d, 0x09, 0xf8, 0x19, 0x2c, 0x51, 0xf2, 0x2a, 0x24, 0x01, 0x33, 0x03, 0x66, 0x51, 0x66, + 0xbe, 0x24, 0x97, 0x02, 0x79, 0x61, 0x27, 0xf3, 0xf6, 0x6a, 0x7d, 0xe6, 0x80, 0x5c, 0xe2, 0x82, + 0xb2, 0x68, 0x72, 0x83, 0x03, 0x72, 0x89, 0xca, 0x10, 0x89, 0x4c, 0xe2, 0xd9, 0xc2, 0x65, 0x7a, + 0xd4, 0x25, 0xaf, 0xf4, 0x86, 0x67, 0x73, 0x87, 0x23, 0x28, 0xf6, 0xd4, 0xb2, 0xc4, 0x36, 0xc5, + 0xae, 0xc4, 0xc5, 0xe7, 0xb6, 0xf5, 0xa4, 0xe8, 0x71, 0x7d, 0x2c, 0x76, 0x85, 0xa1, 0xaf, 0x50, + 0xa1, 0x03, 0x28, 0x04, 0x61, 0xa7, 0x43, 0x02, 0x36, 0x40, 0x4b, 0x4f, 0x8c, 0xb6, 0x38, 0x70, + 0x15, 0x1a, 0xfd, 0x6f, 0x29, 0xd0, 0x31, 0xb1, 0xec, 0x53, 0x87, 0x75, 0x1d, 0xef, 0xc4, 0x6b, + 0x13, 0xca, 0x2c, 0xc7, 0x63, 0x97, 0x35, 0x8f, 0x11, 0xfa, 0xda, 0x72, 0xe5, 0x45, 0x3d, 0x83, + 0x45, 0x4a, 0x2c, 0xdb, 0x1c, 0x14, 0x82, 0xca, 0xe4, 0xf7, 0x63, 0x4b, 0xf2, 0x6a, 0xd9, 0xea, + 0xba, 0xed, 0xad, 0x56, 0x64, 0xa4, 0xe2, 0x9d, 0xe7, 0xae, 0x03, 0x21, 0xc2, 0x80, 0xc8, 0x85, + 0x13, 0x30, 0xc7, 0xeb, 0xc4, 0xf0, 0xa6, 0x27, 0xc7, 0x5b, 0x8a, 0xdc, 0x07, 0x0a, 0xfd, 0x01, + 0xac, 0xb6, 0xa8, 0xe5, 0x05, 0x56, 0x9b, 0x39, 0xbe, 0x57, 0x39, 0xf3, 0x29, 0x23, 0x32, 0x69, + 0xf4, 0xaf, 0x61, 0x39, 0xa6, 0x3a, 0x0e, 0x03, 0x15, 0xfb, 0x5d, 0x80, 0x7e, 0x18, 0x74, 0x09, + 0x31, 0xd9, 0x85, 0xa7, 0x8e, 0x53, 0x4a, 0xb8, 0xc1, 0x98, 0x73, 0x94, 0xbf, 0xd2, 0xaf, 0x75, + 0xe1, 0xe9, 0xbf, 0x86, 0xfb, 0x31, 0x3d, 0x26, 0x8c, 0x5e, 0x4a, 0xf4, 0x7d, 0x98, 0xa3, 0xc4, + 0x0a, 0x7c, 0x89, 0xbc, 0xb8, 0xfd, 0xc9, 0xdd, 0xc8, 0xc2, 0x13, 0x0b, 0x07, 0xb5, 0x88, 0x72, + 0xd7, 0x35, 0x58, 0x19, 0xb1, 0xeb, 0xbb, 0x96, 0x5c, 0x42, 0xff, 0x74, 0x44, 0xd3, 0x64, 0x16, + 0x0b, 0x03, 0xb9, 0xf8, 0x0a, 0xcc, 0xf0, 0x9a, 0x4b, 0xc5, 0x6a, 0x8e, 0x0b, 0xf4, 0x26, 0x14, + 0x4f, 0xa9, 0xc3, 0x08, 0x8f, 0xad, 0xc7, 0xa4, 0xed, 0xe7, 0x90, 0x71, 0xc4, 0xcf, 0x40, 0x4b, + 0x6d, 0xcc, 0x6c, 0xe6, 0xb6, 0x1f, 0x24, 0xec, 0x54, 0x3a, 0x44, 0x1d, 0x49, 0xd9, 0x3f, 0x4b, + 0xcf, 0x4f, 0x17, 0x67, 0xf4, 0x3f, 0xa5, 0x14, 0x6a, 0xcb, 0xf7, 0x1b, 0xae, 0xaa, 0xd4, 0x0a, + 0x64, 0xbf, 0x57, 0xaa, 0x0c, 0xbd, 0x50, 0x1d, 0x8a, 0x56, 0x9b, 0x85, 0x96, 0xfb, 0xfd, 0x92, + 0xa4, 0x20, 0x9d, 0x87, 0x29, 0xb2, 0x0c, 0xa8, 0xd1, 0xc7, 0xe4, 0x55, 0xe8, 0x50, 0x12, 0xb4, + 0x2e, 0x3c, 0x79, 0x89, 0x4d, 0x58, 0xde, 0xf5, 0x3d, 0xdb, 0xe1, 0x57, 0xb8, 0x67, 0x39, 0xae, + 0xca, 0x1a, 0xf4, 0x33, 0x58, 0x50, 0xab, 0xbf, 0xb6, 0xdc, 0x90, 0xa8, 0x33, 0x24, 0x35, 0xbd, + 0xe7, 0x5c, 0x8f, 0x73, 0xd2, 0x5a, 0xfc, 0xd0, 0xff, 0x92, 0x02, 0x24, 0x7b, 0x21, 0xf9, 0x86, + 0xb4, 0xa3, 0x4c, 0x44, 0x25, 0xc8, 0xf4, 0x48, 0x10, 0x58, 0x1d, 0x32, 0x12, 0x9a, 0x48, 0x88, + 0x7e, 0x0e, 0x59, 0xd5, 0x38, 0x88, 0xad, 0x8e, 0x3a, 0xb6, 0xcb, 0x46, 0xf7, 0x35, 0x70, 0x40, + 0x4f, 0x60, 0x3e, 0xaa, 0x0b, 0xd5, 0x5d, 0xde, 0xe5, 0x3c, 0xb0, 0xd7, 0x7f, 0x0c, 0xd9, 0x26, + 0xf1, 0x26, 0xdb, 0xa6, 0x0a, 0xfb, 0x2b, 0x58, 0xae, 0xf4, 0xce, 0x9c, 0x4e, 0xe8, 0x87, 0x01, + 0x26, 0x41, 0xe8, 0xb2, 0xc9, 0x0e, 0xf9, 0x39, 0xe4, 0xce, 0xa9, 0xd5, 0xef, 0x13, 0xdb, 0x24, + 0x94, 0xde, 0x71, 0x4c, 0x01, 0x87, 0x41, 0x19, 0x1b, 0x94, 0xea, 0xab, 0xbc, 0x8d, 0xbf, 0x60, + 0xfb, 0xd4, 0x0f, 0xfb, 0x55, 0xe2, 0x92, 0x41, 0x89, 0x9b, 0xb0, 0xa2, 0x38, 0x6f, 0xd7, 0xa7, + 0x34, 0xec, 0xf3, 0x60, 0xca, 0xdd, 0x7c, 0x00, 0x59, 0x31, 0x3b, 0x98, 0x37, 0xeb, 0x61, 0x5e, + 0x88, 0x8f, 0x82, 0x0e, 0xd2, 0x21, 0xdb, 0xa7, 0x7e, 0x9b, 0x04, 0x81, 0xba, 0xf5, 0xf9, 0x41, + 0x99, 0x47, 0x62, 0xbd, 0x09, 0x48, 0x2d, 0x10, 0x4f, 0xf2, 0x5f, 0x00, 0x28, 0x72, 0x8e, 0x08, + 0x69, 0x76, 0xa7, 0xa4, 0x08, 0x29, 0xab, 0xec, 0x05, 0x25, 0x0d, 0x7f, 0xf0, 0x80, 0xc9, 0x4f, + 0x9b, 0x73, 0x5c, 0x93, 0xf9, 0xf4, 0x36, 0xc7, 0x05, 0x5c, 0x9a, 0xc0, 0x71, 0xc2, 0x5a, 0x72, + 0x9c, 0xfa, 0xc4, 0x19, 0x61, 0x5b, 0xb3, 0xf5, 0xff, 0x2f, 0x42, 0x4e, 0x00, 0x54, 0x09, 0xb3, + 0x1c, 0x17, 0x1d, 0x43, 0xd1, 0xf3, 0x99, 0x39, 0x32, 0x31, 0xc8, 0x1c, 0xfe, 0x38, 0xe1, 0xae, + 0x13, 0xa6, 0x16, 0xbc, 0xe8, 0x8d, 0x08, 0xd1, 0x11, 0x14, 0x24, 0xf9, 0x72, 0xdc, 0x17, 0x7c, + 0xc3, 0x2a, 0x78, 0x3f, 0x18, 0x47, 0x3b, 0x23, 0x07, 0xc3, 0x79, 0x1a, 0x97, 0xa1, 0xe7, 0x80, + 0x24, 0xdc, 0x4b, 0x72, 0x69, 0x46, 0x14, 0xa7, 0x12, 0x77, 0x73, 0x1c, 0xe2, 0x4d, 0x02, 0xc7, + 0x45, 0x7a, 0x43, 0x8c, 0x7e, 0x03, 0x1b, 0x82, 0xa9, 0xce, 0x05, 0xa1, 0x99, 0xe1, 0x90, 0xd1, + 0x4c, 0x47, 0x51, 0x9a, 0xa2, 0xcb, 0xc7, 0x89, 0xa3, 0xd3, 0xbb, 0xa8, 0x10, 0xbf, 0x4f, 0xef, + 0xb2, 0x41, 0x5f, 0xc3, 0x3d, 0x36, 0xec, 0xca, 0xa6, 0x25, 0xa9, 0x48, 0xcc, 0x3f, 0xb9, 0xed, + 0x47, 0x77, 0xb3, 0x40, 0x9c, 0xb7, 0x30, 0x62, 0xb7, 0x14, 0x08, 0x43, 0x31, 0x0e, 0xce, 0x79, + 0x48, 0x9b, 0x13, 0xc8, 0x3f, 0xbc, 0x1b, 0x79, 0x40, 0x7b, 0xb8, 0xc0, 0x46, 0xa5, 0xe8, 0x04, + 0x96, 0xe2, 0x98, 0x94, 0x33, 0x91, 0x96, 0x19, 0x1b, 0x87, 0x44, 0xba, 0xc3, 0xf1, 0x6d, 0x09, + 0x31, 0xfa, 0x25, 0xc4, 0x0f, 0xc0, 0xc7, 0x2b, 0x16, 0x06, 0xda, 0xbc, 0xc0, 0x7d, 0x07, 0x19, + 0xc6, 0xa8, 0x0c, 0xc7, 0xf7, 0x26, 0xe5, 0x68, 0x0f, 0x16, 0xce, 0x39, 0xdf, 0x98, 0x92, 0x87, + 0xb4, 0xac, 0xc0, 0xfc, 0x30, 0x01, 0xf3, 0x26, 0xd9, 0xe1, 0xdc, 0xf9, 0x50, 0x82, 0xf6, 0x21, + 0x2f, 0x71, 0x98, 0xef, 0x9b, 0xbe, 0x6b, 0x6b, 0x70, 0x37, 0x50, 0xac, 0xf4, 0x15, 0x90, 0x94, + 0xf0, 0xca, 0xf0, 0xfb, 0x26, 0x55, 0xd4, 0x22, 0xc6, 0x89, 0xdc, 0xd8, 0xca, 0xb8, 0xcd, 0x41, + 0x38, 0xef, 0xc7, 0x65, 0x3c, 0xc8, 0xed, 0x88, 0x92, 0xcc, 0x17, 0x82, 0x93, 0xb4, 0x85, 0xb1, + 0x41, 0x4e, 0x62, 0x2f, 0x5c, 0x68, 0x8f, 0x4a, 0xd1, 0x21, 0x2c, 0xca, 0x56, 0x40, 0x15, 0x23, + 0x69, 0xf9, 0xb1, 0x3b, 0xbc, 0xcd, 0x5c, 0x38, 0xef, 0xc6, 0x65, 0x7c, 0x87, 0x9e, 0x6f, 0x13, + 0x33, 0x1c, 0x8e, 0xed, 0xda, 0xe2, 0xd8, 0x1d, 0x26, 0x0d, 0xf8, 0xb8, 0xe0, 0x8d, 0x4a, 0xd1, + 0xa7, 0x90, 0x0e, 0x88, 0x67, 0x6b, 0x05, 0x81, 0xf3, 0x5e, 0x02, 0xce, 0x80, 0xa1, 0xb0, 0xb0, + 0x94, 0x1d, 0xe4, 0x05, 0x33, 0x3b, 0x9c, 0x0f, 0x4c, 0x5b, 0x12, 0x82, 0x56, 0xbc, 0xa3, 0x83, + 0x24, 0x70, 0x07, 0xef, 0x20, 0xa3, 0x62, 0x9e, 0xb9, 0x51, 0x5b, 0x6f, 0x0f, 0xe8, 0x44, 0x5b, + 0x1a, 0x9b, 0xb9, 0xc9, 0xd4, 0x83, 0x97, 0xe8, 0x4d, 0xb9, 0x68, 0xa1, 0x0a, 0x39, 0xca, 0x39, + 0x34, 0xbe, 0x85, 0xde, 0x22, 0x1c, 0x3e, 0x48, 0xc7, 0x64, 0x37, 0x4b, 0x8c, 0x8a, 0xd9, 0x50, + 0x5b, 0x99, 0xa4, 0xc4, 0x62, 0x73, 0xe4, 0x48, 0x89, 0x49, 0x39, 0x0f, 0xb0, 0x15, 0x91, 0xbb, + 0x49, 0x05, 0xbb, 0x6b, 0x6b, 0x63, 0x03, 0x9c, 0x34, 0x07, 0xe0, 0x82, 0x35, 0x2a, 0xe5, 0x87, + 0x97, 0xc4, 0x36, 0xe4, 0x8f, 0x87, 0x63, 0x0f, 0x7f, 0x9b, 0x18, 0x71, 0x3e, 0x88, 0xcb, 0xd0, + 0x37, 0xf0, 0xb0, 0x6b, 0x79, 0xb6, 0xcb, 0xdf, 0x40, 0xbc, 0xe1, 0xf0, 0x24, 0xe2, 0xa5, 0x67, + 0x0a, 0x62, 0xd7, 0xde, 0x13, 0xd0, 0x3f, 0x4a, 0x80, 0x7e, 0x2a, 0xbd, 0x70, 0xe4, 0x34, 0x28, + 0x43, 0xad, 0x3b, 0x46, 0xf3, 0x24, 0xfd, 0xe6, 0xdb, 0xf5, 0xd4, 0xb3, 0xf4, 0xfc, 0x83, 0xe2, + 0x9a, 0xfe, 0x89, 0xe0, 0xd9, 0x63, 0x3f, 0x10, 0xe5, 0x85, 0xd6, 0x60, 0xd6, 0xf1, 0x6c, 0x72, + 0xa1, 0xe8, 0x5f, 0x4e, 0x0e, 0x52, 0xa4, 0xff, 0x79, 0x06, 0x66, 0x27, 0x1b, 0x8a, 0x7e, 0x35, + 0x4a, 0x1a, 0x94, 0x88, 0xd7, 0xa8, 0x60, 0xc3, 0xc5, 0xc4, 0xfb, 0x19, 0x09, 0xa5, 0x30, 0x56, + 0x90, 0x88, 0xdd, 0xd2, 0xa0, 0x5d, 0xc8, 0x87, 0x1e, 0xb9, 0xe8, 0xfb, 0x01, 0xb1, 0x45, 0x77, + 0x4a, 0x4f, 0xf2, 0xd8, 0xc1, 0x0b, 0x03, 0x27, 0xde, 0x95, 0xca, 0x90, 0xf3, 0xa9, 0xd3, 0x71, + 0x3c, 0x93, 0x57, 0xae, 0xe0, 0xb3, 0xd9, 0x9d, 0x45, 0xbe, 0xe6, 0xdb, 0xab, 0xf5, 0x39, 0x5e, + 0xe3, 0xb5, 0x2a, 0x06, 0x69, 0xc2, 0x7f, 0xa1, 0x9f, 0xc2, 0x9c, 0x2d, 0x66, 0x11, 0xc5, 0x50, + 0xa5, 0x71, 0x33, 0x9e, 0x9c, 0x58, 0xb0, 0xb2, 0x46, 0x3f, 0x89, 0x6e, 0x34, 0x73, 0x97, 0x5b, + 0x14, 0x00, 0x75, 0xd7, 0xe8, 0x31, 0xcc, 0x78, 0xfe, 0xb9, 0xe2, 0x97, 0x89, 0x1e, 0x08, 0xdc, + 0xfe, 0x49, 0xfa, 0xf7, 0xdf, 0xae, 0x4f, 0xa9, 0x89, 0xf6, 0x08, 0x56, 0x4f, 0xbc, 0x9b, 0x39, + 0x20, 0xe3, 0xb7, 0x0d, 0xe9, 0xbe, 0x41, 0xe9, 0x1d, 0xaf, 0x00, 0x61, 0xa7, 0x90, 0x85, 0xad, + 0xfe, 0xf7, 0x14, 0x68, 0xe3, 0x72, 0x6d, 0xdc, 0x0b, 0x0d, 0x9d, 0xc2, 0x1c, 0xcf, 0x61, 0xc7, + 0x56, 0x7f, 0x29, 0x7c, 0xf1, 0xef, 0xab, 0xf5, 0xcf, 0x3a, 0x0e, 0xeb, 0x86, 0x67, 0x5b, 0x6d, + 0xbf, 0x57, 0x1e, 0x2c, 0x6c, 0x9f, 0x95, 0x13, 0xfe, 0xab, 0x0a, 0x43, 0xc7, 0xde, 0x3a, 0x39, + 0xa9, 0x55, 0xaf, 0xaf, 0xd6, 0x67, 0x5b, 0x17, 0x5e, 0xad, 0x8a, 0x67, 0xd9, 0x85, 0x57, 0xb3, + 0xd1, 0x17, 0x90, 0x8b, 0x65, 0x86, 0x9a, 0xb3, 0xde, 0x95, 0x01, 0x71, 0x97, 0x47, 0xff, 0x4a, + 0xdd, 0x78, 0x89, 0x0e, 0x5e, 0xac, 0x48, 0x83, 0x65, 0x6c, 0xb4, 0xf0, 0x97, 0x26, 0x36, 0x2a, + 0xcd, 0x46, 0xdd, 0x3c, 0xa9, 0x1f, 0xd4, 0x1b, 0xa7, 0xf5, 0xe2, 0x14, 0x5a, 0x85, 0x7b, 0x52, + 0x73, 0x8a, 0x6b, 0x2d, 0xc3, 0x6c, 0x35, 0x1a, 0x66, 0xe3, 0xb0, 0x5a, 0x4c, 0xa1, 0x15, 0x40, + 0x52, 0x51, 0x35, 0x0e, 0x8d, 0x96, 0x61, 0xe2, 0x4a, 0x7d, 0xdf, 0x28, 0x4e, 0x0f, 0xe5, 0x4d, + 0x03, 0xd7, 0x2a, 0x87, 0xb5, 0xaf, 0x2a, 0x3b, 0x87, 0x46, 0x71, 0x06, 0x3d, 0x80, 0xfb, 0x52, + 0x7e, 0xdc, 0x68, 0x36, 0x6b, 0x3b, 0x87, 0x86, 0x89, 0x8d, 0xe3, 0xc3, 0xca, 0x97, 0xc5, 0x34, + 0x7a, 0x08, 0xab, 0x52, 0xd5, 0xe2, 0xb6, 0xe6, 0x1e, 0x6e, 0x1c, 0x99, 0x7b, 0x27, 0xad, 0x13, + 0x6c, 0x14, 0x67, 0x51, 0x09, 0xd6, 0xe2, 0xca, 0xe7, 0x06, 0x6e, 0xd6, 0x1a, 0x75, 0x73, 0xf7, + 0xa9, 0x58, 0x6f, 0x6e, 0x2d, 0xfd, 0xdb, 0x3f, 0x96, 0xa6, 0x1e, 0x3d, 0x01, 0x74, 0xbb, 0xa2, + 0xd0, 0x3c, 0xa4, 0xeb, 0x8d, 0xba, 0x51, 0x9c, 0x42, 0x39, 0xc8, 0xec, 0x54, 0x76, 0x0f, 0x1a, + 0x7b, 0x7b, 0xc5, 0x14, 0xca, 0x43, 0xb6, 0x76, 0x74, 0x64, 0x54, 0x6b, 0x95, 0x96, 0x51, 0x9c, + 0xde, 0xf9, 0xe0, 0xcd, 0x7f, 0x4b, 0x53, 0x6f, 0xae, 0x4b, 0xa9, 0x7f, 0x5c, 0x97, 0x52, 0xff, + 0xbc, 0x2e, 0xa5, 0xfe, 0x73, 0x5d, 0x4a, 0xfd, 0xee, 0x7f, 0xa5, 0xa9, 0xaf, 0x32, 0xea, 0x4e, + 0xbf, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x35, 0x2f, 0x18, 0x25, 0xd6, 0x14, 0x00, 0x00, } diff --git a/pkg/roachpb/errors.proto b/pkg/roachpb/errors.proto index 13236a7b5331..cc7c5f5eaf47 100644 --- a/pkg/roachpb/errors.proto +++ b/pkg/roachpb/errors.proto @@ -126,6 +126,11 @@ enum TransactionRetryReason { // that the write timestamp on the table descriptor is after the // transaction timestamp. RETRY_TABLE_FROM_FUTURE = 5; + // A transaction using a table descriptor with a specific version + // for some commands, sees a new table version that it cannot + // use for more commands. Retry until it can run the entire + // transaction with one table version. + RETRY_TABLE_VERSION_CHANGE = 6; } // A TransactionRetryError indicates that the transaction must be diff --git a/pkg/sql/crdb_internal.go b/pkg/sql/crdb_internal.go index 837529d13ce9..6f813139aedb 100644 --- a/pkg/sql/crdb_internal.go +++ b/pkg/sql/crdb_internal.go @@ -221,7 +221,7 @@ CREATE TABLE crdb_internal.leases ( parent_id INT NOT NULL, expiration TIMESTAMP NOT NULL, released BOOL NOT NULL, - deleted BOOL NOT NULL + dropped BOOL NOT NULL ); `, populate: func(_ context.Context, p *planner, addRow func(...parser.Datum) error) error { @@ -238,7 +238,7 @@ CREATE TABLE crdb_internal.leases ( ts.mu.Lock() defer ts.mu.Unlock() - deleted := parser.MakeDBool(parser.DBool(ts.deleted)) + dropped := parser.MakeDBool(parser.DBool(ts.dropped)) for _, state := range ts.active.data { if !userCanSeeDescriptor(&state.TableDescriptor, p.session.User) { @@ -252,7 +252,7 @@ CREATE TABLE crdb_internal.leases ( parser.NewDInt(parser.DInt(int64(state.ParentID))), &expCopy, parser.MakeDBool(parser.DBool(state.released)), - deleted, + dropped, ); err != nil { return err } diff --git a/pkg/sql/drop.go b/pkg/sql/drop.go index 30a6186cf057..f824292aad0a 100644 --- a/pkg/sql/drop.go +++ b/pkg/sql/drop.go @@ -739,7 +739,6 @@ func (n *dropTableNode) Start(ctx context.Context) error { if err != nil { return err } - // Log a Drop Table event for this table. This is an auditable log event // and is recorded in the same transaction as the table descriptor // update. diff --git a/pkg/sql/lease.go b/pkg/sql/lease.go index b96cad79acdc..ab7d34ba2ff4 100644 --- a/pkg/sql/lease.go +++ b/pkg/sql/lease.go @@ -61,9 +61,7 @@ type LeaseState struct { // This descriptor is immutable and can be shared by many goroutines. // Care must be taken to not modify it. sqlbase.TableDescriptor - // The timestamp of the transaction that wrote the table descriptor. - creationTime hlc.Timestamp - expiration parser.DTimestamp + expiration parser.DTimestamp // mu protects refcount and released mu syncutil.Mutex @@ -144,22 +142,13 @@ func (s LeaseStore) Acquire( minVersion sqlbase.DescriptorVersion, minExpirationTime parser.DTimestamp, ) (*LeaseState, error) { - if log.V(2) { - log.Infof(ctx, "acquiring lease from store on table %d", tableID) - } - lease := &LeaseState{testingKnobs: s.testingKnobs} - expiration := time.Unix(0, s.clock.Now().WallTime).Add(jitteredLeaseDuration()) - expiration = expiration.Round(time.Microsecond) - if !minExpirationTime.IsZero() && expiration.Before(minExpirationTime.Time) { - expiration = minExpirationTime.Time - } - lease.expiration = parser.DTimestamp{Time: expiration} - - // This transaction reads the latest table descriptor and updated the lease - // table with the version of the table descriptor. It is read the table - // descriptor and writing to the lease table, and will thus conflict with - // a schema change that is updating the table version by writing to the - // table descriptor and reading (scanning) the lease table. + log.VEventf(ctx, 2, "acquiring lease from store on table %d", tableID) + var lease *LeaseState + // This transaction reads the latest table descriptor and updates the lease + // table with the version of the table descriptor. It reads the table + // descriptor and writes to the lease table, and will thus conflict with + // a schema change that updates the table version and scans the + // lease table. // // Serializability guarantees that either this transaction will run first // or the schema change. If this transaction runs first it will write a @@ -168,7 +157,7 @@ func (s LeaseStore) Acquire( // increment the table version and the latest version will be seen by // this transaction. err := s.db.Txn(ctx, func(ctx context.Context, txn *client.Txn) error { - tableDesc, timestamp, err := sqlbase.GetTableDescAndTimestampFromID(ctx, txn, tableID) + tableDesc, err := sqlbase.GetTableDescFromID(ctx, txn, tableID) if err != nil { return err } @@ -176,13 +165,23 @@ func (s LeaseStore) Acquire( return err } tableDesc.MaybeUpgradeFormatVersion() + + // Transaction timestamp guaranteed to be set by now. Use the transaction timestamp + // so that we can guarantee that the lease can be used for the expiration time. + lease = &LeaseState{testingKnobs: s.testingKnobs} + expiration := time.Unix(0, txn.Proto().Timestamp.WallTime).Add(jitteredLeaseDuration()) + expiration = expiration.Round(time.Microsecond) + if !minExpirationTime.IsZero() && expiration.Before(minExpirationTime.Time) { + expiration = minExpirationTime.Time + } + lease.expiration = parser.DTimestamp{Time: expiration} + // Once the descriptor is set it is immutable and care must be taken // to not modify it. lease.TableDescriptor = *tableDesc - lease.creationTime = timestamp // Update the hlc so that any future transaction doesn't need to // be restarted. - s.clock.Update(timestamp) + s.clock.Update(tableDesc.ModificationTime) // ValidateTable instead of Validate, even though we have a txn available, // so we don't block reads waiting for this lease. @@ -215,9 +214,7 @@ func (s LeaseStore) Acquire( // Release a previously acquired table descriptor lease. func (s LeaseStore) Release(ctx context.Context, stopper *stop.Stopper, lease *LeaseState) { - if log.V(2) { - log.Infof(ctx, "releasing lease from store on table %d", lease.ID) - } + log.VEventf(ctx, 2, "releasing lease from store on table %d", lease.ID) retryOptions := base.DefaultRetryOptions() retryOptions.Closer = stopper.ShouldQuiesce() firstAttempt := true @@ -265,37 +262,25 @@ func (s LeaseStore) Release(ctx context.Context, stopper *stop.Stopper, lease *L func (s LeaseStore) WaitForOneVersion( ctx context.Context, tableID sqlbase.ID, retryOpts retry.Options, ) (sqlbase.DescriptorVersion, error) { - desc := &sqlbase.Descriptor{} - descKey := sqlbase.MakeDescMetadataKey(tableID) - var tableDesc *sqlbase.TableDescriptor + var version sqlbase.DescriptorVersion for r := retry.Start(retryOpts); r.Next(); { - // Get the current version of the table descriptor non-transactionally. - // - // TODO(pmattis): Do an inconsistent read here? - if err := s.db.GetProto(context.TODO(), descKey, desc); err != nil { - return 0, err - } - tableDesc = desc.GetTable() - if tableDesc == nil { - return 0, errors.Errorf("ID %d is not a table", tableID) - } - // Check to see if there are any leases that still exist on the previous - // version of the descriptor. - now := s.clock.Now() - count, err := s.countLeases(ctx, tableDesc.ID, tableDesc.Version-1, now.GoTime()) - if err != nil { - return 0, err - } - if count == 0 { - break + err := s.db.Txn(ctx, func(ctx context.Context, txn *client.Txn) error { + desc, err := s.reportPrevVersionInUse(ctx, txn, tableID) + if err == nil { + tableDesc := desc.GetTable() + version = tableDesc.Version + } + return err + }) + if err != errPrevVersionInUse { + return version, err } - log.Infof(context.TODO(), "publish (count leases): descID=%d name=%s version=%d count=%d", - tableDesc.ID, tableDesc.Name, tableDesc.Version-1, count) } - return tableDesc.Version, nil + panic("not reached") } var errDidntUpdateDescriptor = errors.New("didn't update the table descriptor") +var errPrevVersionInUse = errors.New("previous table descriptor version in use") // Publish updates a table descriptor. It also maintains the invariant that // there are at most two versions of the descriptor out in the wild at any time @@ -313,40 +298,16 @@ func (s LeaseStore) Publish( update func(*sqlbase.TableDescriptor) error, logEvent func(*client.Txn) error, ) (*sqlbase.Descriptor, error) { - errLeaseVersionChanged := errors.New("lease version changed") - // Retry while getting errLeaseVersionChanged. + var desc *sqlbase.Descriptor + // Retry while getting errPrevVersionInUse. for r := retry.Start(base.DefaultRetryOptions()); r.Next(); { - // Wait until there are no unexpired leases on the previous version - // of the table. - expectedVersion, err := s.WaitForOneVersion(ctx, tableID, base.DefaultRetryOptions()) - if err != nil { - return nil, err - } - - desc := &sqlbase.Descriptor{} - // There should be only one version of the descriptor, but it's - // a race now to update to the next version. - err = s.db.Txn(ctx, func(ctx context.Context, txn *client.Txn) error { - descKey := sqlbase.MakeDescMetadataKey(tableID) - - // Re-read the current version of the table descriptor, this time - // transactionally. - if err := txn.GetProto(ctx, descKey, desc); err != nil { + err := s.db.Txn(ctx, func(ctx context.Context, txn *client.Txn) error { + var err error + desc, err = s.reportPrevVersionInUse(ctx, txn, tableID) + if err != nil { return err } tableDesc := desc.GetTable() - if tableDesc == nil { - return errors.Errorf("ID %d is not a table", tableID) - } - if expectedVersion != tableDesc.Version { - // The version changed out from under us. Someone else must be - // performing a schema change operation. - if log.V(3) { - log.Infof(ctx, "publish (version changed): %d != %d", expectedVersion, tableDesc.Version) - } - return errLeaseVersionChanged - } - // Run the update closure. version := tableDesc.Version if err := update(tableDesc); err != nil { @@ -358,10 +319,10 @@ func (s LeaseStore) Publish( } tableDesc.Version++ - now := s.clock.Now() - tableDesc.ModificationTime = now + txnTimestamp := txn.Proto().Timestamp + tableDesc.ModificationTime = txnTimestamp log.Infof(ctx, "publish: descID=%d (%s) version=%d mtime=%s", - tableDesc.ID, tableDesc.Name, tableDesc.Version, now.GoTime()) + tableDesc.ID, tableDesc.Name, tableDesc.Version, txnTimestamp.GoTime()) if err := tableDesc.ValidateTable(); err != nil { return err } @@ -371,7 +332,7 @@ func (s LeaseStore) Publish( return err } b := txn.NewBatch() - b.Put(descKey, desc) + b.Put(sqlbase.MakeDescMetadataKey(tableID), desc) if logEvent != nil { // If an event log is required for this update, ensure that the // descriptor change occurs first in the transaction. This is @@ -394,7 +355,7 @@ func (s LeaseStore) Publish( switch err { case nil, errDidntUpdateDescriptor: return desc, nil - case errLeaseVersionChanged: + case errPrevVersionInUse: // will loop around to retry default: return nil, err @@ -404,25 +365,37 @@ func (s LeaseStore) Publish( panic("not reached") } -// countLeases returns the number of unexpired leases for a particular version -// of a descriptor. -func (s LeaseStore) countLeases( - ctx context.Context, descID sqlbase.ID, version sqlbase.DescriptorVersion, expiration time.Time, -) (int, error) { - var count int - err := s.db.Txn(ctx, func(ctx context.Context, txn *client.Txn) error { - p := makeInternalPlanner("leases-count", txn, security.RootUser, s.memMetrics) - defer finishInternalPlanner(p) - const countLeases = `SELECT COUNT(version) FROM system.lease ` + - `WHERE descID = $1 AND version = $2 AND expiration > $3` - values, err := p.QueryRow(ctx, countLeases, descID, int(version), expiration) - if err != nil { - return err - } - count = int(parser.MustBeDInt(values[0])) - return nil - }) - return count, err +// Reads the table descriptor using the transaction and reports if +// the prevVersion of the table descriptor is still in use. +func (s LeaseStore) reportPrevVersionInUse( + ctx context.Context, txn *client.Txn, tableID sqlbase.ID, +) (*sqlbase.Descriptor, error) { + descKey := sqlbase.MakeDescMetadataKey(tableID) + desc := &sqlbase.Descriptor{} + if err := txn.GetProto(ctx, descKey, desc); err != nil { + return nil, err + } + tableDesc := desc.GetTable() + if tableDesc == nil { + return nil, errors.Errorf("ID %d is not a table", tableID) + } + prevVersion := tableDesc.Version - 1 + // Wait until there are no unexpired leases on the previous version + // of the table. + p := makeInternalPlanner("leases-count", txn, security.RootUser, s.memMetrics) + defer finishInternalPlanner(p) + const countLeases = `SELECT COUNT(version) FROM system.lease ` + + `WHERE descID = $1 AND version = $2 AND expiration > $3` + values, err := p.QueryRow(ctx, countLeases, tableID, int(prevVersion), txn.Proto().Timestamp.GoTime().Add(-1*time.Microsecond)) + if err != nil { + return nil, err + } + if count := int(parser.MustBeDInt(values[0])); count != 0 { + log.Infof(ctx, "publish (count leases): descID=%d name=%s version=%d count=%d", + tableDesc.ID, tableDesc.Name, prevVersion, count) + return nil, errPrevVersionInUse + } + return desc, nil } // leaseSet maintains an ordered set of LeaseState objects. It supports @@ -538,10 +511,10 @@ type tableState struct { // nil if there is no lease acquisition in progress for the table. If // non-nil, the channel will be closed when lease acquisition completes. acquiring chan struct{} - // Indicates that the table has been deleted, or has an outstanding deletion. + // Indicates that the table has been dropped, or has an outstanding deletion. // If set, leases are released from the store as soon as their refcount drops // to 0, as opposed to waiting until they expire. - deleted bool + dropped bool } // acquire returns a lease at the specified version. The lease will have its @@ -690,7 +663,7 @@ func (t *tableState) acquireWait() bool { } // If the lease cannot be obtained because the descriptor is in the process of -// being deleted, the error will be errDescriptorDeleted. +// being dropped, the error will be errDescriptordropped. // minExpirationTime, if not set to the zero value, will be used as a lower // bound on the expiration of the new lease. This can be used to eliminate the // jitter in the expiration time, and guarantee that we get a lease that will be @@ -741,9 +714,9 @@ func (t *tableState) release(lease *LeaseState, m *LeaseManager) error { // when the refcount drops to 0). If so, we'll need to mark the lease as // released. removeOnceDereferenced := m.LeaseStore.testingKnobs.RemoveOnceDereferenced || - // Release from the store if the table has been deleted; no leases + // Release from the store if the table has been dropped; no leases // can be acquired any more. - t.deleted || + t.dropped || // Release from the store if the LeaseManager is draining. m.isDraining() || // Release from the store if the lease is not for the latest @@ -813,10 +786,10 @@ func (t *tableState) purgeOldLeases( return nil } - releaseInactives := func(deleted bool) { + releaseInactives := func(dropped bool) { t.mu.Lock() defer t.mu.Unlock() - t.deleted = deleted + t.dropped = dropped t.releaseInactiveLeases(m) } @@ -829,12 +802,12 @@ func (t *tableState) purgeOldLeases( lease, err := t.acquire(ctx, minVersion, m) if err != nil { if err == errTableDropped { - releaseInactives(true /* deleted */) + releaseInactives(true /* dropped */) return nil } return err } - releaseInactives(false /* deleted */) + releaseInactives(false /* dropped */) if lease == nil { return nil } @@ -957,7 +930,7 @@ func (c *tableNameCache) remove(lease *LeaseState) { if !ok { // Table for lease not found in table name cache. This can happen if we had // a more recent lease on the table in the tableNameCache, then the table - // gets deleted, then the more recent lease is remove()d - which clears the + // gets dropped, then the more recent lease is remove()d - which clears the // cache. return } @@ -1255,7 +1228,7 @@ func (m *LeaseManager) RefreshLeases(s *stop.Stopper, db *client.DB, gossip *gos continue } if log.V(2) { - log.Infof(ctx, "%s: refreshing lease table: %d (%s), version: %d, deleted: %t", + log.Infof(ctx, "%s: refreshing lease table: %d (%s), version: %d, dropped: %t", kv.Key, table.ID, table.Name, table.Version, table.Dropped()) } // Try to refresh the table lease to one >= this version. diff --git a/pkg/sql/lease_internal_test.go b/pkg/sql/lease_internal_test.go index fcebf888554d..27f54efa4871 100644 --- a/pkg/sql/lease_internal_test.go +++ b/pkg/sql/lease_internal_test.go @@ -333,7 +333,7 @@ CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR); // Test that there's no deadlock between AcquireByName and Release. // We used to have one due to lock inversion between the tableNameCache lock and // the leaseState lock, triggered when the same lease was Release()d after the -// table had been deleted (which means it's removed from the tableNameCache) and +// table had been dropped (which means it's removed from the tableNameCache) and // AcquireByName()d at the same time. func TestReleaseAcquireByNameDeadlock(t *testing.T) { defer leaktest.AfterTest(t)() @@ -369,10 +369,10 @@ CREATE TABLE t.test (k CHAR PRIMARY KEY, v CHAR); t.Fatal(err) } - // Pretend the table has been deleted, so that when we release leases on it, + // Pretend the table has been dropped, so that when we release leases on it, // they are removed from the tableNameCache too. tableState := leaseManager.findTableState(tableDesc.ID, true) - tableState.deleted = true + tableState.dropped = true // Try to trigger the race repeatedly: race an AcquireByName against a // Release. diff --git a/pkg/sql/sqlbase/structured.go b/pkg/sql/sqlbase/structured.go index 45d9278dd3bc..679af02bca6a 100644 --- a/pkg/sql/sqlbase/structured.go +++ b/pkg/sql/sqlbase/structured.go @@ -30,7 +30,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/sql/parser" "github.com/cockroachdb/cockroach/pkg/util/encoding" - "github.com/cockroachdb/cockroach/pkg/util/hlc" ) // ID, ColumnID, FamilyID, and IndexID are all uint32, but are each given a @@ -59,6 +58,9 @@ type IndexID parser.IndexID // DescriptorVersion is a custom type for TableDescriptor Versions. type DescriptorVersion uint32 +// InvalidDescriptorVersion is the uninitialised descriptor version. +const InvalidDescriptorVersion DescriptorVersion = 0 + // FormatVersion is a custom type for TableDescriptor versions of the sql to // key:value mapping. //go:generate stringer -type=FormatVersion @@ -156,33 +158,18 @@ func GetDatabaseDescFromID( // ID passed in using an existing txn. Returns an error if the // descriptor doesn't exist or if it exists and is not a table. func GetTableDescFromID(ctx context.Context, txn *client.Txn, id ID) (*TableDescriptor, error) { - table, _, err := GetTableDescAndTimestampFromID(ctx, txn, id) - return table, err -} - -// GetTableDescAndTimestampFromID retrieves the table descriptor for the table -// ID passed in using an existing txn. Returns an error if the -// descriptor doesn't exist or if it exists and is not a table. -func GetTableDescAndTimestampFromID( - ctx context.Context, txn *client.Txn, id ID, -) (*TableDescriptor, hlc.Timestamp, error) { descKey := MakeDescMetadataKey(id) - kv, err := txn.Get(ctx, descKey) - if err != nil { - return nil, hlc.Timestamp{}, err - } - desc := &Descriptor{} - if err := kv.ValueProto(desc); err != nil { - return nil, hlc.Timestamp{}, err + if err := txn.GetProto(ctx, descKey, desc); err != nil { + return nil, err } table := desc.GetTable() if table == nil { - return nil, hlc.Timestamp{}, ErrDescriptorNotFound + return nil, ErrDescriptorNotFound } - return table, kv.Value.Timestamp, nil + return table, nil } // RunOverAllColumns applies its argument fn to each of the column IDs in desc. @@ -495,7 +482,7 @@ func (desc *TableDescriptor) AllocateIDs() error { if desc.NextColumnID == 0 { desc.NextColumnID = 1 } - if desc.Version == 0 { + if desc.Version == InvalidDescriptorVersion { desc.Version = 1 } if desc.NextMutationID == InvalidMutationID { diff --git a/pkg/sql/sqlbase/structured.pb.go b/pkg/sql/sqlbase/structured.pb.go index 9bb5c2a4b859..7996f6456ad6 100644 --- a/pkg/sql/sqlbase/structured.pb.go +++ b/pkg/sql/sqlbase/structured.pb.go @@ -747,7 +747,15 @@ type TableDescriptor struct { Version DescriptorVersion `protobuf:"varint,5,opt,name=version,casttype=DescriptorVersion" json:"version"` // See comment above. UpVersion bool `protobuf:"varint,6,opt,name=up_version,json=upVersion" json:"up_version"` - // Last modification time of the table descriptor. + // The timestamp of the last transaction that incremented the version of + // the table descriptor. Note that modifying the table descriptor and + // setting the up_version bit without incrementing the version does not + // change the modification_time. This time can be used as a time lower bound + // for a valid use of a table lease holding a particular version of + // a table descriptor, without worrying about the table descriptor being + // modified to queue up schema changes or write schema change checkpoints + // that do change the write timestamp of the table descriptor at the mvcc + // layer. Only incrementing the table version truly modifies it! ModificationTime cockroach_util_hlc.Timestamp `protobuf:"bytes,7,opt,name=modification_time,json=modificationTime" json:"modification_time"` Columns []ColumnDescriptor `protobuf:"bytes,8,rep,name=columns" json:"columns"` // next_column_id is used to ensure that deleted column ids are not reused. diff --git a/pkg/sql/sqlbase/structured.proto b/pkg/sql/sqlbase/structured.proto index cd78d41af00c..6015eecaf3e5 100644 --- a/pkg/sql/sqlbase/structured.proto +++ b/pkg/sql/sqlbase/structured.proto @@ -411,7 +411,15 @@ message TableDescriptor { optional uint32 version = 5 [(gogoproto.nullable) = false, (gogoproto.casttype) = "DescriptorVersion"]; // See comment above. optional bool up_version = 6 [(gogoproto.nullable) = false]; - // Last modification time of the table descriptor. + // The timestamp of the last transaction that incremented the version of + // the table descriptor. Note that modifying the table descriptor and + // setting the up_version bit without incrementing the version does not + // change the modification_time. This time can be used as a time lower bound + // for a valid use of a table lease holding a particular version of + // a table descriptor, without worrying about the table descriptor being + // modified to queue up schema changes or write schema change checkpoints + // that do change the write timestamp of the table descriptor at the mvcc + // layer. Only incrementing the table version truly modifies it! optional util.hlc.Timestamp modification_time = 7 [(gogoproto.nullable) = false]; repeated ColumnDescriptor columns = 8 [(gogoproto.nullable) = false]; // next_column_id is used to ensure that deleted column ids are not reused. diff --git a/pkg/sql/table.go b/pkg/sql/table.go index 4677d60602fb..9fc91ad413ae 100644 --- a/pkg/sql/table.go +++ b/pkg/sql/table.go @@ -278,14 +278,27 @@ func (lc *LeaseCollection) getUncommittedTable( return nil, nil } +func newTableVersionChangeError( + ctx context.Context, txn *client.Txn, version, newVersion sqlbase.DescriptorVersion, +) error { + retryErr := roachpb.NewErrorWithTxn(roachpb.NewTransactionRetryError(roachpb.RETRY_TABLE_VERSION_CHANGE), txn.Proto()) + txn.UpdateStateOnRetryableErr(ctx, *retryErr) + err := roachpb.NewHandledRetryableTxnError( + retryErr.GetDetail().Error(), txn.Proto().ID, *txn.Proto()) + log.Infof(ctx, "transaction using table version %d encountered table with future version %d", + version, newVersion) + return err +} + func newTableFromFutureError( - ctx context.Context, txn *client.Txn, timestamp, creationTime hlc.Timestamp, + ctx context.Context, txn *client.Txn, timestamp, modificationTime hlc.Timestamp, ) error { retryErr := roachpb.NewErrorWithTxn(roachpb.NewTransactionRetryError(roachpb.RETRY_TABLE_FROM_FUTURE), txn.Proto()) txn.UpdateStateOnRetryableErr(ctx, *retryErr) err := roachpb.NewHandledRetryableTxnError( retryErr.GetDetail().Error(), txn.Proto().ID, *txn.Proto()) - log.Infof(ctx, "transaction at time %s encountered table with future timestamp %s", timestamp, creationTime) + log.Infof(ctx, "transaction at time %s encountered table with future timestamp %s", + timestamp, modificationTime) return err } @@ -337,7 +350,7 @@ func (lc *LeaseCollection) getTableLease( return nil, err } else if table != nil { if log.V(2) { - log.Infof(ctx, "found uncommitted table %d", table.ID) + log.Infof(ctx, "returning an uncommitted table %s", table.Name) } return table, nil } @@ -359,7 +372,7 @@ func (lc *LeaseCollection) getTableLease( } // If we didn't find a lease or the lease is about to expire, acquire one. - if lease == nil || lc.removeLeaseIfExpiring(ctx, txn, lease) { + if version, ok := lc.removeLeaseIfExpiring(ctx, txn, lease); ok { var err error lease, err = lc.leaseMgr.AcquireByName(ctx, dbID, tn.Table()) if err != nil { @@ -370,13 +383,19 @@ func (lc *LeaseCollection) getTableLease( } return nil, err } + if version != sqlbase.InvalidDescriptorVersion && version != lease.Version { + if err := lc.leaseMgr.Release(lease); err != nil { + log.Warning(ctx, err) + } + return nil, newTableVersionChangeError(ctx, txn, version, lease.Version) + } timestamp := txn.OrigTimestamp() - if creationTime := lease.creationTime; timestamp.Less(creationTime) { - // Push the transaction forward. + modificationTime := lease.TableDescriptor.ModificationTime + if timestamp.Less(modificationTime) { if err := lc.leaseMgr.Release(lease); err != nil { log.Warning(ctx, err) } - return nil, newTableFromFutureError(ctx, txn, timestamp, creationTime) + return nil, newTableFromFutureError(ctx, txn, timestamp, modificationTime) } lc.leases = append(lc.leases, lease) if log.V(2) { @@ -411,7 +430,7 @@ func (lc *LeaseCollection) getTableLeaseByID( for _, table := range lc.uncommittedTables { if table.ID == tableID { if log.V(2) { - log.Infof(ctx, "found uncommitted table %d", tableID) + log.Infof(ctx, "returning uncommitted table %s", table.Name) } if table.Dropped() { return nil, sqlbase.NewUndefinedTableError(fmt.Sprintf("", tableID)) @@ -427,14 +446,14 @@ func (lc *LeaseCollection) getTableLeaseByID( if l.ID == tableID { lease = l if log.V(2) { - log.Infof(ctx, "found lease in cache for table %d", tableID) + log.Infof(ctx, "found lease in cache for table %s", l.Name) } break } } // If we didn't find a lease or the lease is about to expire, acquire one. - if lease == nil || lc.removeLeaseIfExpiring(ctx, txn, lease) { + if version, ok := lc.removeLeaseIfExpiring(ctx, txn, lease); ok { var err error lease, err = lc.leaseMgr.Acquire(ctx, tableID, 0) if err != nil { @@ -445,13 +464,19 @@ func (lc *LeaseCollection) getTableLeaseByID( } return nil, err } + if version != sqlbase.InvalidDescriptorVersion && version != lease.Version { + if err := lc.leaseMgr.Release(lease); err != nil { + log.Warning(ctx, err) + } + return nil, newTableVersionChangeError(ctx, txn, version, lease.Version) + } timestamp := txn.OrigTimestamp() - if creationTime := lease.creationTime; timestamp.Less(creationTime) { - // Push the transaction forward. + modificationTime := lease.TableDescriptor.ModificationTime + if timestamp.Less(modificationTime) { if err := lc.leaseMgr.Release(lease); err != nil { log.Warning(ctx, err) } - return nil, newTableFromFutureError(ctx, txn, timestamp, creationTime) + return nil, newTableFromFutureError(ctx, txn, timestamp, modificationTime) } lc.leases = append(lc.leases, lease) // If the lease we just acquired expires before the txn's deadline, reduce @@ -465,9 +490,13 @@ func (lc *LeaseCollection) getTableLeaseByID( // The method also resets the transaction deadline. func (lc *LeaseCollection) removeLeaseIfExpiring( ctx context.Context, txn *client.Txn, lease *LeaseState, -) bool { - if lease == nil || lease.hasSomeLifeLeft(lc.leaseMgr.clock) { - return false +) (sqlbase.DescriptorVersion, bool) { + if lease == nil { + return sqlbase.InvalidDescriptorVersion, true + } + version := lease.Version + if lease.hasSomeLifeLeft(lc.leaseMgr.clock) { + return version, false } // Remove the lease from session.leases. @@ -480,7 +509,7 @@ func (lc *LeaseCollection) removeLeaseIfExpiring( } if idx == -1 { log.Warningf(ctx, "lease (%s) not found", lease) - return false + return version, false } lc.leases[idx] = lc.leases[len(lc.leases)-1] lc.leases[len(lc.leases)-1] = nil @@ -495,7 +524,7 @@ func (lc *LeaseCollection) removeLeaseIfExpiring( for _, l := range lc.leases { txn.UpdateDeadlineMaybe(hlc.Timestamp{WallTime: l.Expiration().UnixNano()}) } - return true + return version, true } // getTableNames retrieves the list of qualified names of tables diff --git a/pkg/sql/table_test.go b/pkg/sql/table_test.go index 3564849f74f0..4b49d26f264b 100644 --- a/pkg/sql/table_test.go +++ b/pkg/sql/table_test.go @@ -282,18 +282,24 @@ func TestRemoveLeaseIfExpiring(t *testing.T) { var txn client.Txn - if lc.removeLeaseIfExpiring(context.TODO(), &txn, nil) { - t.Error("expected false with nil input") + if version, ok := lc.removeLeaseIfExpiring( + context.TODO(), &txn, nil, + ); !ok || version != sqlbase.InvalidDescriptorVersion { + t.Errorf("expected %v with a nil lease at version: %d", ok, version) } // Add a lease to the planner. d := int64(LeaseDuration) - l1 := &LeaseState{expiration: parser.DTimestamp{Time: time.Unix(0, mc.UnixNano()+d+1)}} + const expectedVersion = 5 + l1 := &LeaseState{ + TableDescriptor: sqlbase.TableDescriptor{Version: expectedVersion}, + expiration: parser.DTimestamp{Time: time.Unix(0, mc.UnixNano()+d+1)}, + } lc.leases = append(lc.leases, l1) et := hlc.Timestamp{WallTime: l1.Expiration().UnixNano()} txn.UpdateDeadlineMaybe(et) - if lc.removeLeaseIfExpiring(context.TODO(), &txn, l1) { + if _, ok := lc.removeLeaseIfExpiring(context.TODO(), &txn, l1); ok { t.Error("expected false with a non-expiring lease") } if d := *txn.GetDeadline(); d != et { @@ -304,10 +310,15 @@ func TestRemoveLeaseIfExpiring(t *testing.T) { mc.Increment(d + 1) // Add another lease. - l2 := &LeaseState{expiration: parser.DTimestamp{Time: time.Unix(0, mc.UnixNano()+d+1)}} + l2 := &LeaseState{ + TableDescriptor: sqlbase.TableDescriptor{Version: expectedVersion + 1}, + expiration: parser.DTimestamp{Time: time.Unix(0, mc.UnixNano()+d+1)}, + } lc.leases = append(lc.leases, l2) - if !lc.removeLeaseIfExpiring(context.TODO(), &txn, l1) { - t.Error("expected true with an expiring lease") + if version, ok := lc.removeLeaseIfExpiring( + context.TODO(), &txn, l1, + ); !ok || version != expectedVersion { + t.Errorf("expected %v with an expiring lease at version: %d", ok, version) } et = hlc.Timestamp{WallTime: l2.Expiration().UnixNano()} txn.UpdateDeadlineMaybe(et) diff --git a/pkg/sql/txn_restart_test.go b/pkg/sql/txn_restart_test.go index 13d99e20ac82..cd0c444ec760 100644 --- a/pkg/sql/txn_restart_test.go +++ b/pkg/sql/txn_restart_test.go @@ -26,6 +26,7 @@ import ( "strings" "sync/atomic" "testing" + "time" "github.com/lib/pq" "github.com/pkg/errors" @@ -1406,7 +1407,7 @@ func TestDistSQLRetryableError(t *testing.T) { // Test that a transaction cannot use a table lease referencing a table // descriptor written after the transaction timestamp. -func TestCantUseFutureTableLease(t *testing.T) { +func TestRetryTableFromFuture(t *testing.T) { defer leaktest.AfterTest(t)() params, _ := createTestServerParams() retries := 0 @@ -1438,6 +1439,7 @@ func TestCantUseFutureTableLease(t *testing.T) { if _, err := sqlDB.Exec(` CREATE DATABASE d; CREATE TABLE d.kv (k CHAR PRIMARY KEY, v CHAR); +CREATE TABLE d.read (k CHAR PRIMARY KEY, v CHAR); `); err != nil { t.Fatal(err) } @@ -1473,12 +1475,21 @@ CREATE TABLE d.kv (k CHAR PRIMARY KEY, v CHAR); t.Fatal(err) } + // Read from a dumb table so that a transaction ID is allocated the + // the transaction and we don't see "retryable error from another txn" + // + // TODO(vivek): Remove once we assign a Txn ID at the start of a + // transaction instead of in the TxnCoordinator. + rows, err := tx.Query(`SELECT * FROM d.read;`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + _, err = tx.Exec(`INSERT INTO d.kv (k,v) VALUES ('c', 'd');`) if i == 0 { - // TODO(vivek): use isRetryableErr once we assign a Txn ID - // at the start of a transaction instead of in the TxnCoordinator. - if !testutils.IsError(err, "retryable error from another txn") { + if !isRetryableErr(err) { t.Fatalf("err = %v", err) } @@ -1501,3 +1512,95 @@ CREATE TABLE d.kv (k CHAR PRIMARY KEY, v CHAR); t.Fatalf("%d retries", retries) } } + +// Test that a transaction cannot use two table leases for the same table +// at different versions. +func TestRetryTableVersionChange(t *testing.T) { + defer leaktest.AfterTest(t)() + params, _ := createTestServerParams() + // Set the minimum lease duration such that the next lease acquisition will + // require the lease to be reacquired. + savedLeaseDuration, savedMinLeaseDuration := sql.LeaseDuration, sql.MinLeaseDuration + defer func() { + sql.LeaseDuration, sql.MinLeaseDuration = savedLeaseDuration, savedMinLeaseDuration + }() + sql.MinLeaseDuration = 50 * time.Millisecond + sql.LeaseDuration = 2 * sql.MinLeaseDuration + + s, sqlDB, _ := serverutils.StartServer(t, params) + defer s.Stopper().Stop(context.TODO()) + + if _, err := sqlDB.Exec(` +CREATE DATABASE d; +CREATE TABLE d.kv (k CHAR PRIMARY KEY, v CHAR); +`); err != nil { + t.Fatal(err) + } + + retried := false + for i := 0; i < 2; i++ { + tx, err := sqlDB.Begin() + if err != nil { + t.Fatal(err) + } + + rows, err := tx.Query(`SELECT * FROM d.kv;`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + + // Increment the descriptor version. + if i == 0 { + func() { + leaseMgr := s.LeaseManager().(*sql.LeaseManager) + var version sqlbase.DescriptorVersion + id := sqlbase.ID(51) + ctx := context.TODO() + if _, err := leaseMgr.Publish(ctx, id, func(table *sqlbase.TableDescriptor) error { + // Publish nothing; only update the version. + version = table.Version + return nil + }, nil); err != nil { + t.Fatal(err) + } + // Grab a lease at the latest version so that we are confident + // that all future leases will be taken at the latest version. + lease, err := leaseMgr.Acquire(ctx, id, version+1) + if err != nil { + t.Fatal(err) + } + if err := leaseMgr.Release(lease); err != nil { + t.Fatal(err) + } + + time.Sleep(sql.MinLeaseDuration) + }() + } + + rows, err = tx.Query(`SELECT * FROM d.kv;`) + if i == 0 { + if !isRetryableErr(err) { + t.Fatalf("err = %v", err) + } + + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + retried = true + continue + } + if err != nil { + t.Fatal(err) + } + defer rows.Close() + + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + } + + if !retried { + t.Fatal("not retried") + } +}