--v2-deprecation flag: opt-in mode to validate that store-v2 has no user-content

server/config/config.go

@@ -192,6 +192,9 @@ type ServerConfig struct {
 	// ExperimentalBootstrapDefragThresholdMegabytes is the minimum number of megabytes needed to be freed for etcd server to
 	// consider running defrag during bootstrap. Needs to be set to non-zero value to take effect.
 	ExperimentalBootstrapDefragThresholdMegabytes uint `json:"experimental-bootstrap-defrag-threshold-megabytes"`
+
+	// V2Deprecation defines a phase of v2store deprecation process.
+	V2Deprecation V2DeprecationEnum `json:"v2-deprecation"`
 }
 
 // VerifyBootstrap sanity-checks the initial config for bootstrap case

server/config/v2_deprecation.go

@@ -0,0 +1,50 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+type V2DeprecationEnum string
+
+const (
+	// Default in v3.5.  Issues a warning if v2store have meaningful content.
+	V2_DEPR_0_NOT_YET = V2DeprecationEnum("not-yet")
+	// Default in v3.6.  Meaningful v2 state is not allowed.
+	// The V2 files are maintained for v3.5 rollback.
+	V2_DEPR_1_WRITE_ONLY = V2DeprecationEnum("write-only")
+	// V2store is WIPED if found !!!
+	V2_DEPR_1_WRITE_ONLY_DROP = V2DeprecationEnum("write-only-drop-data")
+	// V2store is neither written nor read. Usage of this configuration is blocking
+	// ability to rollback to etcd v3.5.
+	V2_DEPR_2_GONE = V2DeprecationEnum("gone")
+
+	V2_DEPR_DEFAULT = V2_DEPR_0_NOT_YET
+)
+
+func (e V2DeprecationEnum) IsAtLeast(v2d V2DeprecationEnum) bool {
+	return e.level() >= v2d.level()
+}
+
+func (e V2DeprecationEnum) level() int {
+	switch e {
+	case V2_DEPR_0_NOT_YET:
+		return 0
+	case V2_DEPR_1_WRITE_ONLY:
+		return 1
+	case V2_DEPR_1_WRITE_ONLY_DROP:
+		return 2
+	case V2_DEPR_2_GONE:
+		return 3
+	}
+	panic("Unknown V2DeprecationEnum: " + e)
+}

server/config/v2_deprecation_test.go

@@ -0,0 +1,41 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import "testing"
+
+func TestV2DeprecationEnum_IsAtLeast(t *testing.T) {
+	tests := []struct {
+		e    V2DeprecationEnum
+		v2d  V2DeprecationEnum
+		want bool
+	}{
+		{V2_DEPR_0_NOT_YET, V2_DEPR_0_NOT_YET, true},
+		{V2_DEPR_0_NOT_YET, V2_DEPR_1_WRITE_ONLY_DROP, false},
+		{V2_DEPR_0_NOT_YET, V2_DEPR_2_GONE, false},
+		{V2_DEPR_2_GONE, V2_DEPR_1_WRITE_ONLY_DROP, true},
+		{V2_DEPR_2_GONE, V2_DEPR_0_NOT_YET, true},
+		{V2_DEPR_2_GONE, V2_DEPR_2_GONE, true},
+		{V2_DEPR_1_WRITE_ONLY, V2_DEPR_1_WRITE_ONLY_DROP, false},
+		{V2_DEPR_1_WRITE_ONLY_DROP, V2_DEPR_1_WRITE_ONLY, true},
+	}
+	for _, tt := range tests {
+		t.Run(string(tt.e)+" >= "+string(tt.v2d), func(t *testing.T) {
+			if got := tt.e.IsAtLeast(tt.v2d); got != tt.want {
+				t.Errorf("IsAtLeast() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

server/embed/config.go

@@ -33,6 +33,7 @@ import (
 	"go.etcd.io/etcd/client/pkg/v3/types"
 	"go.etcd.io/etcd/pkg/v3/flags"
 	"go.etcd.io/etcd/pkg/v3/netutil"
+	"go.etcd.io/etcd/server/v3/config"
 	"go.etcd.io/etcd/server/v3/etcdserver"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v3compactor"
 
@@ -400,6 +401,9 @@ type Config struct {
 
 	// ExperimentalTxnModeWriteWithSharedBuffer enables write transaction to use a shared buffer in its readonly check operations.
 	ExperimentalTxnModeWriteWithSharedBuffer bool `json:"experimental-txn-mode-write-with-shared-buffer"`
+
+	// V2Deprecation describes phase of API & Storage V2 support
+	V2Deprecation config.V2DeprecationEnum `json:"v2-deprecation"`
 }
 
 // configYAML holds the config suitable for yaml parsing
@@ -494,6 +498,8 @@ func NewConfig() *Config {
 		ExperimentalDowngradeCheckTime:           DefaultDowngradeCheckTime,
 		ExperimentalMemoryMlock:                  false,
 		ExperimentalTxnModeWriteWithSharedBuffer: true,
+
+		V2Deprecation: config.V2_DEPR_DEFAULT,
 	}
 	cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name)
 	return cfg
@@ -795,6 +801,13 @@ func (cfg Config) InitialClusterFromName(name string) (ret string) {
 func (cfg Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew }
 func (cfg Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) }
 
+func (cfg Config) V2DeprecationEffective() config.V2DeprecationEnum {
+	if cfg.V2Deprecation == "" {
+		return config.V2_DEPR_DEFAULT
+	}
+	return cfg.V2Deprecation
+}
+
 func (cfg Config) defaultPeerHost() bool {
 	return len(cfg.APUrls) == 1 && cfg.APUrls[0].String() == DefaultInitialAdvertisePeerURLs
 }

server/embed/etcd.go

@@ -226,6 +226,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 		ExperimentalMemoryMlock:                  cfg.ExperimentalMemoryMlock,
 		ExperimentalTxnModeWriteWithSharedBuffer: cfg.ExperimentalTxnModeWriteWithSharedBuffer,
 		ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes,
+		V2Deprecation: cfg.V2DeprecationEffective(),
 	}
 
 	if srvcfg.ExperimentalEnableDistributedTracing {
@@ -696,6 +697,9 @@ func (e *Etcd) serveClients() (err error) {
 	// Start a client server goroutine for each listen address
 	var h http.Handler
 	if e.Config().EnableV2 {
+		if e.Config().V2DeprecationEffective().IsAtLeast(config.V2_DEPR_1_WRITE_ONLY) {
+			return fmt.Errorf("--enable-v2 and --v2-deprecation=%s are mutually exclusive", e.Config().V2DeprecationEffective())
+		}
 		e.cfg.logger.Warn("Flag `enable-v2` is deprecated and will get removed in etcd 3.6.")
 		if len(e.Config().ExperimentalEnableV2V3) > 0 {
 			e.cfg.logger.Warn("Flag `experimental-enable-v2v3` is deprecated and will get removed in etcd 3.6.")

server/etcdmain/config.go

@@ -27,6 +27,7 @@ import (
 	"go.etcd.io/etcd/api/v3/version"
 	"go.etcd.io/etcd/client/pkg/v3/logutil"
 	"go.etcd.io/etcd/pkg/v3/flags"
+	cconfig "go.etcd.io/etcd/server/v3/config"
 	"go.etcd.io/etcd/server/v3/embed"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp"
 
@@ -86,10 +87,11 @@ type config struct {
 
 // configFlags has the set of flags used for command line parsing a Config
 type configFlags struct {
-	flagSet      *flag.FlagSet
-	clusterState *flags.SelectiveStringValue
-	fallback     *flags.SelectiveStringValue
-	proxy        *flags.SelectiveStringValue
+	flagSet       *flag.FlagSet
+	clusterState  *flags.SelectiveStringValue
+	fallback      *flags.SelectiveStringValue
+	proxy         *flags.SelectiveStringValue
+	v2deprecation *flags.SelectiveStringsValue
 }
 
 func newConfig() *config {
@@ -119,6 +121,11 @@ func newConfig() *config {
 			proxyFlagReadonly,
 			proxyFlagOn,
 		),
+		v2deprecation: flags.NewSelectiveStringsValue(
+			string(cconfig.V2_DEPR_0_NOT_YET),
+			string(cconfig.V2_DEPR_1_WRITE_ONLY),
+			string(cconfig.V2_DEPR_1_WRITE_ONLY_DROP),
+			string(cconfig.V2_DEPR_2_GONE)),
 	}
 
 	fs := cfg.cf.flagSet
@@ -190,9 +197,13 @@ func newConfig() *config {
 	fs.Var(cfg.cf.clusterState, "initial-cluster-state", "Initial cluster state ('new' or 'existing').")
 
 	fs.BoolVar(&cfg.ec.StrictReconfigCheck, "strict-reconfig-check", cfg.ec.StrictReconfigCheck, "Reject reconfiguration requests that would cause quorum loss.")
-	fs.BoolVar(&cfg.ec.EnableV2, "enable-v2", cfg.ec.EnableV2, "Accept etcd V2 client requests. Deprecated in v3.5. Will be decommission in v3.6.")
+
 	fs.BoolVar(&cfg.ec.PreVote, "pre-vote", cfg.ec.PreVote, "Enable to run an additional Raft election phase.")
 
+	fs.BoolVar(&cfg.ec.EnableV2, "enable-v2", cfg.ec.EnableV2, "Accept etcd V2 client requests. Deprecated in v3.5. Will be decommission in v3.6.")
+	fs.StringVar(&cfg.ec.ExperimentalEnableV2V3, "experimental-enable-v2v3", cfg.ec.ExperimentalEnableV2V3, "v3 prefix for serving emulated v2 state. Deprecated in 3.5. Will be decomissioned in 3.6.")
+	fs.Var(cfg.cf.v2deprecation, "v2-deprecation", fmt.Sprintf("v2store deprecation stage: %q. ", cfg.cf.proxy.Valids()))
+
 	// proxy
 	fs.Var(cfg.cf.proxy, "proxy", fmt.Sprintf("Valid values include %q", cfg.cf.proxy.Valids()))
 	fs.UintVar(&cfg.cp.ProxyFailureWaitMs, "proxy-failure-wait", cfg.cp.ProxyFailureWaitMs, "Time (in milliseconds) an endpoint will be held in a failed state.")
@@ -268,7 +279,7 @@ func newConfig() *config {
 	// experimental
 	fs.BoolVar(&cfg.ec.ExperimentalInitialCorruptCheck, "experimental-initial-corrupt-check", cfg.ec.ExperimentalInitialCorruptCheck, "Enable to check data corruption before serving any client/peer traffic.")
 	fs.DurationVar(&cfg.ec.ExperimentalCorruptCheckTime, "experimental-corrupt-check-time", cfg.ec.ExperimentalCorruptCheckTime, "Duration of time between cluster corruption check passes.")
-	fs.StringVar(&cfg.ec.ExperimentalEnableV2V3, "experimental-enable-v2v3", cfg.ec.ExperimentalEnableV2V3, "v3 prefix for serving emulated v2 state. Deprecated in 3.5. Will be decomissioned in 3.6.")
+
 	fs.BoolVar(&cfg.ec.ExperimentalEnableLeaseCheckpoint, "experimental-enable-lease-checkpoint", false, "Enable to persist lease remaining TTL to prevent indefinite auto-renewal of long lived leases.")
 	fs.IntVar(&cfg.ec.ExperimentalCompactionBatchLimit, "experimental-compaction-batch-limit", cfg.ec.ExperimentalCompactionBatchLimit, "Sets the maximum revisions deleted in each compaction batch.")
 	fs.DurationVar(&cfg.ec.ExperimentalWatchProgressNotifyInterval, "experimental-watch-progress-notify-interval", cfg.ec.ExperimentalWatchProgressNotifyInterval, "Duration of periodic watch progress notifications.")
@@ -331,6 +342,11 @@ func (cfg *config) parse(arguments []string) error {
 	} else {
 		err = cfg.configFromCmdLine()
 	}
+
+	if cfg.ec.V2Deprecation == "" {
+		cfg.ec.V2Deprecation = cconfig.V2_DEPR_DEFAULT
+	}
+
 	// now logger is set up
 	return err
 }
@@ -385,6 +401,8 @@ func (cfg *config) configFromCmdLine() error {
 	cfg.cp.Fallback = cfg.cf.fallback.String()
 	cfg.cp.Proxy = cfg.cf.proxy.String()
 
+	cfg.ec.V2Deprecation = cconfig.V2DeprecationEnum(cfg.cf.v2deprecation.String())
+
 	// disable default advertise-client-urls if lcurls is set
 	missingAC := flags.IsSet(cfg.cf.flagSet, "listen-client-urls") && !flags.IsSet(cfg.cf.flagSet, "advertise-client-urls")
 	if !cfg.mayBeProxy() && missingAC {

server/etcdmain/help.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"strconv"
 
+	cconfig "go.etcd.io/etcd/server/v3/config"
 	"go.etcd.io/etcd/server/v3/embed"
 	"golang.org/x/crypto/bcrypt"
 )
@@ -124,6 +125,13 @@ Clustering:
     Interpret 'auto-compaction-retention' one of: periodic|revision. 'periodic' for duration based retention, defaulting to hours if no time unit is provided (e.g. '5m'). 'revision' for revision number based retention.
   --enable-v2 '` + strconv.FormatBool(embed.DefaultEnableV2) + `'
     Accept etcd V2 client requests. Deprecated and to be decommissioned in v3.6.
+  --v2-deprecation '` + string(cconfig.V2_DEPR_DEFAULT) + `'
+    Phase of v2store deprecation. Allows to opt-in for higher compatibility mode.
+    Supported values:
+      'not-yet'                // Issues a warning if v2store have meaningful content (default in v3.5)
+      'write-only'             // Custom v2 state is not allowed (planned default in v3.6)
+      'write-only-drop-data'   // Custom v2 state will get DELETED !
+      'gone'                   // v2store is not maintained any longer. (planned default in v3.7)
 
 Security:
   --cert-file ''

server/etcdserver/api/membership/store.go

@@ -31,7 +31,7 @@ const (
 	attributesSuffix     = "attributes"
 	raftAttributesSuffix = "raftAttributes"
 
-	// the prefix for stroing membership related information in store provided by store pkg.
+	// the prefix for storing membership related information in store provided by store pkg.
 	storePrefix = "/0"
 )
 

server/etcdserver/api/membership/storev2.go

@@ -0,0 +1,36 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package membership
+
+import (
+	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
+)
+
+// IsMetaStoreOnly verifies if the given `store` contains only
+// a meta-information (members, version) that can be recovered from the
+// backend (storev3) as well as opposed to user-data.
+func IsMetaStoreOnly(store v2store.Store) (bool, error) {
+	event, err := store.Get("/", true, false)
+	if err != nil {
+		return false, err
+	}
+	for _, n := range event.Node.Nodes {
+		if n.Key != storePrefix && n.Nodes.Len() > 0 {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}

server/etcdserver/api/membership/storev2_test.go

@@ -0,0 +1,54 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package membership
+
+import (
+	"testing"
+
+	"github.com/coreos/go-semver/semver"
+	"github.com/stretchr/testify/assert"
+	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
+	"go.uber.org/zap/zaptest"
+)
+
+func TestIsMetaStoreOnly(t *testing.T) {
+	lg := zaptest.NewLogger(t)
+	s := v2store.New("/0", "/1")
+
+	metaOnly, err := IsMetaStoreOnly(s)
+	assert.NoError(t, err)
+	assert.True(t, metaOnly, "Just created v2store should be meta-only")
+
+	mustSaveClusterVersionToStore(lg, s, semver.New("3.5.17"))
+	metaOnly, err = IsMetaStoreOnly(s)
+	assert.NoError(t, err)
+	assert.True(t, metaOnly, "Just created v2store should be meta-only")
+
+	mustSaveMemberToStore(lg, s, &Member{ID: 0x00abcd})
+	metaOnly, err = IsMetaStoreOnly(s)
+	assert.NoError(t, err)
+	assert.True(t, metaOnly, "Just created v2store should be meta-only")
+
+	_, err = s.Create("/1/foo", false, "v1", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent})
+	assert.NoError(t, err)
+	metaOnly, err = IsMetaStoreOnly(s)
+	assert.NoError(t, err)
+	assert.False(t, metaOnly, "Just created v2store should be meta-only")
+
+	_, err = s.Delete("/1/foo", false, false)
+	assert.NoError(t, err)
+	assert.NoError(t, err)
+	assert.False(t, metaOnly, "Just created v2store should be meta-only")
+}