From f3b4a3e578aae9cce6a690c4c7401ae8f537af36 Mon Sep 17 00:00:00 2001 From: Piotr Tabor Date: Fri, 30 Apr 2021 23:09:18 +0200 Subject: [PATCH 1/4] Detecting whether v2store is "empty" (metadata only). --- server/etcdserver/api/membership/store.go | 2 +- server/etcdserver/api/membership/storev2.go | 36 +++++++++++++ .../etcdserver/api/membership/storev2_test.go | 54 +++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 server/etcdserver/api/membership/storev2.go create mode 100644 server/etcdserver/api/membership/storev2_test.go diff --git a/server/etcdserver/api/membership/store.go b/server/etcdserver/api/membership/store.go index 96b20cb0e69..30b4c241c5e 100644 --- a/server/etcdserver/api/membership/store.go +++ b/server/etcdserver/api/membership/store.go @@ -31,7 +31,7 @@ const ( attributesSuffix = "attributes" raftAttributesSuffix = "raftAttributes" - // the prefix for stroing membership related information in store provided by store pkg. + // the prefix for storing membership related information in store provided by store pkg. storePrefix = "/0" ) diff --git a/server/etcdserver/api/membership/storev2.go b/server/etcdserver/api/membership/storev2.go new file mode 100644 index 00000000000..8505c63f367 --- /dev/null +++ b/server/etcdserver/api/membership/storev2.go @@ -0,0 +1,36 @@ +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package membership + +import ( + "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" +) + +// IsMetaStoreOnly verifies if the given `store` contains only +// a meta-information (members, version) that can be recovered from the +// backend (storev3) as well as opposed to user-data. +func IsMetaStoreOnly(store v2store.Store) (bool, error) { + event, err := store.Get("/", true, false) + if err != nil { + return false, err + } + for _, n := range event.Node.Nodes { + if n.Key != storePrefix && n.Nodes.Len() > 0 { + return false, nil + } + } + + return true, nil +} diff --git a/server/etcdserver/api/membership/storev2_test.go b/server/etcdserver/api/membership/storev2_test.go new file mode 100644 index 00000000000..29eb5a7d54a --- /dev/null +++ b/server/etcdserver/api/membership/storev2_test.go @@ -0,0 +1,54 @@ +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package membership + +import ( + "testing" + + "github.com/coreos/go-semver/semver" + "github.com/stretchr/testify/assert" + "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" + "go.uber.org/zap/zaptest" +) + +func TestIsMetaStoreOnly(t *testing.T) { + lg := zaptest.NewLogger(t) + s := v2store.New("/0", "/1") + + metaOnly, err := IsMetaStoreOnly(s) + assert.NoError(t, err) + assert.True(t, metaOnly, "Just created v2store should be meta-only") + + mustSaveClusterVersionToStore(lg, s, semver.New("3.5.17")) + metaOnly, err = IsMetaStoreOnly(s) + assert.NoError(t, err) + assert.True(t, metaOnly, "Just created v2store should be meta-only") + + mustSaveMemberToStore(lg, s, &Member{ID: 0x00abcd}) + metaOnly, err = IsMetaStoreOnly(s) + assert.NoError(t, err) + assert.True(t, metaOnly, "Just created v2store should be meta-only") + + _, err = s.Create("/1/foo", false, "v1", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) + assert.NoError(t, err) + metaOnly, err = IsMetaStoreOnly(s) + assert.NoError(t, err) + assert.False(t, metaOnly, "Just created v2store should be meta-only") + + _, err = s.Delete("/1/foo", false, false) + assert.NoError(t, err) + assert.NoError(t, err) + assert.False(t, metaOnly, "Just created v2store should be meta-only") +} From 7c508741b31ef08f3d0d45d1629fe24aa935f656 Mon Sep 17 00:00:00 2001 From: Piotr Tabor Date: Tue, 4 May 2021 11:16:10 +0200 Subject: [PATCH 2/4] Adding --v2-deprecation flag. --- server/embed/config.go | 13 ++++++++ server/embed/v2_deprecation.go | 46 +++++++++++++++++++++++++++++ server/embed/v2_deprecation_test.go | 41 +++++++++++++++++++++++++ server/etcdmain/config.go | 29 ++++++++++++++---- server/etcdmain/help.go | 7 +++++ 5 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 server/embed/v2_deprecation.go create mode 100644 server/embed/v2_deprecation_test.go diff --git a/server/embed/config.go b/server/embed/config.go index f50ff166357..bf24b048309 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -400,6 +400,9 @@ type Config struct { // ExperimentalTxnModeWriteWithSharedBuffer enables write transaction to use a shared buffer in its readonly check operations. ExperimentalTxnModeWriteWithSharedBuffer bool `json:"experimental-txn-mode-write-with-shared-buffer"` + + // V2Deprecation describes phase of API & Storage V2 support + V2Deprecation V2DeprecationEnum `json:"v2-deprecation"` } // configYAML holds the config suitable for yaml parsing @@ -494,6 +497,8 @@ func NewConfig() *Config { ExperimentalDowngradeCheckTime: DefaultDowngradeCheckTime, ExperimentalMemoryMlock: false, ExperimentalTxnModeWriteWithSharedBuffer: true, + + V2Deprecation: V2_DEPR_DEFAULT, } cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name) return cfg @@ -795,6 +800,14 @@ func (cfg Config) InitialClusterFromName(name string) (ret string) { func (cfg Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew } func (cfg Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) } +func (cfg Config) V2DeprecationEffective() V2DeprecationEnum { + if cfg.V2Deprecation == "" { + return V2_DEPR_DEFAULT + } else { + return cfg.V2Deprecation + } +} + func (cfg Config) defaultPeerHost() bool { return len(cfg.APUrls) == 1 && cfg.APUrls[0].String() == DefaultInitialAdvertisePeerURLs } diff --git a/server/embed/v2_deprecation.go b/server/embed/v2_deprecation.go new file mode 100644 index 00000000000..2fb9ba37412 --- /dev/null +++ b/server/embed/v2_deprecation.go @@ -0,0 +1,46 @@ +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package embed + +type V2DeprecationEnum string + +const ( +// Default in v3.5. Issues a warning if v2store have meaningful content. +V2_DEPR_0_NOT_YET = V2DeprecationEnum("not-yet") +// Default in v3.6. Meaningful v2 state is not allowed. +// The V2 files are maintained for v3.5 rollback. +V2_DEPR_1_WRITE_ONLY = V2DeprecationEnum("write-only") +// V2store is WIPED if found !!! +V2_DEPR_1_WRITE_ONLY_DROP = V2DeprecationEnum("write-only-drop-data") +// V2store is neither written nor read. Usage of this configuration is blocking +// ability to rollback to etcd v3.5. +V2_DEPR_2_GONE = V2DeprecationEnum("gone") + +V2_DEPR_DEFAULT = V2_DEPR_0_NOT_YET +) + +func (e V2DeprecationEnum) IsAtLeast(v2d V2DeprecationEnum) bool { + return e.level() >= v2d.level() +} + +func (e V2DeprecationEnum) level() int { + switch e { + case V2_DEPR_0_NOT_YET: return 0 + case V2_DEPR_1_WRITE_ONLY: return 1 + case V2_DEPR_1_WRITE_ONLY_DROP: return 2 + case V2_DEPR_2_GONE: return 3 + } + panic("Unknown V2DeprecationEnum: " + e) +} \ No newline at end of file diff --git a/server/embed/v2_deprecation_test.go b/server/embed/v2_deprecation_test.go new file mode 100644 index 00000000000..f9533bd5a1a --- /dev/null +++ b/server/embed/v2_deprecation_test.go @@ -0,0 +1,41 @@ +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package embed + +import "testing" + +func TestV2DeprecationEnum_IsAtLeast(t *testing.T) { + tests := []struct { + e V2DeprecationEnum + v2d V2DeprecationEnum + want bool + }{ + {V2_DEPR_0_NOT_YET, V2_DEPR_0_NOT_YET, true}, + {V2_DEPR_0_NOT_YET, V2_DEPR_1_WRITE_ONLY_DROP, false}, + {V2_DEPR_0_NOT_YET, V2_DEPR_2_GONE, false}, + {V2_DEPR_2_GONE, V2_DEPR_1_WRITE_ONLY_DROP, true}, + {V2_DEPR_2_GONE, V2_DEPR_0_NOT_YET, true}, + {V2_DEPR_2_GONE, V2_DEPR_2_GONE, true}, + {V2_DEPR_1_WRITE_ONLY, V2_DEPR_1_WRITE_ONLY_DROP, false}, + {V2_DEPR_1_WRITE_ONLY_DROP, V2_DEPR_1_WRITE_ONLY, true}, + } + for _, tt := range tests { + t.Run(string(tt.e) + " >= " + string(tt.v2d), func(t *testing.T) { + if got := tt.e.IsAtLeast(tt.v2d); got != tt.want { + t.Errorf("IsAtLeast() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index fefb230f215..0072109d691 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -86,10 +86,11 @@ type config struct { // configFlags has the set of flags used for command line parsing a Config type configFlags struct { - flagSet *flag.FlagSet - clusterState *flags.SelectiveStringValue - fallback *flags.SelectiveStringValue - proxy *flags.SelectiveStringValue + flagSet *flag.FlagSet + clusterState *flags.SelectiveStringValue + fallback *flags.SelectiveStringValue + proxy *flags.SelectiveStringValue + v2deprecation *flags.SelectiveStringsValue } func newConfig() *config { @@ -119,6 +120,11 @@ func newConfig() *config { proxyFlagReadonly, proxyFlagOn, ), + v2deprecation: flags.NewSelectiveStringsValue( + string(embed.V2_DEPR_0_NOT_YET), + string(embed.V2_DEPR_1_WRITE_ONLY), + string(embed.V2_DEPR_1_WRITE_ONLY_DROP), + string(embed.V2_DEPR_2_GONE)), } fs := cfg.cf.flagSet @@ -190,9 +196,13 @@ func newConfig() *config { fs.Var(cfg.cf.clusterState, "initial-cluster-state", "Initial cluster state ('new' or 'existing').") fs.BoolVar(&cfg.ec.StrictReconfigCheck, "strict-reconfig-check", cfg.ec.StrictReconfigCheck, "Reject reconfiguration requests that would cause quorum loss.") - fs.BoolVar(&cfg.ec.EnableV2, "enable-v2", cfg.ec.EnableV2, "Accept etcd V2 client requests. Deprecated in v3.5. Will be decommission in v3.6.") + fs.BoolVar(&cfg.ec.PreVote, "pre-vote", cfg.ec.PreVote, "Enable to run an additional Raft election phase.") + fs.BoolVar(&cfg.ec.EnableV2, "enable-v2", cfg.ec.EnableV2, "Accept etcd V2 client requests. Deprecated in v3.5. Will be decommission in v3.6.") + fs.StringVar(&cfg.ec.ExperimentalEnableV2V3, "experimental-enable-v2v3", cfg.ec.ExperimentalEnableV2V3, "v3 prefix for serving emulated v2 state. Deprecated in 3.5. Will be decomissioned in 3.6.") + fs.Var(cfg.cf.v2deprecation, "v2-deprecation", fmt.Sprintf("v2store deprecation stage: %q. ", cfg.cf.proxy.Valids())) + // proxy fs.Var(cfg.cf.proxy, "proxy", fmt.Sprintf("Valid values include %q", cfg.cf.proxy.Valids())) fs.UintVar(&cfg.cp.ProxyFailureWaitMs, "proxy-failure-wait", cfg.cp.ProxyFailureWaitMs, "Time (in milliseconds) an endpoint will be held in a failed state.") @@ -268,7 +278,7 @@ func newConfig() *config { // experimental fs.BoolVar(&cfg.ec.ExperimentalInitialCorruptCheck, "experimental-initial-corrupt-check", cfg.ec.ExperimentalInitialCorruptCheck, "Enable to check data corruption before serving any client/peer traffic.") fs.DurationVar(&cfg.ec.ExperimentalCorruptCheckTime, "experimental-corrupt-check-time", cfg.ec.ExperimentalCorruptCheckTime, "Duration of time between cluster corruption check passes.") - fs.StringVar(&cfg.ec.ExperimentalEnableV2V3, "experimental-enable-v2v3", cfg.ec.ExperimentalEnableV2V3, "v3 prefix for serving emulated v2 state. Deprecated in 3.5. Will be decomissioned in 3.6.") + fs.BoolVar(&cfg.ec.ExperimentalEnableLeaseCheckpoint, "experimental-enable-lease-checkpoint", false, "Enable to persist lease remaining TTL to prevent indefinite auto-renewal of long lived leases.") fs.IntVar(&cfg.ec.ExperimentalCompactionBatchLimit, "experimental-compaction-batch-limit", cfg.ec.ExperimentalCompactionBatchLimit, "Sets the maximum revisions deleted in each compaction batch.") fs.DurationVar(&cfg.ec.ExperimentalWatchProgressNotifyInterval, "experimental-watch-progress-notify-interval", cfg.ec.ExperimentalWatchProgressNotifyInterval, "Duration of periodic watch progress notifications.") @@ -331,6 +341,11 @@ func (cfg *config) parse(arguments []string) error { } else { err = cfg.configFromCmdLine() } + + if cfg.ec.V2Deprecation == "" { + cfg.ec.V2Deprecation = embed.V2_DEPR_DEFAULT + } + // now logger is set up return err } @@ -385,6 +400,8 @@ func (cfg *config) configFromCmdLine() error { cfg.cp.Fallback = cfg.cf.fallback.String() cfg.cp.Proxy = cfg.cf.proxy.String() + cfg.ec.V2Deprecation = embed.V2DeprecationEnum(cfg.cf.v2deprecation.String()) + // disable default advertise-client-urls if lcurls is set missingAC := flags.IsSet(cfg.cf.flagSet, "listen-client-urls") && !flags.IsSet(cfg.cf.flagSet, "advertise-client-urls") if !cfg.mayBeProxy() && missingAC { diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index b63282e8950..ca8ac039da8 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -124,6 +124,13 @@ Clustering: Interpret 'auto-compaction-retention' one of: periodic|revision. 'periodic' for duration based retention, defaulting to hours if no time unit is provided (e.g. '5m'). 'revision' for revision number based retention. --enable-v2 '` + strconv.FormatBool(embed.DefaultEnableV2) + `' Accept etcd V2 client requests. Deprecated and to be decommissioned in v3.6. + --v2-deprecation '` + string(embed.V2_DEPR_DEFAULT) + `' + Phase of v2store deprecation. Allows to optin for higher compatibility mode. + Supported values: + 'not-yet' // Issues a warning if v2store have meaningful content (default in v3.5) + 'write-only' // Custom v2 state is not allowed (planned default in v3.6) + 'write-only-drop-data' // Custom v2 state will get DELETED ! + 'gone' // v2store is not maintained any longer. (planned default in v3.7) Security: --cert-file '' From ead81df9488262e767bf3375414a5c39d6f1a33a Mon Sep 17 00:00:00 2001 From: Piotr Tabor Date: Mon, 10 May 2021 17:44:46 +0200 Subject: [PATCH 3/4] Disallow -v2-deprecation>'not-yet' combined with --enable-v2 --- server/config/config.go | 3 ++ server/{embed => config}/v2_deprecation.go | 40 ++++++++++--------- .../{embed => config}/v2_deprecation_test.go | 4 +- server/embed/config.go | 12 +++--- server/embed/etcd.go | 4 ++ server/etcdmain/config.go | 13 +++--- server/etcdmain/help.go | 5 ++- server/etcdserver/server.go | 28 +++++++++++++ tests/integration/cluster.go | 2 + 9 files changed, 77 insertions(+), 34 deletions(-) rename server/{embed => config}/v2_deprecation.go (50%) rename server/{embed => config}/v2_deprecation_test.go (94%) diff --git a/server/config/config.go b/server/config/config.go index 7874d8ee0a2..021b42f3588 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -192,6 +192,9 @@ type ServerConfig struct { // ExperimentalBootstrapDefragThresholdMegabytes is the minimum number of megabytes needed to be freed for etcd server to // consider running defrag during bootstrap. Needs to be set to non-zero value to take effect. ExperimentalBootstrapDefragThresholdMegabytes uint `json:"experimental-bootstrap-defrag-threshold-megabytes"` + + // V2Deprecation defines a phase of v2store deprecation process. + V2Deprecation V2DeprecationEnum `json:"v2-deprecation"` } // VerifyBootstrap sanity-checks the initial config for bootstrap case diff --git a/server/embed/v2_deprecation.go b/server/config/v2_deprecation.go similarity index 50% rename from server/embed/v2_deprecation.go rename to server/config/v2_deprecation.go index 2fb9ba37412..828bd9a8f43 100644 --- a/server/embed/v2_deprecation.go +++ b/server/config/v2_deprecation.go @@ -12,35 +12,39 @@ // See the License for the specific language governing permissions and // limitations under the License. -package embed +package config type V2DeprecationEnum string const ( -// Default in v3.5. Issues a warning if v2store have meaningful content. -V2_DEPR_0_NOT_YET = V2DeprecationEnum("not-yet") -// Default in v3.6. Meaningful v2 state is not allowed. -// The V2 files are maintained for v3.5 rollback. -V2_DEPR_1_WRITE_ONLY = V2DeprecationEnum("write-only") -// V2store is WIPED if found !!! -V2_DEPR_1_WRITE_ONLY_DROP = V2DeprecationEnum("write-only-drop-data") -// V2store is neither written nor read. Usage of this configuration is blocking -// ability to rollback to etcd v3.5. -V2_DEPR_2_GONE = V2DeprecationEnum("gone") + // Default in v3.5. Issues a warning if v2store have meaningful content. + V2_DEPR_0_NOT_YET = V2DeprecationEnum("not-yet") + // Default in v3.6. Meaningful v2 state is not allowed. + // The V2 files are maintained for v3.5 rollback. + V2_DEPR_1_WRITE_ONLY = V2DeprecationEnum("write-only") + // V2store is WIPED if found !!! + V2_DEPR_1_WRITE_ONLY_DROP = V2DeprecationEnum("write-only-drop-data") + // V2store is neither written nor read. Usage of this configuration is blocking + // ability to rollback to etcd v3.5. + V2_DEPR_2_GONE = V2DeprecationEnum("gone") -V2_DEPR_DEFAULT = V2_DEPR_0_NOT_YET + V2_DEPR_DEFAULT = V2_DEPR_0_NOT_YET ) func (e V2DeprecationEnum) IsAtLeast(v2d V2DeprecationEnum) bool { - return e.level() >= v2d.level() + return e.level() >= v2d.level() } func (e V2DeprecationEnum) level() int { switch e { - case V2_DEPR_0_NOT_YET: return 0 - case V2_DEPR_1_WRITE_ONLY: return 1 - case V2_DEPR_1_WRITE_ONLY_DROP: return 2 - case V2_DEPR_2_GONE: return 3 + case V2_DEPR_0_NOT_YET: + return 0 + case V2_DEPR_1_WRITE_ONLY: + return 1 + case V2_DEPR_1_WRITE_ONLY_DROP: + return 2 + case V2_DEPR_2_GONE: + return 3 } panic("Unknown V2DeprecationEnum: " + e) -} \ No newline at end of file +} diff --git a/server/embed/v2_deprecation_test.go b/server/config/v2_deprecation_test.go similarity index 94% rename from server/embed/v2_deprecation_test.go rename to server/config/v2_deprecation_test.go index f9533bd5a1a..c8d911d6076 100644 --- a/server/embed/v2_deprecation_test.go +++ b/server/config/v2_deprecation_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package embed +package config import "testing" @@ -32,7 +32,7 @@ func TestV2DeprecationEnum_IsAtLeast(t *testing.T) { {V2_DEPR_1_WRITE_ONLY_DROP, V2_DEPR_1_WRITE_ONLY, true}, } for _, tt := range tests { - t.Run(string(tt.e) + " >= " + string(tt.v2d), func(t *testing.T) { + t.Run(string(tt.e)+" >= "+string(tt.v2d), func(t *testing.T) { if got := tt.e.IsAtLeast(tt.v2d); got != tt.want { t.Errorf("IsAtLeast() = %v, want %v", got, tt.want) } diff --git a/server/embed/config.go b/server/embed/config.go index bf24b048309..9e8583a49a6 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -33,6 +33,7 @@ import ( "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/flags" "go.etcd.io/etcd/pkg/v3/netutil" + "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api/v3compactor" @@ -402,7 +403,7 @@ type Config struct { ExperimentalTxnModeWriteWithSharedBuffer bool `json:"experimental-txn-mode-write-with-shared-buffer"` // V2Deprecation describes phase of API & Storage V2 support - V2Deprecation V2DeprecationEnum `json:"v2-deprecation"` + V2Deprecation config.V2DeprecationEnum `json:"v2-deprecation"` } // configYAML holds the config suitable for yaml parsing @@ -498,7 +499,7 @@ func NewConfig() *Config { ExperimentalMemoryMlock: false, ExperimentalTxnModeWriteWithSharedBuffer: true, - V2Deprecation: V2_DEPR_DEFAULT, + V2Deprecation: config.V2_DEPR_DEFAULT, } cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name) return cfg @@ -800,12 +801,11 @@ func (cfg Config) InitialClusterFromName(name string) (ret string) { func (cfg Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew } func (cfg Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) } -func (cfg Config) V2DeprecationEffective() V2DeprecationEnum { +func (cfg Config) V2DeprecationEffective() config.V2DeprecationEnum { if cfg.V2Deprecation == "" { - return V2_DEPR_DEFAULT - } else { - return cfg.V2Deprecation + return config.V2_DEPR_DEFAULT } + return cfg.V2Deprecation } func (cfg Config) defaultPeerHost() bool { diff --git a/server/embed/etcd.go b/server/embed/etcd.go index d3809157393..79b473c6fb2 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -226,6 +226,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { ExperimentalMemoryMlock: cfg.ExperimentalMemoryMlock, ExperimentalTxnModeWriteWithSharedBuffer: cfg.ExperimentalTxnModeWriteWithSharedBuffer, ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes, + V2Deprecation: cfg.V2DeprecationEffective(), } if srvcfg.ExperimentalEnableDistributedTracing { @@ -696,6 +697,9 @@ func (e *Etcd) serveClients() (err error) { // Start a client server goroutine for each listen address var h http.Handler if e.Config().EnableV2 { + if e.Config().V2DeprecationEffective().IsAtLeast(config.V2_DEPR_1_WRITE_ONLY) { + return fmt.Errorf("--enable-v2 and --v2-deprecation=%s are mutually exclusive", e.Config().V2DeprecationEffective()) + } e.cfg.logger.Warn("Flag `enable-v2` is deprecated and will get removed in etcd 3.6.") if len(e.Config().ExperimentalEnableV2V3) > 0 { e.cfg.logger.Warn("Flag `experimental-enable-v2v3` is deprecated and will get removed in etcd 3.6.") diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index 0072109d691..f9c91d9f9c5 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -27,6 +27,7 @@ import ( "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/pkg/v3/flags" + cconfig "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" @@ -121,10 +122,10 @@ func newConfig() *config { proxyFlagOn, ), v2deprecation: flags.NewSelectiveStringsValue( - string(embed.V2_DEPR_0_NOT_YET), - string(embed.V2_DEPR_1_WRITE_ONLY), - string(embed.V2_DEPR_1_WRITE_ONLY_DROP), - string(embed.V2_DEPR_2_GONE)), + string(cconfig.V2_DEPR_0_NOT_YET), + string(cconfig.V2_DEPR_1_WRITE_ONLY), + string(cconfig.V2_DEPR_1_WRITE_ONLY_DROP), + string(cconfig.V2_DEPR_2_GONE)), } fs := cfg.cf.flagSet @@ -343,7 +344,7 @@ func (cfg *config) parse(arguments []string) error { } if cfg.ec.V2Deprecation == "" { - cfg.ec.V2Deprecation = embed.V2_DEPR_DEFAULT + cfg.ec.V2Deprecation = cconfig.V2_DEPR_DEFAULT } // now logger is set up @@ -400,7 +401,7 @@ func (cfg *config) configFromCmdLine() error { cfg.cp.Fallback = cfg.cf.fallback.String() cfg.cp.Proxy = cfg.cf.proxy.String() - cfg.ec.V2Deprecation = embed.V2DeprecationEnum(cfg.cf.v2deprecation.String()) + cfg.ec.V2Deprecation = cconfig.V2DeprecationEnum(cfg.cf.v2deprecation.String()) // disable default advertise-client-urls if lcurls is set missingAC := flags.IsSet(cfg.cf.flagSet, "listen-client-urls") && !flags.IsSet(cfg.cf.flagSet, "advertise-client-urls") diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index ca8ac039da8..dc5b55fae7e 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -18,6 +18,7 @@ import ( "fmt" "strconv" + cconfig "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/embed" "golang.org/x/crypto/bcrypt" ) @@ -124,8 +125,8 @@ Clustering: Interpret 'auto-compaction-retention' one of: periodic|revision. 'periodic' for duration based retention, defaulting to hours if no time unit is provided (e.g. '5m'). 'revision' for revision number based retention. --enable-v2 '` + strconv.FormatBool(embed.DefaultEnableV2) + `' Accept etcd V2 client requests. Deprecated and to be decommissioned in v3.6. - --v2-deprecation '` + string(embed.V2_DEPR_DEFAULT) + `' - Phase of v2store deprecation. Allows to optin for higher compatibility mode. + --v2-deprecation '` + string(cconfig.V2_DEPR_DEFAULT) + `' + Phase of v2store deprecation. Allows to opt-in for higher compatibility mode. Supported values: 'not-yet' // Issues a warning if v2store have meaningful content (default in v3.5) 'write-only' // Custom v2 state is not allowed (planned default in v3.6) diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go index ab62888c500..791740df97a 100644 --- a/server/etcdserver/server.go +++ b/server/etcdserver/server.go @@ -479,6 +479,11 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { cfg.Logger.Panic("failed to recover from snapshot", zap.Error(err)) } + if err = assertNoV2StoreContent(cfg.Logger, st, cfg.V2Deprecation); err != nil { + cfg.Logger.Error("illegal v2store content", zap.Error(err)) + return nil, err + } + cfg.Logger.Info( "recovered v2 store from snapshot", zap.Uint64("snapshot-index", snapshot.Metadata.Index), @@ -496,6 +501,8 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { zap.Int64("backend-size-in-use-bytes", s2), zap.String("backend-size-in-use", humanize.Bytes(uint64(s2))), ) + } else { + cfg.Logger.Info("No snapshot found. Recovering WAL from scratch!") } if !cfg.ForceNewCluster { @@ -662,6 +669,23 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { return srv, nil } +// assertNoV2StoreContent -> depending on the deprecation stage, warns or report an error +// if the v2store contains custom content. +func assertNoV2StoreContent(lg *zap.Logger, st v2store.Store, deprecationStage config.V2DeprecationEnum) error { + metaOnly, err := membership.IsMetaStoreOnly(st) + if err != nil { + return err + } + if metaOnly { + return nil + } + if deprecationStage.IsAtLeast(config.V2_DEPR_1_WRITE_ONLY) { + return fmt.Errorf("detected disallowed custom content in v2store for stage --v2-deprecation=%s", deprecationStage) + } + lg.Warn("detected custom v2store content. Etcd v3.5 is the last version allowing to access it using API v2. Please remove the content.") + return nil +} + func (s *EtcdServer) Logger() *zap.Logger { s.lgMu.RLock() l := s.lg @@ -1252,6 +1276,10 @@ func (s *EtcdServer) applySnapshot(ep *etcdProgress, apply *apply) { lg.Panic("failed to restore v2 store", zap.Error(err)) } + if err := assertNoV2StoreContent(lg, s.v2store, s.Cfg.V2Deprecation); err != nil { + lg.Panic("illegal v2store content", zap.Error(err)) + } + lg.Info("restored v2 store") s.cluster.SetBackend(newbe) diff --git a/tests/integration/cluster.go b/tests/integration/cluster.go index 755d8520a66..c5302e42920 100644 --- a/tests/integration/cluster.go +++ b/tests/integration/cluster.go @@ -706,6 +706,8 @@ func mustNewMember(t testutil.TB, mcfg memberConfig) *member { m.InitialCorruptCheck = true m.WarningApplyDuration = embed.DefaultWarningApplyDuration + m.V2Deprecation = config.V2_DEPR_DEFAULT + m.Logger = memberLogger(t, mcfg.name) t.Cleanup(func() { // if we didn't cleanup the logger, the consecutive test From 79e3d7bd3eb67daa90f17fa40b5521ae8f3b1f28 Mon Sep 17 00:00:00 2001 From: Piotr Tabor Date: Wed, 12 May 2021 16:00:06 +0200 Subject: [PATCH 4/4] Add e2e tests for --v2-deprecation flag. --- tests/e2e/cluster_proxy_test.go | 9 +-- tests/e2e/cluster_test.go | 20 ++++-- tests/e2e/ctl_v3_migrate_test.go | 8 --- tests/e2e/ctl_v3_test.go | 2 +- tests/e2e/main_test.go | 14 +--- tests/e2e/testing.go | 38 +++++++++++ tests/e2e/v2store_deprecation_test.go | 98 +++++++++++++++++++++++++++ 7 files changed, 161 insertions(+), 28 deletions(-) create mode 100644 tests/e2e/testing.go create mode 100644 tests/e2e/v2store_deprecation_test.go diff --git a/tests/e2e/cluster_proxy_test.go b/tests/e2e/cluster_proxy_test.go index 953e68946b5..7a5740d1498 100644 --- a/tests/e2e/cluster_proxy_test.go +++ b/tests/e2e/cluster_proxy_test.go @@ -22,6 +22,7 @@ import ( "net" "net/url" "os" + "path" "strconv" "strings" @@ -268,10 +269,10 @@ func newProxyV3Proc(cfg *etcdServerProcessConfig) *proxyV3Proc { // Configure certificates for connection proxy ---> server. // This certificate must NOT have CN set. tlsArgs = append(tlsArgs, - "--cert", "../fixtures/client-nocn.crt", - "--key", "../fixtures/client-nocn.key.insecure", - "--cacert", "../fixtures/ca.crt", - "--client-crl-file", "../fixtures/revoke.crl") + "--cert", path.Join(fixturesDir, "client-nocn.crt"), + "--key", path.Join(fixturesDir, "client-nocn.key.insecure"), + "--cacert", path.Join(fixturesDir, "ca.crt"), + "--client-crl-file", path.Join(fixturesDir, "revoke.crl")) } return &proxyV3Proc{ proxyProc{ diff --git a/tests/e2e/cluster_test.go b/tests/e2e/cluster_test.go index 2a3298fb0b1..cc47121fa21 100644 --- a/tests/e2e/cluster_test.go +++ b/tests/e2e/cluster_test.go @@ -18,17 +18,23 @@ import ( "fmt" "net/url" "os" + "path" "strings" "testing" "time" "go.etcd.io/etcd/server/v3/etcdserver" + "go.etcd.io/etcd/tests/v3/integration" ) const etcdProcessBasePort = 20000 type clientConnType int +var ( + fixturesDir = integration.MustAbsPath("../fixtures") +) + const ( clientNonTLS clientConnType = iota clientTLS @@ -113,9 +119,10 @@ func newConfigClientTLSCertAuthWithNoCN() *etcdProcessClusterConfig { func newConfigJWT() *etcdProcessClusterConfig { return &etcdProcessClusterConfig{ - clusterSize: 1, - initialToken: "new", - authTokenOpts: "jwt,pub-key=../fixtures/server.crt,priv-key=../fixtures/server.key.insecure,sign-method=RS256,ttl=1s", + clusterSize: 1, + initialToken: "new", + authTokenOpts: "jwt,pub-key=" + path.Join(fixturesDir, "server.crt") + + ",priv-key=" + path.Join(fixturesDir, "server.key.insecure") + ",sign-method=RS256,ttl=1s", } } @@ -161,6 +168,7 @@ type etcdProcessClusterConfig struct { enableV2 bool initialCorruptCheck bool authTokenOpts string + v2deprecation string rollingStart bool } @@ -246,7 +254,7 @@ func (cfg *etcdProcessClusterConfig) etcdServerProcessConfigs(tb testing.TB) []* } purl := url.URL{Scheme: cfg.peerScheme(), Host: fmt.Sprintf("localhost:%d", port+1)} - name := fmt.Sprintf("test-%s-%d", tb.Name(), i) + name := fmt.Sprintf("test-%d", i) dataDirPath := cfg.dataDirPath if cfg.dataDirPath == "" { dataDirPath = tb.TempDir() @@ -296,6 +304,10 @@ func (cfg *etcdProcessClusterConfig) etcdServerProcessConfigs(tb testing.TB) []* args = append(args, "--auth-token", cfg.authTokenOpts) } + if cfg.v2deprecation != "" { + args = append(args, "--v2-deprecation", cfg.v2deprecation) + } + etcdCfgs[i] = &etcdServerProcessConfig{ execPath: cfg.execPath, args: args, diff --git a/tests/e2e/ctl_v3_migrate_test.go b/tests/e2e/ctl_v3_migrate_test.go index bdaeaf22bec..c36b5a38c61 100644 --- a/tests/e2e/ctl_v3_migrate_test.go +++ b/tests/e2e/ctl_v3_migrate_test.go @@ -21,17 +21,9 @@ import ( "testing" "time" - "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/v3" - "go.etcd.io/etcd/server/v3/verify" ) -func BeforeTest(t testing.TB) { - skipInShortMode(t) - testutil.BeforeTest(t) - os.Setenv(verify.ENV_VERIFY, verify.ENV_VERIFY_ALL_VALUE) -} - func TestCtlV3Migrate(t *testing.T) { BeforeTest(t) diff --git a/tests/e2e/ctl_v3_test.go b/tests/e2e/ctl_v3_test.go index d1743e08881..f6eeeecbfed 100644 --- a/tests/e2e/ctl_v3_test.go +++ b/tests/e2e/ctl_v3_test.go @@ -30,7 +30,7 @@ import ( func TestCtlV3Version(t *testing.T) { testCtl(t, versionTest) } func TestClusterVersion(t *testing.T) { - skipInShortMode(t) + BeforeTest(t) tests := []struct { name string diff --git a/tests/e2e/main_test.go b/tests/e2e/main_test.go index 75adb1e8e0f..ddb7ae4ba7b 100644 --- a/tests/e2e/main_test.go +++ b/tests/e2e/main_test.go @@ -6,13 +6,12 @@ package e2e import ( "flag" - "log" "os" - "path/filepath" "runtime" "testing" "go.etcd.io/etcd/client/pkg/v3/testutil" + "go.etcd.io/etcd/tests/v3/integration" ) var ( @@ -38,15 +37,8 @@ func TestMain(m *testing.M) { os.Setenv("ETCD_UNSUPPORTED_ARCH", runtime.GOARCH) os.Unsetenv("ETCDCTL_API") - binDirDef, err := filepath.Abs("../../bin") - if err != nil { - log.Fatal(err) - } - - certDirDef, err := filepath.Abs("../fixtures") - if err != nil { - log.Fatal(err) - } + binDirDef := integration.MustAbsPath("../../bin") + certDirDef := fixturesDir flag.StringVar(&binDir, "bin-dir", binDirDef, "The directory for store etcd and etcdctl binaries.") flag.StringVar(&certDir, "cert-dir", certDirDef, "The directory for store certificate files.") diff --git a/tests/e2e/testing.go b/tests/e2e/testing.go new file mode 100644 index 00000000000..a36c075dbd7 --- /dev/null +++ b/tests/e2e/testing.go @@ -0,0 +1,38 @@ +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "go.etcd.io/etcd/client/pkg/v3/testutil" + "go.etcd.io/etcd/server/v3/verify" +) + +func BeforeTest(t testing.TB) { + skipInShortMode(t) + testutil.BeforeTest(t) + os.Setenv(verify.ENV_VERIFY, verify.ENV_VERIFY_ALL_VALUE) + + path, err := os.Getwd() + assert.NoError(t, err) + tempDir := t.TempDir() + assert.NoError(t, os.Chdir(tempDir)) + t.Logf("Changing working directory to: %s", tempDir) + + t.Cleanup(func() { assert.NoError(t, os.Chdir(path)) }) +} diff --git a/tests/e2e/v2store_deprecation_test.go b/tests/e2e/v2store_deprecation_test.go new file mode 100644 index 00000000000..06ad555d07a --- /dev/null +++ b/tests/e2e/v2store_deprecation_test.go @@ -0,0 +1,98 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func createV2store(t testing.TB, dataDirPath string) { + t.Log("Creating not-yet v2-deprecated etcd") + + cfg := configStandalone(etcdProcessClusterConfig{enableV2: true, dataDirPath: dataDirPath, snapshotCount: 5}) + epc, err := newEtcdProcessCluster(t, cfg) + assert.NoError(t, err) + + defer func() { + assert.NoError(t, epc.Stop()) + }() + + // We need to exceed 'snapshotCount' such that v2 snapshot is dumped. + for i := 0; i < 10; i++ { + if err := cURLPut(epc, cURLReq{ + endpoint: "/v2/keys/foo", value: "bar" + fmt.Sprint(i), + expected: `{"action":"set","node":{"key":"/foo","value":"bar` + fmt.Sprint(i)}); err != nil { + t.Fatalf("failed put with curl (%v)", err) + } + } +} + +func assertVerifyCanStartV2deprecationNotYet(t testing.TB, dataDirPath string) { + t.Log("verify: possible to start etcd with --v2-deprecation=not-yet mode") + + cfg := configStandalone(etcdProcessClusterConfig{enableV2: true, dataDirPath: dataDirPath, v2deprecation: "not-yet", keepDataDir: true}) + epc, err := newEtcdProcessCluster(t, cfg) + assert.NoError(t, err) + + defer func() { + assert.NoError(t, epc.Stop()) + }() + + if err := cURLGet(epc, cURLReq{ + endpoint: "/v2/keys/foo", + expected: `{"action":"get","node":{"key":"/foo","value":"bar9","modifiedIndex":13,"createdIndex":13}}`}); err != nil { + t.Fatalf("failed get with curl (%v)", err) + } + +} + +func assertVerifyCannotStartV2deprecationWriteOnly(t testing.TB, dataDirPath string) { + t.Log("Verify its infeasible to start etcd with --v2-deprecation=write-only mode") + proc, err := spawnCmd([]string{binDir + "/etcd", "--v2-deprecation=write-only", "--data-dir=" + dataDirPath}) + assert.NoError(t, err) + + _, err = proc.Expect("detected disallowed custom content in v2store for stage --v2-deprecation=write-only") + assert.NoError(t, err) +} + +func TestV2Deprecation(t *testing.T) { + BeforeTest(t) + dataDirPath := t.TempDir() + + t.Run("create-storev2-data", func(t *testing.T) { + createV2store(t, dataDirPath) + }) + + t.Run("--v2-deprecation=write-only fails", func(t *testing.T) { + assertVerifyCannotStartV2deprecationWriteOnly(t, dataDirPath) + }) + + t.Run("--v2-deprecation=not-yet succeeds", func(t *testing.T) { + assertVerifyCanStartV2deprecationNotYet(t, dataDirPath) + }) + +} + +func TestV2DeprecationWriteOnlyNoV2Api(t *testing.T) { + BeforeTest(t) + proc, err := spawnCmd([]string{binDir + "/etcd", "--v2-deprecation=write-only", "--enable-v2"}) + assert.NoError(t, err) + + _, err = proc.Expect("--enable-v2 and --v2-deprecation=write-only are mutually exclusive") + assert.NoError(t, err) +}