From 6a3018fcef09e6d39b09d91b26bff9419b99c8c4 Mon Sep 17 00:00:00 2001 From: Thomas Gosteli Date: Thu, 12 Sep 2024 08:43:57 +0200 Subject: [PATCH 01/41] docs: remove warning that etcd 3.5.0 was not yet released Signed-off-by: Thomas Gosteli --- client/v3/README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/client/v3/README.md b/client/v3/README.md index 1e037d7eb6b..16c0fe888cf 100644 --- a/client/v3/README.md +++ b/client/v3/README.md @@ -11,13 +11,6 @@ go get go.etcd.io/etcd/client/v3 ``` -Warning: As etcd 3.5.0 was not yet released, the command above does not work. -After first pre-release of 3.5.0 [#12498](https://github.com/etcd-io/etcd/issues/12498), -etcd can be referenced using: -``` -go get go.etcd.io/etcd/client/v3@v3.5.0-pre -``` - ## Get started Create client using `clientv3.New`: From b56619e1df8e431fbfeea35213a23f4221ac9088 Mon Sep 17 00:00:00 2001 From: James Blair Date: Mon, 16 Sep 2024 11:31:30 +1200 Subject: [PATCH 02/41] Ensure commits are signed in scripts/release.sh. Signed-off-by: James Blair --- scripts/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release.sh b/scripts/release.sh index 94b7452b851..993ccff8f8e 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -154,7 +154,7 @@ main() { # shellcheck disable=SC2038,SC2046,SC2185 run git add $(find -name go.mod ! -path './release/*'| xargs) run git diff --staged | cat - run git commit -m "version: bump up to ${VERSION}" + run git commit --signoff --message "version: bump up to ${VERSION}" run git diff --staged | cat fi From 6021768694835bcfd526b9b913218b172f6c29b5 Mon Sep 17 00:00:00 2001 From: Agni Date: Wed, 2 Oct 2024 23:39:04 +0530 Subject: [PATCH 03/41] Bump go version to 1.22.8 Signed-off-by: Agni --- .go-version | 2 +- api/go.mod | 2 +- client/pkg/go.mod | 2 +- client/v2/go.mod | 2 +- client/v3/go.mod | 2 +- etcdctl/go.mod | 2 +- etcdutl/go.mod | 2 +- go.mod | 2 +- pkg/go.mod | 2 +- raft/go.mod | 2 +- server/go.mod | 2 +- tests/go.mod | 2 +- tools/mod/go.mod | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.go-version b/.go-version index 87b26e8b1aa..229a27c6f20 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.22.7 +1.22.8 diff --git a/api/go.mod b/api/go.mod index b359a329c1b..58c4a0d4d78 100644 --- a/api/go.mod +++ b/api/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/api/v3 go 1.22 -toolchain go1.22.7 +toolchain go1.22.8 require ( github.com/coreos/go-semver v0.3.0 diff --git a/client/pkg/go.mod b/client/pkg/go.mod index ca8b517a303..7897dc27d8b 100644 --- a/client/pkg/go.mod +++ b/client/pkg/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/client/pkg/v3 go 1.22 -toolchain go1.22.7 +toolchain go1.22.8 require ( github.com/coreos/go-systemd/v22 v22.3.2 diff --git a/client/v2/go.mod b/client/v2/go.mod index 5395ee2fca2..86b4f8d6027 100644 --- a/client/v2/go.mod +++ b/client/v2/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/client/v2 go 1.22 -toolchain go1.22.7 +toolchain go1.22.8 require ( github.com/json-iterator/go v1.1.11 diff --git a/client/v3/go.mod b/client/v3/go.mod index a55e60cd6fb..c0bfa1a8f4e 100644 --- a/client/v3/go.mod +++ b/client/v3/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/client/v3 go 1.22 -toolchain go1.22.7 +toolchain go1.22.8 require ( github.com/dustin/go-humanize v1.0.0 diff --git a/etcdctl/go.mod b/etcdctl/go.mod index 4cabf764268..d81fc1b4599 100644 --- a/etcdctl/go.mod +++ b/etcdctl/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/etcdctl/v3 go 1.22 -toolchain go1.22.7 +toolchain go1.22.8 require ( github.com/bgentry/speakeasy v0.1.0 diff --git a/etcdutl/go.mod b/etcdutl/go.mod index d6031e021e7..07c01725597 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/etcdutl/v3 go 1.22 -toolchain go1.22.7 +toolchain go1.22.8 replace ( go.etcd.io/etcd/api/v3 => ../api diff --git a/go.mod b/go.mod index 702857ca523..b74535ad866 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/v3 go 1.22 -toolchain go1.22.7 +toolchain go1.22.8 replace ( go.etcd.io/etcd/api/v3 => ./api diff --git a/pkg/go.mod b/pkg/go.mod index 8af2ccbc2f7..535a540f12a 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/pkg/v3 go 1.22 -toolchain go1.22.7 +toolchain go1.22.8 require ( github.com/creack/pty v1.1.11 diff --git a/raft/go.mod b/raft/go.mod index 1cf1ac3a327..0ccf2e36906 100644 --- a/raft/go.mod +++ b/raft/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/raft/v3 go 1.22 -toolchain go1.22.7 +toolchain go1.22.8 require ( github.com/cockroachdb/datadriven v1.0.2 diff --git a/server/go.mod b/server/go.mod index 74b4facbfdc..a6c68184639 100644 --- a/server/go.mod +++ b/server/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/server/v3 go 1.22 -toolchain go1.22.7 +toolchain go1.22.8 require ( github.com/coreos/go-semver v0.3.0 diff --git a/tests/go.mod b/tests/go.mod index 70b42cb9380..e17619fbb9c 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/tests/v3 go 1.22 -toolchain go1.22.7 +toolchain go1.22.8 replace ( go.etcd.io/etcd/api/v3 => ../api diff --git a/tools/mod/go.mod b/tools/mod/go.mod index 1fa8217a70b..ce99f83263b 100644 --- a/tools/mod/go.mod +++ b/tools/mod/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/tools/v3 go 1.22 -toolchain go1.22.7 +toolchain go1.22.8 require ( github.com/alexkohler/nakedret v1.0.0 From 2dbfe58ac93f517a2c129d6bf83be089a5ab69f6 Mon Sep 17 00:00:00 2001 From: Ivan Valdes Date: Wed, 2 Oct 2024 12:52:41 -0700 Subject: [PATCH 04/41] github/workflows: use ECR mirror for Trivy's DB GitHub Container Registry is returning a TOOMANYREQUESTS error. Switch to AWS ECR mirror, as suggested in https://github.com/aquasecurity/trivy-action/issues/389. Signed-off-by: Ivan Valdes --- .github/workflows/release.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ed4a4cfd77c..5139db4876d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -64,6 +64,11 @@ jobs: severity: 'CRITICAL,HIGH' format: 'sarif' output: 'trivy-results-${{ matrix.platforms }}.sarif' + env: + # Use AWS' ECR mirror for the trivy-db image, as GitHub's Container + # Registry is returning a TOOMANYREQUESTS error. + # Ref: https://github.com/aquasecurity/trivy-action/issues/389 + TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2' - name: upload scan results uses: github/codeql-action/upload-sarif@956f09c2ef1926b580554b9014cfb8a51abf89dd # v2.16.6 with: From e5905727aeead866678b9b8235be45f79da3628b Mon Sep 17 00:00:00 2001 From: Oleg Guba Date: Mon, 14 Oct 2024 14:16:22 -0700 Subject: [PATCH 05/41] [serverWatchStream] terminate recvLoop on sws.close() Signed-off-by: Oleg Guba --- server/etcdserver/api/v3rpc/watch.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/etcdserver/api/v3rpc/watch.go b/server/etcdserver/api/v3rpc/watch.go index ddbcd231bf0..0b0284ec3e0 100644 --- a/server/etcdserver/api/v3rpc/watch.go +++ b/server/etcdserver/api/v3rpc/watch.go @@ -345,11 +345,17 @@ func (sws *serverWatchStream) recvLoop() error { id := uv.CancelRequest.WatchId err := sws.watchStream.Cancel(mvcc.WatchID(id)) if err == nil { - sws.ctrlStream <- &pb.WatchResponse{ + wr := &pb.WatchResponse{ Header: sws.newResponseHeader(sws.watchStream.Rev()), WatchId: id, Canceled: true, } + select { + case sws.ctrlStream <- wr: + case <-sws.closec: + return nil + } + sws.mu.Lock() delete(sws.progress, mvcc.WatchID(id)) delete(sws.prevKV, mvcc.WatchID(id)) From dea4eb3e9189502ddc0204e5ba0fc1e6760397ba Mon Sep 17 00:00:00 2001 From: Shyam Jeedigunta Date: Mon, 28 Oct 2024 12:15:23 -0700 Subject: [PATCH 06/41] [3.5] Fix risk of a partial write txn being applied Signed-off-by: Shyam Jeedigunta --- server/etcdserver/apply.go | 9 +++++---- server/etcdserver/apply_test.go | 31 +++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/server/etcdserver/apply.go b/server/etcdserver/apply.go index 2ef151259c1..9a3f18d0f41 100644 --- a/server/etcdserver/apply.go +++ b/server/etcdserver/apply.go @@ -478,10 +478,11 @@ func (a *applierV3backend) Txn(ctx context.Context, rt *pb.TxnRequest) (*pb.TxnR _, err := a.applyTxn(ctx, txn, rt, txnPath, txnResp) if err != nil { if isWrite { - // end txn to release locks before panic - txn.End() - // When txn with write operations starts it has to be successful - // We don't have a way to recover state in case of write failure + // CAUTION: When a txn performing write operations starts, we always expect it to be successful. + // If a write failure is seen we SHOULD NOT try to recover the server, but crash with a panic to make the failure explicit. + // Trying to silently recover (e.g by ignoring the failed txn or calling txn.End() early) poses serious risks: + // - violation of transaction atomicity if some write operations have been partially executed + // - data inconsistency across different etcd members if they applied the txn asymmetrically lg.Panic("unexpected error during txn with writes", zap.Error(err)) } else { lg.Error("unexpected error during readonly txn", zap.Error(err)) diff --git a/server/etcdserver/apply_test.go b/server/etcdserver/apply_test.go index 65704fd4e78..b968b89fea3 100644 --- a/server/etcdserver/apply_test.go +++ b/server/etcdserver/apply_test.go @@ -2,6 +2,10 @@ package etcdserver import ( "context" + "crypto/sha256" + "github.com/stretchr/testify/require" + "io" + "os" "strings" "sync" "testing" @@ -55,8 +59,7 @@ func TestReadonlyTxnError(t *testing.T) { } func TestWriteTxnPanic(t *testing.T) { - b, _ := betesting.NewDefaultTmpBackend(t) - defer betesting.Close(t, b) + b, bePath := betesting.NewDefaultTmpBackend(t) s := mvcc.New(zap.NewExample(), b, &lease.FakeLessor{}, mvcc.StoreConfig{}) defer s.Close() @@ -92,5 +95,29 @@ func TestWriteTxnPanic(t *testing.T) { }, } + // compute DB file hash before applying the txn + dbHashBefore, err := computeFileHash(bePath) + require.NoErrorf(t, err, "failed to compute DB file hash before txn") + + // we verify the following properties below: + // 1. server panics after a write txn aply fails (invariant: server should never try to move on from a failed write) + // 2. no writes from the txn are applied to the backend (invariant: failed write should have no side-effect on DB state besides panic) assert.Panics(t, func() { a.Txn(ctx, txn) }, "Expected panic in Txn with writes") + dbHashAfter, err := computeFileHash(bePath) + require.NoErrorf(t, err, "failed to compute DB file hash after txn") + require.Equalf(t, dbHashBefore, dbHashAfter, "mismatch in DB hash before and after failed write txn") +} + +func computeFileHash(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", err + } + defer file.Close() + + h := sha256.New() + if _, err := io.Copy(h, file); err != nil { + return "", err + } + return string(h.Sum(nil)), nil } From 17ab1fa6d2d1f041e548d8bf6190f82e9db321a8 Mon Sep 17 00:00:00 2001 From: James Blair Date: Tue, 29 Oct 2024 12:18:35 +1300 Subject: [PATCH 07/41] Remove defunct actuated arm64 workflows. Signed-off-by: James Blair --- .github/workflows/tests.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 1340e1851ed..6f9f6398592 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -13,13 +13,3 @@ jobs: 'linux-unit-4-cpu-race', 'linux-386-unit-1-cpu', 'all-build']" - arm64: - uses: ./.github/workflows/tests-template.yaml - with: - arch: arm64 - runs-on: actuated-arm64-8cpu-32gb - targets: "['linux-test-smoke', - 'linux-integration-1-cpu', - 'linux-integration-2-cpu', - 'linux-integration-4-cpu', - 'linux-unit-4-cpu-race']" From 5ec08a2369d310018684bfa1589b8cef5b56cd90 Mon Sep 17 00:00:00 2001 From: Thomas Gosteli Date: Tue, 29 Oct 2024 14:53:47 +0100 Subject: [PATCH 08/41] chore(e2e): backport TestNoErrorLogsDuringNormalOperations test Signed-off-by: Thomas Gosteli --- tests/e2e/logging_test.go | 91 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 tests/e2e/logging_test.go diff --git a/tests/e2e/logging_test.go b/tests/e2e/logging_test.go new file mode 100644 index 00000000000..87c8adb6532 --- /dev/null +++ b/tests/e2e/logging_test.go @@ -0,0 +1,91 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "go.etcd.io/etcd/tests/v3/framework/e2e" +) + +func TestNoErrorLogsDuringNormalOperations(t *testing.T) { + tests := []struct { + name string + clusterSize int + allowedErrors map[string]bool + }{ + { + name: "single node cluster", + clusterSize: 1, + allowedErrors: map[string]bool{}, + }, + { + name: "three node cluster", + clusterSize: 3, + allowedErrors: map[string]bool{}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + e2e.BeforeTest(t) + + epc, err := e2e.NewEtcdProcessCluster(t, + &e2e.EtcdProcessClusterConfig{ + LogLevel: "debug", + ClusterSize: tc.clusterSize, + }, + ) + require.NoError(t, err) + defer epc.Close() + + require.Lenf(t, epc.Procs, tc.clusterSize, "embedded etcd cluster process count is not as expected") + + // Collect the handle of logs before closing the processes. + var logHandles []e2e.LogsExpect + for i := range tc.clusterSize { + logHandles = append(logHandles, epc.Procs[i].Logs()) + } + + time.Sleep(time.Second) + err = epc.Close() + require.NoErrorf(t, err, "closing etcd processes") + + // Now that the processes are closed we can collect all log lines. This must happen after closing, else we + // might not get all log lines. + var lines []string + for _, h := range logHandles { + lines = append(lines, h.Lines()...) + } + require.NotEmptyf(t, lines, "expected at least one log line") + + var entry logEntry + for _, line := range lines { + err := json.Unmarshal([]byte(line), &entry) + require.NoErrorf(t, err, "parse log line as json, line: %s", line) + + if tc.allowedErrors[entry.Message] { + continue + } + + require.NotEqualf(t, "error", entry.Level, "error level log message found: %s", line) + } + }) + } +} From ae6655f11e965c983e6b04a5603f4dc2aac9bd87 Mon Sep 17 00:00:00 2001 From: Thomas Gosteli Date: Fri, 1 Nov 2024 12:23:48 +0000 Subject: [PATCH 09/41] fix(defrag): handle no space left error Signed-off-by: Thomas Gosteli --- server/mvcc/backend/backend.go | 12 +++++--- tests/e2e/defrag_no_space_test.go | 49 +++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/defrag_no_space_test.go diff --git a/server/mvcc/backend/backend.go b/server/mvcc/backend/backend.go index 7d77da12fd6..077146b82ee 100644 --- a/server/mvcc/backend/backend.go +++ b/server/mvcc/backend/backend.go @@ -477,10 +477,6 @@ func (b *backend) defrag() error { b.readTx.Lock() defer b.readTx.Unlock() - b.batchTx.unsafeCommit(true) - - b.batchTx.tx = nil - // Create a temporary file to ensure we start with a clean slate. // Snapshotter.cleanupSnapdir cleans up any of these that are found during startup. dir := filepath.Dir(b.db.Path()) @@ -488,11 +484,14 @@ func (b *backend) defrag() error { if err != nil { return err } + options := bolt.Options{} if boltOpenOptions != nil { options = *boltOpenOptions } options.OpenFile = func(_ string, _ int, _ os.FileMode) (file *os.File, err error) { + // gofail: var defragNoSpace string + // return nil, fmt.Errorf(defragNoSpace) return temp, nil } // Don't load tmp db into memory regardless of opening options @@ -515,6 +514,11 @@ func (b *backend) defrag() error { zap.String("current-db-size-in-use", humanize.Bytes(uint64(sizeInUse1))), ) } + + // Commit/stop and then reset current transactions (including the readTx) + b.batchTx.unsafeCommit(true) + b.batchTx.tx = nil + // gofail: var defragBeforeCopy struct{} err = defragdb(b.db, tmpdb, defragLimit) if err != nil { diff --git a/tests/e2e/defrag_no_space_test.go b/tests/e2e/defrag_no_space_test.go new file mode 100644 index 00000000000..810136f156e --- /dev/null +++ b/tests/e2e/defrag_no_space_test.go @@ -0,0 +1,49 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "go.etcd.io/etcd/tests/v3/framework/config" + "go.etcd.io/etcd/tests/v3/framework/e2e" +) + +func TestDefragNoSpace(t *testing.T) { + e2e.BeforeTest(t) + + clus, err := e2e.NewEtcdProcessCluster(context.TODO(), t, + e2e.WithClusterSize(1), + e2e.WithGoFailEnabled(true), + ) + require.NoError(t, err) + t.Cleanup(func() { clus.Stop() }) + + member := clus.Procs[0] + + require.NoError(t, member.Failpoints().SetupHTTP(context.Background(), "defragNoSpace", `return("no space")`)) + require.ErrorContains(t, member.Etcdctl().Defragment(context.Background(), config.DefragOption{Timeout: time.Minute}), "no space") + + // Make sure etcd continues to run even after the failed defrag attempt + require.NoError(t, member.Etcdctl().Put(context.Background(), "foo", "bar", config.PutOptions{})) + value, err := member.Etcdctl().Get(context.Background(), "foo", config.GetOptions{}) + require.NoError(t, err) + require.Len(t, value.Kvs, 1) + require.Equal(t, "bar", string(value.Kvs[0].Value)) +} From 1d175079a8d094eaee2e1e09e181ce69d82def7b Mon Sep 17 00:00:00 2001 From: Thomas Gosteli Date: Wed, 6 Nov 2024 11:47:18 +0100 Subject: [PATCH 10/41] fix(defrag): handle defragdb failure Signed-off-by: Thomas Gosteli --- server/mvcc/backend/backend.go | 12 +++++- tests/e2e/defrag_no_space_test.go | 62 +++++++++++++++++++++---------- 2 files changed, 52 insertions(+), 22 deletions(-) diff --git a/server/mvcc/backend/backend.go b/server/mvcc/backend/backend.go index 077146b82ee..d04126d94ae 100644 --- a/server/mvcc/backend/backend.go +++ b/server/mvcc/backend/backend.go @@ -490,8 +490,8 @@ func (b *backend) defrag() error { options = *boltOpenOptions } options.OpenFile = func(_ string, _ int, _ os.FileMode) (file *os.File, err error) { - // gofail: var defragNoSpace string - // return nil, fmt.Errorf(defragNoSpace) + // gofail: var defragOpenFileError string + // return nil, fmt.Errorf(defragOpenFileError) return temp, nil } // Don't load tmp db into memory regardless of opening options @@ -526,6 +526,11 @@ func (b *backend) defrag() error { if rmErr := os.RemoveAll(tmpdb.Path()); rmErr != nil { b.lg.Error("failed to remove db.tmp after defragmentation completed", zap.Error(rmErr)) } + + // restore the bbolt transactions if defragmentation fails + b.batchTx.tx = b.unsafeBegin(true) + b.readTx.tx = b.unsafeBegin(false) + return err } @@ -578,6 +583,9 @@ func (b *backend) defrag() error { } func defragdb(odb, tmpdb *bolt.DB, limit int) error { + // gofail: var defragdbFail string + // return fmt.Errorf(defragdbFail) + // open a tx on tmpdb for writes tmptx, err := tmpdb.Begin(true) if err != nil { diff --git a/tests/e2e/defrag_no_space_test.go b/tests/e2e/defrag_no_space_test.go index 810136f156e..f6ceabe667b 100644 --- a/tests/e2e/defrag_no_space_test.go +++ b/tests/e2e/defrag_no_space_test.go @@ -16,6 +16,7 @@ package e2e import ( "context" + "fmt" "testing" "time" @@ -26,24 +27,45 @@ import ( ) func TestDefragNoSpace(t *testing.T) { - e2e.BeforeTest(t) - - clus, err := e2e.NewEtcdProcessCluster(context.TODO(), t, - e2e.WithClusterSize(1), - e2e.WithGoFailEnabled(true), - ) - require.NoError(t, err) - t.Cleanup(func() { clus.Stop() }) - - member := clus.Procs[0] - - require.NoError(t, member.Failpoints().SetupHTTP(context.Background(), "defragNoSpace", `return("no space")`)) - require.ErrorContains(t, member.Etcdctl().Defragment(context.Background(), config.DefragOption{Timeout: time.Minute}), "no space") - - // Make sure etcd continues to run even after the failed defrag attempt - require.NoError(t, member.Etcdctl().Put(context.Background(), "foo", "bar", config.PutOptions{})) - value, err := member.Etcdctl().Get(context.Background(), "foo", config.GetOptions{}) - require.NoError(t, err) - require.Len(t, value.Kvs, 1) - require.Equal(t, "bar", string(value.Kvs[0].Value)) + tests := []struct { + name string + failpoint string + err string + }{ + { + name: "no space (#18810) - can't open/create new bbolt db", + failpoint: "defragOpenFileError", + err: "no space", + }, + { + name: "defragdb failure", + failpoint: "defragdbFail", + err: "some random error", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + e2e.BeforeTest(t) + + clus, err := e2e.NewEtcdProcessCluster(context.TODO(), t, + e2e.WithClusterSize(1), + e2e.WithGoFailEnabled(true), + ) + require.NoError(t, err) + t.Cleanup(func() { clus.Stop() }) + + member := clus.Procs[0] + + require.NoError(t, member.Failpoints().SetupHTTP(context.Background(), tc.failpoint, fmt.Sprintf(`return("%s")`, tc.err))) + require.ErrorContains(t, member.Etcdctl().Defragment(context.Background(), config.DefragOption{Timeout: time.Minute}), tc.err) + + // Make sure etcd continues to run even after the failed defrag attempt + require.NoError(t, member.Etcdctl().Put(context.Background(), "foo", "bar", config.PutOptions{})) + value, err := member.Etcdctl().Get(context.Background(), "foo", config.GetOptions{}) + require.NoError(t, err) + require.Len(t, value.Kvs, 1) + require.Equal(t, "bar", string(value.Kvs[0].Value)) + }) + } } From f26ff912a98d08d8c9ac94563a99051c75c4a493 Mon Sep 17 00:00:00 2001 From: Thomas Gosteli Date: Wed, 6 Nov 2024 14:15:07 +0100 Subject: [PATCH 11/41] chore(e2e): adapt defrag tests for 3.5 Signed-off-by: Thomas Gosteli --- tests/e2e/defrag_no_space_test.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/e2e/defrag_no_space_test.go b/tests/e2e/defrag_no_space_test.go index f6ceabe667b..1f5c2b50894 100644 --- a/tests/e2e/defrag_no_space_test.go +++ b/tests/e2e/defrag_no_space_test.go @@ -22,7 +22,6 @@ import ( "github.com/stretchr/testify/require" - "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" ) @@ -48,21 +47,25 @@ func TestDefragNoSpace(t *testing.T) { t.Run(tc.name, func(t *testing.T) { e2e.BeforeTest(t) - clus, err := e2e.NewEtcdProcessCluster(context.TODO(), t, - e2e.WithClusterSize(1), - e2e.WithGoFailEnabled(true), + clus, err := e2e.NewEtcdProcessCluster(t, + &e2e.EtcdProcessClusterConfig{ + ClusterSize: 1, + LogLevel: "debug", + GoFailEnabled: true, + }, ) require.NoError(t, err) t.Cleanup(func() { clus.Stop() }) member := clus.Procs[0] + etcdctl := member.Etcdctl(e2e.ClientNonTLS, false, false) require.NoError(t, member.Failpoints().SetupHTTP(context.Background(), tc.failpoint, fmt.Sprintf(`return("%s")`, tc.err))) - require.ErrorContains(t, member.Etcdctl().Defragment(context.Background(), config.DefragOption{Timeout: time.Minute}), tc.err) + require.ErrorContains(t, etcdctl.Defragment(time.Minute), tc.err) // Make sure etcd continues to run even after the failed defrag attempt - require.NoError(t, member.Etcdctl().Put(context.Background(), "foo", "bar", config.PutOptions{})) - value, err := member.Etcdctl().Get(context.Background(), "foo", config.GetOptions{}) + require.NoError(t, etcdctl.Put("foo", "bar")) + value, err := etcdctl.Get("foo") require.NoError(t, err) require.Len(t, value.Kvs, 1) require.Equal(t, "bar", string(value.Kvs[0].Value)) From bceca16a9b41d65f39a6e93468ccc194112d32d9 Mon Sep 17 00:00:00 2001 From: Ivan Valdes Date: Wed, 6 Nov 2024 21:33:43 -0800 Subject: [PATCH 12/41] Bump go toolchain to 1.22.9 Signed-off-by: Ivan Valdes --- .go-version | 2 +- api/go.mod | 2 +- client/pkg/go.mod | 2 +- client/v2/go.mod | 2 +- client/v3/go.mod | 2 +- etcdctl/go.mod | 2 +- etcdutl/go.mod | 2 +- go.mod | 2 +- pkg/go.mod | 2 +- raft/go.mod | 2 +- server/go.mod | 2 +- tests/go.mod | 2 +- tools/mod/go.mod | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.go-version b/.go-version index 229a27c6f20..d28b1eb8f3f 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.22.8 +1.22.9 diff --git a/api/go.mod b/api/go.mod index 58c4a0d4d78..8a943509d25 100644 --- a/api/go.mod +++ b/api/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/api/v3 go 1.22 -toolchain go1.22.8 +toolchain go1.22.9 require ( github.com/coreos/go-semver v0.3.0 diff --git a/client/pkg/go.mod b/client/pkg/go.mod index 7897dc27d8b..d9a43579d70 100644 --- a/client/pkg/go.mod +++ b/client/pkg/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/client/pkg/v3 go 1.22 -toolchain go1.22.8 +toolchain go1.22.9 require ( github.com/coreos/go-systemd/v22 v22.3.2 diff --git a/client/v2/go.mod b/client/v2/go.mod index 86b4f8d6027..a47616d4ef2 100644 --- a/client/v2/go.mod +++ b/client/v2/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/client/v2 go 1.22 -toolchain go1.22.8 +toolchain go1.22.9 require ( github.com/json-iterator/go v1.1.11 diff --git a/client/v3/go.mod b/client/v3/go.mod index c0bfa1a8f4e..51935d38779 100644 --- a/client/v3/go.mod +++ b/client/v3/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/client/v3 go 1.22 -toolchain go1.22.8 +toolchain go1.22.9 require ( github.com/dustin/go-humanize v1.0.0 diff --git a/etcdctl/go.mod b/etcdctl/go.mod index d81fc1b4599..e99f913216c 100644 --- a/etcdctl/go.mod +++ b/etcdctl/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/etcdctl/v3 go 1.22 -toolchain go1.22.8 +toolchain go1.22.9 require ( github.com/bgentry/speakeasy v0.1.0 diff --git a/etcdutl/go.mod b/etcdutl/go.mod index 07c01725597..48a4135983a 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/etcdutl/v3 go 1.22 -toolchain go1.22.8 +toolchain go1.22.9 replace ( go.etcd.io/etcd/api/v3 => ../api diff --git a/go.mod b/go.mod index b74535ad866..d2faff4bb63 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/v3 go 1.22 -toolchain go1.22.8 +toolchain go1.22.9 replace ( go.etcd.io/etcd/api/v3 => ./api diff --git a/pkg/go.mod b/pkg/go.mod index 535a540f12a..ac0059b4353 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/pkg/v3 go 1.22 -toolchain go1.22.8 +toolchain go1.22.9 require ( github.com/creack/pty v1.1.11 diff --git a/raft/go.mod b/raft/go.mod index 0ccf2e36906..6e26b1c27d9 100644 --- a/raft/go.mod +++ b/raft/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/raft/v3 go 1.22 -toolchain go1.22.8 +toolchain go1.22.9 require ( github.com/cockroachdb/datadriven v1.0.2 diff --git a/server/go.mod b/server/go.mod index a6c68184639..8a7f7330eac 100644 --- a/server/go.mod +++ b/server/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/server/v3 go 1.22 -toolchain go1.22.8 +toolchain go1.22.9 require ( github.com/coreos/go-semver v0.3.0 diff --git a/tests/go.mod b/tests/go.mod index e17619fbb9c..14c3d8b0717 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/tests/v3 go 1.22 -toolchain go1.22.8 +toolchain go1.22.9 replace ( go.etcd.io/etcd/api/v3 => ../api diff --git a/tools/mod/go.mod b/tools/mod/go.mod index ce99f83263b..81dcb0fd4d7 100644 --- a/tools/mod/go.mod +++ b/tools/mod/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/tools/v3 go 1.22 -toolchain go1.22.8 +toolchain go1.22.9 require ( github.com/alexkohler/nakedret v1.0.0 From b2ba488269f1f081667c0ae11fe775bb080f9357 Mon Sep 17 00:00:00 2001 From: Thomas Gosteli Date: Thu, 7 Nov 2024 10:19:15 +0100 Subject: [PATCH 13/41] fix(defrag): close temp file in case of error Signed-off-by: Thomas Gosteli --- server/mvcc/backend/backend.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/mvcc/backend/backend.go b/server/mvcc/backend/backend.go index d04126d94ae..d278e2be097 100644 --- a/server/mvcc/backend/backend.go +++ b/server/mvcc/backend/backend.go @@ -499,6 +499,15 @@ func (b *backend) defrag() error { tdbp := temp.Name() tmpdb, err := bolt.Open(tdbp, 0600, &options) if err != nil { + temp.Close() + if rmErr := os.Remove(temp.Name()); rmErr != nil && b.lg != nil { + b.lg.Error( + "failed to remove temporary file", + zap.String("path", temp.Name()), + zap.Error(rmErr), + ) + } + return err } From 76275d62735d785e779a0729da23de03839fee22 Mon Sep 17 00:00:00 2001 From: Ivan Valdes Date: Wed, 30 Oct 2024 16:27:57 -0700 Subject: [PATCH 14/41] release: properly change working dir if tmp location already exists Currently, the script won't change directories into the previously cloned repository if the temporary location exists. This may be an issue when testing first with a dry run and later with the actual release. Signed-off-by: Ivan Valdes --- scripts/release.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 993ccff8f8e..efc7bfb1b5d 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -86,10 +86,12 @@ main() { # Set up release directory. local reldir="/tmp/etcd-release-${VERSION}" log_callout "Preparing temporary directory: ${reldir}" - if [ ! -d "${reldir}/etcd" ] && [ "${IN_PLACE}" == 0 ]; then - mkdir -p "${reldir}" - cd "${reldir}" - run git clone "${REPOSITORY}" --branch "${BRANCH}" --depth 1 + if [ "${IN_PLACE}" == 0 ]; then + if [ ! -d "${reldir}/etcd" ]; then + mkdir -p "${reldir}" + cd "${reldir}" + run git clone "${REPOSITORY}" --branch "${BRANCH}" --depth 1 + fi run cd "${reldir}/etcd" || exit 2 run git checkout "${BRANCH}" || exit 2 run git pull origin From cb970e5c1ab8a3296d87889369bd9731a61cef42 Mon Sep 17 00:00:00 2001 From: Ivan Valdes Date: Mon, 11 Nov 2024 15:26:47 -0700 Subject: [PATCH 15/41] release: use GitHub's gh to create GitHub release Backport pull request #18649. Signed-off-by: Ivan Valdes --- .github/workflows/release.yaml | 2 +- scripts/release.sh | 96 +++++++++++++++++++++++++++++++--- scripts/release_notes.tpl.txt | 91 ++++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 9 deletions(-) create mode 100644 scripts/release_notes.tpl.txt diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5139db4876d..a88e6d18397 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -26,7 +26,7 @@ jobs: Name-Email: github-action@etcd.io Expire-Date: 0 EOF - DRY_RUN=true ./scripts/release --no-upload --no-docker-push --in-place 3.5.99 + DRY_RUN=true ./scripts/release --no-upload --no-docker-push --no-gh-release --in-place 3.5.99 - name: test-image run: | VERSION=3.5.99 ./scripts/test_images.sh diff --git a/scripts/release.sh b/scripts/release.sh index 993ccff8f8e..fe254dbaeea 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -33,15 +33,15 @@ help() { echo "WARNING: This does not perform the 'Add API capabilities', 'Performance testing' " echo " or 'Documentation' steps. These steps must be performed manually BEFORE running this tool." echo "" - echo "WARNING: This script does not sign releases, publish releases to github or sent announcement" - echo " emails. These steps must be performed manually AFTER running this tool." + echo "WARNING: This script does not send announcement emails. This step must be performed manually AFTER running this tool." echo "" echo " args:" echo " version: version of etcd to release, e.g. 'v3.2.18'" echo " flags:" - echo " --no-upload: skip gs://etcd binary artifact uploads." - echo " --no-docker-push: skip docker image pushes." echo " --in-place: build binaries using current branch." + echo " --no-docker-push: skip docker image pushes." + echo " --no-gh-release: skip creating the GitHub release using gh." + echo " --no-upload: skip gs://etcd binary artifact uploads." echo "" echo "One can perform a (dry-run) test release from any (uncommitted) branch using:" echo " DRY_RUN=true REPOSITORY=\`pwd\` BRANCH='local-branch-name' ./scripts/release 3.5.0-foobar.2" @@ -119,6 +119,21 @@ main() { exit 1 fi + if [ "${NO_GH_RELEASE}" == 1 ]; then + log_callout "Skipping gh verification, --no-gh-release is set" + else + # Check that gh is installed and logged in. + log_callout "Check gh installation" + if ! command -v gh >/dev/null; then + log_error "Cannot find gh. Please follow the installation instructions at https://github.com/cli/cli#installation" + exit 1 + fi + if ! gh auth status &>/dev/null; then + log_error "GitHub authentication failed for gh. Please run gh auth login." + exit 1 + fi + fi + # If the release tag does not already exist remotely, create it. log_callout "Create tag if not present" if [ "${remote_tag_exists}" -eq 0 ]; then @@ -314,10 +329,70 @@ main() { exit 1 fi - # TODO: signing process - log_warning "" - log_warning "WARNING: The release has not been signed and published to github. This must be done manually." - log_warning "" + if [ "${DRY_RUN}" == "true" ] || [ "${NO_GH_RELEASE}" == 1 ]; then + log_warning "" + log_warning "WARNING: Skipping creating GitHub release, --no-gh-release is set." + log_warning "WARNING: If not running on DRY_MODE, please do the GitHub release manually." + log_warning "" + else + local gh_repo + local release_notes_temp_file + local release_url + local gh_release_args=() + + # For the main branch (v3.6), we should mark the release as a prerelease. + # The release-3.5 (v3.5) branch, should be marked as latest. And release-3.4 (v3.4) + # should be left without any additional mark (therefore, it doesn't need a special argument). + if [ "${BRANCH}" = "main" ]; then + gh_release_args=(--prerelease) + elif [ "${BRANCH}" = "release-3.5" ]; then + gh_release_args=(--latest) + fi + + if [ "${REPOSITORY}" = "$(pwd)" ]; then + gh_repo=$(git remote get-url origin) + else + gh_repo="${REPOSITORY}" + fi + + gh_repo=$(echo "${gh_repo}" | sed 's/^[^@]\+@//' | sed 's/https\?:\/\///' | sed 's/\.git$//' | tr ':' '/') + log_callout "Creating GitHub release for ${RELEASE_VERSION} on ${gh_repo}" + + release_notes_temp_file=$(mktemp) + + local release_version=${RELEASE_VERSION#v} # Remove the v prefix from the release version (i.e., v3.6.1 -> 3.6.1) + local release_version_major_minor=${release_version%.*} # Remove the patch from the version (i.e., 3.6) + local release_version_major=${release_version_major_minor%.*} # Extract the major (i.e., 3) + local release_version_minor=${release_version_major_minor/*./} # Extract the minor (i.e., 6) + + # Disable sellcheck SC2016, the single quoted syntax for sed is intentional. + # shellcheck disable=SC2016 + sed 's/${RELEASE_VERSION}/'"${RELEASE_VERSION}"'/g' ./scripts/release_notes.tpl.txt | + sed 's/${RELEASE_VERSION_MAJOR_MINOR}/'"${release_version_major_minor}"'/g' | + sed 's/${RELEASE_VERSION_MAJOR}/'"${release_version_major}"'/g' | + sed 's/${RELEASE_VERSION_MINOR}/'"${release_version_minor}"'/g' > "${release_notes_temp_file}" + + if ! gh --repo "${gh_repo}" release view "${RELEASE_VERSION}" &>/dev/null; then + maybe_run gh release create "${RELEASE_VERSION}" \ + --repo "${gh_repo}" \ + --draft \ + --title "${RELEASE_VERSION}" \ + --notes-file "${release_notes_temp_file}" \ + "${gh_release_args[@]}" + fi + + # Upload files one by one, as gh doesn't support passing globs as input. + maybe_run find ./release '(' -name '*.tar.gz' -o -name '*.zip' ')' -exec \ + gh --repo "${gh_repo}" release upload "${RELEASE_VERSION}" {} --clobber \; + maybe_run gh --repo "${gh_repo}" release upload "${RELEASE_VERSION}" ./release/SHA256SUMS --clobber + + release_url=$(gh --repo "${gh_repo}" release view "${RELEASE_VERSION}" --json url --jq '.url') + + log_warning "" + log_warning "WARNING: The GitHub release for ${RELEASE_VERSION} has been created as a draft, please go to ${release_url} and release it." + log_warning "" + fi + log_success "Success." exit 0 } @@ -326,6 +401,7 @@ POSITIONAL=() NO_UPLOAD=0 NO_DOCKER_PUSH=0 IN_PLACE=0 +NO_GH_RELEASE=0 while test $# -gt 0; do case "$1" in @@ -346,6 +422,10 @@ while test $# -gt 0; do NO_DOCKER_PUSH=1 shift ;; + --no-gh-release) + NO_GH_RELEASE=1 + shift + ;; *) POSITIONAL+=("$1") # save it in an array for later shift # past argument diff --git a/scripts/release_notes.tpl.txt b/scripts/release_notes.tpl.txt new file mode 100644 index 00000000000..1038a504d6f --- /dev/null +++ b/scripts/release_notes.tpl.txt @@ -0,0 +1,91 @@ +Please check out [CHANGELOG](https://github.com/etcd-io/etcd/blob/main/CHANGELOG/CHANGELOG-${RELEASE_VERSION_MAJOR_MINOR}.md) for a full list of changes. And make sure to read [upgrade guide](https://etcd.io/docs/v${RELEASE_VERSION_MAJOR_MINOR}/upgrades/upgrade_${RELEASE_VERSION_MAJOR}_${RELEASE_VERSION_MINOR}/) before upgrading etcd (there may be breaking changes). + +For installation guides, please check out [play.etcd.io](http://play.etcd.io) and [operating etcd](https://etcd.io/docs/v${RELEASE_VERSION_MAJOR_MINOR}/op-guide/). Latest support status for common architectures and operating systems can be found at [supported platforms](https://etcd.io/docs/v${RELEASE_VERSION_MAJOR_MINOR}/op-guide/supported-platform/). + +###### Linux + +```sh +ETCD_VER=${RELEASE_VERSION} + +# choose either URL +GOOGLE_URL=https://storage.googleapis.com/etcd +GITHUB_URL=https://github.com/etcd-io/etcd/releases/download +DOWNLOAD_URL=${GOOGLE_URL} + +rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz +rm -rf /tmp/etcd-download-test && mkdir -p /tmp/etcd-download-test + +curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz +tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1 +rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz + +/tmp/etcd-download-test/etcd --version +/tmp/etcd-download-test/etcdctl version +/tmp/etcd-download-test/etcdutl version + +# start a local etcd server +/tmp/etcd-download-test/etcd + +# write,read to etcd +/tmp/etcd-download-test/etcdctl --endpoints=localhost:2379 put foo bar +/tmp/etcd-download-test/etcdctl --endpoints=localhost:2379 get foo +``` + +###### macOS (Darwin) + +```sh +ETCD_VER=${RELEASE_VERSION} + +# choose either URL +GOOGLE_URL=https://storage.googleapis.com/etcd +GITHUB_URL=https://github.com/etcd-io/etcd/releases/download +DOWNLOAD_URL=${GOOGLE_URL} + +rm -f /tmp/etcd-${ETCD_VER}-darwin-amd64.zip +rm -rf /tmp/etcd-download-test && mkdir -p /tmp/etcd-download-test + +curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-darwin-amd64.zip -o /tmp/etcd-${ETCD_VER}-darwin-amd64.zip +unzip /tmp/etcd-${ETCD_VER}-darwin-amd64.zip -d /tmp && rm -f /tmp/etcd-${ETCD_VER}-darwin-amd64.zip +mv /tmp/etcd-${ETCD_VER}-darwin-amd64/* /tmp/etcd-download-test && rm -rf mv /tmp/etcd-${ETCD_VER}-darwin-amd64 + +/tmp/etcd-download-test/etcd --version +/tmp/etcd-download-test/etcdctl version +/tmp/etcd-download-test/etcdutl version +``` + +###### Docker + +etcd uses [`gcr.io/etcd-development/etcd`](https://gcr.io/etcd-development/etcd) as a primary container registry, and [`quay.io/coreos/etcd`](https://quay.io/coreos/etcd) as secondary. + +```sh +ETCD_VER=${RELEASE_VERSION} + +rm -rf /tmp/etcd-data.tmp && mkdir -p /tmp/etcd-data.tmp && \ + docker rmi gcr.io/etcd-development/etcd:${ETCD_VER} || true && \ + docker run \ + -p 2379:2379 \ + -p 2380:2380 \ + --mount type=bind,source=/tmp/etcd-data.tmp,destination=/etcd-data \ + --name etcd-gcr-${ETCD_VER} \ + gcr.io/etcd-development/etcd:${ETCD_VER} \ + /usr/local/bin/etcd \ + --name s1 \ + --data-dir /etcd-data \ + --listen-client-urls http://0.0.0.0:2379 \ + --advertise-client-urls http://0.0.0.0:2379 \ + --listen-peer-urls http://0.0.0.0:2380 \ + --initial-advertise-peer-urls http://0.0.0.0:2380 \ + --initial-cluster s1=http://0.0.0.0:2380 \ + --initial-cluster-token tkn \ + --initial-cluster-state new \ + --log-level info \ + --logger zap \ + --log-outputs stderr + +docker exec etcd-gcr-${ETCD_VER}/usr/local/bin/etcd --version +docker exec etcd-gcr-${ETCD_VER}/usr/local/bin/etcdctl version +docker exec etcd-gcr-${ETCD_VER}/usr/local/bin/etcdutl version +docker exec etcd-gcr-${ETCD_VER}/usr/local/bin/etcdctl endpoint health +docker exec etcd-gcr-${ETCD_VER}/usr/local/bin/etcdctl put foo bar +docker exec etcd-gcr-${ETCD_VER}/usr/local/bin/etcdctl get foo +``` From 507c0de87bd5034e3de4ab76ebf96b54dae0cd52 Mon Sep 17 00:00:00 2001 From: Ivan Valdes Date: Tue, 12 Nov 2024 09:30:31 -0700 Subject: [PATCH 16/41] version: bump up to 3.5.17 Signed-off-by: Ivan Valdes --- api/version/version.go | 2 +- client/v2/go.mod | 4 ++-- client/v3/go.mod | 4 ++-- etcdctl/go.mod | 16 ++++++++-------- etcdutl/go.mod | 14 +++++++------- go.mod | 20 ++++++++++---------- pkg/go.mod | 2 +- raft/go.mod | 2 +- server/go.mod | 12 ++++++------ tests/go.mod | 16 ++++++++-------- 10 files changed, 46 insertions(+), 46 deletions(-) diff --git a/api/version/version.go b/api/version/version.go index 36719c19208..ca6efc51367 100644 --- a/api/version/version.go +++ b/api/version/version.go @@ -26,7 +26,7 @@ import ( var ( // MinClusterVersion is the min cluster version this etcd binary is compatible with. MinClusterVersion = "3.0.0" - Version = "3.5.16" + Version = "3.5.17" APIVersion = "unknown" // Git SHA Value will be set during build diff --git a/client/v2/go.mod b/client/v2/go.mod index a47616d4ef2..163bc5491e0 100644 --- a/client/v2/go.mod +++ b/client/v2/go.mod @@ -7,8 +7,8 @@ toolchain go1.22.9 require ( github.com/json-iterator/go v1.1.11 github.com/modern-go/reflect2 v1.0.1 - go.etcd.io/etcd/api/v3 v3.5.16 - go.etcd.io/etcd/client/pkg/v3 v3.5.16 + go.etcd.io/etcd/api/v3 v3.5.17 + go.etcd.io/etcd/client/pkg/v3 v3.5.17 ) require ( diff --git a/client/v3/go.mod b/client/v3/go.mod index 51935d38779..5578133b596 100644 --- a/client/v3/go.mod +++ b/client/v3/go.mod @@ -8,8 +8,8 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/prometheus/client_golang v1.11.1 - go.etcd.io/etcd/api/v3 v3.5.16 - go.etcd.io/etcd/client/pkg/v3 v3.5.16 + go.etcd.io/etcd/api/v3 v3.5.17 + go.etcd.io/etcd/client/pkg/v3 v3.5.17 go.uber.org/zap v1.17.0 google.golang.org/grpc v1.59.0 sigs.k8s.io/yaml v1.2.0 diff --git a/etcdctl/go.mod b/etcdctl/go.mod index e99f913216c..9cb819749d5 100644 --- a/etcdctl/go.mod +++ b/etcdctl/go.mod @@ -11,12 +11,12 @@ require ( github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 github.com/urfave/cli v1.22.4 - go.etcd.io/etcd/api/v3 v3.5.16 - go.etcd.io/etcd/client/pkg/v3 v3.5.16 - go.etcd.io/etcd/client/v2 v2.305.16 - go.etcd.io/etcd/client/v3 v3.5.16 - go.etcd.io/etcd/etcdutl/v3 v3.5.16 - go.etcd.io/etcd/pkg/v3 v3.5.16 + go.etcd.io/etcd/api/v3 v3.5.17 + go.etcd.io/etcd/client/pkg/v3 v3.5.17 + go.etcd.io/etcd/client/v2 v2.305.17 + go.etcd.io/etcd/client/v3 v3.5.17 + go.etcd.io/etcd/etcdutl/v3 v3.5.17 + go.etcd.io/etcd/pkg/v3 v3.5.17 go.uber.org/zap v1.17.0 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba google.golang.org/grpc v1.59.0 @@ -50,8 +50,8 @@ require ( github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/bbolt v1.3.11 // indirect - go.etcd.io/etcd/raft/v3 v3.5.16 // indirect - go.etcd.io/etcd/server/v3 v3.5.16 // indirect + go.etcd.io/etcd/raft/v3 v3.5.17 // indirect + go.etcd.io/etcd/server/v3 v3.5.17 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect go.opentelemetry.io/otel v1.20.0 // indirect go.opentelemetry.io/otel/metric v1.20.0 // indirect diff --git a/etcdutl/go.mod b/etcdutl/go.mod index 48a4135983a..f9be8731078 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -27,12 +27,12 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/spf13/cobra v1.1.3 go.etcd.io/bbolt v1.3.11 - go.etcd.io/etcd/api/v3 v3.5.16 - go.etcd.io/etcd/client/pkg/v3 v3.5.16 - go.etcd.io/etcd/client/v3 v3.5.16 - go.etcd.io/etcd/pkg/v3 v3.5.16 - go.etcd.io/etcd/raft/v3 v3.5.16 - go.etcd.io/etcd/server/v3 v3.5.16 + go.etcd.io/etcd/api/v3 v3.5.17 + go.etcd.io/etcd/client/pkg/v3 v3.5.17 + go.etcd.io/etcd/client/v3 v3.5.17 + go.etcd.io/etcd/pkg/v3 v3.5.17 + go.etcd.io/etcd/raft/v3 v3.5.17 + go.etcd.io/etcd/server/v3 v3.5.17 go.uber.org/zap v1.17.0 ) @@ -60,7 +60,7 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect - go.etcd.io/etcd/client/v2 v2.305.16 // indirect + go.etcd.io/etcd/client/v2 v2.305.17 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect go.opentelemetry.io/otel v1.20.0 // indirect go.opentelemetry.io/otel/metric v1.20.0 // indirect diff --git a/go.mod b/go.mod index d2faff4bb63..19f50365e2d 100644 --- a/go.mod +++ b/go.mod @@ -22,16 +22,16 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/spf13/cobra v1.1.3 go.etcd.io/bbolt v1.3.11 - go.etcd.io/etcd/api/v3 v3.5.16 - go.etcd.io/etcd/client/pkg/v3 v3.5.16 - go.etcd.io/etcd/client/v2 v2.305.16 - go.etcd.io/etcd/client/v3 v3.5.16 - go.etcd.io/etcd/etcdctl/v3 v3.5.16 - go.etcd.io/etcd/etcdutl/v3 v3.5.16 - go.etcd.io/etcd/pkg/v3 v3.5.16 - go.etcd.io/etcd/raft/v3 v3.5.16 - go.etcd.io/etcd/server/v3 v3.5.16 - go.etcd.io/etcd/tests/v3 v3.5.16 + go.etcd.io/etcd/api/v3 v3.5.17 + go.etcd.io/etcd/client/pkg/v3 v3.5.17 + go.etcd.io/etcd/client/v2 v2.305.17 + go.etcd.io/etcd/client/v3 v3.5.17 + go.etcd.io/etcd/etcdctl/v3 v3.5.17 + go.etcd.io/etcd/etcdutl/v3 v3.5.17 + go.etcd.io/etcd/pkg/v3 v3.5.17 + go.etcd.io/etcd/raft/v3 v3.5.17 + go.etcd.io/etcd/server/v3 v3.5.17 + go.etcd.io/etcd/tests/v3 v3.5.17 go.uber.org/zap v1.17.0 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba google.golang.org/grpc v1.59.0 diff --git a/pkg/go.mod b/pkg/go.mod index ac0059b4353..d18ace3e31a 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -10,7 +10,7 @@ require ( github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 - go.etcd.io/etcd/client/pkg/v3 v3.5.16 + go.etcd.io/etcd/client/pkg/v3 v3.5.17 go.uber.org/zap v1.17.0 google.golang.org/grpc v1.59.0 ) diff --git a/raft/go.mod b/raft/go.mod index 6e26b1c27d9..9e8b2cc43de 100644 --- a/raft/go.mod +++ b/raft/go.mod @@ -8,7 +8,7 @@ require ( github.com/cockroachdb/datadriven v1.0.2 github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.5.4 - go.etcd.io/etcd/client/pkg/v3 v3.5.16 + go.etcd.io/etcd/client/pkg/v3 v3.5.17 ) require ( diff --git a/server/go.mod b/server/go.mod index 8a7f7330eac..eef29947efe 100644 --- a/server/go.mod +++ b/server/go.mod @@ -26,12 +26,12 @@ require ( github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go.etcd.io/bbolt v1.3.11 - go.etcd.io/etcd/api/v3 v3.5.16 - go.etcd.io/etcd/client/pkg/v3 v3.5.16 - go.etcd.io/etcd/client/v2 v2.305.16 - go.etcd.io/etcd/client/v3 v3.5.16 - go.etcd.io/etcd/pkg/v3 v3.5.16 - go.etcd.io/etcd/raft/v3 v3.5.16 + go.etcd.io/etcd/api/v3 v3.5.17 + go.etcd.io/etcd/client/pkg/v3 v3.5.17 + go.etcd.io/etcd/client/v2 v2.305.17 + go.etcd.io/etcd/client/v3 v3.5.17 + go.etcd.io/etcd/pkg/v3 v3.5.17 + go.etcd.io/etcd/raft/v3 v3.5.17 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 go.opentelemetry.io/otel v1.20.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 diff --git a/tests/go.mod b/tests/go.mod index 14c3d8b0717..969036a158e 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -30,14 +30,14 @@ require ( github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 - go.etcd.io/etcd/api/v3 v3.5.16 - go.etcd.io/etcd/client/pkg/v3 v3.5.16 - go.etcd.io/etcd/client/v2 v2.305.16 - go.etcd.io/etcd/client/v3 v3.5.16 - go.etcd.io/etcd/etcdutl/v3 v3.5.16 - go.etcd.io/etcd/pkg/v3 v3.5.16 - go.etcd.io/etcd/raft/v3 v3.5.16 - go.etcd.io/etcd/server/v3 v3.5.16 + go.etcd.io/etcd/api/v3 v3.5.17 + go.etcd.io/etcd/client/pkg/v3 v3.5.17 + go.etcd.io/etcd/client/v2 v2.305.17 + go.etcd.io/etcd/client/v3 v3.5.17 + go.etcd.io/etcd/etcdutl/v3 v3.5.17 + go.etcd.io/etcd/pkg/v3 v3.5.17 + go.etcd.io/etcd/raft/v3 v3.5.17 + go.etcd.io/etcd/server/v3 v3.5.17 go.etcd.io/gofail v0.2.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 go.opentelemetry.io/otel v1.20.0 From e61b49583d8ad4d3f98d2d2a541d5595cc39275a Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Fri, 15 Nov 2024 13:16:44 +0100 Subject: [PATCH 17/41] Bump golang-jwt/jwt to 4.5.1 to address GO-2024-3250 fixes #18898 Signed-off-by: Thomas Jungblut --- etcdctl/go.mod | 2 +- etcdctl/go.sum | 4 ++-- etcdutl/go.mod | 2 +- etcdutl/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- server/go.mod | 2 +- server/go.sum | 4 ++-- tests/go.mod | 2 +- tests/go.sum | 4 ++-- 10 files changed, 15 insertions(+), 15 deletions(-) diff --git a/etcdctl/go.mod b/etcdctl/go.mod index 9cb819749d5..a8182492622 100644 --- a/etcdctl/go.mod +++ b/etcdctl/go.mod @@ -32,7 +32,7 @@ require ( github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.4.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.1 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.0.1 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect diff --git a/etcdctl/go.sum b/etcdctl/go.sum index 83d758328c4..456bf9da962 100644 --- a/etcdctl/go.sum +++ b/etcdctl/go.sum @@ -85,8 +85,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= -github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= diff --git a/etcdutl/go.mod b/etcdutl/go.mod index f9be8731078..0be081578eb 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -44,7 +44,7 @@ require ( github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.4.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.1 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.0.1 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect diff --git a/etcdutl/go.sum b/etcdutl/go.sum index 4854537fe03..a6ab545c968 100644 --- a/etcdutl/go.sum +++ b/etcdutl/go.sum @@ -81,8 +81,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= -github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= diff --git a/go.mod b/go.mod index 19f50365e2d..3c43a08a9fd 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.4.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.0.1 // indirect diff --git a/go.sum b/go.sum index 4ff7b4eab74..15d5614d234 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= -github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= diff --git a/server/go.mod b/server/go.mod index eef29947efe..8427235fd72 100644 --- a/server/go.mod +++ b/server/go.mod @@ -9,7 +9,7 @@ require ( github.com/coreos/go-systemd/v22 v22.3.2 github.com/dustin/go-humanize v1.0.0 github.com/gogo/protobuf v1.3.2 - github.com/golang-jwt/jwt/v4 v4.4.2 + github.com/golang-jwt/jwt/v4 v4.5.1 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da github.com/golang/protobuf v1.5.4 github.com/google/btree v1.0.1 diff --git a/server/go.sum b/server/go.sum index d6b6646714a..392fd8c1696 100644 --- a/server/go.sum +++ b/server/go.sum @@ -91,8 +91,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= -github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= diff --git a/tests/go.mod b/tests/go.mod index 969036a158e..e799ee746cb 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -61,7 +61,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang-jwt/jwt/v4 v4.4.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/btree v1.0.1 // indirect github.com/gorilla/websocket v1.4.2 // indirect diff --git a/tests/go.sum b/tests/go.sum index 805f7aaf587..c82a51185e4 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -93,8 +93,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= -github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= From c84e84c58265b986f9160c7a3861008b92df0c54 Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Thu, 31 Oct 2024 13:46:41 +0100 Subject: [PATCH 18/41] add tls min/max version to grpc proxy This adds the min and max TLS version support from #13506 and #15156 to the grpc proxy. Fixes #13506 Signed-off-by: Thomas Jungblut --- server/etcdmain/grpc_proxy.go | 73 +++++++++++++++++++++++++------- tests/e2e/etcd_grpcproxy_test.go | 33 +++++++++++++++ 2 files changed, 91 insertions(+), 15 deletions(-) diff --git a/server/etcdmain/grpc_proxy.go b/server/etcdmain/grpc_proxy.go index 8622605b0af..db672fe56d4 100644 --- a/server/etcdmain/grpc_proxy.go +++ b/server/etcdmain/grpc_proxy.go @@ -78,13 +78,16 @@ var ( // tls for clients connecting to proxy - grpcProxyListenCA string - grpcProxyListenCert string - grpcProxyListenKey string - grpcProxyListenCipherSuites []string - grpcProxyListenAutoTLS bool - grpcProxyListenCRL string - selfSignedCertValidity uint + grpcProxyListenCA string + grpcProxyListenCert string + grpcProxyListenKey string + grpcProxyListenCipherSuites []string + grpcProxyListenAutoTLS bool + grpcProxyListenCRL string + grpcProxyListenTLSMinVersion string + grpcProxyListenTLSMaxVersion string + + selfSignedCertValidity uint grpcProxyAdvertiseClientURL string grpcProxyResolverPrefix string @@ -166,6 +169,8 @@ func newGRPCProxyStartCommand() *cobra.Command { cmd.Flags().BoolVar(&grpcProxyListenAutoTLS, "auto-tls", false, "proxy TLS using generated certificates") cmd.Flags().StringVar(&grpcProxyListenCRL, "client-crl-file", "", "proxy client certificate revocation list file.") cmd.Flags().UintVar(&selfSignedCertValidity, "self-signed-cert-validity", 1, "The validity period of the proxy certificates, unit is year") + cmd.Flags().StringVar(&grpcProxyListenTLSMinVersion, "tls-min-version", string(tlsutil.TLSVersion12), "Minimum TLS version supported by grpc proxy. Possible values: TLS1.2, TLS1.3.") + cmd.Flags().StringVar(&grpcProxyListenTLSMaxVersion, "tls-max-version", string(tlsutil.TLSVersionDefault), "Maximum TLS version supported by grpc proxy. Possible values: TLS1.2, TLS1.3 (empty defers to Go).") // experimental flags cmd.Flags().BoolVar(&grpcProxyEnableOrdering, "experimental-serializable-ordering", false, "Ensure serializable reads have monotonically increasing store revisions across endpoints.") @@ -197,13 +202,6 @@ func startGRPCProxy(cmd *cobra.Command, args []string) { // The empty CN is required for grpcProxyCert. // Please see https://github.com/etcd-io/etcd/issues/11970#issuecomment-687875315 for more context. tlsInfo := newTLS(grpcProxyListenCA, grpcProxyListenCert, grpcProxyListenKey, false) - if len(grpcProxyListenCipherSuites) > 0 { - cs, err := tlsutil.GetCipherSuites(grpcProxyListenCipherSuites) - if err != nil { - log.Fatal(err) - } - tlsInfo.CipherSuites = cs - } if tlsInfo == nil && grpcProxyListenAutoTLS { host := []string{"https://" + grpcProxyListenAddr} dir := filepath.Join(grpcProxyDataDir, "fixtures", "proxy") @@ -213,10 +211,32 @@ func startGRPCProxy(cmd *cobra.Command, args []string) { } tlsInfo = &autoTLS } - if tlsInfo != nil { + if len(grpcProxyListenCipherSuites) > 0 { + cs, err := tlsutil.GetCipherSuites(grpcProxyListenCipherSuites) + if err != nil { + log.Fatal(err) + } + tlsInfo.CipherSuites = cs + } + if grpcProxyListenTLSMinVersion != "" { + version, err := tlsutil.GetTLSVersion(grpcProxyListenTLSMinVersion) + if err != nil { + log.Fatal(err) + } + tlsInfo.MinVersion = version + } + if grpcProxyListenTLSMaxVersion != "" { + version, err := tlsutil.GetTLSVersion(grpcProxyListenTLSMaxVersion) + if err != nil { + log.Fatal(err) + } + tlsInfo.MaxVersion = version + } + lg.Info("gRPC proxy server TLS", zap.String("tls-info", fmt.Sprintf("%+v", tlsInfo))) } + m := mustListenCMux(lg, tlsInfo) grpcl := m.Match(cmux.HTTP2()) defer func() { @@ -290,6 +310,29 @@ func checkArgs() { fmt.Fprintln(os.Stderr, fmt.Errorf("selfSignedCertValidity is invalid,it should be greater than 0")) os.Exit(1) } + + minVersion, err := tlsutil.GetTLSVersion(grpcProxyListenTLSMinVersion) + if err != nil { + fmt.Fprintln(os.Stderr, fmt.Errorf("tls-min-version is invalid: %w", err)) + os.Exit(1) + } + maxVersion, err := tlsutil.GetTLSVersion(grpcProxyListenTLSMaxVersion) + if err != nil { + fmt.Fprintln(os.Stderr, fmt.Errorf("tls-max-version is invalid: %w", err)) + os.Exit(1) + } + + // maxVersion == 0 means that Go selects the highest available version. + if maxVersion != 0 && minVersion > maxVersion { + fmt.Fprintln(os.Stderr, fmt.Errorf("min version (%s) is greater than max version (%s)", grpcProxyListenTLSMinVersion, grpcProxyListenTLSMaxVersion)) + os.Exit(1) + } + + // Check if user attempted to configure ciphers for TLS1.3 only: Go does not support that currently. + if minVersion == tls.VersionTLS13 && len(grpcProxyListenCipherSuites) > 0 { + fmt.Fprintln(os.Stderr, fmt.Errorf("cipher suites cannot be configured when only TLS1.3 is enabled")) + os.Exit(1) + } } func mustNewClient(lg *zap.Logger) *clientv3.Client { diff --git a/tests/e2e/etcd_grpcproxy_test.go b/tests/e2e/etcd_grpcproxy_test.go index 70c5fff5ffe..077b1db189c 100644 --- a/tests/e2e/etcd_grpcproxy_test.go +++ b/tests/e2e/etcd_grpcproxy_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" @@ -126,6 +127,38 @@ func TestGrpcProxyAutoSync(t *testing.T) { require.NoError(t, proxyProc.Stop()) } +func TestGrpcProxyTLSVersions(t *testing.T) { + e2e.SkipInShortMode(t) + + epc, err := e2e.NewEtcdProcessCluster(t, e2e.NewConfigClientBoth()) + require.NoError(t, err) + defer func() { + assert.NoError(t, epc.Close()) + }() + + var ( + node1ClientURL = epc.Procs[0].Config().ClientHttpUrl + proxyClientURL = "127.0.0.1:42379" + ) + + // Run independent grpc-proxy instance + proxyProc, err := e2e.SpawnCmd([]string{e2e.BinDir + "/etcd", "grpc-proxy", "start", + "--advertise-client-url", proxyClientURL, + "--listen-addr", proxyClientURL, + "--endpoints", node1ClientURL, + "--endpoints-auto-sync-interval", "1s", + "--cert-file", e2e.CertPath2, + "--key-file", e2e.PrivateKeyPath2, + "--tls-min-version", "TLS1.2", + "--tls-max-version", "TLS1.3", + }, nil) + require.NoError(t, err) + defer proxyProc.Stop() + + _, err = proxyProc.Expect("listening for gRPC proxy client requests") + require.NoError(t, err) +} + func runEtcdNode(name, dataDir, clientURL, peerURL, clusterState, initialCluster string) (*expect.ExpectProcess, error) { proc, err := e2e.SpawnCmd([]string{e2e.BinPath, "--name", name, From 83045bca64bbdfbf0406f54e5b82bd8468fa6748 Mon Sep 17 00:00:00 2001 From: KT <416432526@qq.com> Date: Thu, 14 Nov 2024 21:57:50 +0800 Subject: [PATCH 19/41] fix runtime error: comparing uncomparable type Signed-off-by: KT <416432526@qq.com> --- client/v3/lease.go | 12 +++++++- .../integration/clientv3/lease/lease_test.go | 29 ++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/client/v3/lease.go b/client/v3/lease.go index 4e7d1caf831..4877ee94962 100644 --- a/client/v3/lease.go +++ b/client/v3/lease.go @@ -263,6 +263,12 @@ func (l *lessor) Leases(ctx context.Context) (*LeaseLeasesResponse, error) { return nil, ContextError(ctx, err) } +// To identify the context passed to `KeepAlive`, a key/value pair is +// attached to the context. The key is a `keepAliveCtxKey` object, and +// the value is the pointer to the context object itself, ensuring +// uniqueness as each context has a unique memory address. +type keepAliveCtxKey struct{} + func (l *lessor) KeepAlive(ctx context.Context, id LeaseID) (<-chan *LeaseKeepAliveResponse, error) { ch := make(chan *LeaseKeepAliveResponse, LeaseResponseChSize) @@ -277,6 +283,10 @@ func (l *lessor) KeepAlive(ctx context.Context, id LeaseID) (<-chan *LeaseKeepAl default: } ka, ok := l.keepAlives[id] + + if ctx.Done() != nil { + ctx = context.WithValue(ctx, keepAliveCtxKey{}, &ctx) + } if !ok { // create fresh keep alive ka = &keepAlive{ @@ -347,7 +357,7 @@ func (l *lessor) keepAliveCtxCloser(ctx context.Context, id LeaseID, donec <-cha // close channel and remove context if still associated with keep alive for i, c := range ka.ctxs { - if c == ctx { + if c.Value(keepAliveCtxKey{}) == ctx.Value(keepAliveCtxKey{}) { close(ka.chs[i]) ka.ctxs = append(ka.ctxs[:i], ka.ctxs[i+1:]...) ka.chs = append(ka.chs[:i], ka.chs[i+1:]...) diff --git a/tests/integration/clientv3/lease/lease_test.go b/tests/integration/clientv3/lease/lease_test.go index ef7065eb592..cd783952dcd 100644 --- a/tests/integration/clientv3/lease/lease_test.go +++ b/tests/integration/clientv3/lease/lease_test.go @@ -133,7 +133,14 @@ func TestLeaseKeepAlive(t *testing.T) { t.Errorf("failed to create lease %v", err) } - rc, kerr := lapi.KeepAlive(context.Background(), resp.ID) + type uncomparableCtx struct { + context.Context + _ func() + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rc, kerr := lapi.KeepAlive(uncomparableCtx{Context: ctx}, resp.ID) if kerr != nil { t.Errorf("failed to keepalive lease %v", kerr) } @@ -151,6 +158,26 @@ func TestLeaseKeepAlive(t *testing.T) { t.Errorf("ID = %x, want %x", kresp.ID, resp.ID) } + ctx2, cancel2 := context.WithCancel(context.Background()) + rc2, kerr2 := lapi.KeepAlive(uncomparableCtx{Context: ctx2}, resp.ID) + if kerr2 != nil { + t.Errorf("failed to keepalive lease %v", kerr2) + } + + cancel2() + + _, ok = <-rc2 + if ok { + t.Errorf("chan is not closed, want cancel stop keepalive") + } + + select { + case <-rc: + // cancel2() should not affect first keepalive + t.Errorf("chan is closed, want keepalive continue") + default: + } + lapi.Close() _, ok = <-rc From 28a19f761b0540c7007e139a01e816635ee076ac Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Tue, 3 Dec 2024 16:29:13 +0000 Subject: [PATCH 20/41] Print warning message for deprecated flags if set Signed-off-by: Benjamin Wang --- server/etcdmain/config.go | 41 +++++++++++++++++---- server/etcdmain/help.go | 2 +- tests/e2e/etcd_config_test.go | 69 +++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 8 deletions(-) diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index 8b22ccf21ee..7ee18b86f43 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -36,6 +36,8 @@ import ( "sigs.k8s.io/yaml" ) +const deprecatedWarningMessage = "--%s is deprecated in 3.5 and will be decommissioned in 3.6." + var ( proxyFlagOff = "off" proxyFlagReadonly = "readonly" @@ -62,6 +64,17 @@ var ( "test.coverprofile", "test.outputdir", } + + deprecatedFlags = map[string]struct{}{ + "enable-v2": struct{}{}, + "experimental-enable-v2v3": struct{}{}, + "proxy": struct{}{}, + "proxy-failure-wait": struct{}{}, + "proxy-refresh-interval": struct{}{}, + "proxy-dial-timeout": struct{}{}, + "proxy-write-timeout": struct{}{}, + "proxy-read-timeout": struct{}{}, + } ) type configProxy struct { @@ -207,17 +220,17 @@ func newConfig() *config { fs.BoolVar(&cfg.ec.PreVote, "pre-vote", cfg.ec.PreVote, "Enable to run an additional Raft election phase.") - fs.BoolVar(&cfg.ec.EnableV2, "enable-v2", cfg.ec.EnableV2, "Accept etcd V2 client requests. Deprecated in v3.5. Will be decommission in v3.6.") - fs.StringVar(&cfg.ec.ExperimentalEnableV2V3, "experimental-enable-v2v3", cfg.ec.ExperimentalEnableV2V3, "v3 prefix for serving emulated v2 state. Deprecated in 3.5. Will be decomissioned in 3.6.") + fs.BoolVar(&cfg.ec.EnableV2, "enable-v2", cfg.ec.EnableV2, "Accept etcd V2 client requests. Deprecated in v3.5 and will be decommissioned in v3.6.") + fs.StringVar(&cfg.ec.ExperimentalEnableV2V3, "experimental-enable-v2v3", cfg.ec.ExperimentalEnableV2V3, "v3 prefix for serving emulated v2 state. Deprecated in 3.5 and will be decommissioned in 3.6.") fs.Var(cfg.cf.v2deprecation, "v2-deprecation", fmt.Sprintf("v2store deprecation stage: %q. ", cfg.cf.proxy.Valids())) // proxy fs.Var(cfg.cf.proxy, "proxy", fmt.Sprintf("Valid values include %q", cfg.cf.proxy.Valids())) - fs.UintVar(&cfg.cp.ProxyFailureWaitMs, "proxy-failure-wait", cfg.cp.ProxyFailureWaitMs, "Time (in milliseconds) an endpoint will be held in a failed state.") - fs.UintVar(&cfg.cp.ProxyRefreshIntervalMs, "proxy-refresh-interval", cfg.cp.ProxyRefreshIntervalMs, "Time (in milliseconds) of the endpoints refresh interval.") - fs.UintVar(&cfg.cp.ProxyDialTimeoutMs, "proxy-dial-timeout", cfg.cp.ProxyDialTimeoutMs, "Time (in milliseconds) for a dial to timeout.") - fs.UintVar(&cfg.cp.ProxyWriteTimeoutMs, "proxy-write-timeout", cfg.cp.ProxyWriteTimeoutMs, "Time (in milliseconds) for a write to timeout.") - fs.UintVar(&cfg.cp.ProxyReadTimeoutMs, "proxy-read-timeout", cfg.cp.ProxyReadTimeoutMs, "Time (in milliseconds) for a read to timeout.") + fs.UintVar(&cfg.cp.ProxyFailureWaitMs, "proxy-failure-wait", cfg.cp.ProxyFailureWaitMs, "Time (in milliseconds) an endpoint will be held in a failed state. Deprecated in 3.5 and will be decommissioned in 3.6.") + fs.UintVar(&cfg.cp.ProxyRefreshIntervalMs, "proxy-refresh-interval", cfg.cp.ProxyRefreshIntervalMs, "Time (in milliseconds) of the endpoints refresh interval. Deprecated in 3.5 and will be decommissioned in 3.6.") + fs.UintVar(&cfg.cp.ProxyDialTimeoutMs, "proxy-dial-timeout", cfg.cp.ProxyDialTimeoutMs, "Time (in milliseconds) for a dial to timeout. Deprecated in 3.5 and will be decommissioned in 3.6.") + fs.UintVar(&cfg.cp.ProxyWriteTimeoutMs, "proxy-write-timeout", cfg.cp.ProxyWriteTimeoutMs, "Time (in milliseconds) for a write to timeout. Deprecated in 3.5 and will be decommissioned in 3.6.") + fs.UintVar(&cfg.cp.ProxyReadTimeoutMs, "proxy-read-timeout", cfg.cp.ProxyReadTimeoutMs, "Time (in milliseconds) for a read to timeout. Deprecated in 3.5 and will be decommissioned in 3.6.") // security fs.StringVar(&cfg.ec.ClientTLSInfo.CertFile, "cert-file", "", "Path to the client server TLS cert file.") @@ -364,6 +377,20 @@ func (cfg *config) parse(arguments []string) error { cfg.ec.V2Deprecation = cconfig.V2_DEPR_DEFAULT } + var warningsForDeprecatedFlags []string + cfg.cf.flagSet.Visit(func(f *flag.Flag) { + if _, ok := deprecatedFlags[f.Name]; ok { + warningsForDeprecatedFlags = append(warningsForDeprecatedFlags, fmt.Sprintf(deprecatedWarningMessage, f.Name)) + } + }) + if len(warningsForDeprecatedFlags) > 0 { + if lg := cfg.ec.GetLogger(); lg != nil { + for _, msg := range warningsForDeprecatedFlags { + lg.Warn(msg) + } + } + } + // now logger is set up return err } diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index 03e81cc8dba..f6e778ee6dc 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -237,7 +237,7 @@ Experimental distributed tracing: --experimental-distributed-tracing-sampling-rate '0' Number of samples to collect per million spans for distributed tracing. Disabled by default. -v2 Proxy (to be deprecated in v3.6): +v2 Proxy (Deprecated and to be decommissioned in v3.6): --proxy 'off' Proxy mode setting ('off', 'readonly' or 'on'). --proxy-failure-wait 5000 diff --git a/tests/e2e/etcd_config_test.go b/tests/e2e/etcd_config_test.go index 7dc7d5d31ef..a2eb3b4cc1a 100644 --- a/tests/e2e/etcd_config_test.go +++ b/tests/e2e/etcd_config_test.go @@ -549,5 +549,74 @@ func TestEtcdTLSVersion(t *testing.T) { assert.NoError(t, err) assert.NoError(t, e2e.WaitReadyExpectProc(proc, e2e.EtcdServerReadyLines), "did not receive expected output from etcd process") assert.NoError(t, proc.Stop()) +} + +// TestEtcdDeprecatedFlags checks that etcd will print warning messages if deprecated flags are set. +func TestEtcdDeprecatedFlags(t *testing.T) { + e2e.SkipInShortMode(t) + + commonArgs := []string{ + e2e.BinDir + "/etcd", + "--name", "e1", + } + + deprecatedWarningMessage := "--%s is deprecated in 3.5 and will be decommissioned in 3.6." + testCases := []struct { + name string + args []string + expectedMsg string + }{ + { + name: "enable-v2", + args: append(commonArgs, "--enable-v2"), + expectedMsg: fmt.Sprintf(deprecatedWarningMessage, "enable-v2"), + }, + { + name: "experimental-enable-v2v3", + args: append(commonArgs, "--experimental-enable-v2v3", "v3prefix"), + expectedMsg: fmt.Sprintf(deprecatedWarningMessage, "experimental-enable-v2v3"), + }, + { + name: "proxy", + args: append(commonArgs, "--proxy", "off"), + expectedMsg: fmt.Sprintf(deprecatedWarningMessage, "proxy"), + }, + { + name: "proxy-failure-wait", + args: append(commonArgs, "--proxy-failure-wait", "10"), + expectedMsg: fmt.Sprintf(deprecatedWarningMessage, "proxy-failure-wait"), + }, + { + name: "proxy-refresh-interval", + args: append(commonArgs, "--proxy-refresh-interval", "10"), + expectedMsg: fmt.Sprintf(deprecatedWarningMessage, "proxy-refresh-interval"), + }, + { + name: "proxy-dial-timeout", + args: append(commonArgs, "--proxy-dial-timeout", "10"), + expectedMsg: fmt.Sprintf(deprecatedWarningMessage, "proxy-dial-timeout"), + }, + { + name: "proxy-write-timeout", + args: append(commonArgs, "--proxy-write-timeout", "10"), + expectedMsg: fmt.Sprintf(deprecatedWarningMessage, "proxy-write-timeout"), + }, + { + name: "proxy-read-timeout", + args: append(commonArgs, "--proxy-read-timeout", "10"), + expectedMsg: fmt.Sprintf(deprecatedWarningMessage, "proxy-read-timeout"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + proc, err := e2e.SpawnCmd( + tc.args, nil, + ) + require.NoError(t, err) + require.NoError(t, e2e.WaitReadyExpectProc(proc, []string{tc.expectedMsg})) + require.NoError(t, proc.Stop()) + }) + } } From 727c9b6535ed9957092a70dddf248c0d7b8dfcdd Mon Sep 17 00:00:00 2001 From: Anurag De Date: Wed, 4 Dec 2024 14:13:14 +0530 Subject: [PATCH 21/41] Bump go toolchain to 1.22.9 Signed-off-by: Anurag De --- .go-version | 2 +- api/go.mod | 2 +- client/pkg/go.mod | 2 +- client/v2/go.mod | 2 +- client/v3/go.mod | 2 +- etcdctl/go.mod | 2 +- etcdutl/go.mod | 2 +- go.mod | 2 +- pkg/go.mod | 2 +- raft/go.mod | 2 +- server/go.mod | 2 +- tests/go.mod | 2 +- tools/mod/go.mod | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.go-version b/.go-version index d28b1eb8f3f..f5ae945712e 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.22.9 +1.22.10 diff --git a/api/go.mod b/api/go.mod index 8a943509d25..b5a9ab40f0d 100644 --- a/api/go.mod +++ b/api/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/api/v3 go 1.22 -toolchain go1.22.9 +toolchain go1.22.10 require ( github.com/coreos/go-semver v0.3.0 diff --git a/client/pkg/go.mod b/client/pkg/go.mod index d9a43579d70..28f826dce88 100644 --- a/client/pkg/go.mod +++ b/client/pkg/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/client/pkg/v3 go 1.22 -toolchain go1.22.9 +toolchain go1.22.10 require ( github.com/coreos/go-systemd/v22 v22.3.2 diff --git a/client/v2/go.mod b/client/v2/go.mod index 163bc5491e0..767a3f4a78c 100644 --- a/client/v2/go.mod +++ b/client/v2/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/client/v2 go 1.22 -toolchain go1.22.9 +toolchain go1.22.10 require ( github.com/json-iterator/go v1.1.11 diff --git a/client/v3/go.mod b/client/v3/go.mod index 5578133b596..f99b7b2498a 100644 --- a/client/v3/go.mod +++ b/client/v3/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/client/v3 go 1.22 -toolchain go1.22.9 +toolchain go1.22.10 require ( github.com/dustin/go-humanize v1.0.0 diff --git a/etcdctl/go.mod b/etcdctl/go.mod index a8182492622..f3be7de757f 100644 --- a/etcdctl/go.mod +++ b/etcdctl/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/etcdctl/v3 go 1.22 -toolchain go1.22.9 +toolchain go1.22.10 require ( github.com/bgentry/speakeasy v0.1.0 diff --git a/etcdutl/go.mod b/etcdutl/go.mod index 0be081578eb..41724120de9 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/etcdutl/v3 go 1.22 -toolchain go1.22.9 +toolchain go1.22.10 replace ( go.etcd.io/etcd/api/v3 => ../api diff --git a/go.mod b/go.mod index 3c43a08a9fd..c33b4704d76 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/v3 go 1.22 -toolchain go1.22.9 +toolchain go1.22.10 replace ( go.etcd.io/etcd/api/v3 => ./api diff --git a/pkg/go.mod b/pkg/go.mod index d18ace3e31a..8fe669ca52a 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/pkg/v3 go 1.22 -toolchain go1.22.9 +toolchain go1.22.10 require ( github.com/creack/pty v1.1.11 diff --git a/raft/go.mod b/raft/go.mod index 9e8b2cc43de..d5bb99af0ec 100644 --- a/raft/go.mod +++ b/raft/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/raft/v3 go 1.22 -toolchain go1.22.9 +toolchain go1.22.10 require ( github.com/cockroachdb/datadriven v1.0.2 diff --git a/server/go.mod b/server/go.mod index 8427235fd72..74e7a99d727 100644 --- a/server/go.mod +++ b/server/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/server/v3 go 1.22 -toolchain go1.22.9 +toolchain go1.22.10 require ( github.com/coreos/go-semver v0.3.0 diff --git a/tests/go.mod b/tests/go.mod index e799ee746cb..6808879fef1 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/tests/v3 go 1.22 -toolchain go1.22.9 +toolchain go1.22.10 replace ( go.etcd.io/etcd/api/v3 => ../api diff --git a/tools/mod/go.mod b/tools/mod/go.mod index 81dcb0fd4d7..6e6dd703d5c 100644 --- a/tools/mod/go.mod +++ b/tools/mod/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/tools/v3 go 1.22 -toolchain go1.22.9 +toolchain go1.22.10 require ( github.com/alexkohler/nakedret v1.0.0 From 5d284fda82d6e5a7a5921da9ebcf70173004db55 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Sat, 28 Dec 2024 22:12:28 +0000 Subject: [PATCH 22/41] *: support custom content check offline in v2store Part of #18993 Signed-off-by: Wei Fu --- etcdutl/ctl.go | 1 + etcdutl/etcdutl/check_command.go | 111 ++++++++++++++++++++++++++ tests/e2e/v2store_deprecation_test.go | 57 +++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 etcdutl/etcdutl/check_command.go diff --git a/etcdutl/ctl.go b/etcdutl/ctl.go index a044547c63c..dbdc5cc7810 100644 --- a/etcdutl/ctl.go +++ b/etcdutl/ctl.go @@ -41,6 +41,7 @@ func init() { etcdutl.NewDefragCommand(), etcdutl.NewSnapshotCommand(), etcdutl.NewVersionCommand(), + etcdutl.NewCheckCommand(), ) } diff --git a/etcdutl/etcdutl/check_command.go b/etcdutl/etcdutl/check_command.go new file mode 100644 index 00000000000..d596dc9f9a6 --- /dev/null +++ b/etcdutl/etcdutl/check_command.go @@ -0,0 +1,111 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcdutl + +import ( + "errors" + "fmt" + "path/filepath" + + "github.com/spf13/cobra" + + "go.etcd.io/etcd/pkg/v3/cobrautl" + "go.etcd.io/etcd/server/v3/etcdserver" + "go.etcd.io/etcd/server/v3/etcdserver/api/membership" + "go.etcd.io/etcd/server/v3/etcdserver/api/snap" + "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" + "go.etcd.io/etcd/server/v3/wal" +) + +// NewCheckCommand returns the cobra command for "check". +func NewCheckCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "check ", + Short: "commands for checking properties", + } + cmd.AddCommand(NewCheckV2StoreCommand()) + return cmd +} + +var ( + argCheckV2StoreDataDir string +) + +// NewCheckV2StoreCommand returns the cobra command for "check v2store". +func NewCheckV2StoreCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "v2store", + Short: "Check custom content in v2store", + Run: checkV2StoreRunFunc, + } + cmd.Flags().StringVar(&argCheckV2StoreDataDir, "data-dir", "", "Required. A data directory not in use by etcd.") + cmd.MarkFlagRequired("data-dir") + return cmd +} + +func checkV2StoreRunFunc(_ *cobra.Command, _ []string) { + err := checkV2StoreDataDir(argCheckV2StoreDataDir) + if err != nil { + cobrautl.ExitWithError(cobrautl.ExitError, err) + } + fmt.Println("No custom content found in v2store.") +} + +func checkV2StoreDataDir(dataDir string) error { + var ( + lg = GetLogger() + + walDir = filepath.Join(dataDir, "member", "wal") + snapDir = filepath.Join(dataDir, "member", "snap") + ) + + walSnaps, err := wal.ValidSnapshotEntries(lg, walDir) + if err != nil { + if errors.Is(err, wal.ErrFileNotFound) { + return nil + } + return err + } + + ss := snap.New(lg, snapDir) + snapshot, err := ss.LoadNewestAvailable(walSnaps) + if err != nil { + if errors.Is(err, snap.ErrNoSnapshot) { + return nil + } + return err + } + if snapshot == nil { + return nil + } + + st := v2store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix) + + if err := st.Recovery(snapshot.Data); err != nil { + return fmt.Errorf("failed to recover v2store from snapshot: %w", err) + } + return assertNoV2StoreContent(st) +} + +func assertNoV2StoreContent(st v2store.Store) error { + metaOnly, err := membership.IsMetaStoreOnly(st) + if err != nil { + return err + } + if metaOnly { + return nil + } + return fmt.Errorf("detected custom content in v2store") +} diff --git a/tests/e2e/v2store_deprecation_test.go b/tests/e2e/v2store_deprecation_test.go index 52dec549b07..a89b1a4fc40 100644 --- a/tests/e2e/v2store_deprecation_test.go +++ b/tests/e2e/v2store_deprecation_test.go @@ -97,3 +97,60 @@ func TestV2DeprecationWriteOnlyNoV2Api(t *testing.T) { _, err = proc.Expect("--enable-v2 and --v2-deprecation=write-only are mutually exclusive") assert.NoError(t, err) } + +func TestV2DeprecationCheckCustomContentOffline(t *testing.T) { + e2e.BeforeTest(t) + + t.Run("WithCustomContent", func(t *testing.T) { + dataDirPath := t.TempDir() + + createV2store(t, dataDirPath) + + assertVerifyCheckCustomContentOffline(t, dataDirPath) + }) + + t.Run("WithoutCustomContent", func(t *testing.T) { + dataDirPath := "" + + func() { + cCtx := getDefaultCtlCtx(t) + + cfg := cCtx.cfg + cfg.ClusterSize = 3 + cfg.SnapshotCount = 5 + cfg.EnableV2 = true + + // create a cluster with 3 members + epc, err := e2e.NewEtcdProcessCluster(t, &cfg) + assert.NoError(t, err) + + cCtx.epc = epc + dataDirPath = epc.Procs[0].Config().DataDirPath + + defer func() { + assert.NoError(t, epc.Stop()) + }() + + // create key-values with v3 api + for i := 0; i < 10; i++ { + assert.NoError(t, ctlV3Put(cCtx, fmt.Sprintf("key%d", i), fmt.Sprintf("value%d", i), "")) + } + }() + + proc, err := e2e.SpawnCmd([]string{e2e.BinDir + "/etcdutl", "check", "v2store", "--data-dir=" + dataDirPath}, nil) + assert.NoError(t, err) + + _, err = proc.Expect("No custom content found in v2store") + assert.NoError(t, err) + }) +} + +func assertVerifyCheckCustomContentOffline(t *testing.T, dataDirPath string) { + t.Logf("Checking custom content in v2store - %s", dataDirPath) + + proc, err := e2e.SpawnCmd([]string{e2e.BinDir + "/etcdutl", "check", "v2store", "--data-dir=" + dataDirPath}, nil) + assert.NoError(t, err) + + _, err = proc.Expect("detected custom content in v2store") + assert.NoError(t, err) +} From c1ab521a28d1347b0f676f00e869ae4e58e424e2 Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Wed, 8 Jan 2025 10:16:50 +0000 Subject: [PATCH 23/41] Fix mixVersion test case: ensure a snapshot to be sent out Signed-off-by: Benjamin Wang --- server/etcdserver/server.go | 2 +- tests/e2e/etcd_mix_versions_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go index 2655d47d670..d60c8da64fc 100644 --- a/server/etcdserver/server.go +++ b/server/etcdserver/server.go @@ -623,7 +623,7 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { srv.kv = mvcc.New(srv.Logger(), srv.be, srv.lessor, mvccStoreConfig) kvindex := ci.ConsistentIndex() - srv.lg.Debug("restore consistentIndex", zap.Uint64("index", kvindex)) + srv.lg.Info("restore consistentIndex", zap.Uint64("index", kvindex)) if beExist { // TODO: remove kvindex != 0 checking when we do not expect users to upgrade diff --git a/tests/e2e/etcd_mix_versions_test.go b/tests/e2e/etcd_mix_versions_test.go index e66c73b913d..cb41cfe5ce5 100644 --- a/tests/e2e/etcd_mix_versions_test.go +++ b/tests/e2e/etcd_mix_versions_test.go @@ -149,9 +149,9 @@ func mixVersionsSnapshotTestByMockPartition(t *testing.T, cfg e2e.EtcdProcessClu require.NoError(t, err) time.Sleep(2 * time.Second) - t.Log("Writing 20 keys to the cluster (more than SnapshotCount entries to trigger at least a snapshot)") + t.Log("Writing 30 keys to the cluster (more than SnapshotCount entries to trigger at least a snapshot)") etcdctl := epc.Procs[0].Etcdctl(e2e.ClientNonTLS, false, false) - writeKVs(t, etcdctl, 0, 20) + writeKVs(t, etcdctl, 0, 30) t.Log("Verify logs to check leader has saved snapshot") leaderEPC := epc.Procs[epc.WaitLeader(t)] From c5d14b46162ffb6038caa61f1f2f125e32f8e6a4 Mon Sep 17 00:00:00 2001 From: Ivan Valdes Date: Thu, 9 Jan 2025 12:19:24 -0800 Subject: [PATCH 24/41] bump golang.org/x/crypto from 0.21.0 to 0.32.0 Remediation for CVE-2024-45337. Signed-off-by: Ivan Valdes --- api/go.mod | 6 +++--- api/go.sum | 12 ++++++------ client/pkg/go.mod | 2 +- client/pkg/go.sum | 4 ++-- client/v3/go.mod | 6 +++--- client/v3/go.sum | 12 ++++++------ etcdctl/go.mod | 8 ++++---- etcdctl/go.sum | 20 ++++++++++---------- etcdutl/go.mod | 8 ++++---- etcdutl/go.sum | 20 ++++++++++---------- go.mod | 8 ++++---- go.sum | 20 ++++++++++---------- pkg/go.mod | 6 +++--- pkg/go.sum | 12 ++++++------ server/go.mod | 8 ++++---- server/go.sum | 20 ++++++++++---------- tests/go.mod | 10 +++++----- tests/go.sum | 20 ++++++++++---------- tools/mod/go.mod | 11 ++++++----- tools/mod/go.sum | 34 ++++++++++++++++------------------ 20 files changed, 123 insertions(+), 124 deletions(-) diff --git a/api/go.mod b/api/go.mod index b5a9ab40f0d..30a55d22385 100644 --- a/api/go.mod +++ b/api/go.mod @@ -14,9 +14,9 @@ require ( ) require ( - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/api/go.sum b/api/go.sum index 743046d4a36..cffeac3875b 100644 --- a/api/go.sum +++ b/api/go.sum @@ -53,8 +53,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -68,12 +68,12 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/client/pkg/go.mod b/client/pkg/go.mod index 28f826dce88..94a26c4e711 100644 --- a/client/pkg/go.mod +++ b/client/pkg/go.mod @@ -8,7 +8,7 @@ require ( github.com/coreos/go-systemd/v22 v22.3.2 github.com/stretchr/testify v1.9.0 go.uber.org/zap v1.17.0 - golang.org/x/sys v0.18.0 + golang.org/x/sys v0.29.0 ) require ( diff --git a/client/pkg/go.sum b/client/pkg/go.sum index 1d2244852ef..3cd3c13ed47 100644 --- a/client/pkg/go.sum +++ b/client/pkg/go.sum @@ -19,8 +19,8 @@ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= diff --git a/client/v3/go.mod b/client/v3/go.mod index f99b7b2498a..96d1b6145b2 100644 --- a/client/v3/go.mod +++ b/client/v3/go.mod @@ -28,9 +28,9 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect diff --git a/client/v3/go.sum b/client/v3/go.sum index 854a57ecf36..a56ea6984d8 100644 --- a/client/v3/go.sum +++ b/client/v3/go.sum @@ -135,8 +135,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -156,13 +156,13 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/etcdctl/go.mod b/etcdctl/go.mod index f3be7de757f..a8d6451245c 100644 --- a/etcdctl/go.mod +++ b/etcdctl/go.mod @@ -58,10 +58,10 @@ require ( go.opentelemetry.io/otel/trace v1.20.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/crypto v0.32.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect diff --git a/etcdctl/go.sum b/etcdctl/go.sum index 456bf9da962..ca44ecc2bf8 100644 --- a/etcdctl/go.sum +++ b/etcdctl/go.sum @@ -302,8 +302,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -342,8 +342,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -357,8 +357,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -380,14 +380,14 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= diff --git a/etcdutl/go.mod b/etcdutl/go.mod index 41724120de9..d3109711bfe 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -67,10 +67,10 @@ require ( go.opentelemetry.io/otel/trace v1.20.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/crypto v0.32.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect diff --git a/etcdutl/go.sum b/etcdutl/go.sum index a6ab545c968..6a934576847 100644 --- a/etcdutl/go.sum +++ b/etcdutl/go.sum @@ -292,8 +292,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -332,8 +332,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -347,8 +347,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -370,14 +370,14 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= diff --git a/go.mod b/go.mod index c33b4704d76..6176427dcfc 100644 --- a/go.mod +++ b/go.mod @@ -84,10 +84,10 @@ require ( go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/crypto v0.32.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect diff --git a/go.sum b/go.sum index 15d5614d234..0b7c1aac914 100644 --- a/go.sum +++ b/go.sum @@ -348,8 +348,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -390,8 +390,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -406,8 +406,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -432,14 +432,14 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= diff --git a/pkg/go.mod b/pkg/go.mod index 8fe669ca52a..b26718093fd 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -22,9 +22,9 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/pkg/go.sum b/pkg/go.sum index 5f96c6562c3..bf022aa9286 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -226,8 +226,8 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -249,13 +249,13 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/server/go.mod b/server/go.mod index 74e7a99d727..2ea3af37988 100644 --- a/server/go.mod +++ b/server/go.mod @@ -38,8 +38,8 @@ require ( go.opentelemetry.io/otel/sdk v1.20.0 go.uber.org/multierr v1.6.0 go.uber.org/zap v1.17.0 - golang.org/x/crypto v0.21.0 - golang.org/x/net v0.23.0 + golang.org/x/crypto v0.32.0 + golang.org/x/net v0.25.0 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d google.golang.org/grpc v1.59.0 @@ -71,8 +71,8 @@ require ( go.opentelemetry.io/otel/trace v1.20.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.7.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/server/go.sum b/server/go.sum index 392fd8c1696..e896cd2eb89 100644 --- a/server/go.sum +++ b/server/go.sum @@ -336,8 +336,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -378,8 +378,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -394,8 +394,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -418,14 +418,14 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= diff --git a/tests/go.mod b/tests/go.mod index 6808879fef1..b20d8250426 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -45,8 +45,8 @@ require ( go.opentelemetry.io/otel/trace v1.20.0 go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/zap v1.17.0 - golang.org/x/crypto v0.21.0 - golang.org/x/sync v0.5.0 + golang.org/x/crypto v0.32.0 + golang.org/x/sync v0.10.0 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba google.golang.org/grpc v1.59.0 gopkg.in/yaml.v2 v2.4.0 @@ -84,9 +84,9 @@ require ( go.opentelemetry.io/otel/metric v1.20.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect diff --git a/tests/go.sum b/tests/go.sum index c82a51185e4..aeb9f4e16ac 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -340,8 +340,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -382,8 +382,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -398,8 +398,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -422,14 +422,14 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= diff --git a/tools/mod/go.mod b/tools/mod/go.mod index 6e6dd703d5c..07395b3d841 100644 --- a/tools/mod/go.mod +++ b/tools/mod/go.mod @@ -58,11 +58,12 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/trustmaster/go-aspell v0.0.0-20200701131845-c2b1f55bec8f // indirect go.mongodb.org/mongo-driver v1.3.0 // indirect - golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.6.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect diff --git a/tools/mod/go.sum b/tools/mod/go.sum index fb9afa6328a..d058218d030 100644 --- a/tools/mod/go.sum +++ b/tools/mod/go.sum @@ -161,8 +161,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -344,8 +344,8 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -360,8 +360,6 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/gofail v0.1.0 h1:XItAMIhOojXFQMgrxjnd2EIIHun/d5qL0Pf7FzVTkFg= -go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M= go.etcd.io/gofail v0.2.0 h1:p19drv16FKK345a09a1iubchlw/vmRuksmRzgBIGjcA= go.etcd.io/gofail v0.2.0/go.mod h1:nL3ILMGfkXTekKI3clMBNazKnjUZjYLKmBHzsVAnC1o= go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116 h1:QQiUXlqz+d96jyNG71NE+IGTgOK6Xlhdx+PzvfbLHlQ= @@ -405,8 +403,8 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -428,8 +426,8 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -442,8 +440,8 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -472,14 +470,14 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -511,8 +509,8 @@ golang.org/x/tools v0.0.0-20200225230052-807dcd883420/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From f627f57e27ce7b6b12b3ab02bfa0d082644a85e7 Mon Sep 17 00:00:00 2001 From: Ivan Valdes Date: Thu, 9 Jan 2025 15:06:20 -0800 Subject: [PATCH 25/41] bump golang.org/x/net from 0.25.0 to 0.34.0 Remediation for CVE-2024-45338. Signed-off-by: Ivan Valdes --- api/go.mod | 2 +- api/go.sum | 4 ++-- client/v3/go.mod | 2 +- client/v3/go.sum | 4 ++-- etcdctl/go.mod | 2 +- etcdctl/go.sum | 4 ++-- etcdutl/go.mod | 2 +- etcdutl/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- pkg/go.mod | 2 +- pkg/go.sum | 4 ++-- server/go.mod | 2 +- server/go.sum | 4 ++-- tests/go.mod | 2 +- tests/go.sum | 4 ++-- tools/mod/go.mod | 2 +- tools/mod/go.sum | 4 ++-- 18 files changed, 27 insertions(+), 27 deletions(-) diff --git a/api/go.mod b/api/go.mod index 30a55d22385..471c0417c3c 100644 --- a/api/go.mod +++ b/api/go.mod @@ -14,7 +14,7 @@ require ( ) require ( - golang.org/x/net v0.25.0 // indirect + golang.org/x/net v0.34.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect diff --git a/api/go.sum b/api/go.sum index cffeac3875b..05e53dd961c 100644 --- a/api/go.sum +++ b/api/go.sum @@ -53,8 +53,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/client/v3/go.mod b/client/v3/go.mod index 96d1b6145b2..2a2dfd5a50d 100644 --- a/client/v3/go.mod +++ b/client/v3/go.mod @@ -28,7 +28,7 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/net v0.25.0 // indirect + golang.org/x/net v0.34.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect diff --git a/client/v3/go.sum b/client/v3/go.sum index a56ea6984d8..b5a4e41f863 100644 --- a/client/v3/go.sum +++ b/client/v3/go.sum @@ -135,8 +135,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/etcdctl/go.mod b/etcdctl/go.mod index a8d6451245c..82f5749f2d3 100644 --- a/etcdctl/go.mod +++ b/etcdctl/go.mod @@ -59,7 +59,7 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/crypto v0.32.0 // indirect - golang.org/x/net v0.25.0 // indirect + golang.org/x/net v0.34.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect diff --git a/etcdctl/go.sum b/etcdctl/go.sum index ca44ecc2bf8..2afdd56ec40 100644 --- a/etcdctl/go.sum +++ b/etcdctl/go.sum @@ -342,8 +342,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/etcdutl/go.mod b/etcdutl/go.mod index d3109711bfe..3f4577db6a8 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -68,7 +68,7 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/crypto v0.32.0 // indirect - golang.org/x/net v0.25.0 // indirect + golang.org/x/net v0.34.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect diff --git a/etcdutl/go.sum b/etcdutl/go.sum index 6a934576847..f024e624eae 100644 --- a/etcdutl/go.sum +++ b/etcdutl/go.sum @@ -332,8 +332,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/go.mod b/go.mod index 6176427dcfc..ac978f5951e 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,7 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/crypto v0.32.0 // indirect - golang.org/x/net v0.25.0 // indirect + golang.org/x/net v0.34.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect diff --git a/go.sum b/go.sum index 0b7c1aac914..a63721f0e9d 100644 --- a/go.sum +++ b/go.sum @@ -390,8 +390,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/pkg/go.mod b/pkg/go.mod index b26718093fd..bef66a50178 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -22,7 +22,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/net v0.25.0 // indirect + golang.org/x/net v0.34.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect diff --git a/pkg/go.sum b/pkg/go.sum index bf022aa9286..8d27fafd5d9 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -226,8 +226,8 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/server/go.mod b/server/go.mod index 2ea3af37988..09f3c23aee4 100644 --- a/server/go.mod +++ b/server/go.mod @@ -39,7 +39,7 @@ require ( go.uber.org/multierr v1.6.0 go.uber.org/zap v1.17.0 golang.org/x/crypto v0.32.0 - golang.org/x/net v0.25.0 + golang.org/x/net v0.34.0 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d google.golang.org/grpc v1.59.0 diff --git a/server/go.sum b/server/go.sum index e896cd2eb89..e5f6fa072ce 100644 --- a/server/go.sum +++ b/server/go.sum @@ -378,8 +378,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/tests/go.mod b/tests/go.mod index b20d8250426..bdc21bb587e 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -84,7 +84,7 @@ require ( go.opentelemetry.io/otel/metric v1.20.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/net v0.25.0 // indirect + golang.org/x/net v0.34.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect diff --git a/tests/go.sum b/tests/go.sum index aeb9f4e16ac..2be79b19f2e 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -382,8 +382,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/tools/mod/go.mod b/tools/mod/go.mod index 07395b3d841..aad37b370c9 100644 --- a/tools/mod/go.mod +++ b/tools/mod/go.mod @@ -59,7 +59,7 @@ require ( github.com/trustmaster/go-aspell v0.0.0-20200701131845-c2b1f55bec8f // indirect go.mongodb.org/mongo-driver v1.3.0 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.25.0 // indirect + golang.org/x/net v0.34.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect diff --git a/tools/mod/go.sum b/tools/mod/go.sum index d058218d030..013a8bc5dba 100644 --- a/tools/mod/go.sum +++ b/tools/mod/go.sum @@ -426,8 +426,8 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= From 8943d8fc13e80cba274427bbb68e0a7296d7067c Mon Sep 17 00:00:00 2001 From: Ivan Valdes Date: Thu, 9 Jan 2025 16:26:04 -0800 Subject: [PATCH 26/41] Update release_notes.tpl.txt Add space between ${ETCD_VER} and Path Chery-pick commit 519a2e3a272903d888138e8518ea48333364edf5 / pull request #19153. Co-authored-by: sunfeilong <31331079+sunfeilong@users.noreply.github.com> Signed-off-by: Ivan Valdes --- scripts/release_notes.tpl.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/release_notes.tpl.txt b/scripts/release_notes.tpl.txt index 1038a504d6f..67701d9886e 100644 --- a/scripts/release_notes.tpl.txt +++ b/scripts/release_notes.tpl.txt @@ -82,10 +82,10 @@ rm -rf /tmp/etcd-data.tmp && mkdir -p /tmp/etcd-data.tmp && \ --logger zap \ --log-outputs stderr -docker exec etcd-gcr-${ETCD_VER}/usr/local/bin/etcd --version -docker exec etcd-gcr-${ETCD_VER}/usr/local/bin/etcdctl version -docker exec etcd-gcr-${ETCD_VER}/usr/local/bin/etcdutl version -docker exec etcd-gcr-${ETCD_VER}/usr/local/bin/etcdctl endpoint health -docker exec etcd-gcr-${ETCD_VER}/usr/local/bin/etcdctl put foo bar -docker exec etcd-gcr-${ETCD_VER}/usr/local/bin/etcdctl get foo +docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcd --version +docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcdctl version +docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcdutl version +docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcdctl endpoint health +docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcdctl put foo bar +docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcdctl get foo ``` From b9a7aed91ba09fa0398fecd8a79ee996e98f41b1 Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Thu, 12 Dec 2024 21:45:28 +0000 Subject: [PATCH 27/41] Do not wait for ready notify if the server is stopping Signed-off-by: Benjamin Wang --- server/embed/serve.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/embed/serve.go b/server/embed/serve.go index 91ec6e9a376..8af26a8227c 100644 --- a/server/embed/serve.go +++ b/server/embed/serve.go @@ -16,6 +16,7 @@ package embed import ( "context" + "errors" "fmt" "io/ioutil" defaultLog "log" @@ -100,6 +101,12 @@ func (sctx *serveCtx) serve( logger := defaultLog.New(ioutil.Discard, "etcdhttp", 0) <-s.ReadyNotify() + select { + case <-s.StoppingNotify(): + return errors.New("server is stopping") + case <-s.ReadyNotify(): + } + sctx.lg.Info("ready to serve client requests") m := cmux.New(sctx.l) From 9b241941202bb939a29248bfaaab5f7b91fe8cd3 Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Fri, 13 Dec 2024 10:44:14 +0000 Subject: [PATCH 28/41] Remove duplicated <-s.ReadyNotify() Signed-off-by: Benjamin Wang --- server/embed/serve.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server/embed/serve.go b/server/embed/serve.go index 8af26a8227c..99abe515db3 100644 --- a/server/embed/serve.go +++ b/server/embed/serve.go @@ -99,7 +99,6 @@ func (sctx *serveCtx) serve( splitHttp bool, gopts ...grpc.ServerOption) (err error) { logger := defaultLog.New(ioutil.Discard, "etcdhttp", 0) - <-s.ReadyNotify() select { case <-s.StoppingNotify(): From e5b7e089fc54f835005067a6c5dd098fb7f31a1a Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Fri, 10 Jan 2025 16:16:10 +0000 Subject: [PATCH 29/41] etcdutl/etcdutl: use datadir package to build wal/snapdir It's follow-up for https://github.com/etcd-io/etcd/pull/19113#discussion_r1910213211 Signed-off-by: Wei Fu --- etcdutl/etcdutl/check_command.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etcdutl/etcdutl/check_command.go b/etcdutl/etcdutl/check_command.go index d596dc9f9a6..2ea6fb29ae2 100644 --- a/etcdutl/etcdutl/check_command.go +++ b/etcdutl/etcdutl/check_command.go @@ -17,11 +17,11 @@ package etcdutl import ( "errors" "fmt" - "path/filepath" "github.com/spf13/cobra" "go.etcd.io/etcd/pkg/v3/cobrautl" + "go.etcd.io/etcd/server/v3/datadir" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" @@ -67,8 +67,8 @@ func checkV2StoreDataDir(dataDir string) error { var ( lg = GetLogger() - walDir = filepath.Join(dataDir, "member", "wal") - snapDir = filepath.Join(dataDir, "member", "snap") + walDir = datadir.ToWalDir(dataDir) + snapDir = datadir.ToSnapDir(dataDir) ) walSnaps, err := wal.ValidSnapshotEntries(lg, walDir) From 80b0a73ee2fe74923f87e2b9aaf675b54e752cf5 Mon Sep 17 00:00:00 2001 From: Joshua Zhang Date: Sat, 11 Jan 2025 07:17:00 +0000 Subject: [PATCH 30/41] Avoid deadlock in etcd.Close when stopping during bootstrapping Signed-off-by: Joshua Zhang --- server/embed/etcd.go | 7 +++- server/embed/serve.go | 13 +++++- server/etcdserver/server.go | 1 + tests/integration/embed/embed_test.go | 58 ++++++++++++++++++++++++++- 4 files changed, 75 insertions(+), 4 deletions(-) diff --git a/server/embed/etcd.go b/server/embed/etcd.go index b2c7fee4482..10ed426d6b2 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -86,6 +86,7 @@ type Etcd struct { errc chan error closeOnce sync.Once + wg sync.WaitGroup } type peerListener struct { @@ -111,7 +112,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { if !serving { // errored before starting gRPC server for serveCtx.serversC for _, sctx := range e.sctxs { - close(sctx.serversC) + sctx.close() } } e.Close() @@ -436,6 +437,7 @@ func (e *Etcd) Close() { } } if e.errc != nil { + e.wg.Wait() close(e.errc) } } @@ -880,6 +882,9 @@ func (e *Etcd) serveMetrics() (err error) { } func (e *Etcd) errHandler(err error) { + e.wg.Add(1) + defer e.wg.Done() + select { case <-e.stopc: return diff --git a/server/embed/serve.go b/server/embed/serve.go index 99abe515db3..1a46f51af93 100644 --- a/server/embed/serve.go +++ b/server/embed/serve.go @@ -23,6 +23,7 @@ import ( "net" "net/http" "strings" + "sync" etcdservergw "go.etcd.io/etcd/api/v3/etcdserverpb/gw" "go.etcd.io/etcd/client/pkg/v3/transport" @@ -65,6 +66,7 @@ type serveCtx struct { userHandlers map[string]http.Handler serviceRegister func(*grpc.Server) serversC chan *servers + closeOnce sync.Once } type servers struct { @@ -100,6 +102,9 @@ func (sctx *serveCtx) serve( gopts ...grpc.ServerOption) (err error) { logger := defaultLog.New(ioutil.Discard, "etcdhttp", 0) + // Make sure serversC is closed even if we prematurely exit the function. + defer sctx.close() + select { case <-s.StoppingNotify(): return errors.New("server is stopping") @@ -119,8 +124,6 @@ func (sctx *serveCtx) serve( servElection := v3election.NewElectionServer(v3c) servLock := v3lock.NewLockServer(v3c) - // Make sure serversC is closed even if we prematurely exit the function. - defer close(sctx.serversC) var gwmux *gw.ServeMux if s.Cfg.EnableGRPCGateway { // GRPC gateway connects to grpc server via connection provided by grpc dial. @@ -503,3 +506,9 @@ func (sctx *serveCtx) registerTrace() { evf := func(w http.ResponseWriter, r *http.Request) { trace.RenderEvents(w, r, true) } sctx.registerUserHandler("/debug/events", http.HandlerFunc(evf)) } + +func (sctx *serveCtx) close() { + sctx.closeOnce.Do(func() { + close(sctx.serversC) + }) +} diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go index d60c8da64fc..513ef6bd7b8 100644 --- a/server/etcdserver/server.go +++ b/server/etcdserver/server.go @@ -2130,6 +2130,7 @@ func (s *EtcdServer) publish(timeout time.Duration) { Val: string(b), } + // gofail: var beforePublishing struct{} for { ctx, cancel := context.WithTimeout(s.ctx, timeout) _, err := s.Do(ctx, req) diff --git a/tests/integration/embed/embed_test.go b/tests/integration/embed/embed_test.go index 27da5bf473b..e45c7ade229 100644 --- a/tests/integration/embed/embed_test.go +++ b/tests/integration/embed/embed_test.go @@ -30,11 +30,14 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/pkg/v3/transport" - "go.etcd.io/etcd/client/v3" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/tests/v3/integration" + gofail "go.etcd.io/gofail/runtime" ) var ( @@ -210,3 +213,56 @@ func setupEmbedCfg(cfg *embed.Config, curls []url.URL, purls []url.URL) { } cfg.InitialCluster = cfg.InitialCluster[1:] } + +func TestEmbedEtcdStopDuringBootstrapping(t *testing.T) { + if len(gofail.List()) == 0 { + t.Skip("please run 'make gofail-enable' before running the test") + } + + fpName := "beforePublishing" + require.NoError(t, gofail.Enable(fpName, `sleep("2s")`)) + t.Cleanup(func() { + terr := gofail.Disable(fpName) + if terr != nil && terr != gofail.ErrDisabled { + t.Fatalf("failed to disable %s: %v", fpName, terr) + } + }) + + done := make(chan struct{}) + go func() { + defer close(done) + + cfg := embed.NewConfig() + urls := newEmbedURLs(false, 2) + setupEmbedCfg(cfg, []url.URL{urls[0]}, []url.URL{urls[1]}) + cfg.Dir = filepath.Join(t.TempDir(), "embed-etcd") + + e, err := embed.StartEtcd(cfg) + if err != nil { + t.Errorf("Failed to start etcd, got error %v", err) + } + defer e.Close() + + go func() { + time.Sleep(time.Second) + e.Server.Stop() + t.Log("Stopped server during bootstrapping") + }() + + select { + case <-e.Server.ReadyNotify(): + t.Log("Server is ready!") + case <-e.Server.StopNotify(): + t.Log("Server is stopped") + case <-time.After(20 * time.Second): + e.Server.Stop() // trigger a shutdown + t.Error("Server took too long to start!") + } + }() + + select { + case <-done: + case <-time.After(10 * time.Second): + t.Error("timeout in bootstrapping etcd") + } +} From 3ff20e6f60efc1f8b1b315d0dfffb527ad78455d Mon Sep 17 00:00:00 2001 From: Ivan Valdes Date: Thu, 16 Jan 2025 15:27:00 -0800 Subject: [PATCH 31/41] Bump go toolchain to 1.22.11 Signed-off-by: Ivan Valdes --- .go-version | 2 +- api/go.mod | 2 +- client/pkg/go.mod | 2 +- client/v2/go.mod | 2 +- client/v3/go.mod | 2 +- etcdctl/go.mod | 2 +- etcdutl/go.mod | 2 +- go.mod | 2 +- pkg/go.mod | 2 +- raft/go.mod | 2 +- server/go.mod | 2 +- tests/go.mod | 2 +- tools/mod/go.mod | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.go-version b/.go-version index f5ae945712e..3dfc2ecf954 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.22.10 +1.22.11 diff --git a/api/go.mod b/api/go.mod index 471c0417c3c..ece3ae71b03 100644 --- a/api/go.mod +++ b/api/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/api/v3 go 1.22 -toolchain go1.22.10 +toolchain go1.22.11 require ( github.com/coreos/go-semver v0.3.0 diff --git a/client/pkg/go.mod b/client/pkg/go.mod index 94a26c4e711..dabb14428fa 100644 --- a/client/pkg/go.mod +++ b/client/pkg/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/client/pkg/v3 go 1.22 -toolchain go1.22.10 +toolchain go1.22.11 require ( github.com/coreos/go-systemd/v22 v22.3.2 diff --git a/client/v2/go.mod b/client/v2/go.mod index 767a3f4a78c..b07fcf148a8 100644 --- a/client/v2/go.mod +++ b/client/v2/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/client/v2 go 1.22 -toolchain go1.22.10 +toolchain go1.22.11 require ( github.com/json-iterator/go v1.1.11 diff --git a/client/v3/go.mod b/client/v3/go.mod index 2a2dfd5a50d..5fbe0f0cf9c 100644 --- a/client/v3/go.mod +++ b/client/v3/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/client/v3 go 1.22 -toolchain go1.22.10 +toolchain go1.22.11 require ( github.com/dustin/go-humanize v1.0.0 diff --git a/etcdctl/go.mod b/etcdctl/go.mod index 82f5749f2d3..88fd423319a 100644 --- a/etcdctl/go.mod +++ b/etcdctl/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/etcdctl/v3 go 1.22 -toolchain go1.22.10 +toolchain go1.22.11 require ( github.com/bgentry/speakeasy v0.1.0 diff --git a/etcdutl/go.mod b/etcdutl/go.mod index 3f4577db6a8..e610c03069d 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/etcdutl/v3 go 1.22 -toolchain go1.22.10 +toolchain go1.22.11 replace ( go.etcd.io/etcd/api/v3 => ../api diff --git a/go.mod b/go.mod index ac978f5951e..2a7c92a88cc 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/v3 go 1.22 -toolchain go1.22.10 +toolchain go1.22.11 replace ( go.etcd.io/etcd/api/v3 => ./api diff --git a/pkg/go.mod b/pkg/go.mod index bef66a50178..8f41a489525 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/pkg/v3 go 1.22 -toolchain go1.22.10 +toolchain go1.22.11 require ( github.com/creack/pty v1.1.11 diff --git a/raft/go.mod b/raft/go.mod index d5bb99af0ec..6348b3f61d7 100644 --- a/raft/go.mod +++ b/raft/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/raft/v3 go 1.22 -toolchain go1.22.10 +toolchain go1.22.11 require ( github.com/cockroachdb/datadriven v1.0.2 diff --git a/server/go.mod b/server/go.mod index 09f3c23aee4..0d1887ad75b 100644 --- a/server/go.mod +++ b/server/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/server/v3 go 1.22 -toolchain go1.22.10 +toolchain go1.22.11 require ( github.com/coreos/go-semver v0.3.0 diff --git a/tests/go.mod b/tests/go.mod index bdc21bb587e..18d8f35f315 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/tests/v3 go 1.22 -toolchain go1.22.10 +toolchain go1.22.11 replace ( go.etcd.io/etcd/api/v3 => ../api diff --git a/tools/mod/go.mod b/tools/mod/go.mod index aad37b370c9..8eafa473f61 100644 --- a/tools/mod/go.mod +++ b/tools/mod/go.mod @@ -2,7 +2,7 @@ module go.etcd.io/etcd/tools/v3 go 1.22 -toolchain go1.22.10 +toolchain go1.22.11 require ( github.com/alexkohler/nakedret v1.0.0 From c6fcd2701db8ead4be062c7381c87c43ccdb10d0 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Mon, 13 Jan 2025 17:30:40 -0500 Subject: [PATCH 32/41] mvcc: restore tombstone index if it's first revision The tombstone could be the only one available revision in database. It happens when all historical revisions have been deleted in previous compactions. Since tombstone revision is still in database, we should restore it as valid key index. Otherwise, we lost that event. Signed-off-by: Wei Fu (cherry picked from commit d8b41925749ec85b0320839968a2218c4f9dc73f) Signed-off-by: Wei Fu --- server/mvcc/key_index.go | 9 ++++ server/mvcc/key_index_test.go | 42 +++++++++++++++++++ server/mvcc/kv_test.go | 23 ++++++++-- server/mvcc/kvstore.go | 8 +++- tests/e2e/watch_test.go | 79 +++++++++++++++++++++++++++++++++++ 5 files changed, 156 insertions(+), 5 deletions(-) diff --git a/server/mvcc/key_index.go b/server/mvcc/key_index.go index d38b0933d92..dfb5dcbee26 100644 --- a/server/mvcc/key_index.go +++ b/server/mvcc/key_index.go @@ -119,6 +119,15 @@ func (ki *keyIndex) restore(lg *zap.Logger, created, modified revision, ver int6 keysGauge.Inc() } +// restoreTombstone is used to restore a tombstone revision, which is the only +// revision so far for a key. We don't know the creating revision (i.e. already +// compacted) of the key, so set it empty. +func (ki *keyIndex) restoreTombstone(lg *zap.Logger, main, sub int64) { + ki.restore(lg, revision{}, revision{main, sub}, 1) + ki.generations = append(ki.generations, generation{}) + keysGauge.Dec() +} + // tombstone puts a revision, pointing to a tombstone, to the keyIndex. // It also creates a new empty generation in the keyIndex. // It returns ErrRevisionNotFound when tombstone on an empty generation. diff --git a/server/mvcc/key_index_test.go b/server/mvcc/key_index_test.go index 814e252fd4a..693514266f9 100644 --- a/server/mvcc/key_index_test.go +++ b/server/mvcc/key_index_test.go @@ -19,9 +19,51 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/zap" + "go.uber.org/zap/zaptest" ) +func TestRestoreTombstone(t *testing.T) { + lg := zaptest.NewLogger(t) + + // restore from tombstone + // + // key: "foo" + // modified: 16 + // "created": 16 + // generations: + // {empty} + // {{16, 0}(t)[0]} + // + ki := &keyIndex{key: []byte("foo")} + ki.restoreTombstone(lg, 16, 0) + + // get should return not found + for retAt := 16; retAt <= 20; retAt++ { + _, _, _, err := ki.get(lg, int64(retAt)) + require.ErrorIs(t, err, ErrRevisionNotFound) + } + + // doCompact should keep that tombstone + availables := map[revision]struct{}{} + ki.doCompact(16, availables) + require.Len(t, availables, 1) + _, ok := availables[revision{main: 16}] + require.True(t, ok) + + // should be able to put new revisions + ki.put(lg, 17, 0) + ki.put(lg, 18, 0) + revs := ki.since(lg, 16) + require.Equal(t, []revision{{16, 0}, {17, 0}, {18, 0}}, revs) + + // compaction should remove restored tombstone + ki.compact(lg, 17, map[revision]struct{}{}) + require.Len(t, ki.generations, 1) + require.Equal(t, []revision{{17, 0}, {18, 0}}, ki.generations[0].revs) +} + func TestKeyIndexGet(t *testing.T) { // key: "foo" // rev: 16 diff --git a/server/mvcc/kv_test.go b/server/mvcc/kv_test.go index 524a1346ebd..7ccd719fb07 100644 --- a/server/mvcc/kv_test.go +++ b/server/mvcc/kv_test.go @@ -31,7 +31,9 @@ import ( "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/assert" "go.uber.org/zap" + "go.uber.org/zap/zaptest" ) // Functional tests for features implemented in v3 store. It treats v3 store @@ -620,6 +622,8 @@ func TestKVHash(t *testing.T) { } func TestKVRestore(t *testing.T) { + compactBatchLimit := 5 + tests := []func(kv KV){ func(kv KV) { kv.Put([]byte("foo"), []byte("bar0"), 1) @@ -637,10 +641,23 @@ func TestKVRestore(t *testing.T) { kv.Put([]byte("foo"), []byte("bar1"), 2) kv.Compact(traceutil.TODO(), 1) }, + func(kv KV) { // after restore, foo1 key only has tombstone revision + kv.Put([]byte("foo1"), []byte("bar1"), 0) + kv.Put([]byte("foo2"), []byte("bar2"), 0) + kv.Put([]byte("foo3"), []byte("bar3"), 0) + kv.Put([]byte("foo4"), []byte("bar4"), 0) + kv.Put([]byte("foo5"), []byte("bar5"), 0) + _, delAtRev := kv.DeleteRange([]byte("foo1"), nil) + assert.Equal(t, int64(7), delAtRev) + + // after compaction and restore, foo1 key only has tombstone revision + ch, _ := kv.Compact(traceutil.TODO(), delAtRev) + <-ch + }, } for i, tt := range tests { b, tmpPath := betesting.NewDefaultTmpBackend(t) - s := NewStore(zap.NewExample(), b, &lease.FakeLessor{}, StoreConfig{}) + s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{CompactionBatchLimit: compactBatchLimit}) tt(s) var kvss [][]mvccpb.KeyValue for k := int64(0); k < 10; k++ { @@ -651,8 +668,8 @@ func TestKVRestore(t *testing.T) { keysBefore := readGaugeInt(keysGauge) s.Close() - // ns should recover the the previous state from backend. - ns := NewStore(zap.NewExample(), b, &lease.FakeLessor{}, StoreConfig{}) + // ns should recover the previous state from backend. + ns := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{CompactionBatchLimit: compactBatchLimit}) if keysRestore := readGaugeInt(keysGauge); keysBefore != keysRestore { t.Errorf("#%d: got %d key count, expected %d", i, keysRestore, keysBefore) diff --git a/server/mvcc/kvstore.go b/server/mvcc/kvstore.go index fb81af6ec74..f0450537db3 100644 --- a/server/mvcc/kvstore.go +++ b/server/mvcc/kvstore.go @@ -484,8 +484,12 @@ func restoreIntoIndex(lg *zap.Logger, idx index) (chan<- revKeyValue, <-chan int continue } ki.put(lg, rev.main, rev.sub) - } else if !isTombstone(rkv.key) { - ki.restore(lg, revision{rkv.kv.CreateRevision, 0}, rev, rkv.kv.Version) + } else { + if isTombstone(rkv.key) { + ki.restoreTombstone(lg, rev.main, rev.sub) + } else { + ki.restore(lg, revision{rkv.kv.CreateRevision, 0}, rev, rkv.kv.Version) + } idx.Insert(ki) kiCache[rkv.kstr] = ki } diff --git a/tests/e2e/watch_test.go b/tests/e2e/watch_test.go index d09347f5bdd..3fcb5bf81b4 100644 --- a/tests/e2e/watch_test.go +++ b/tests/e2e/watch_test.go @@ -482,3 +482,82 @@ func testStartWatcherFromCompactedRevision(t *testing.T, performCompactOnTombsto } } } + +// TestResumeCompactionOnTombstone verifies whether a deletion event is preserved +// when etcd restarts and resumes compaction on a key that only has a tombstone revision. +func TestResumeCompactionOnTombstone(t *testing.T) { + e2e.BeforeTest(t) + + ctx := context.Background() + compactBatchLimit := 5 + + cfg := e2e.EtcdProcessClusterConfig{ + GoFailEnabled: true, + ClusterSize: 1, + IsClientAutoTLS: true, + ClientTLS: e2e.ClientTLS, + CompactionBatchLimit: compactBatchLimit, + WatchProcessNotifyInterval: 100 * time.Millisecond, + } + clus, err := e2e.NewEtcdProcessCluster(t, &cfg) + require.NoError(t, err) + defer clus.Close() + + c1 := newClient(t, clus.EndpointsGRPC(), cfg.ClientTLS, cfg.IsClientAutoTLS) + defer c1.Close() + + keyPrefix := "/key-" + for i := 0; i < compactBatchLimit; i++ { + key := fmt.Sprintf("%s%d", keyPrefix, i) + value := fmt.Sprintf("%d", i) + + t.Logf("PUT key=%s, val=%s", key, value) + _, err = c1.KV.Put(ctx, key, value) + require.NoError(t, err) + } + + firstKey := keyPrefix + "0" + t.Logf("DELETE key=%s", firstKey) + deleteResp, err := c1.KV.Delete(ctx, firstKey) + require.NoError(t, err) + + var deleteEvent *clientv3.Event + select { + case watchResp := <-c1.Watch(ctx, firstKey, clientv3.WithRev(deleteResp.Header.Revision)): + require.Len(t, watchResp.Events, 1) + + require.Equal(t, mvccpb.DELETE, watchResp.Events[0].Type) + deletedKey := string(watchResp.Events[0].Kv.Key) + require.Equal(t, firstKey, deletedKey) + + deleteEvent = watchResp.Events[0] + case <-time.After(100 * time.Millisecond): + t.Fatal("timed out getting watch response") + } + + require.NoError(t, clus.Procs[0].Failpoints().SetupHTTP(ctx, "compactBeforeSetFinishedCompact", `panic`)) + + t.Logf("COMPACT rev=%d", deleteResp.Header.Revision) + _, err = c1.KV.Compact(ctx, deleteResp.Header.Revision, clientv3.WithCompactPhysical()) + require.Error(t, err) + + require.Error(t, clus.Procs[0].Stop()) + // NOTE: The proc panics and exit code is 2. It's impossible to restart + // that etcd proc because last exit code is 2 and Restart() refuses to + // start new one. Using IsRunning() function is to cleanup status. + require.False(t, clus.Procs[0].IsRunning()) + require.NoError(t, clus.Restart()) + + c2 := newClient(t, clus.EndpointsGRPC(), cfg.ClientTLS, cfg.IsClientAutoTLS) + defer c2.Close() + + watchChan := c2.Watch(ctx, firstKey, clientv3.WithRev(deleteResp.Header.Revision)) + select { + case watchResp := <-watchChan: + require.Equal(t, []*clientv3.Event{deleteEvent}, watchResp.Events) + case <-time.After(100 * time.Millisecond): + // we care only about the first response, but have an + // escape hatch in case the watch response is delayed. + t.Fatal("timed out getting watch response") + } +} From 0c8e7b19b648a056c9e8400c591f2414402d0419 Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Wed, 22 Jan 2025 15:10:06 +0000 Subject: [PATCH 33/41] Ensure all goroutines created by StartEtcd to exit before closing the errc Signed-off-by: Benjamin Wang --- server/embed/etcd.go | 46 ++++++++++++++++++++++++++++++++++++------- server/embed/serve.go | 34 ++++++++++++++++++++++++++------ 2 files changed, 67 insertions(+), 13 deletions(-) diff --git a/server/embed/etcd.go b/server/embed/etcd.go index 10ed426d6b2..b894458b758 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -81,12 +81,23 @@ type Etcd struct { Server *etcdserver.EtcdServer - cfg Config - stopc chan struct{} - errc chan error + cfg Config + // closeOnce is to ensure `stopc` is closed only once, no matter + // how many times the Close() method is called. closeOnce sync.Once - wg sync.WaitGroup + // stopc is used to notify the sub goroutines not to send + // any errors to `errc`. + stopc chan struct{} + // errc is used to receive error from sub goroutines (including + // client handler, peer handler and metrics handler). It's closed + // after all these sub goroutines exit (checked via `wg`). Writers + // should avoid writing after `stopc` is closed by selecting on + // reading from `stopc`. + errc chan error + + // wg is used to track the lifecycle of all sub goroutines created by `StartEtcd`. + wg sync.WaitGroup } type peerListener struct { @@ -368,6 +379,24 @@ func (e *Etcd) Config() Config { // Close gracefully shuts down all servers/listeners. // Client requests will be terminated with request timeout. // After timeout, enforce remaning requests be closed immediately. +// +// The rough workflow to shut down etcd: +// 1. close the `stopc` channel, so that all error handlers (child +// goroutines) won't send back any errors anymore; +// 2. stop the http and grpc servers gracefully, within request timeout; +// 3. close all client and metrics listeners, so that etcd server +// stops receiving any new connection; +// 4. call the cancel function to close the gateway context, so that +// all gateway connections are closed. +// 5. stop etcd server gracefully, and ensure the main raft loop +// goroutine is stopped; +// 6. stop all peer listeners, so that it stops receiving peer connections +// and messages (wait up to 1-second); +// 7. wait for all child goroutines (i.e. client handlers, peer handlers +// and metrics handlers) to exit; +// 8. close the `errc` channel to release the resource. Note that it's only +// safe to close the `errc` after step 7 above is done, otherwise the +// child goroutines may send errors back to already closed `errc` channel. func (e *Etcd) Close() { fields := []zap.Field{ zap.String("name", e.cfg.Name), @@ -597,7 +626,9 @@ func (e *Etcd) servePeers() (err error) { // start peer servers in a goroutine for _, pl := range e.Peers { + e.wg.Add(1) go func(l *peerListener) { + defer e.wg.Done() u := l.Addr().String() e.cfg.logger.Info( "serving peer traffic", @@ -781,7 +812,9 @@ func (e *Etcd) serveClients() (err error) { // start client servers in each goroutine for _, sctx := range e.sctxs { + e.wg.Add(1) go func(s *serveCtx) { + defer e.wg.Done() e.errHandler(s.serve(e.Server, &e.cfg.ClientTLSInfo, h, e.errHandler, e.grpcGatewayDial(splitHttp), splitHttp, gopts...)) }(sctx) } @@ -869,7 +902,9 @@ func (e *Etcd) serveMetrics() (err error) { return err } e.metricsListeners = append(e.metricsListeners, ml) + e.wg.Add(1) go func(u url.URL, ln net.Listener) { + defer e.wg.Done() e.cfg.logger.Info( "serving metrics", zap.String("address", u.String()), @@ -882,9 +917,6 @@ func (e *Etcd) serveMetrics() (err error) { } func (e *Etcd) errHandler(err error) { - e.wg.Add(1) - defer e.wg.Done() - select { case <-e.stopc: return diff --git a/server/embed/serve.go b/server/embed/serve.go index 1a46f51af93..55ffe3743f4 100644 --- a/server/embed/serve.go +++ b/server/embed/serve.go @@ -60,13 +60,23 @@ type serveCtx struct { insecure bool httpOnly bool + // ctx is used to control the grpc gateway. Terminate the grpc gateway + // by calling `cancel` when shutting down the etcd. ctx context.Context cancel context.CancelFunc userHandlers map[string]http.Handler serviceRegister func(*grpc.Server) - serversC chan *servers - closeOnce sync.Once + + // serversC is used to receive the http and grpc server objects (created + // in `serve`), both of which will be closed when shutting down the etcd. + // Close it when `serve` returns or when etcd fails to bootstrap. + serversC chan *servers + // closeOnce is to ensure `serversC` is closed only once. + closeOnce sync.Once + + // wg is used to track the lifecycle of all sub goroutines created by `serve`. + wg sync.WaitGroup } type servers struct { @@ -180,13 +190,17 @@ func (sctx *serveCtx) serve( server = m.Serve httpl := m.Match(cmux.HTTP1()) + sctx.wg.Add(1) go func(srvhttp *http.Server, tlsLis net.Listener) { + defer sctx.wg.Done() errHandler(srvhttp.Serve(tlsLis)) }(srv, httpl) if grpcEnabled { grpcl := m.Match(cmux.HTTP2()) + sctx.wg.Add(1) go func(gs *grpc.Server, l net.Listener) { + defer sctx.wg.Done() errHandler(gs.Serve(l)) }(gs, grpcl) } @@ -246,11 +260,13 @@ func (sctx *serveCtx) serve( } else { server = m.Serve - tlsl, err := transport.NewTLSListener(m.Match(cmux.Any()), tlsinfo) - if err != nil { - return err + tlsl, tlsErr := transport.NewTLSListener(m.Match(cmux.Any()), tlsinfo) + if tlsErr != nil { + return tlsErr } + sctx.wg.Add(1) go func(srvhttp *http.Server, tlsl net.Listener) { + defer sctx.wg.Done() errHandler(srvhttp.Serve(tlsl)) }(srv, tlsl) } @@ -263,7 +279,11 @@ func (sctx *serveCtx) serve( ) } - return server() + err = server() + sctx.close() + // ensure all goroutines, which are created by this method, to complete before this method returns. + sctx.wg.Wait() + return err } func configureHttpServer(srv *http.Server, cfg config.ServerConfig) error { @@ -316,7 +336,9 @@ func (sctx *serveCtx) registerGateway(dial func(ctx context.Context) (*grpc.Clie return nil, err } } + sctx.wg.Add(1) go func() { + defer sctx.wg.Done() <-ctx.Done() if cerr := conn.Close(); cerr != nil { sctx.lg.Warn( From 5bca08ec10d2ffcc658a5002381a3e5600d7132f Mon Sep 17 00:00:00 2001 From: James Blair Date: Sat, 25 Jan 2025 08:14:51 +1300 Subject: [PATCH 34/41] version: bump up to 3.5.18 Signed-off-by: James Blair --- api/version/version.go | 2 +- client/v2/go.mod | 4 ++-- client/v3/go.mod | 4 ++-- etcdctl/go.mod | 16 ++++++++-------- etcdutl/go.mod | 14 +++++++------- go.mod | 20 ++++++++++---------- pkg/go.mod | 2 +- raft/go.mod | 2 +- server/go.mod | 12 ++++++------ tests/go.mod | 16 ++++++++-------- 10 files changed, 46 insertions(+), 46 deletions(-) diff --git a/api/version/version.go b/api/version/version.go index ca6efc51367..15c99a2c282 100644 --- a/api/version/version.go +++ b/api/version/version.go @@ -26,7 +26,7 @@ import ( var ( // MinClusterVersion is the min cluster version this etcd binary is compatible with. MinClusterVersion = "3.0.0" - Version = "3.5.17" + Version = "3.5.18" APIVersion = "unknown" // Git SHA Value will be set during build diff --git a/client/v2/go.mod b/client/v2/go.mod index b07fcf148a8..58084b21ecb 100644 --- a/client/v2/go.mod +++ b/client/v2/go.mod @@ -7,8 +7,8 @@ toolchain go1.22.11 require ( github.com/json-iterator/go v1.1.11 github.com/modern-go/reflect2 v1.0.1 - go.etcd.io/etcd/api/v3 v3.5.17 - go.etcd.io/etcd/client/pkg/v3 v3.5.17 + go.etcd.io/etcd/api/v3 v3.5.18 + go.etcd.io/etcd/client/pkg/v3 v3.5.18 ) require ( diff --git a/client/v3/go.mod b/client/v3/go.mod index 5fbe0f0cf9c..e1254b7bbc9 100644 --- a/client/v3/go.mod +++ b/client/v3/go.mod @@ -8,8 +8,8 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/prometheus/client_golang v1.11.1 - go.etcd.io/etcd/api/v3 v3.5.17 - go.etcd.io/etcd/client/pkg/v3 v3.5.17 + go.etcd.io/etcd/api/v3 v3.5.18 + go.etcd.io/etcd/client/pkg/v3 v3.5.18 go.uber.org/zap v1.17.0 google.golang.org/grpc v1.59.0 sigs.k8s.io/yaml v1.2.0 diff --git a/etcdctl/go.mod b/etcdctl/go.mod index 88fd423319a..ef19c42eaa3 100644 --- a/etcdctl/go.mod +++ b/etcdctl/go.mod @@ -11,12 +11,12 @@ require ( github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 github.com/urfave/cli v1.22.4 - go.etcd.io/etcd/api/v3 v3.5.17 - go.etcd.io/etcd/client/pkg/v3 v3.5.17 - go.etcd.io/etcd/client/v2 v2.305.17 - go.etcd.io/etcd/client/v3 v3.5.17 - go.etcd.io/etcd/etcdutl/v3 v3.5.17 - go.etcd.io/etcd/pkg/v3 v3.5.17 + go.etcd.io/etcd/api/v3 v3.5.18 + go.etcd.io/etcd/client/pkg/v3 v3.5.18 + go.etcd.io/etcd/client/v2 v2.305.18 + go.etcd.io/etcd/client/v3 v3.5.18 + go.etcd.io/etcd/etcdutl/v3 v3.5.18 + go.etcd.io/etcd/pkg/v3 v3.5.18 go.uber.org/zap v1.17.0 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba google.golang.org/grpc v1.59.0 @@ -50,8 +50,8 @@ require ( github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/bbolt v1.3.11 // indirect - go.etcd.io/etcd/raft/v3 v3.5.17 // indirect - go.etcd.io/etcd/server/v3 v3.5.17 // indirect + go.etcd.io/etcd/raft/v3 v3.5.18 // indirect + go.etcd.io/etcd/server/v3 v3.5.18 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect go.opentelemetry.io/otel v1.20.0 // indirect go.opentelemetry.io/otel/metric v1.20.0 // indirect diff --git a/etcdutl/go.mod b/etcdutl/go.mod index e610c03069d..43fccd4403c 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -27,12 +27,12 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/spf13/cobra v1.1.3 go.etcd.io/bbolt v1.3.11 - go.etcd.io/etcd/api/v3 v3.5.17 - go.etcd.io/etcd/client/pkg/v3 v3.5.17 - go.etcd.io/etcd/client/v3 v3.5.17 - go.etcd.io/etcd/pkg/v3 v3.5.17 - go.etcd.io/etcd/raft/v3 v3.5.17 - go.etcd.io/etcd/server/v3 v3.5.17 + go.etcd.io/etcd/api/v3 v3.5.18 + go.etcd.io/etcd/client/pkg/v3 v3.5.18 + go.etcd.io/etcd/client/v3 v3.5.18 + go.etcd.io/etcd/pkg/v3 v3.5.18 + go.etcd.io/etcd/raft/v3 v3.5.18 + go.etcd.io/etcd/server/v3 v3.5.18 go.uber.org/zap v1.17.0 ) @@ -60,7 +60,7 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect - go.etcd.io/etcd/client/v2 v2.305.17 // indirect + go.etcd.io/etcd/client/v2 v2.305.18 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect go.opentelemetry.io/otel v1.20.0 // indirect go.opentelemetry.io/otel/metric v1.20.0 // indirect diff --git a/go.mod b/go.mod index 2a7c92a88cc..5a7b225e5ef 100644 --- a/go.mod +++ b/go.mod @@ -22,16 +22,16 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/spf13/cobra v1.1.3 go.etcd.io/bbolt v1.3.11 - go.etcd.io/etcd/api/v3 v3.5.17 - go.etcd.io/etcd/client/pkg/v3 v3.5.17 - go.etcd.io/etcd/client/v2 v2.305.17 - go.etcd.io/etcd/client/v3 v3.5.17 - go.etcd.io/etcd/etcdctl/v3 v3.5.17 - go.etcd.io/etcd/etcdutl/v3 v3.5.17 - go.etcd.io/etcd/pkg/v3 v3.5.17 - go.etcd.io/etcd/raft/v3 v3.5.17 - go.etcd.io/etcd/server/v3 v3.5.17 - go.etcd.io/etcd/tests/v3 v3.5.17 + go.etcd.io/etcd/api/v3 v3.5.18 + go.etcd.io/etcd/client/pkg/v3 v3.5.18 + go.etcd.io/etcd/client/v2 v2.305.18 + go.etcd.io/etcd/client/v3 v3.5.18 + go.etcd.io/etcd/etcdctl/v3 v3.5.18 + go.etcd.io/etcd/etcdutl/v3 v3.5.18 + go.etcd.io/etcd/pkg/v3 v3.5.18 + go.etcd.io/etcd/raft/v3 v3.5.18 + go.etcd.io/etcd/server/v3 v3.5.18 + go.etcd.io/etcd/tests/v3 v3.5.18 go.uber.org/zap v1.17.0 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba google.golang.org/grpc v1.59.0 diff --git a/pkg/go.mod b/pkg/go.mod index 8f41a489525..f146e432aa8 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -10,7 +10,7 @@ require ( github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 - go.etcd.io/etcd/client/pkg/v3 v3.5.17 + go.etcd.io/etcd/client/pkg/v3 v3.5.18 go.uber.org/zap v1.17.0 google.golang.org/grpc v1.59.0 ) diff --git a/raft/go.mod b/raft/go.mod index 6348b3f61d7..16296cb87ba 100644 --- a/raft/go.mod +++ b/raft/go.mod @@ -8,7 +8,7 @@ require ( github.com/cockroachdb/datadriven v1.0.2 github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.5.4 - go.etcd.io/etcd/client/pkg/v3 v3.5.17 + go.etcd.io/etcd/client/pkg/v3 v3.5.18 ) require ( diff --git a/server/go.mod b/server/go.mod index 0d1887ad75b..ecc41bda77f 100644 --- a/server/go.mod +++ b/server/go.mod @@ -26,12 +26,12 @@ require ( github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go.etcd.io/bbolt v1.3.11 - go.etcd.io/etcd/api/v3 v3.5.17 - go.etcd.io/etcd/client/pkg/v3 v3.5.17 - go.etcd.io/etcd/client/v2 v2.305.17 - go.etcd.io/etcd/client/v3 v3.5.17 - go.etcd.io/etcd/pkg/v3 v3.5.17 - go.etcd.io/etcd/raft/v3 v3.5.17 + go.etcd.io/etcd/api/v3 v3.5.18 + go.etcd.io/etcd/client/pkg/v3 v3.5.18 + go.etcd.io/etcd/client/v2 v2.305.18 + go.etcd.io/etcd/client/v3 v3.5.18 + go.etcd.io/etcd/pkg/v3 v3.5.18 + go.etcd.io/etcd/raft/v3 v3.5.18 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 go.opentelemetry.io/otel v1.20.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 diff --git a/tests/go.mod b/tests/go.mod index 18d8f35f315..f4fc7fa4613 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -30,14 +30,14 @@ require ( github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 - go.etcd.io/etcd/api/v3 v3.5.17 - go.etcd.io/etcd/client/pkg/v3 v3.5.17 - go.etcd.io/etcd/client/v2 v2.305.17 - go.etcd.io/etcd/client/v3 v3.5.17 - go.etcd.io/etcd/etcdutl/v3 v3.5.17 - go.etcd.io/etcd/pkg/v3 v3.5.17 - go.etcd.io/etcd/raft/v3 v3.5.17 - go.etcd.io/etcd/server/v3 v3.5.17 + go.etcd.io/etcd/api/v3 v3.5.18 + go.etcd.io/etcd/client/pkg/v3 v3.5.18 + go.etcd.io/etcd/client/v2 v2.305.18 + go.etcd.io/etcd/client/v3 v3.5.18 + go.etcd.io/etcd/etcdutl/v3 v3.5.18 + go.etcd.io/etcd/pkg/v3 v3.5.18 + go.etcd.io/etcd/raft/v3 v3.5.18 + go.etcd.io/etcd/server/v3 v3.5.18 go.etcd.io/gofail v0.2.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 go.opentelemetry.io/otel v1.20.0 From 04f775ed52d2650c37154c74cf54403a0fb8e88f Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Thu, 12 Sep 2024 09:47:59 +0200 Subject: [PATCH 35/41] DOWNSTREAM: : ETCD-656: Automate datadir move after quorum-restore This PR will add the notion of cluster ID into the initial cluster discovery process. This allows us to automatically archive a data directory when we detect the cluster identifier changing. The cluster identifier will only change when we are running a restore operation. The detection requires that the revision.json (created by the revision monitor sidecar) contains the cluster id. The cluster identifier is also stored in the local WAL, which is much more expensive to parse. We're going to only fallback to it when we could not get the cluster id from the revision.json for any reason. Otherwise the WAL stays untouched, no repair operations are attempted when it is found corrupted. Signed-off-by: Thomas Jungblut --- .../initial-cluster.go | 393 ++++++++++++++++++ .../initial-cluster_test.go | 113 +++++ .../discover-etcd-initial-cluster/walutil.go | 58 +++ 3 files changed, 564 insertions(+) create mode 100644 openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster.go create mode 100644 openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster_test.go create mode 100644 openshift-tools/pkg/discover-etcd-initial-cluster/walutil.go diff --git a/openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster.go b/openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster.go new file mode 100644 index 00000000000..0c6bf6163e9 --- /dev/null +++ b/openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster.go @@ -0,0 +1,393 @@ +package discover_etcd_initial_cluster + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "go.uber.org/zap" + "net/url" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "google.golang.org/grpc" + + "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/client/pkg/v3/transport" + "go.etcd.io/etcd/client/v3" +) + +type DiscoverEtcdInitialClusterOptions struct { + // TargetPeerURLHost is the host portion of the peer URL. It is used to match on. (either IP or hostname) + TargetPeerURLHost string + // TargetPeerURLScheme is the host scheme of the peer URL. + TargetPeerURLScheme string + // TargetPeerURLPort is the host port of the peer URL. + TargetPeerURLPort string + // TargetName is the name to assign to this peer if we create it. + TargetName string + + // CABundleFile is the file to use to trust the etcd server + CABundleFile string + // ClientCertFile is the client cert to use to authenticate this binary to etcd + ClientCertFile string + // ClientKeyFile is the client key to use to authenticate this binary to etcd + ClientKeyFile string + // Endpoints is a list of all the endpoints to use to try to contact etcd + Endpoints []string + + // DataDir is the directory created when etcd starts the first time + DataDir string +} + +type revisionStruct struct { + ClusterId uint64 `json:"clusterId,omitempty"` +} + +func NewDiscoverEtcdInitialCluster() *DiscoverEtcdInitialClusterOptions { + return &DiscoverEtcdInitialClusterOptions{ + TargetPeerURLScheme: "https", + TargetPeerURLPort: "2380", + } +} + +func NewDiscoverEtcdInitialClusterCommand() *cobra.Command { + o := NewDiscoverEtcdInitialCluster() + + cmd := &cobra.Command{ + Use: "discover-etcd-initial-cluster", + Short: "output the value for ETCD_INITIAL_CLUSTER in openshift etcd static pod", + Long: `output the value for ETCD_INITIAL_CLUSTER in openshift etcd static pod + +Please see docs for more details: +https://github.com/openshift/cluster-etcd-operator/tree/master/docs/discover-etcd-initial-cluster.md +`, + Run: func(cmd *cobra.Command, args []string) { + if err := o.Validate(); err != nil { + fmt.Fprint(os.Stderr, err) + os.Exit(1) + } + + if err := o.Run(); err != nil { + fmt.Fprint(os.Stderr, err) + os.Exit(1) + } + }, + } + o.BindFlags(cmd.Flags()) + + return cmd +} + +func (o *DiscoverEtcdInitialClusterOptions) BindFlags(flags *pflag.FlagSet) { + flags.StringVar(&o.CABundleFile, "cacert", o.CABundleFile, "file to use to verify the identity of the etcd server") + flags.StringVar(&o.ClientCertFile, "cert", o.ClientCertFile, "client cert to use to authenticate this binary to etcd") + flags.StringVar(&o.ClientKeyFile, "key", o.ClientKeyFile, "client key to use to authenticate this binary to etcd") + flags.StringSliceVar(&o.Endpoints, "endpoints", o.Endpoints, "list of all the endpoints to use to try to contact etcd") + flags.StringVar(&o.DataDir, "data-dir", o.DataDir, "dir to stat for existence of the member directory") + flags.StringVar(&o.TargetPeerURLHost, "target-peer-url-host", o.TargetPeerURLHost, "host portion of the peer URL. It is used to match on. (either IP or hostname)") + flags.StringVar(&o.TargetName, "target-name", o.TargetName, "name to assign to this peer if we create it") +} + +func (o *DiscoverEtcdInitialClusterOptions) Validate() error { + if len(o.CABundleFile) == 0 { + return fmt.Errorf("missing --cacert") + } + if len(o.ClientCertFile) == 0 { + return fmt.Errorf("missing --cert") + } + if len(o.ClientKeyFile) == 0 { + return fmt.Errorf("missing --key") + } + if len(o.Endpoints) == 0 { + return fmt.Errorf("missing --endpoints") + } + if len(o.DataDir) == 0 { + return fmt.Errorf("missing --data-dir") + } + if len(o.TargetPeerURLHost) == 0 { + return fmt.Errorf("missing --target-peer-url-host") + } + if len(o.TargetName) == 0 { + return fmt.Errorf("missing --target-name") + } + if len(o.TargetPeerURLPort) == 0 { + return fmt.Errorf("missing TargetPeerURLPort") + } + if len(o.TargetPeerURLScheme) == 0 { + return fmt.Errorf("missing TargetPeerURLScheme") + } + return nil +} + +func (o *DiscoverEtcdInitialClusterOptions) Run() error { + var dataDirExists bool + // check if dataDir structure exists + _, err := os.Stat(filepath.Join(o.DataDir, "member/snap")) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + fmt.Fprintf(os.Stderr, "dataDir is present on %s\n", o.TargetName) + dataDirExists = true + } + + client, err := o.getClient() + + // Condition: create client fail with dataDir + // Possible reasons for this condition. + // 1.) single node etcd cluster + // 2.) transient networking problem + // 3.) on and off flow + // Result: start etcd with empty initial config + if err != nil && dataDirExists { + fmt.Fprintf(os.Stderr, "failed to create etcd client, but the server is already initialized as member %q before, starting as etcd member: %v", o.TargetName, err.Error()) + return nil + } + // Condition: create client fail, no dataDir + // Possible reasons for the condition include transient network partition. + // Result: return error and restart container + if err != nil { + return fmt.Errorf("failed to create etcd client: %v", err) + } + defer client.Close() + + localClusterIdentifier, err := o.findLocalClusterIdentifier() + if err != nil { + return fmt.Errorf("could not find local cluster id: %w", err) + } + + // the startupProbe waits for 180s, so we are giving 135s to this process and 45s to the etcd that runs after us. + for i := 0; i < 135; i++ { + fmt.Fprintf(os.Stderr, "#### attempt %d\n", i) + + // Check member list on each iteration for changes. + cluster, err := client.Cluster.(clientv3.NonLinearizeableMemberLister).NonLinearizeableMemberList(context.TODO()) + if err != nil { + fmt.Fprintf(os.Stderr, "member list request failed: %v", err) + continue + } + fmt.Fprintf(os.Stderr, "Live Cluster ID: [%s], local: [%s] \n", + strconv.FormatUint(cluster.Header.ClusterId, 16), + strconv.FormatUint(localClusterIdentifier, 16)) + logCurrentMembership(cluster.Members) + + mismatchingClusterId := localClusterIdentifier != 0 && cluster.Header.ClusterId != localClusterIdentifier + + initialCluster, memberFound, err := o.getInitialCluster( + cluster.Members, + dataDirExists, + mismatchingClusterId) + if err != nil && memberFound { + return err + } + // If member is not yet part of the cluster print to stderr and retry. + if err != nil && !memberFound { + fmt.Fprintf(os.Stderr, " %s\n#### sleeping...\n", err.Error()) + time.Sleep(1 * time.Second) + continue + } + // Empty string value for initialCluster is valid. + // this is important to go as the only resulting string to stdout, as it is creating an env var in the pod yaml + fmt.Println(initialCluster) + + return nil + } + return fmt.Errorf("timed out") +} + +func (o *DiscoverEtcdInitialClusterOptions) getInitialCluster(members []*etcdserverpb.Member, dataDirExists, mismatchingClusterId bool) (string, bool, error) { + target := url.URL{ + Scheme: o.TargetPeerURLScheme, + Host: fmt.Sprintf("%s:%s", o.TargetPeerURLHost, o.TargetPeerURLPort), + } + + targetMember, memberFound := checkTargetMember(target, members) + + // Condition: unstarted member found, no dataDir + // This member is part of the cluster but has not yet started. We know this because the name is populated at + // runtime which this member does not have. + // Result: populate initial cluster so etcd can communicate with peers during startup + if memberFound && targetMember.Name == "" && !dataDirExists { + return formatInitialCluster(o.TargetName, targetMember, members), memberFound, nil + } + + // Condition: unstarted member found with dataDir + // This member is part of the cluster but has not yet started, yet has a dataDir. + // Result: archive old dataDir and return error which will restart container + if memberFound && targetMember.Name == "" && dataDirExists { + archivedDir, err := archiveDataDir(o.DataDir) + if err != nil { + return "", memberFound, err + } + return "", memberFound, fmt.Errorf("member %q is unstarted but previous members dataDir exists: archiving to %q", target.String(), archivedDir) + } + + // Condition: started member found with dataDir + // Result: start etcd with empty initial config + if memberFound && dataDirExists { + return "", memberFound, nil + } + + // Condition: started member found, no dataDir + // A member is not actually gone forever unless it is removed from cluster with MemberRemove or the dataDir is destroyed. Since + // this is the latter. Do not let etcd start and report the condition as an error. + // Result: return error and restart container + if memberFound && !dataDirExists { + return "", memberFound, fmt.Errorf("member %q dataDir has been destroyed and must be removed from the cluster", target.String()) + } + + // Condition: member not found with dataDir + // The member has been removed from the cluster, likely from a restore operation. + // Result: if the cluster ID does not match anymore, we're archiving the data dir and returning a signal to start etcd. + // If it matches, there is a scaling problem and the datadir must be removed manually. + if !memberFound && dataDirExists { + if mismatchingClusterId { + _, err := archiveDataDir(o.DataDir) + if err != nil { + return "", memberFound, err + } + + return "", true, nil + } + + return "", memberFound, fmt.Errorf("member %q not found in member list but dataDir exists, check operator logs for possible scaling problems\n", target.String()) + } + + // Condition: member not found, no dataDir + // The member list does not reflect the target member as it is waiting to be scaled up. + // Result: retry + if !memberFound && !dataDirExists { + return "", memberFound, fmt.Errorf("member %q not found in member list, check operator logs for possible scaling problems", target.String()) + } + + return "", memberFound, nil +} + +func (o *DiscoverEtcdInitialClusterOptions) findLocalClusterIdentifier() (uint64, error) { + // we favor the revision.json as its fastest to parse, if we error here we can still rely on the WAL parsing (slow) + fromRevisionFile, revErr := o.findLocalClusterIdFromRevFile() + // zero can also be returned when the revision file does not contain the ClusterId attribute + if revErr != nil || fromRevisionFile == 0 { + fmt.Fprintf(os.Stderr, "could not parse revision.json, falling back to WAL parsing. Err=%v", revErr) + fromWal, walErr := o.findLocalClusterIdFromWal() + if walErr != nil { + return 0, fmt.Errorf("couldn't find cluster id in WAL or revision: %v", errors.Join(walErr, revErr)) + } + return fromWal, nil + } + + return fromRevisionFile, nil +} + +func (o *DiscoverEtcdInitialClusterOptions) findLocalClusterIdFromRevFile() (uint64, error) { + content, err := os.ReadFile(path.Join(o.DataDir, "revision.json")) + if err != nil { + return 0, err + } + + result := revisionStruct{} + err = json.Unmarshal(content, &result) + if err != nil { + return 0, err + } + + return result.ClusterId, nil +} + +func (o *DiscoverEtcdInitialClusterOptions) findLocalClusterIdFromWal() (uint64, error) { + x, err := readClusterIdFromWAL(zap.NewNop(), o.DataDir) + return uint64(x), err +} + +func (o *DiscoverEtcdInitialClusterOptions) getClient() (*clientv3.Client, error) { + dialOptions := []grpc.DialOption{ + grpc.WithBlock(), // block until the underlying connection is up + } + + tlsInfo := transport.TLSInfo{ + CertFile: o.ClientCertFile, + KeyFile: o.ClientKeyFile, + TrustedCAFile: o.CABundleFile, + } + tlsConfig, err := tlsInfo.ClientConfig() + if err != nil { + return nil, err + } + + cfg := &clientv3.Config{ + DialOptions: dialOptions, + Endpoints: o.Endpoints, + DialTimeout: 2 * time.Second, // fail fast + TLS: tlsConfig, + } + + return clientv3.New(*cfg) +} + +func archiveDataDir(dataDir string) (string, error) { + // for testing + if strings.HasPrefix(dataDir, "/tmp") { + return "/tmp-removed-archive", nil + } + sourceDir := filepath.Join(dataDir, "member") + targetDir := filepath.Join(sourceDir + "-removed-archive-" + time.Now().Format("2006-01-02-030405")) + + fmt.Fprintf(os.Stderr, "attempting to archive %s to %s", sourceDir, targetDir) + if err := os.Rename(sourceDir, targetDir); err != nil { + return "", err + } + fmt.Fprintf(os.Stdout, "moved datadir successfully to %s\n", targetDir) + return targetDir, nil +} + +func stringifyMember(member *etcdserverpb.Member) string { + return fmt.Sprintf("{name=%q, peerURLs=[%s}, clientURLs=[%s]", member.Name, strings.Join(member.PeerURLs, ","), strings.Join(member.ClientURLs, ",")) +} + +// checkTargetMember populates the target member if it is part of the member list and print member details into etcd log. +func checkTargetMember(target url.URL, members []*etcdserverpb.Member) (*etcdserverpb.Member, bool) { + for _, member := range members { + for _, peerURL := range member.PeerURLs { + if peerURL == target.String() { + fmt.Fprintf(os.Stderr, " target=%s\n", stringifyMember(member)) + return member, true + } + } + } + return nil, false +} + +// logCurrentMembership prints the current etcd membership to the etcd logs. +func logCurrentMembership(members []*etcdserverpb.Member) { + for _, member := range members { + fmt.Fprintf(os.Stderr, " member=%s\n", stringifyMember(member)) + } + return +} + +// formatInitialCluster populates the initial cluster comma delimited string in the format =. +func formatInitialCluster(targetName string, target *etcdserverpb.Member, members []*etcdserverpb.Member) string { + var initialCluster []string + for _, member := range members { + if member.Name == "" { // this is the signal for whether or not a given peer is started + continue + } + for _, peerURL := range member.PeerURLs { + initialCluster = append(initialCluster, fmt.Sprintf("%s=%s", member.Name, peerURL)) + } + } + if target.Name == "" { + // Adding unstarted member to the end of list + initialCluster = append(initialCluster, fmt.Sprintf("%s=%s", targetName, target.PeerURLs[0])) + } + + return strings.Join(initialCluster, ",") +} diff --git a/openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster_test.go b/openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster_test.go new file mode 100644 index 00000000000..e6128be5af1 --- /dev/null +++ b/openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster_test.go @@ -0,0 +1,113 @@ +package discover_etcd_initial_cluster + +import ( + "regexp" + "testing" + + "go.etcd.io/etcd/api/v3/etcdserverpb" +) + +var ( + emptyInitialCluster = "" + startedEtcdMember = &etcdserverpb.Member{Name: "etcd-0", PeerURLs: []string{"https://etcd-0:2380"}} + unstartedEtcdMember = &etcdserverpb.Member{Name: "", PeerURLs: []string{"https://etcd-0:2380"}} + notFoundEtcdMember = &etcdserverpb.Member{Name: "not-found", PeerURLs: []string{"https://not-found:2380"}} +) + +func Test_ensureValidMember(t *testing.T) { + tests := map[string]struct { + member *etcdserverpb.Member + dataDirExists bool + wantMemberFound bool + mismatchingClusterId bool + wantInitialCluster string + wantErr bool + wantErrString string + }{ + "started member found no dataDir": { + member: startedEtcdMember, + wantMemberFound: true, + dataDirExists: false, + wantInitialCluster: emptyInitialCluster, + wantErr: true, + wantErrString: "dataDir has been destroyed and must be removed from the cluster", + }, + "started member found with dataDir": { + member: startedEtcdMember, + wantMemberFound: true, + dataDirExists: true, + wantInitialCluster: emptyInitialCluster, + wantErr: false, + }, + "member not found with dataDir": { + member: notFoundEtcdMember, + wantMemberFound: false, + dataDirExists: true, + wantInitialCluster: emptyInitialCluster, + wantErr: true, + wantErrString: "check operator logs for possible scaling problems", + }, + "member not found with dataDir and mismatching clusterid": { + member: notFoundEtcdMember, + wantMemberFound: true, + mismatchingClusterId: true, + dataDirExists: true, + wantInitialCluster: "", + wantErr: false, + }, + "member not found no dataDir": { + member: notFoundEtcdMember, + wantMemberFound: false, + dataDirExists: false, + wantInitialCluster: emptyInitialCluster, + wantErr: true, + wantErrString: "check operator logs for possible scaling problems", + }, + "unstarted member found with dataDir": { + member: unstartedEtcdMember, + wantMemberFound: true, + dataDirExists: true, + wantInitialCluster: emptyInitialCluster, + wantErr: true, + wantErrString: "previous members dataDir exists: archiving", + }, + "unstarted member found no dataDir": { + member: unstartedEtcdMember, + wantMemberFound: true, + dataDirExists: false, + wantInitialCluster: "etcd-0=https://etcd-0:2380", + wantErr: false, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + o := DiscoverEtcdInitialClusterOptions{ + TargetPeerURLHost: "etcd-0", + TargetPeerURLScheme: "https", + TargetPeerURLPort: "2380", + TargetName: "etcd-0", + DataDir: "/tmp", + } + gotInitialCluster, gotMemberFound, err := o.getInitialCluster([]*etcdserverpb.Member{test.member}, test.dataDirExists, test.mismatchingClusterId) + if gotInitialCluster != test.wantInitialCluster { + t.Fatalf("initialCluster: want: %q, got: %q", test.wantInitialCluster, gotInitialCluster) + } + if err != nil && !test.wantErr { + t.Fatalf("unexpected error: %v", err) + } + if err == nil && test.wantErr { + t.Fatal("expected error got nil") + } + if gotMemberFound != test.wantMemberFound { + t.Fatalf("memberFound: want %v, got %v", gotMemberFound, test.wantMemberFound) + } + if test.wantErrString != "" { + regex := regexp.MustCompile(test.wantErrString) + if len(regex.FindAll([]byte(err.Error()), -1)) != 1 { + t.Fatalf("unexpected error wanted %q in %q", test.wantErrString, err.Error()) + } + } + }) + } + +} diff --git a/openshift-tools/pkg/discover-etcd-initial-cluster/walutil.go b/openshift-tools/pkg/discover-etcd-initial-cluster/walutil.go new file mode 100644 index 00000000000..a7695b20711 --- /dev/null +++ b/openshift-tools/pkg/discover-etcd-initial-cluster/walutil.go @@ -0,0 +1,58 @@ +package discover_etcd_initial_cluster + +import ( + "errors" + "go.etcd.io/etcd/server/v3/datadir" + "go.etcd.io/etcd/server/v3/etcdserver/api/snap" + "path/filepath" + + pb "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/client/pkg/v3/types" + "go.etcd.io/etcd/pkg/v3/pbutil" + "go.etcd.io/etcd/server/v3/wal" + "go.etcd.io/etcd/server/v3/wal/walpb" + + "go.uber.org/zap" +) + +func readClusterIdFromWAL(lg *zap.Logger, dataDir string) (cid types.ID, err error) { + walDir := datadir.ToWalDir(dataDir) + snapDir := filepath.Join(datadir.ToMemberDir(dataDir), "snap") + + // Find a snapshot to start/restart a raft node + ss := snap.New(lg, snapDir) + + var walSnaps []walpb.Snapshot + walSnaps, err = wal.ValidSnapshotEntries(lg, walDir) + if err != nil { + return 0, err + } + + snapshot, err := ss.LoadNewestAvailable(walSnaps) + if err != nil && !errors.Is(err, snap.ErrNoSnapshot) { + return 0, err + } + + var walSnap walpb.Snapshot + if snapshot != nil { + walSnap.Index, walSnap.Term = snapshot.Metadata.Index, snapshot.Metadata.Term + } + + w, err := wal.Open(lg, walDir, walSnap) + if err != nil { + return 0, err + } + + defer func() { + err = errors.Join(err, w.Close()) + }() + + walMeta, _, _, err := w.ReadAll() + if err != nil { + return 0, err + } + + var metadata pb.Metadata + pbutil.MustUnmarshal(&metadata, walMeta) + return types.ID(metadata.ClusterID), nil +} From 778cbb72c407b7471d52993cf16b476632efeb3e Mon Sep 17 00:00:00 2001 From: Mustafa Elbehery Date: Thu, 26 Sep 2024 15:13:05 +0200 Subject: [PATCH 36/41] NO-JIRA: use golang 1.22 image --- .ci-operator.yaml | 4 ++ Dockerfile.art | 26 ++++++++++++ Dockerfile.installer | 48 ++++++++++++++++++++++ Dockerfile.installer.art | 87 ++++++++++++++++++++++++++++++++++++++++ Dockerfile.rhel | 24 +++++++++++ 5 files changed, 189 insertions(+) create mode 100644 .ci-operator.yaml create mode 100644 Dockerfile.art create mode 100644 Dockerfile.installer create mode 100644 Dockerfile.installer.art create mode 100644 Dockerfile.rhel diff --git a/.ci-operator.yaml b/.ci-operator.yaml new file mode 100644 index 00000000000..64887a08b41 --- /dev/null +++ b/.ci-operator.yaml @@ -0,0 +1,4 @@ +build_root_image: + name: release + namespace: openshift + tag: rhel-9-release-golang-1.22-openshift-4.18 diff --git a/Dockerfile.art b/Dockerfile.art new file mode 100644 index 00000000000..be01f479e97 --- /dev/null +++ b/Dockerfile.art @@ -0,0 +1,26 @@ +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS builder + +COPY $REMOTE_SOURCES $REMOTE_SOURCES_DIR +WORKDIR $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app +RUN ls -lR $REMOTE_SOURCES_DIR +RUN cat $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env +RUN source $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env && GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh + +RUN mkdir -p /go/src/go.etcd.io/ +RUN ln -s $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app /go/src/go.etcd.io/etcd + +# stage 2 (note: any changes should reflect in Dockerfile.rhel) +FROM registry.ci.openshift.org/ocp/4.18:base-rhel9 + +ENTRYPOINT ["/usr/bin/etcd"] + +RUN yum install --setopt=tsflags=nodocs -y jq && yum clean all && rm -rf /var/cache/yum/* + +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcd /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcdctl /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcdutl /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/discover-etcd-initial-cluster /usr/bin/ + +LABEL io.k8s.display-name="etcd server" \ + io.k8s.description="etcd is a distributed key-value store which stores the persistent master state for Kubernetes and OpenShift." \ + maintainer="Sam Batschelet " diff --git a/Dockerfile.installer b/Dockerfile.installer new file mode 100644 index 00000000000..ae2c7a4d122 --- /dev/null +++ b/Dockerfile.installer @@ -0,0 +1,48 @@ +# This Dockerfile builds an image containing Mac and Linux ARM64/AMD64 versions of the etcd. +# The resulting image is used to build the statically-linked openshift-installer binary. + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS macbuilder +ENV GO_COMPLIANCE_EXCLUDE=".*" +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS macarmbuilder +ENV GO_COMPLIANCE_EXCLUDE=".*" +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS linuxbuilder +ENV GO_COMPLIANCE_EXCLUDE=".*" +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS linuxarmbuilder +ENV GO_COMPLIANCE_EXCLUDE=".*" +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS builder +ENV GO_COMPLIANCE_EXCLUDE=".*" +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN CGO_ENABLED=0 GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh +RUN mkdir -p /usr/share/openshift/$(go env GOOS)/$(go env GOHOSTARCH) && \ + mv bin/etcd /usr/share/openshift/$(go env GOOS)/$(go env GOHOSTARCH)/ + +# stage 2 +FROM registry.ci.openshift.org/ocp/4.18:base-rhel9 + +RUN yum install --setopt=tsflags=nodocs -y jq && yum clean all && rm -rf /var/cache/yum/* + +COPY --from=macbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/darwin/amd64/etcd +COPY --from=macarmbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/darwin/arm64/etcd +COPY --from=linuxbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/linux/amd64/etcd +COPY --from=linuxarmbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/linux/arm64/etcd +COPY --from=builder /usr/share/openshift/ /usr/share/openshift/ + +# This image is not an operator, it is only used as part of the build pipeline +LABEL io.openshift.release.operator=false diff --git a/Dockerfile.installer.art b/Dockerfile.installer.art new file mode 100644 index 00000000000..bbe1dbd6f33 --- /dev/null +++ b/Dockerfile.installer.art @@ -0,0 +1,87 @@ +# This Dockerfile builds an image containing Mac and Linux ARM64/AMD64 versions of the etcd. +# The resulting image is used to build the statically-linked openshift-installer binary. + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS macbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" +COPY $REMOTE_SOURCES $REMOTE_SOURCES_DIR +WORKDIR $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app +RUN cat $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env +RUN mkdir -p /go/src/go.etcd.io/ +RUN ln -s $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app /go/src/go.etcd.io/etcd +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN source $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env \ + && export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS macarmbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" +COPY $REMOTE_SOURCES $REMOTE_SOURCES_DIR +WORKDIR $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app +RUN cat $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env +RUN mkdir -p /go/src/go.etcd.io/ +RUN ln -s $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app /go/src/go.etcd.io/etcd +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN source $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env \ + && export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS linuxbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" +COPY $REMOTE_SOURCES $REMOTE_SOURCES_DIR +WORKDIR $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app +RUN cat $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env +RUN mkdir -p /go/src/go.etcd.io/ +RUN ln -s $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app /go/src/go.etcd.io/etcd +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN source $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env \ + && export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS linuxarmbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" +COPY $REMOTE_SOURCES $REMOTE_SOURCES_DIR +WORKDIR $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app +RUN cat $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env +RUN mkdir -p /go/src/go.etcd.io/ +RUN ln -s $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app /go/src/go.etcd.io/etcd +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN source $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env \ + && export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=linux GOARCH=arm64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS builder +ENV GO_COMPLIANCE_EXCLUDE=".*" +COPY $REMOTE_SOURCES $REMOTE_SOURCES_DIR +WORKDIR $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app +RUN cat $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env +RUN mkdir -p /go/src/go.etcd.io/ +RUN ln -s $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app /go/src/go.etcd.io/etcd +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN source $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env \ + && export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 ./build.sh +RUN mkdir -p /usr/share/openshift/$(go env GOOS)/$(go env GOHOSTARCH) && \ + mv bin/etcd /usr/share/openshift/$(go env GOOS)/$(go env GOHOSTARCH)/ + +# stage 2 +FROM registry.ci.openshift.org/ocp/4.18:base-rhel9 + +RUN yum install --setopt=tsflags=nodocs -y jq && yum clean all && rm -rf /var/cache/yum/* + +COPY --from=macbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/darwin/amd64/etcd +COPY --from=macarmbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/darwin/arm64/etcd +COPY --from=linuxbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/linux/amd64/etcd +COPY --from=linuxarmbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/linux/arm64/etcd +COPY --from=builder /usr/share/openshift/ /usr/share/openshift/ + +# This image is not an operator, it is only used as part of the build pipeline +LABEL io.openshift.release.operator=false diff --git a/Dockerfile.rhel b/Dockerfile.rhel new file mode 100644 index 00000000000..b500e1edce3 --- /dev/null +++ b/Dockerfile.rhel @@ -0,0 +1,24 @@ +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS builder + +WORKDIR /go/src/go.etcd.io/etcd + + +COPY . . + +RUN GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh + +# stage 2 (note: any changes should reflect in Dockerfile.art) +FROM registry.ci.openshift.org/ocp/4.18:base-rhel9 + +ENTRYPOINT ["/usr/bin/etcd"] + +RUN yum install --setopt=tsflags=nodocs -y jq && yum clean all && rm -rf /var/cache/yum/* + +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcd /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcdctl /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcdutl /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/discover-etcd-initial-cluster /usr/bin/ + +LABEL io.k8s.display-name="etcd server" \ + io.k8s.description="etcd is a distributed key-value store which stores the persistent master state for Kubernetes and OpenShift." \ + maintainer="Sam Batschelet " From c0c66eea9c869d4edd96e08d50264e08b14befa9 Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Fri, 27 Oct 2023 15:05:10 +0200 Subject: [PATCH 37/41] DOWNSTREAM : resolve merge conflicts --- CHANGELOG/CHANGELOG-3.6.md | 0 OWNERS | 21 ++- REBASE.openshift.md | 87 +++++++++ ROADMAP.md | 21 --- build.sh | 13 ++ client/v3/patch_cluster.go | 24 +++ etcdctl/ctlv3/command/check.go | 7 +- openshift-hack/rebase.sh | 174 ++++++++++++++++++ .../discover-etcd-initial-cluster/main.go | 35 ++++ raft/OWNERS | 20 +- server/config/config.go | 3 + server/embed/config.go | 4 + server/embed/etcd.go | 4 +- server/etcdmain/config.go | 2 + server/etcdmain/grpc_proxy.go | 2 +- server/etcdmain/help.go | 4 +- server/etcdserver/api/membership/cluster.go | 74 +++++--- .../etcdserver/api/membership/cluster_opts.go | 43 +++++ .../etcdserver/api/membership/cluster_test.go | 35 +++- server/etcdserver/raft.go | 4 +- server/etcdserver/server.go | 17 +- tests/integration/clientv3/cluster_test.go | 33 ++-- tests/integration/cluster.go | 9 +- 23 files changed, 535 insertions(+), 101 deletions(-) create mode 100644 CHANGELOG/CHANGELOG-3.6.md create mode 100644 REBASE.openshift.md delete mode 100644 ROADMAP.md create mode 100644 client/v3/patch_cluster.go create mode 100755 openshift-hack/rebase.sh create mode 100644 openshift-tools/discover-etcd-initial-cluster/main.go create mode 100644 server/etcdserver/api/membership/cluster_opts.go diff --git a/CHANGELOG/CHANGELOG-3.6.md b/CHANGELOG/CHANGELOG-3.6.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/OWNERS b/OWNERS index 6685e9a2467..4a00b7e4aa0 100644 --- a/OWNERS +++ b/OWNERS @@ -1,12 +1,13 @@ -# See the OWNERS docs at https://go.k8s.io/owners - approvers: - - ahrtr # Benjamin Wang - - jmhbnz # James Blair - - serathius # Marek Siarkowicz - - spzala # Sahdev Zala - - wenjiaswe # Wenjia Zhang +- deads2k +- hasbro17 +- dusk125 +- Elbehery +- tjungblu reviewers: - - fuweid # Wei Fu - - ivanvc # Ivan Valdes - - siyuanfoundation # Siyuan Zhang \ No newline at end of file +- deads2k +- dusk125 +- hasbro17 +- Elbehery +- tjungblu +component: "Etcd" diff --git a/REBASE.openshift.md b/REBASE.openshift.md new file mode 100644 index 00000000000..51b27ae23a6 --- /dev/null +++ b/REBASE.openshift.md @@ -0,0 +1,87 @@ +# Maintaining openshift/etcd + +OpenShift is based on upstream etcd. With every release of etcd that is +intended to be shipped as OCP, it is necessary to incorporate the upstream changes +while ensuring that our downstream customizations are maintained. + +## Maintaining this document + +An openshift/etcd rebase is a complex process involving many manual and +potentially error-prone steps. If, while performing a rebase, you find areas where +the documented procedure is unclear or missing detail, please update this document +and include the change in the rebase PR. This will ensure that the instructions are +as comprehensive and accurate as possible for the person performing the next +rebase. + +## Getting started + +Before incorporating upstream changes you may want to: + +- Read this document +- Find the best tool for resolving merge conflicts +- Use diff3 conflict resolution strategy + (https://blog.nilbus.com/take-the-pain-out-of-git-conflict-resolution-use-diff3/) +- Teach Git to remember how you’ve resolved a conflict so that the next time it can + resolve it automatically (https://git-scm.com/book/en/v2/Git-Tools-Rerere) + +## Preparing the local repo clone + +Clone from a personal fork of etcd via a pushable (ssh) url: + +``` +git clone git@github.com:/etcd +``` + +## Updating with `rebase.sh` + +To finally rebase, a script that will merge and rebase along the happy path without automatic conflict resolution and at the end will create a PR for you. + +Here are the steps: +1. Create a new OCPBUGS JIRA ticket with the respective OpenShift version to rebase. Please include the change logs in the ticket description. You can clone a previous rebase we did in [OCPBUGS-947](https://issues.redhat.com/browse/OCPBUGS-947) and adjust. +2. It's best to start off with a fresh fork of [openshift/etcd](https://github.com/openshift/etcd/). Stay on the master branch. +3. This script requires `jq`, `git`, `podman` and `bash`. `gh` is optional. +4. In the root dir of that fork run: +``` +openshift-hack/rebase.sh --etcd-tag=v3.5.4 --openshift-release=openshift-4.12 --jira-id=666 +``` + +where `etcd-tag` is the [etcd-io/etcd](https://github.com/etcd-io/etcd/) release tag, the `openshift-release` +is the OpenShift release branch in [openshift/etcd](https://github.com/openshift/etcd/) and the `jira-id` is the +number of the OCPBUGS ticket created in step (1). + +5. In case of conflicts, it will ask you to step into another shell to resolve those. The script will continue by committing the resolution with `UPSTREAM: `. +6. At the end, there will be a "rebase-$VERSION" branch pushed to your fork. +7. If you have `gh` installed and are logged in, it will attempt to create a PR for you by opening a web browser. + +## Building and testing + +- Build the code with `make` +- Test the code with `make test` + +## Payload testing + +After all the above are green and your PR pre-submits are too, you can start with payload testing. This is to ensure the nightly jobs won't break on etcd after the merge, they also test all of OpenShift (including upgrades) well enough. + +You should run those two: + +> /payload 4.x nightly informing +> +> /payload 4.x nightly blocking + +Replace 4.x with the respective OpenShift release you're merging against. + +It pays off to inspect some Prometheus metrics (CPU, memory, disk usage) with an upgrade job through PromeCIus. This is to ensure we don't increase resource usage inadvertently. + +## Testing with ClusterBot + +Sometimes it's easier to debug an issue using cluster bot. Here you can simply run the given OpenShift release using your rebase PR: + +> launch openshift/etcd#155 + +This is particularly helpful when you want to test specific providers, for example Bare Metal or VSphere, or just other variants like SNO. + +## Performance Testing + +We currently do not do performance testing after an etcd rebase in OpenShift, however the upstream community and [SIG Scalability in k/k does](https://github.com/etcd-io/etcd/issues/14138#issuecomment-1247665949). + +The OpenShift scalability team also regularly runs performance tests with upcoming 4.y.0 releases. diff --git a/ROADMAP.md b/ROADMAP.md deleted file mode 100644 index d9898166c32..00000000000 --- a/ROADMAP.md +++ /dev/null @@ -1,21 +0,0 @@ -# etcd roadmap - -**work in progress** - -This document defines a high level roadmap for etcd development. - -The dates below should not be considered authoritative, but rather indicative of the projected timeline of the project. The [milestones defined in GitHub](https://github.com/etcd-io/etcd/milestones) represent the most up-to-date and issue-for-issue plans. - -etcd 3.3 is our current stable branch. The roadmap below outlines new features that will be added to etcd, and while subject to change, define what future stable will look like. - -### etcd 3.4 (2019) - -- Stabilization of 3.3 experimental features -- Support/document downgrade -- Snapshot restore as Go library -- Improved client balancer with new gRPC balancer interface -- Improve single-client put performance -- Improve large response handling -- Improve test coverage -- Decrease test runtime -- Migrate to Go module for dependency management diff --git a/build.sh b/build.sh index 05d916ea3cf..7d12a8c7e34 100755 --- a/build.sh +++ b/build.sh @@ -2,6 +2,8 @@ set -euo pipefail +GO_BUILD_FLAGS="${GO_BUILD_FLAGS} ${GOFLAGS:+-$GOFLAGS}" + source ./scripts/test_lib.sh GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound") @@ -93,6 +95,17 @@ etcd_build() { "-ldflags=${GO_LDFLAGS[*]}" \ -o="../${out}/etcdctl" . || return 2 ) || return 2 + + run rm -f "${out}/discover-etcd-initial-cluster" + # shellcheck disable=SC2086 + ( + cd ./openshift-tools/discover-etcd-initial-cluster + run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" "${GO_BUILD_ENV[@]}" go build $GO_BUILD_FLAGS \ + -installsuffix=cgo \ + "-ldflags=${GO_LDFLAGS[*]}" \ + -o="../../${out}/discover-etcd-initial-cluster" . || return 2 + ) || return 2 + # Verify whether symbol we overriden exists # For cross-compiling we cannot run: ${out}/etcd --version | grep -q "Git SHA: ${GIT_SHA}" diff --git a/client/v3/patch_cluster.go b/client/v3/patch_cluster.go new file mode 100644 index 00000000000..b629628850b --- /dev/null +++ b/client/v3/patch_cluster.go @@ -0,0 +1,24 @@ +package clientv3 + +import ( + "context" + pb "go.etcd.io/etcd/api/v3/etcdserverpb" +) + +// NonLinearizeableMemberLister is used by the discover-etcd-initial-cluster command to get a list of members to ensure that *this* +// member has been added to the list. This is needed on restart scenarios when there isn't quorum. We need the first +// two etcd servers to start without quorum having been established by finding themselves in the member list and moving +// past the gate. +type NonLinearizeableMemberLister interface { + // NonLinearizeableMemberList is like MemberList only without linearization. + NonLinearizeableMemberList(ctx context.Context) (*MemberListResponse, error) +} + +func (c *cluster) NonLinearizeableMemberList(ctx context.Context) (*MemberListResponse, error) { + // it is safe to retry on list. + resp, err := c.remote.MemberList(ctx, &pb.MemberListRequest{}, c.callOpts...) + if err == nil { + return (*MemberListResponse)(resp), nil + } + return nil, toErr(ctx, err) +} \ No newline at end of file diff --git a/etcdctl/ctlv3/command/check.go b/etcdctl/ctlv3/command/check.go index a2a5ca31593..b7f9d750f2c 100644 --- a/etcdctl/ctlv3/command/check.go +++ b/etcdctl/ctlv3/command/check.go @@ -107,13 +107,10 @@ var checkDatascaleCfgMap = map[string]checkDatascaleCfg{ // NewCheckCommand returns the cobra command for "check". func NewCheckCommand() *cobra.Command { cc := &cobra.Command{ - Use: "check ", - Short: "commands for checking properties of the etcd cluster", + Use: "check is no longer supported in OpenShift. Performance analysis should be performed using metrics, please see the etcd dashboards", + Short: "command no longer supported in OpenShift", } - cc.AddCommand(NewCheckPerfCommand()) - cc.AddCommand(NewCheckDatascaleCommand()) - return cc } diff --git a/openshift-hack/rebase.sh b/openshift-hack/rebase.sh new file mode 100755 index 00000000000..7e79e2c72dd --- /dev/null +++ b/openshift-hack/rebase.sh @@ -0,0 +1,174 @@ +#!/bin/bash + +# READ FIRST BEFORE USING THIS SCRIPT +# +# This script requires jq, git, podman and bash to work properly (dependencies are checked for you). +# The Github CLI "gh" is optional, but convenient to create a pull request automatically at the end. +# +# The usage is described in /REBASE.openshift.md. + +# validate input args --etcd-tag=v3.5.4 --openshift-release=openshift-4.12 --jira-id=666 +etcd_tag="" +openshift_release="" +jira_id="" + +usage() { + echo "Available arguments:" + echo " --etcd-tag (required) Example: --etcd-tag=v3.4.20" + echo " --openshift-release (required) Example: --openshift-release=openshift-4.12" + echo " --jira-id (optional) creates new PR against openshift/etcd:${openshift-release}: Example: --jira-id=666" +} + +for i in "$@"; do + case $i in + --etcd-tag=*) + etcd_tag="${i#*=}" + shift + ;; + --openshift-release=*) + openshift_release="${i#*=}" + shift + ;; + --jira-id=*) + jira_id="${i#*=}" + shift + ;; + *) + usage + exit 1 + ;; + esac +done + +if [ -z "${etcd_tag}" ]; then + echo "Required argument missing: --etcd-tag" + echo "" + usage + exit 1 +fi + +if [ -z "${openshift_release}" ]; then + echo "Required argument missing: --openshift-release" + echo "" + usage + exit 1 +fi + +echo "Processed arguments are:" +echo "--etcd_tag=${etcd_tag}" +echo "--openshift_release=${openshift_release}" +echo "--jira_id=${jira_id}" + +# prerequisites (check git, podman, ... is present) +if ! command -v git &>/dev/null; then + echo "git not installed, exiting" + exit 1 +fi + +if ! command -v jq &>/dev/null; then + echo "jq not installed, exiting" + exit 1 +fi + +if ! command -v podman &>/dev/null; then + echo "podman not installed, exiting" + exit 1 +fi + +# make sure we're in "etcd" dir, but we also allow openshift-etcd +if [[ $(basename "$PWD") != "etcd" && $(basename "$PWD") != "openshift-etcd" ]]; then + echo "Not in etcd dir, exiting" + exit 1 +fi + +origin=$(git remote get-url origin) +if [[ "$origin" =~ .*etcd-io/etcd.* || "$origin" =~ .*openshift/etcd.* ]]; then + echo "cannot rebase against etcd-io/etcd or openshift/etcd! found: ${origin}, exiting" + exit 1 +fi + +# fetch remote https://github.com/etcd-io/etcd +git remote add upstream git@github.com:etcd-io/etcd.git +git fetch upstream --tags -f +# fetch remote https://github.com/openshift/etcd +git remote add openshift git@github.com:openshift/etcd.git +git fetch openshift + +# clean checkout of the remote openshift release +git branch -D "openshift/$openshift_release" +git checkout --track "openshift/$openshift_release" +git pull openshift "$openshift_release" + +# that should give us the latest (or highest version) etcd tag +# This is a bit experimental for the future, but works across all the current release branches +etcd_forkpoint=$(git tag --merged | sort -V | tail -2 | head -1) +if [[ "$etcd_forkpoint" == "$etcd_tag" ]]; then + echo "forkpoint $etcd_forkpoint matches given etcd tag, no rebase necessary" + exit 1 +fi + +echo "running: \`git rebase --rebase-merges --fork-point $etcd_forkpoint $etcd_tag\`" +git rebase --rebase-merges --fork-point "$etcd_forkpoint" "$etcd_tag" +echo "running: \`git merge $openshift_release\`" +git merge "$openshift_release" + +# shellcheck disable=SC2181 +if [ $? -eq 0 ]; then + echo "No conflicts detected. Automatic merge looks to have succeeded" +else + # commit conflicts + git commit -a + # resolve conflicts + git status + # TODO(tjungblu): we follow-up with a more automated approach: + # - 2/3s of conflicts stem from go.mod/sum, which can be resolved deterministically + # - the large majority of the remainder are vendor/generation conflicts + # - only very few cases require manual intervention due to conflicting business logic + echo "Resolve conflicts manually in another terminal, only then continue" + + # wait for user interaction + read -n 1 -s -r -p "PRESS ANY KEY TO CONTINUE" + + # TODO(tjungblu): verify that the conflicts have been resolved + git commit -am "UPSTREAM: : manually resolve conflicts" +fi + +# ensure we always use the correct openshift release + golang combination +go_mod_go_ver=$(grep -E 'go 1\.[1-9][0-9]?' go.mod | sed -E 's/go (1\.[1-9][0-9]?)/\1/') +tag="rhel-8-release-golang-${go_mod_go_ver}-openshift-${openshift_release#release-}" +echo "> go mod tidy" +podman run -it --rm -v "$(pwd):/go/etcd:Z" \ + --workdir=/go/etcd \ + "registry.ci.openshift.org/openshift/release:$tag" \ + go mod tidy + +# shellcheck disable=SC2181 +if [ $? -ne 0 ]; then + echo "go mod tidy failed, is any dependency missing?" + exit 1 +fi + +git add -A +git commit -m "UPSTREAM: : go mod tidy" + +remote_branch="rebase-$etcd_tag" +git push origin "$openshift_release:$remote_branch" + +XY=$(echo "$etcd_tag" | sed -E "s/v(1\.[0-9]+)\.[0-9]+/\1/") +ver=$(echo "$etcd_tag" | sed "s/\.//g") +link="https://github.com/etcd-io/etcd/blob/master/CHANGELOG/CHANGELOG-$XY.md#$ver" +if [ -n "${jira_id}" ]; then + if command -v gh &>/dev/null; then + XY=$(echo "$etcd_tag" | sed -E "s/v(1\.[0-9]+)\.[0-9]+/\1/") + ver=$(echo "$etcd_tag" | sed "s/\.//g") + link="https://github.com/etcd-io/etcd/blob/master/CHANGELOG/CHANGELOG-$XY.md#$ver" + + # opens a web browser, because we can't properly create PRs against remote repositories with the GH CLI (yet): + # https://github.com/cli/cli/issues/2691 + gh pr create \ + --title "OCPBUGS-$jira_id: Rebase $etcd_tag" \ + --body "CHANGELOG $link" \ + --web + + fi +fi diff --git a/openshift-tools/discover-etcd-initial-cluster/main.go b/openshift-tools/discover-etcd-initial-cluster/main.go new file mode 100644 index 00000000000..544c666d91d --- /dev/null +++ b/openshift-tools/discover-etcd-initial-cluster/main.go @@ -0,0 +1,35 @@ +package main + +import ( + goflag "flag" + "fmt" + "math/rand" + "os" + "strings" + "time" + + "github.com/spf13/pflag" + discover_etcd_initial_cluster "go.etcd.io/etcd/v3/openshift-tools/pkg/discover-etcd-initial-cluster" +) + +// copy from `utilflag "k8s.io/component-base/cli/flag"` +// WordSepNormalizeFunc changes all flags that contain "_" separators +func WordSepNormalizeFunc(f *pflag.FlagSet, name string) pflag.NormalizedName { + if strings.Contains(name, "_") { + return pflag.NormalizedName(strings.Replace(name, "_", "-", -1)) + } + return pflag.NormalizedName(name) +} + +func main() { + rand.Seed(time.Now().UTC().UnixNano()) + + pflag.CommandLine.SetNormalizeFunc(WordSepNormalizeFunc) + pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) + + command := discover_etcd_initial_cluster.NewDiscoverEtcdInitialClusterCommand() + if err := command.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} diff --git a/raft/OWNERS b/raft/OWNERS index ab781066e23..35fc8f68713 100644 --- a/raft/OWNERS +++ b/raft/OWNERS @@ -1,19 +1,5 @@ approvers: -- heyitsanthony -- philips -- fanminshi -- gyuho -- mitake -- jpbetz -- xiang90 -- bdarnell + - hexfusion + - smarterclayton reviewers: -- heyitsanthony -- philips -- fanminshi -- gyuho -- mitake -- jpbetz -- xiang90 -- bdarnell -- tschottdorf + - hexfusion diff --git a/server/config/config.go b/server/config/config.go index bf15b54560e..3d341b1abb9 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -195,6 +195,9 @@ type ServerConfig struct { // consider running defrag during bootstrap. Needs to be set to non-zero value to take effect. ExperimentalBootstrapDefragThresholdMegabytes uint `json:"experimental-bootstrap-defrag-threshold-megabytes"` + // ExperimentalMaxLearners sets a limit to the number of learner members that can exist in the cluster membership. + ExperimentalMaxLearners int `json:"experimental-max-learners"` + // V2Deprecation defines a phase of v2store deprecation process. V2Deprecation V2DeprecationEnum `json:"v2-deprecation"` } diff --git a/server/embed/config.go b/server/embed/config.go index 66785265cd9..30c0d073029 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -37,6 +37,7 @@ import ( "go.etcd.io/etcd/pkg/v3/netutil" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver" + "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/v3compactor" bolt "go.etcd.io/bbolt" @@ -345,6 +346,8 @@ type Config struct { // ExperimentalBootstrapDefragThresholdMegabytes is the minimum number of megabytes needed to be freed for etcd server to // consider running defrag during bootstrap. Needs to be set to non-zero value to take effect. ExperimentalBootstrapDefragThresholdMegabytes uint `json:"experimental-bootstrap-defrag-threshold-megabytes"` + // ExperimentalMaxLearners sets a limit to the number of learner members that can exist in the cluster membership. + ExperimentalMaxLearners int `json:"experimental-max-learners"` // ForceNewCluster starts a new cluster even if previously started; unsafe. ForceNewCluster bool `json:"force-new-cluster"` @@ -523,6 +526,7 @@ func NewConfig() *Config { ExperimentalMemoryMlock: false, ExperimentalTxnModeWriteWithSharedBuffer: true, ExperimentalStopGRPCServiceOnDefrag: false, + ExperimentalMaxLearners: membership.DefaultMaxLearners, ExperimentalCompactHashCheckEnabled: false, ExperimentalCompactHashCheckTime: time.Minute, diff --git a/server/embed/etcd.go b/server/embed/etcd.go index b894458b758..89dd809cfcc 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -238,7 +238,8 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { ExperimentalTxnModeWriteWithSharedBuffer: cfg.ExperimentalTxnModeWriteWithSharedBuffer, ExperimentalStopGRPCServiceOnDefrag: cfg.ExperimentalStopGRPCServiceOnDefrag, ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes, - V2Deprecation: cfg.V2DeprecationEffective(), + ExperimentalMaxLearners: cfg.ExperimentalMaxLearners, + V2Deprecation: cfg.V2DeprecationEffective(), } if srvcfg.ExperimentalEnableDistributedTracing { @@ -368,6 +369,7 @@ func print(lg *zap.Logger, ec Config, sc config.ServerConfig, memberInitialized zap.String("discovery-url", sc.DiscoveryURL), zap.String("discovery-proxy", sc.DiscoveryProxy), zap.String("downgrade-check-interval", sc.DowngradeCheckTime.String()), + zap.Int("max-learners", sc.ExperimentalMaxLearners), ) } diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index 7ee18b86f43..db8535f1fed 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -30,6 +30,7 @@ import ( "go.etcd.io/etcd/pkg/v3/flags" cconfig "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/embed" + "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "go.uber.org/zap" @@ -317,6 +318,7 @@ func newConfig() *config { fs.BoolVar(&cfg.ec.ExperimentalTxnModeWriteWithSharedBuffer, "experimental-txn-mode-write-with-shared-buffer", true, "Enable the write transaction to use a shared buffer in its readonly check operations.") fs.BoolVar(&cfg.ec.ExperimentalStopGRPCServiceOnDefrag, "experimental-stop-grpc-service-on-defrag", cfg.ec.ExperimentalStopGRPCServiceOnDefrag, "Enable etcd gRPC service to stop serving client requests on defragmentation.") fs.UintVar(&cfg.ec.ExperimentalBootstrapDefragThresholdMegabytes, "experimental-bootstrap-defrag-threshold-megabytes", 0, "Enable the defrag during etcd server bootstrap on condition that it will free at least the provided threshold of disk space. Needs to be set to non-zero value to take effect.") + fs.IntVar(&cfg.ec.ExperimentalMaxLearners, "experimental-max-learners", membership.DefaultMaxLearners, "Sets the maximum number of learners that can be available in the cluster membership.") fs.Uint64Var(&cfg.ec.SnapshotCatchUpEntries, "experimental-snapshot-catchup-entries", cfg.ec.SnapshotCatchUpEntries, "(WARNING: Use this flag with caution!) Number of entries for a slow follower to catch up after compacting the raft storage entries.") // unsafe diff --git a/server/etcdmain/grpc_proxy.go b/server/etcdmain/grpc_proxy.go index db672fe56d4..a684d0b5e55 100644 --- a/server/etcdmain/grpc_proxy.go +++ b/server/etcdmain/grpc_proxy.go @@ -436,7 +436,7 @@ func newTLS(ca, cert, key string, requireEmptyCN bool) *transport.TLSInfo { if ca == "" && cert == "" && key == "" { return nil } - return &transport.TLSInfo{TrustedCAFile: ca, CertFile: cert, KeyFile: key, EmptyCN: requireEmptyCN} + return &transport.TLSInfo{TrustedCAFile: ca, CertFile: cert, KeyFile: key} } func mustListenCMux(lg *zap.Logger, tlsinfo *transport.TLSInfo) cmux.CMux { diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index f6e778ee6dc..cd6e2195873 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -279,11 +279,13 @@ Experimental feature: --experimental-memory-mlock Enable to enforce etcd pages (in particular bbolt) to stay in RAM. --experimental-warning-apply-duration '100ms' - Warning is generated if requests take more than this duration. + Warning is generated if requests take more than this duration. --experimental-txn-mode-write-with-shared-buffer 'true' Enable the write transaction to use a shared buffer in its readonly check operations. --experimental-bootstrap-defrag-threshold-megabytes Enable the defrag during etcd server bootstrap on condition that it will free at least the provided threshold of disk space. Needs to be set to non-zero value to take effect. + --experimental-max-learners '1' + Set the max number of learner members allowed in the cluster membership. --experimental-stop-grpc-service-on-defrag Enable etcd gRPC service to stop serving client requests on defragmentation. diff --git a/server/etcdserver/api/membership/cluster.go b/server/etcdserver/api/membership/cluster.go index e177a63029f..63732807d08 100644 --- a/server/etcdserver/api/membership/cluster.go +++ b/server/etcdserver/api/membership/cluster.go @@ -43,8 +43,6 @@ import ( "go.uber.org/zap" ) -const maxLearners = 1 - // RaftCluster is a list of Members that belong to the same raft cluster type RaftCluster struct { lg *zap.Logger @@ -63,6 +61,7 @@ type RaftCluster struct { removed map[types.ID]bool downgradeInfo *DowngradeInfo + maxLearners int } // ConfigChangeContext represents a context for confChange. @@ -83,8 +82,8 @@ const ( // NewClusterFromURLsMap creates a new raft cluster using provided urls map. Currently, it does not support creating // cluster with raft learner member. -func NewClusterFromURLsMap(lg *zap.Logger, token string, urlsmap types.URLsMap) (*RaftCluster, error) { - c := NewCluster(lg) +func NewClusterFromURLsMap(lg *zap.Logger, token string, urlsmap types.URLsMap, opts ...ClusterOption) (*RaftCluster, error) { + c := NewCluster(lg, opts...) for name, urls := range urlsmap { m := NewMember(name, urls, token, nil) if _, ok := c.members[m.ID]; ok { @@ -99,8 +98,8 @@ func NewClusterFromURLsMap(lg *zap.Logger, token string, urlsmap types.URLsMap) return c, nil } -func NewClusterFromMembers(lg *zap.Logger, id types.ID, membs []*Member) *RaftCluster { - c := NewCluster(lg) +func NewClusterFromMembers(lg *zap.Logger, id types.ID, membs []*Member, opts ...ClusterOption) *RaftCluster { + c := NewCluster(lg, opts...) c.cid = id for _, m := range membs { c.members[m.ID] = m @@ -108,15 +107,18 @@ func NewClusterFromMembers(lg *zap.Logger, id types.ID, membs []*Member) *RaftCl return c } -func NewCluster(lg *zap.Logger) *RaftCluster { +func NewCluster(lg *zap.Logger, opts ...ClusterOption) *RaftCluster { if lg == nil { lg = zap.NewNop() } + clOpts := newClusterOpts(opts...) + return &RaftCluster{ lg: lg, members: make(map[types.ID]*Member), removed: make(map[types.ID]bool), downgradeInfo: &DowngradeInfo{Enabled: false}, + maxLearners: clOpts.maxLearners, } } @@ -281,6 +283,7 @@ func (c *RaftCluster) Recover(onSet func(*zap.Logger, *semver.Version)) { zap.String("local-member-id", c.localID.String()), zap.String("recovered-remote-peer-id", m.ID.String()), zap.Strings("recovered-remote-peer-urls", m.PeerURLs), + zap.Bool("recovered-remote-peer-is-learner", m.IsLearner), ) } if c.version != nil { @@ -295,9 +298,9 @@ func (c *RaftCluster) Recover(onSet func(*zap.Logger, *semver.Version)) { // ensures that it is still valid. func (c *RaftCluster) ValidateConfigurationChange(cc raftpb.ConfChange) error { // TODO: this must be switched to backend as well. - members, removed := membersFromStore(c.lg, c.v2store) + membersMap, removedMap := membersFromStore(c.lg, c.v2store) id := types.ID(cc.NodeID) - if removed[id] { + if removedMap[id] { return ErrIDRemoved } switch cc.Type { @@ -308,19 +311,21 @@ func (c *RaftCluster) ValidateConfigurationChange(cc raftpb.ConfChange) error { } if confChangeContext.IsPromote { // promoting a learner member to voting member - if members[id] == nil { + if membersMap[id] == nil { return ErrIDNotFound } - if !members[id].IsLearner { + if !membersMap[id].IsLearner { return ErrMemberNotLearner } } else { // adding a new member - if members[id] != nil { + if membersMap[id] != nil { return ErrIDExists } + var members []*Member urls := make(map[string]bool) - for _, m := range members { + for _, m := range membersMap { + members = append(members, m) for _, u := range m.PeerURLs { urls[u] = true } @@ -331,29 +336,24 @@ func (c *RaftCluster) ValidateConfigurationChange(cc raftpb.ConfChange) error { } } - if confChangeContext.Member.IsLearner { // the new member is a learner - numLearners := 0 - for _, m := range members { - if m.IsLearner { - numLearners++ - } - } - if numLearners+1 > maxLearners { - return ErrTooManyLearners + if confChangeContext.Member.RaftAttributes.IsLearner && cc.Type == raftpb.ConfChangeAddLearnerNode { // the new member is a learner + scaleUpLearners := true + if err := ValidateMaxLearnerConfig(c.maxLearners, members, scaleUpLearners); err != nil { + return err } } } case raftpb.ConfChangeRemoveNode: - if members[id] == nil { + if membersMap[id] == nil { return ErrIDNotFound } case raftpb.ConfChangeUpdateNode: - if members[id] == nil { + if membersMap[id] == nil { return ErrIDNotFound } urls := make(map[string]bool) - for _, m := range members { + for _, m := range membersMap { if m.ID == id { continue } @@ -424,6 +424,7 @@ func (c *RaftCluster) AddMember(m *Member, shouldApplyV3 ShouldApplyV3) { zap.String("local-member-id", c.localID.String()), zap.String("added-peer-id", m.ID.String()), zap.Strings("added-peer-peer-urls", m.PeerURLs), + zap.Bool("added-peer-is-learner", m.IsLearner), ) } @@ -475,6 +476,7 @@ func (c *RaftCluster) RemoveMember(id types.ID, shouldApplyV3 ShouldApplyV3) { zap.String("local-member-id", c.localID.String()), zap.String("removed-remote-peer-id", id.String()), zap.Strings("removed-remote-peer-urls", m.PeerURLs), + zap.Bool("removed-remote-peer-is-learner", m.IsLearner), ) } else { c.lg.Warn( @@ -557,6 +559,7 @@ func (c *RaftCluster) UpdateRaftAttributes(id types.ID, raftAttr RaftAttributes, zap.String("local-member-id", c.localID.String()), zap.String("updated-remote-peer-id", id.String()), zap.Strings("updated-remote-peer-urls", raftAttr.PeerURLs), + zap.Bool("updated-remote-peer-is-learner", raftAttr.IsLearner), ) } @@ -933,3 +936,24 @@ func (c *RaftCluster) PushMembershipToStorage() { } } } + +// ValidateMaxLearnerConfig verifies the existing learner members in the cluster membership and an optional N+1 learner +// scale up are not more than maxLearners. +func ValidateMaxLearnerConfig(maxLearners int, members []*Member, scaleUpLearners bool) error { + numLearners := 0 + for _, m := range members { + if m.IsLearner { + numLearners++ + } + } + // Validate config can accommodate scale up. + if scaleUpLearners { + numLearners++ + } + + if numLearners > maxLearners { + return ErrTooManyLearners + } + + return nil +} diff --git a/server/etcdserver/api/membership/cluster_opts.go b/server/etcdserver/api/membership/cluster_opts.go new file mode 100644 index 00000000000..204fbf04d2c --- /dev/null +++ b/server/etcdserver/api/membership/cluster_opts.go @@ -0,0 +1,43 @@ +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package membership + +const DefaultMaxLearners = 1 + +type ClusterOptions struct { + maxLearners int +} + +// ClusterOption are options which can be applied to the raft cluster. +type ClusterOption func(*ClusterOptions) + +func newClusterOpts(opts ...ClusterOption) *ClusterOptions { + clOpts := &ClusterOptions{} + clOpts.applyOpts(opts) + return clOpts +} + +func (co *ClusterOptions) applyOpts(opts []ClusterOption) { + for _, opt := range opts { + opt(co) + } +} + +// WithMaxLearners sets the maximum number of learners that can exist in the cluster membership. +func WithMaxLearners(max int) ClusterOption { + return func(co *ClusterOptions) { + co.maxLearners = max + } +} diff --git a/server/etcdserver/api/membership/cluster_test.go b/server/etcdserver/api/membership/cluster_test.go index 1793caa6ab9..e152e06a1fa 100644 --- a/server/etcdserver/api/membership/cluster_test.go +++ b/server/etcdserver/api/membership/cluster_test.go @@ -281,10 +281,14 @@ func TestClusterValidateAndAssignIDs(t *testing.T) { } func TestClusterValidateConfigurationChange(t *testing.T) { - cl := NewCluster(zaptest.NewLogger(t)) + cl := NewCluster(zaptest.NewLogger(t), WithMaxLearners(1)) cl.SetStore(v2store.New()) for i := 1; i <= 4; i++ { - attr := RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", i)}} + var isLearner bool + if i == 1 { + isLearner = true + } + attr := RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", i)}, IsLearner: isLearner} cl.AddMember(&Member{ID: types.ID(i), RaftAttributes: attr}, true) } cl.RemoveMember(4, true) @@ -329,6 +333,17 @@ func TestClusterValidateConfigurationChange(t *testing.T) { t.Fatal(err) } + attr = RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 7)}, IsLearner: true} + ctx7, err := json.Marshal(&ConfigChangeContext{Member: Member{ID: types.ID(7), RaftAttributes: attr}}) + if err != nil { + t.Fatal(err) + } + + attr = RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 1)}, IsLearner: true} + ctx8, err := json.Marshal(&ConfigChangeContext{Member: Member{ID: types.ID(1), RaftAttributes: attr}, IsPromote: true}) + if err != nil { + t.Fatal(err) + } tests := []struct { cc raftpb.ConfChange werr error @@ -426,6 +441,22 @@ func TestClusterValidateConfigurationChange(t *testing.T) { }, ErrIDNotFound, }, + { + raftpb.ConfChange{ + Type: raftpb.ConfChangeAddLearnerNode, + NodeID: 7, + Context: ctx7, + }, + ErrTooManyLearners, + }, + { + raftpb.ConfChange{ + Type: raftpb.ConfChangeAddNode, + NodeID: 1, + Context: ctx8, + }, + nil, + }, } for i, tt := range tests { err := cl.ValidateConfigurationChange(tt.cc) diff --git a/server/etcdserver/raft.go b/server/etcdserver/raft.go index c4bf0259797..8338af99d38 100644 --- a/server/etcdserver/raft.go +++ b/server/etcdserver/raft.go @@ -543,7 +543,7 @@ func restartNode(cfg config.ServerConfig, snapshot *raftpb.Snapshot) (types.ID, zap.String("local-member-id", id.String()), zap.Uint64("commit-index", st.Commit), ) - cl := membership.NewCluster(cfg.Logger) + cl := membership.NewCluster(cfg.Logger, membership.WithMaxLearners(cfg.ExperimentalMaxLearners)) cl.SetID(id, cid) s := raft.NewMemoryStorage() if snapshot != nil { @@ -617,7 +617,7 @@ func restartAsStandaloneNode(cfg config.ServerConfig, snapshot *raftpb.Snapshot) zap.Uint64("commit-index", st.Commit), ) - cl := membership.NewCluster(cfg.Logger) + cl := membership.NewCluster(cfg.Logger, membership.WithMaxLearners(cfg.ExperimentalMaxLearners)) cl.SetID(id, cid) s := raft.NewMemoryStorage() if snapshot != nil { diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go index 513ef6bd7b8..e8250b11baf 100644 --- a/server/etcdserver/server.go +++ b/server/etcdserver/server.go @@ -412,7 +412,7 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { if err = cfg.VerifyJoinExisting(); err != nil { return nil, err } - cl, err = membership.NewClusterFromURLsMap(cfg.Logger, cfg.InitialClusterToken, cfg.InitialPeerURLsMap) + cl, err = membership.NewClusterFromURLsMap(cfg.Logger, cfg.InitialClusterToken, cfg.InitialPeerURLsMap, membership.WithMaxLearners(cfg.ExperimentalMaxLearners)) if err != nil { return nil, err } @@ -426,7 +426,10 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { if !isCompatibleWithCluster(cfg.Logger, cl, cl.MemberByName(cfg.Name).ID, prt) { return nil, fmt.Errorf("incompatible with current running cluster") } - + scaleUpLearners := false + if err := membership.ValidateMaxLearnerConfig(cfg.ExperimentalMaxLearners, existingCluster.Members(), scaleUpLearners); err != nil { + return nil, err + } remotes = existingCluster.Members() cl.SetID(types.ID(0), existingCluster.ID()) cl.SetStore(st) @@ -438,7 +441,7 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { if err = cfg.VerifyBootstrap(); err != nil { return nil, err } - cl, err = membership.NewClusterFromURLsMap(cfg.Logger, cfg.InitialClusterToken, cfg.InitialPeerURLsMap) + cl, err = membership.NewClusterFromURLsMap(cfg.Logger, cfg.InitialClusterToken, cfg.InitialPeerURLsMap, membership.WithMaxLearners(cfg.ExperimentalMaxLearners)) if err != nil { return nil, err } @@ -460,7 +463,7 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { if config.CheckDuplicateURL(urlsmap) { return nil, fmt.Errorf("discovery cluster %s has duplicate url", urlsmap) } - if cl, err = membership.NewClusterFromURLsMap(cfg.Logger, cfg.InitialClusterToken, urlsmap); err != nil { + if cl, err = membership.NewClusterFromURLsMap(cfg.Logger, cfg.InitialClusterToken, urlsmap, membership.WithMaxLearners(cfg.ExperimentalMaxLearners)); err != nil { return nil, err } } @@ -541,6 +544,12 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { cl.SetStore(st) cl.SetBackend(be) cl.Recover(api.UpdateCapability) + + scaleUpLearners := false + if err := membership.ValidateMaxLearnerConfig(cfg.ExperimentalMaxLearners, cl.Members(), scaleUpLearners); err != nil { + return nil, err + } + if cl.Version() != nil && !cl.Version().LessThan(semver.Version{Major: 3}) && !beExist { os.RemoveAll(bepath) return nil, fmt.Errorf("database file (%v) of the backend is missing", bepath) diff --git a/tests/integration/clientv3/cluster_test.go b/tests/integration/clientv3/cluster_test.go index eff15cf7d2d..d624a56d7b2 100644 --- a/tests/integration/clientv3/cluster_test.go +++ b/tests/integration/clientv3/cluster_test.go @@ -16,6 +16,7 @@ package clientv3test import ( "context" + "fmt" "math/rand" "reflect" "strings" @@ -376,18 +377,28 @@ func TestMemberPromoteMemberNotExist(t *testing.T) { } } -// TestMaxLearnerInCluster verifies that the maximum number of learners allowed in a cluster is 1 +// TestMaxLearnerInCluster verifies that the maximum number of learners allowed in a cluster func TestMaxLearnerInCluster(t *testing.T) { integration.BeforeTest(t) - // 1. start with a cluster with 3 voting member and 0 learner member - clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) + // 1. start with a cluster with 3 voting member and max learner 2 + clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3, ExperimentalMaxLearners: 2}) defer clus.Terminate(t) - // 2. adding a learner member should succeed - resp1, err := clus.Client(0).MemberAddAsLearner(context.Background(), []string{"http://127.0.0.1:1234"}) + // 2. adding 2 learner members should succeed + for i := 0; i < 2; i++ { + _, err := clus.Client(0).MemberAddAsLearner(context.Background(), []string{fmt.Sprintf("http://127.0.0.1:123%d", i)}) + if err != nil { + t.Fatalf("failed to add learner member %v", err) + } + } + + // ensure client endpoint is voting member + leaderIdx := clus.WaitLeader(t) + capi := clus.Client(leaderIdx) + resp1, err := capi.MemberList(context.Background()) if err != nil { - t.Fatalf("failed to add learner member %v", err) + t.Fatalf("failed to get member list") } numberOfLearners := 0 for _, m := range resp1.Members { @@ -395,12 +406,12 @@ func TestMaxLearnerInCluster(t *testing.T) { numberOfLearners++ } } - if numberOfLearners != 1 { - t.Fatalf("Added 1 learner node to cluster, got %d", numberOfLearners) + if numberOfLearners != 2 { + t.Fatalf("added 2 learner node to cluster, got %d", numberOfLearners) } - // 3. cluster has 3 voting member and 1 learner, adding another learner should fail - _, err = clus.Client(0).MemberAddAsLearner(context.Background(), []string{"http://127.0.0.1:2345"}) + // 3. cluster has 3 voting member and 2 learner, adding another learner should fail + _, err = clus.Client(0).MemberAddAsLearner(context.Background(), []string{"http://127.0.0.1:2342"}) if err == nil { t.Fatalf("expect member add to fail, got no error") } @@ -410,7 +421,7 @@ func TestMaxLearnerInCluster(t *testing.T) { } // 4. cluster has 3 voting member and 1 learner, adding a voting member should succeed - _, err = clus.Client(0).MemberAdd(context.Background(), []string{"http://127.0.0.1:3456"}) + _, err = clus.Client(0).MemberAdd(context.Background(), []string{"http://127.0.0.1:3453"}) if err != nil { t.Errorf("failed to add member %v", err) } diff --git a/tests/integration/cluster.go b/tests/integration/cluster.go index a2e9e0cce1b..57cf23e7156 100644 --- a/tests/integration/cluster.go +++ b/tests/integration/cluster.go @@ -45,6 +45,7 @@ import ( "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp" + "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "go.etcd.io/etcd/server/v3/etcdserver/api/v2http" "go.etcd.io/etcd/server/v3/etcdserver/api/v3client" @@ -171,6 +172,7 @@ type ClusterConfig struct { LeaseCheckpointPersist bool WatchProgressNotifyInterval time.Duration + ExperimentalMaxLearners int CorruptCheckTime time.Duration } @@ -336,6 +338,7 @@ func (c *cluster) mustNewMember(t testutil.TB, memberNumber int64) *member { leaseCheckpointPersist: c.cfg.LeaseCheckpointPersist, leaseCheckpointInterval: c.cfg.LeaseCheckpointInterval, WatchProgressNotifyInterval: c.cfg.WatchProgressNotifyInterval, + ExperimentalMaxLearners: c.cfg.ExperimentalMaxLearners, CorruptCheckTime: c.cfg.CorruptCheckTime, }) m.DiscoveryURL = c.cfg.DiscoveryURL @@ -645,6 +648,7 @@ type memberConfig struct { leaseCheckpointInterval time.Duration leaseCheckpointPersist bool WatchProgressNotifyInterval time.Duration + ExperimentalMaxLearners int CorruptCheckTime time.Duration } @@ -755,7 +759,10 @@ func mustNewMember(t testutil.TB, mcfg memberConfig) *member { m.CorruptCheckTime = mcfg.CorruptCheckTime } m.WarningApplyDuration = embed.DefaultWarningApplyDuration - + m.ExperimentalMaxLearners = membership.DefaultMaxLearners + if mcfg.ExperimentalMaxLearners != 0 { + m.ExperimentalMaxLearners = mcfg.ExperimentalMaxLearners + } m.V2Deprecation = config.V2_DEPR_DEFAULT m.grpcServerRecorder = &grpc_testing.GrpcRecorder{} m.Logger = memberLogger(t, mcfg.name) From 1285d660a81ec72330a55ff29070ac5ded56c377 Mon Sep 17 00:00:00 2001 From: Justin Pierce Date: Mon, 28 Oct 2024 16:11:25 -0400 Subject: [PATCH 38/41] Add support for cachi2 based deps Konflux is replacing RH's internal build system OSBS. OSBS supported a build-time dependency injection system called "cachito". Konflux replaces this with "cachi2" which works differently. REMOTE_SOURCES no longer need to be copied into place and there is no need to source cachito's environment information (Konflux automatically rewrites the Dockerfile to source cachi2/cachi2.env before running the original RUN commands). Additionally, cachito appears to have provided go.sum dependencies whereas cachi2 requires all build-time dependencies in go.mod. Missing dependencies are added to go.mod as // indirect in this change. --- Dockerfile.installer.art-cachi2 | 65 +++++++++++++++++++++++++++++++++ go.mod | 5 +++ 2 files changed, 70 insertions(+) create mode 100644 Dockerfile.installer.art-cachi2 diff --git a/Dockerfile.installer.art-cachi2 b/Dockerfile.installer.art-cachi2 new file mode 100644 index 00000000000..7902bbab57d --- /dev/null +++ b/Dockerfile.installer.art-cachi2 @@ -0,0 +1,65 @@ +# This Dockerfile builds an image containing Mac and Linux ARM64/AMD64 versions of the etcd. +# The resulting image is used to build the statically-linked openshift-installer binary. + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS macbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" + +WORKDIR /go/src/go.etcd.io/etcd +COPY . . + +RUN find $GOPATH +RUN export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS macarmbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" + +WORKDIR /go/src/go.etcd.io/etcd +COPY . . + +RUN export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS linuxbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" + +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS linuxarmbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" + +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=linux GOARCH=arm64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS builder +ENV GO_COMPLIANCE_EXCLUDE=".*" + +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 ./build.sh +RUN mkdir -p /usr/share/openshift/$(go env GOOS)/$(go env GOHOSTARCH) && \ + mv bin/etcd /usr/share/openshift/$(go env GOOS)/$(go env GOHOSTARCH)/ + +# stage 2 +FROM registry.ci.openshift.org/ocp/4.18:base-rhel9 + +RUN yum install --setopt=tsflags=nodocs -y jq && yum clean all && rm -rf /var/cache/yum/* + +COPY --from=macbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/darwin/amd64/etcd +COPY --from=macarmbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/darwin/arm64/etcd +COPY --from=linuxbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/linux/amd64/etcd +COPY --from=linuxarmbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/linux/arm64/etcd +COPY --from=builder /usr/share/openshift/ /usr/share/openshift/ + +# This image is not an operator, it is only used as part of the build pipeline +LABEL io.openshift.release.operator=false diff --git a/go.mod b/go.mod index 5a7b225e5ef..a17d9174826 100644 --- a/go.mod +++ b/go.mod @@ -95,4 +95,9 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect sigs.k8s.io/yaml v1.2.0 // indirect + go.etcd.io/gofail v0.2.0 // indirect + github.com/urfave/cli v1.22.4 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/russross/blackfriday/v2 v2.0.1 //indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 //indirect ) From e7165bf7cb8f5a565c291c87f4f75edd3e219016 Mon Sep 17 00:00:00 2001 From: Justin Pierce Date: Tue, 29 Oct 2024 12:20:43 -0400 Subject: [PATCH 39/41] Migrate Dockerfile.art for Konflux --- Dockerfile.art-cachi2 | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 Dockerfile.art-cachi2 diff --git a/Dockerfile.art-cachi2 b/Dockerfile.art-cachi2 new file mode 100644 index 00000000000..9cf5d998ac4 --- /dev/null +++ b/Dockerfile.art-cachi2 @@ -0,0 +1,21 @@ +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.18 AS builder + +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh + +# stage 2 (note: any changes should reflect in Dockerfile.rhel) +FROM registry.ci.openshift.org/ocp/4.18:base-rhel9 + +ENTRYPOINT ["/usr/bin/etcd"] + +RUN yum install --setopt=tsflags=nodocs -y jq && yum clean all && rm -rf /var/cache/yum/* + +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcd /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcdctl /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcdutl /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/discover-etcd-initial-cluster /usr/bin/ + +LABEL io.k8s.display-name="etcd server" \ + io.k8s.description="etcd is a distributed key-value store which stores the persistent master state for Kubernetes and OpenShift." \ + maintainer="Sam Batschelet " From 01ca4f1a0c515ddecd515c9b8fb382d691a2411e Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Mon, 4 Nov 2024 15:44:54 +0100 Subject: [PATCH 40/41] DOWNSTREAM: : ETCD-696: Add rev bumping to force-new-cluster force-new-cluster seems to have similar watch cache issues as the ordinary snapshot restore. This PR introduces the already existing utl logic as a separate package into the server-side code. This will only introduce a revbump flag, but under the hood implement both rev bumping and compaction markers. Signed-off-by: Thomas Jungblut --- server/config/config.go | 3 +- server/embed/config.go | 3 +- server/embed/etcd.go | 1 + server/etcdmain/config.go | 1 + server/etcdserver/raft.go | 12 ++++++- server/etcdserver/server.go | 2 +- server/revbump/revbump.go | 65 +++++++++++++++++++++++++++++++++++++ server/revbump/revision.go | 46 ++++++++++++++++++++++++++ 8 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 server/revbump/revbump.go create mode 100644 server/revbump/revision.go diff --git a/server/config/config.go b/server/config/config.go index 3d341b1abb9..ca964b7169d 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -152,7 +152,8 @@ type ServerConfig struct { // Logger logs server-side operations. Logger *zap.Logger - ForceNewCluster bool + ForceNewCluster bool + ForceNewClusterBumpAmount uint64 // EnableLeaseCheckpoint enables leader to send regular checkpoints to other members to prevent reset of remaining TTL on leader change. EnableLeaseCheckpoint bool diff --git a/server/embed/config.go b/server/embed/config.go index 30c0d073029..5e0fe252901 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -350,7 +350,8 @@ type Config struct { ExperimentalMaxLearners int `json:"experimental-max-learners"` // ForceNewCluster starts a new cluster even if previously started; unsafe. - ForceNewCluster bool `json:"force-new-cluster"` + ForceNewCluster bool `json:"force-new-cluster"` + ForceNewClusterBumpAmount uint64 `json:"force-new-cluster-bump-amount"` EnablePprof bool `json:"enable-pprof"` Metrics string `json:"metrics"` diff --git a/server/embed/etcd.go b/server/embed/etcd.go index 89dd809cfcc..30cbff3d22a 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -224,6 +224,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { PreVote: cfg.PreVote, Logger: cfg.logger, ForceNewCluster: cfg.ForceNewCluster, + ForceNewClusterBumpAmount: cfg.ForceNewClusterBumpAmount, EnableGRPCGateway: cfg.EnableGRPCGateway, ExperimentalEnableDistributedTracing: cfg.ExperimentalEnableDistributedTracing, UnsafeNoFsync: cfg.UnsafeNoFsync, diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index db8535f1fed..349590ccfdc 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -324,6 +324,7 @@ func newConfig() *config { // unsafe fs.BoolVar(&cfg.ec.UnsafeNoFsync, "unsafe-no-fsync", false, "Disables fsync, unsafe, will cause data loss.") fs.BoolVar(&cfg.ec.ForceNewCluster, "force-new-cluster", false, "Force to create a new one member cluster.") + fs.Uint64Var(&cfg.ec.ForceNewClusterBumpAmount, "force-new-cluster-bump-amount", 0, "How much to increase the latest revision after --force-new-cluster.") // ignored for _, f := range cfg.ignored { diff --git a/server/etcdserver/raft.go b/server/etcdserver/raft.go index 8338af99d38..b022c68fb04 100644 --- a/server/etcdserver/raft.go +++ b/server/etcdserver/raft.go @@ -33,6 +33,8 @@ import ( "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" + "go.etcd.io/etcd/server/v3/mvcc/backend" + "go.etcd.io/etcd/server/v3/revbump" "go.etcd.io/etcd/server/v3/wal" "go.etcd.io/etcd/server/v3/wal/walpb" "go.uber.org/zap" @@ -570,7 +572,8 @@ func restartNode(cfg config.ServerConfig, snapshot *raftpb.Snapshot) (types.ID, return id, cl, n, s, w } -func restartAsStandaloneNode(cfg config.ServerConfig, snapshot *raftpb.Snapshot) (types.ID, *membership.RaftCluster, raft.Node, *raft.MemoryStorage, *wal.WAL) { +func restartAsStandaloneNode(cfg config.ServerConfig, snapshot *raftpb.Snapshot, be backend.Backend) (types.ID, + *membership.RaftCluster, raft.Node, *raft.MemoryStorage, *wal.WAL) { var walsnap walpb.Snapshot if snapshot != nil { walsnap.Index, walsnap.Term = snapshot.Metadata.Index, snapshot.Metadata.Term @@ -610,6 +613,13 @@ func restartAsStandaloneNode(cfg config.ServerConfig, snapshot *raftpb.Snapshot) st.Commit = ents[len(ents)-1].Index } + if cfg.ForceNewClusterBumpAmount > 0 { + err = revbump.UnsafeModifyLastRevision(cfg.Logger, cfg.ForceNewClusterBumpAmount, be) + if err != nil { + cfg.Logger.Fatal("failed to modify last revision", zap.Error(err)) + } + } + cfg.Logger.Info( "forcing restart member", zap.String("cluster-id", cid.String()), diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go index e8250b11baf..6d8097de9d3 100644 --- a/server/etcdserver/server.go +++ b/server/etcdserver/server.go @@ -538,7 +538,7 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { if !cfg.ForceNewCluster { id, cl, n, s, w = restartNode(cfg, snapshot) } else { - id, cl, n, s, w = restartAsStandaloneNode(cfg, snapshot) + id, cl, n, s, w = restartAsStandaloneNode(cfg, snapshot, be) } cl.SetStore(st) diff --git a/server/revbump/revbump.go b/server/revbump/revbump.go new file mode 100644 index 00000000000..ce4ff823b34 --- /dev/null +++ b/server/revbump/revbump.go @@ -0,0 +1,65 @@ +package revbump + +import ( + "go.etcd.io/etcd/server/v3/mvcc" + "go.etcd.io/etcd/server/v3/mvcc/backend" + "go.etcd.io/etcd/server/v3/mvcc/buckets" + "go.uber.org/zap" +) + +func UnsafeModifyLastRevision(lg *zap.Logger, bumpAmount uint64, be backend.Backend) error { + defer be.ForceCommit() + + tx := be.BatchTx() + tx.LockOutsideApply() + defer tx.Unlock() + + latest, err := unsafeGetLatestRevision(tx) + if err != nil { + return err + } + + latest = unsafeBumpRevision(lg, tx, latest, int64(bumpAmount)) + unsafeMarkRevisionCompacted(lg, tx, latest) + return nil +} + +func unsafeBumpRevision(lg *zap.Logger, tx backend.BatchTx, latest revision, amount int64) revision { + lg.Info( + "bumping latest revision", + zap.Int64("latest-revision", latest.main), + zap.Int64("bump-amount", amount), + zap.Int64("new-latest-revision", latest.main+amount), + ) + + latest.main += amount + latest.sub = 0 + k := make([]byte, revBytesLen) + revToBytes(k, latest) + tx.UnsafePut(buckets.Key, k, []byte{}) + + return latest +} + +func unsafeMarkRevisionCompacted(lg *zap.Logger, tx backend.BatchTx, latest revision) { + lg.Info( + "marking revision compacted", + zap.Int64("revision", latest.main), + ) + + mvcc.UnsafeSetScheduledCompact(tx, latest.main) +} + +func unsafeGetLatestRevision(tx backend.BatchTx) (revision, error) { + var latest revision + err := tx.UnsafeForEach(buckets.Key, func(k, _ []byte) (err error) { + rev := bytesToRev(k) + + if rev.GreaterThan(latest) { + latest = rev + } + + return nil + }) + return latest, err +} diff --git a/server/revbump/revision.go b/server/revbump/revision.go new file mode 100644 index 00000000000..875ce537227 --- /dev/null +++ b/server/revbump/revision.go @@ -0,0 +1,46 @@ +package revbump + +import "encoding/binary" + +// revBytesLen is the byte length of a normal revision. +// First 8 bytes is the revision.main in big-endian format. The 9th byte +// is a '_'. The last 8 bytes is the revision.sub in big-endian format. +const revBytesLen = 8 + 1 + 8 +const markedRevBytesLen = revBytesLen + 1 + +// A revision indicates modification of the key-value space. +// The set of changes that share same main revision changes the key-value space atomically. +type revision struct { + // main is the main revision of a set of changes that happen atomically. + main int64 + + // sub is the sub revision of a change in a set of changes that happen + // atomically. Each change has different increasing sub revision in that + // set. + sub int64 +} + +func (a revision) GreaterThan(b revision) bool { + if a.main > b.main { + return true + } + if a.main < b.main { + return false + } + return a.sub > b.sub +} + +// revToBytes should be synced with function in server +// https://github.com/etcd-io/etcd/blob/main/server/storage/mvcc/revision.go +func revToBytes(bytes []byte, rev revision) { + binary.BigEndian.PutUint64(bytes[0:8], uint64(rev.main)) + bytes[8] = '_' + binary.BigEndian.PutUint64(bytes[9:], uint64(rev.sub)) +} + +func bytesToRev(bytes []byte) revision { + return revision{ + main: int64(binary.BigEndian.Uint64(bytes[0:8])), + sub: int64(binary.BigEndian.Uint64(bytes[9:])), + } +} From a5af229ae52440bf0076cdd7eee32861dbba0f96 Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Mon, 4 Nov 2024 16:03:34 +0100 Subject: [PATCH 41/41] DOWNSTREAM: : add tls min/max version to grpc proxy This adds the min and max TLS version support from etcd-io#13506 and etcd-io#15156 to the grpc proxy. Signed-off-by: Thomas Jungblut --- tests/e2e/etcd_grpcproxy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/etcd_grpcproxy_test.go b/tests/e2e/etcd_grpcproxy_test.go index 077b1db189c..84fafba03bc 100644 --- a/tests/e2e/etcd_grpcproxy_test.go +++ b/tests/e2e/etcd_grpcproxy_test.go @@ -127,7 +127,7 @@ func TestGrpcProxyAutoSync(t *testing.T) { require.NoError(t, proxyProc.Stop()) } -func TestGrpcProxyTLSVersions(t *testing.T) { +func TestGrpcProxyTlsVersions(t *testing.T) { e2e.SkipInShortMode(t) epc, err := e2e.NewEtcdProcessCluster(t, e2e.NewConfigClientBoth())