From 1b152071fcc74869188ee0a5a68cf1d670251961 Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Mon, 14 Oct 2024 13:58:37 +0100 Subject: [PATCH 1/8] Investigate intermittent F3 itest failures on CI Repeat F3 itests on CI to investigate intermittent failures. --- .github/workflows/test-f3.yaml | 29 +++++++++++++++++++++++++++ itests/f3_test.go | 36 ++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 .github/workflows/test-f3.yaml diff --git a/.github/workflows/test-f3.yaml b/.github/workflows/test-f3.yaml new file mode 100644 index 00000000000..f1ad3efbb82 --- /dev/null +++ b/.github/workflows/test-f3.yaml @@ -0,0 +1,29 @@ +name: Repeat F3 Integration Tests + +on: + pull_request: + +jobs: + test: + name: ${{ matrix.test }} ${{ matrix.go }} + runs-on: ubuntu-latest + # needs: cache-test-bins + strategy: + fail-fast: false + matrix: + go: + - 1.22 + test: + - TestF3_Enabled + - TestF3_Rebootstrap + - TestF3_PauseAndRebootstrap + - TestF3_Bootstrap + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: ./.github/actions/install-system-dependencies + - uses: ./.github/actions/install-go + - uses: ./.github/actions/make-deps + - name: Repeat ${{ matrix.test }} + run: go test -timeout 30m -count 50 -v --run='^${{ matrix.test }}' --failfast ./itests \ No newline at end of file diff --git a/itests/f3_test.go b/itests/f3_test.go index 68d5e713e6c..40789d444df 100644 --- a/itests/f3_test.go +++ b/itests/f3_test.go @@ -2,6 +2,7 @@ package itests import ( "context" + "sync" "testing" "time" @@ -99,6 +100,7 @@ func TestF3_PauseAndRebootstrap(t *testing.T) { e.ms.UpdateManifest(&cpy) e.waitTillManifestChange(&cpy, 20*time.Second) + e.waitTillF3Instance(0, 200*time.Second) e.waitTillF3Rebootstrap(20 * time.Second) } @@ -196,6 +198,21 @@ func (e *testEnv) waitFor(f func(n *kit.TestFullNode) bool, timeout time.Duratio e.t.Helper() for _, n := range e.minerFullNodes { if !f(n) { + var wg sync.WaitGroup + printProgress := func(n *kit.TestFullNode) { + defer wg.Done() + id, err := n.ID(e.testCtx) + require.NoError(e.t, err) + + progress, err := n.F3GetProgress(e.testCtx) + require.NoError(e.t, err) + e.t.Logf("###### %s -> %v", id, progress) + } + for _, n := range e.minerFullNodes { + wg.Add(1) + go printProgress(n) + } + wg.Wait() return false } } @@ -210,6 +227,13 @@ func (e *testEnv) waitFor(f func(n *kit.TestFullNode) bool, timeout time.Duratio // a miner. The last return value is the manifest sender for the network. func setup(t *testing.T, blocktime time.Duration) *testEnv { manif := lf3.NewManifest(BaseNetworkName+"/1", DefaultFinality, DefaultBootstrapEpoch, blocktime, cid.Undef) + manif.Gpbft.Delta = 250 * time.Millisecond + manif.Gpbft.DeltaBackOffExponent = 1.3 + manif.Gpbft.RebroadcastBackoffBase = manif.Gpbft.Delta * 2 + manif.Gpbft.RebroadcastBackoffMax = manif.Gpbft.RebroadcastBackoffBase * 2 + manif.Gpbft.RebroadcastBackoffExponent = manif.Gpbft.DeltaBackOffExponent + manif.Gpbft.RebroadcastBackoffSpread = 0.2 + return setupWithStaticManifest(t, manif, false) } @@ -276,6 +300,18 @@ func setupWithStaticManifest(t *testing.T, manif *manifest.Manifest, testBootstr require.NoError(t, err) } + m.CertificateExchange.MinimumPollInterval = 200 * time.Millisecond + m.CertificateExchange.MaximumPollInterval = 1 * time.Second + m.Gpbft.Delta = 250 * time.Millisecond + m.Gpbft.DeltaBackOffExponent = 1.3 + m.Gpbft.RebroadcastBackoffBase = m.Gpbft.Delta * 2 + m.Gpbft.RebroadcastBackoffMax = m.Gpbft.RebroadcastBackoffBase * 2 + m.Gpbft.RebroadcastBackoffExponent = m.Gpbft.DeltaBackOffExponent + m.Gpbft.RebroadcastBackoffSpread = 0.2 + + e.ms.UpdateManifest(m) + e.waitTillManifestChange(m, 30*time.Second) + errgrp.Go(func() error { defer func() { require.NoError(t, manifestServerHost.Close()) From 49546f425ee3f4e940a3f556640d056f7083cfce Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Wed, 16 Oct 2024 15:51:48 +0100 Subject: [PATCH 2/8] Fix participation lease removal for wrong network When manifest changes, depending on the timing it is possible for newly generated valid leases to get removed if the sign message loop attempts to sign messages that are as a result of progressing previous network. Here is an example scenario in a specific order that was causing itests to fail: * participants get a lease for network A up to instance 5 * network A progresses to instance 6 * manifest changes the network name to B * participants get a new lease for network B up to instance 5 * sign loop receives a message from network A, instance 6 * `getParticipantsByInstance` lazily removes leases since it only checks the instance. * the node ends up with no participants, and stuck. To fix this: 1) check if participants asked for are within the current network, and if not refuse to participate. 2) check network name, as well as instance, to lazily remove expired leases. --- chain/lf3/f3.go | 2 +- chain/lf3/participation_lease.go | 17 ++++++++++++++--- chain/lf3/participation_lease_test.go | 6 +++--- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/chain/lf3/f3.go b/chain/lf3/f3.go index 1c36ed491bd..9a3fa3d278d 100644 --- a/chain/lf3/f3.go +++ b/chain/lf3/f3.go @@ -155,7 +155,7 @@ func (fff *F3) runSigningLoop(ctx context.Context) { clear(alreadyParticipated) } - participants := fff.leaser.getParticipantsByInstance(mb.Payload.Instance) + participants := fff.leaser.getParticipantsByInstance(mb.NetworkName, mb.Payload.Instance) for _, id := range participants { if _, ok := alreadyParticipated[id]; ok { continue diff --git a/chain/lf3/participation_lease.go b/chain/lf3/participation_lease.go index 24e7afde861..632466a2c78 100644 --- a/chain/lf3/participation_lease.go +++ b/chain/lf3/participation_lease.go @@ -112,15 +112,26 @@ func (l *leaser) participate(ticket api.F3ParticipationTicket) (api.F3Participat return newLease, nil } -func (l *leaser) getParticipantsByInstance(instance uint64) []uint64 { +func (l *leaser) getParticipantsByInstance(network gpbft.NetworkName, instance uint64) []uint64 { l.mutex.Lock() defer l.mutex.Unlock() + currentManifest, _ := l.status() + currentNetwork := currentManifest.NetworkName + if currentNetwork != network { + log.Warnf("no participants for network: current network (%s) does not match requested network (%s) at instance %d", currentNetwork, network, instance) + return nil + } var participants []uint64 for id, lease := range l.leases { - if instance > lease.ToInstance() { + if currentNetwork != lease.Network { + // Lazily delete any lease that does not belong to network, likely acquired from + // prior manifests. + delete(l.leases, id) + log.Warnf("lost F3 participation lease for miner %d at instance %d due to network mismatch: %s != %s", id, instance, currentNetwork, lease.Network) + } else if instance > lease.ToInstance() { // Lazily delete the expired leases. delete(l.leases, id) - log.Warnf("lost F3 participation lease for miner %d", id) + log.Warnf("lost F3 participation lease for miner %d due to instance (%d) > lease to instance (%d)", id, instance, lease.ToInstance()) } else { participants = append(participants, id) } diff --git a/chain/lf3/participation_lease_test.go b/chain/lf3/participation_lease_test.go index 731c1ba5be3..9cf9bee9d66 100644 --- a/chain/lf3/participation_lease_test.go +++ b/chain/lf3/participation_lease_test.go @@ -42,18 +42,18 @@ func TestLeaser(t *testing.T) { require.NoError(t, err) // Both participants should still be valid. - participants := subject.getParticipantsByInstance(11) + participants := subject.getParticipantsByInstance(testManifest.NetworkName, 11) require.Len(t, participants, 2) require.Contains(t, participants, uint64(123)) require.Contains(t, participants, uint64(456)) // After instance 16, only participant 456 should be valid. - participants = subject.getParticipantsByInstance(16) + participants = subject.getParticipantsByInstance(testManifest.NetworkName, 16) require.Len(t, participants, 1) require.Contains(t, participants, uint64(456)) // After instance 17, no participant must have a lease. - participants = subject.getParticipantsByInstance(17) + participants = subject.getParticipantsByInstance(testManifest.NetworkName, 17) require.Empty(t, participants) }) t.Run("expired ticket", func(t *testing.T) { From 4192741a6405e51d5bdf6e317d475f1d5cc3e019 Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Wed, 16 Oct 2024 15:53:03 +0100 Subject: [PATCH 3/8] Add debug capability to F3 itests to print current progress To aid debugging failing tests add option to print progress of all nodes at every eventual assertion, disabled by default. --- itests/f3_test.go | 46 +++++++++++++++------------------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/itests/f3_test.go b/itests/f3_test.go index 40789d444df..9d5d6494ebe 100644 --- a/itests/f3_test.go +++ b/itests/f3_test.go @@ -37,6 +37,7 @@ type testEnv struct { m *manifest.Manifest t *testing.T testCtx context.Context + debug bool } // Test that checks that F3 is enabled successfully, @@ -100,7 +101,6 @@ func TestF3_PauseAndRebootstrap(t *testing.T) { e.ms.UpdateManifest(&cpy) e.waitTillManifestChange(&cpy, 20*time.Second) - e.waitTillF3Instance(0, 200*time.Second) e.waitTillF3Rebootstrap(20 * time.Second) } @@ -196,23 +196,26 @@ func (e *testEnv) waitFor(f func(n *kit.TestFullNode) bool, timeout time.Duratio e.t.Helper() require.Eventually(e.t, func() bool { e.t.Helper() - for _, n := range e.minerFullNodes { - if !f(n) { + defer func() { + if e.debug { var wg sync.WaitGroup - printProgress := func(n *kit.TestFullNode) { + printProgress := func(index int, n *kit.TestFullNode) { defer wg.Done() - id, err := n.ID(e.testCtx) - require.NoError(e.t, err) - - progress, err := n.F3GetProgress(e.testCtx) - require.NoError(e.t, err) - e.t.Logf("###### %s -> %v", id, progress) + if progress, err := n.F3GetProgress(e.testCtx); err != nil { + e.t.Logf("Node #%d progress: err: %v", index, err) + } else { + e.t.Logf("Node #%d progress: %v", index, progress) + } } - for _, n := range e.minerFullNodes { + for i, n := range e.minerFullNodes { wg.Add(1) - go printProgress(n) + go printProgress(i, n) } wg.Wait() + } + }() + for _, n := range e.minerFullNodes { + if !f(n) { return false } } @@ -227,13 +230,6 @@ func (e *testEnv) waitFor(f func(n *kit.TestFullNode) bool, timeout time.Duratio // a miner. The last return value is the manifest sender for the network. func setup(t *testing.T, blocktime time.Duration) *testEnv { manif := lf3.NewManifest(BaseNetworkName+"/1", DefaultFinality, DefaultBootstrapEpoch, blocktime, cid.Undef) - manif.Gpbft.Delta = 250 * time.Millisecond - manif.Gpbft.DeltaBackOffExponent = 1.3 - manif.Gpbft.RebroadcastBackoffBase = manif.Gpbft.Delta * 2 - manif.Gpbft.RebroadcastBackoffMax = manif.Gpbft.RebroadcastBackoffBase * 2 - manif.Gpbft.RebroadcastBackoffExponent = manif.Gpbft.DeltaBackOffExponent - manif.Gpbft.RebroadcastBackoffSpread = 0.2 - return setupWithStaticManifest(t, manif, false) } @@ -300,18 +296,6 @@ func setupWithStaticManifest(t *testing.T, manif *manifest.Manifest, testBootstr require.NoError(t, err) } - m.CertificateExchange.MinimumPollInterval = 200 * time.Millisecond - m.CertificateExchange.MaximumPollInterval = 1 * time.Second - m.Gpbft.Delta = 250 * time.Millisecond - m.Gpbft.DeltaBackOffExponent = 1.3 - m.Gpbft.RebroadcastBackoffBase = m.Gpbft.Delta * 2 - m.Gpbft.RebroadcastBackoffMax = m.Gpbft.RebroadcastBackoffBase * 2 - m.Gpbft.RebroadcastBackoffExponent = m.Gpbft.DeltaBackOffExponent - m.Gpbft.RebroadcastBackoffSpread = 0.2 - - e.ms.UpdateManifest(m) - e.waitTillManifestChange(m, 30*time.Second) - errgrp.Go(func() error { defer func() { require.NoError(t, manifestServerHost.Close()) From 0d3cb660b705a5099ce6b964ac97701ca814e6bf Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Wed, 16 Oct 2024 16:50:41 +0100 Subject: [PATCH 4/8] Shorten GPBFT settings for a more responsive timing Defaults are based on epoch of 30s and real RTT. Shorten Delta and rebroadcast times. --- itests/f3_test.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/itests/f3_test.go b/itests/f3_test.go index 9d5d6494ebe..fb1af08cf5b 100644 --- a/itests/f3_test.go +++ b/itests/f3_test.go @@ -230,6 +230,12 @@ func (e *testEnv) waitFor(f func(n *kit.TestFullNode) bool, timeout time.Duratio // a miner. The last return value is the manifest sender for the network. func setup(t *testing.T, blocktime time.Duration) *testEnv { manif := lf3.NewManifest(BaseNetworkName+"/1", DefaultFinality, DefaultBootstrapEpoch, blocktime, cid.Undef) + manif.Gpbft.Delta = 250 * time.Millisecond + manif.Gpbft.DeltaBackOffExponent = 1.3 + manif.Gpbft.RebroadcastBackoffBase = manif.Gpbft.Delta * 2 + manif.Gpbft.RebroadcastBackoffMax = manif.Gpbft.RebroadcastBackoffBase * 2 + manif.Gpbft.RebroadcastBackoffExponent = manif.Gpbft.DeltaBackOffExponent + return setupWithStaticManifest(t, manif, false) } @@ -295,7 +301,6 @@ func setupWithStaticManifest(t *testing.T, manif *manifest.Manifest, testBootstr err = n.NetConnect(ctx, e.ms.PeerInfo()) require.NoError(t, err) } - errgrp.Go(func() error { defer func() { require.NoError(t, manifestServerHost.Close()) @@ -303,6 +308,16 @@ func setupWithStaticManifest(t *testing.T, manif *manifest.Manifest, testBootstr return e.ms.Run(ctx) }) + // Update initial manifest params to shorten the timeouts and backoff for + // testing, and assert it is consistently applied to all nodes. + e.m.Gpbft.Delta = 250 * time.Millisecond + e.m.Gpbft.DeltaBackOffExponent = 1.3 + e.m.Gpbft.RebroadcastBackoffBase = manif.Gpbft.Delta * 2 + e.m.Gpbft.RebroadcastBackoffMax = manif.Gpbft.RebroadcastBackoffBase * 2 + e.m.Gpbft.RebroadcastBackoffExponent = manif.Gpbft.DeltaBackOffExponent + e.ms.UpdateManifest(m) + e.waitTillManifestChange(m, 20*time.Second) + return e } From dcffa4b688b4c9bb2f206effd945cd24996a5666 Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Wed, 16 Oct 2024 17:12:34 +0100 Subject: [PATCH 5/8] Remove F3 itest repetitions on CI now that saul goodman See proof of the pudding: * https://github.com/filecoin-project/lotus/actions/runs/11369403828/job/31626763159?pr=12597 --- .github/workflows/test-f3.yaml | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 .github/workflows/test-f3.yaml diff --git a/.github/workflows/test-f3.yaml b/.github/workflows/test-f3.yaml deleted file mode 100644 index f1ad3efbb82..00000000000 --- a/.github/workflows/test-f3.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: Repeat F3 Integration Tests - -on: - pull_request: - -jobs: - test: - name: ${{ matrix.test }} ${{ matrix.go }} - runs-on: ubuntu-latest - # needs: cache-test-bins - strategy: - fail-fast: false - matrix: - go: - - 1.22 - test: - - TestF3_Enabled - - TestF3_Rebootstrap - - TestF3_PauseAndRebootstrap - - TestF3_Bootstrap - steps: - - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - uses: ./.github/actions/install-system-dependencies - - uses: ./.github/actions/install-go - - uses: ./.github/actions/make-deps - - name: Repeat ${{ matrix.test }} - run: go test -timeout 30m -count 50 -v --run='^${{ matrix.test }}' --failfast ./itests \ No newline at end of file From 9306a3fce5c5da2a4ccc53f39720a303073adf72 Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Wed, 16 Oct 2024 17:15:09 +0100 Subject: [PATCH 6/8] Update the changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47ecf7d0768..603c0792a5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Fix a bug in the `lotus-shed indexes backfill-events` command that may result in either duplicate events being backfilled where there are existing events (such an operation *should* be idempotent) or events erroneously having duplicate `logIndex` values when queried via ETH APIs. ([filecoin-project/lotus#12567](https://github.com/filecoin-project/lotus/pull/12567)) - Event APIs (Eth events and actor events) should only return reverted events if client queries by specific block hash / tipset. Eth and actor event subscription APIs should always return reverted events to enable accurate observation of real-time changes. ([filecoin-project/lotus#12585](https://github.com/filecoin-project/lotus/pull/12585)) - Add logic to check if the miner's owner address is delegated (f4 address). If it is delegated, the `lotus-shed sectors termination-estimate` command now sends the termination state call using the worker ID. This fix resolves the issue where termination-estimate did not function correctly for miners with delegated owner addresses. ([filecoin-project/lotus#12569](https://github.com/filecoin-project/lotus/pull/12569)) +- Fix a bug in F3 participation API where valid leases may get removed due to dynamic manifest update. ([filecoin-project/lotus#12597](https://github.com/filecoin-project/lotus/pull/12597)) ## Improvements From d4177fc3e8c704c4eaa500457c54d82841bfd043 Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Wed, 16 Oct 2024 18:30:26 +0100 Subject: [PATCH 7/8] Address review comments --- chain/lf3/participation_lease.go | 1 - itests/f3_test.go | 62 ++++++++++++++++++++------------ 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/chain/lf3/participation_lease.go b/chain/lf3/participation_lease.go index 632466a2c78..ef8c1a9074e 100644 --- a/chain/lf3/participation_lease.go +++ b/chain/lf3/participation_lease.go @@ -118,7 +118,6 @@ func (l *leaser) getParticipantsByInstance(network gpbft.NetworkName, instance u currentManifest, _ := l.status() currentNetwork := currentManifest.NetworkName if currentNetwork != network { - log.Warnf("no participants for network: current network (%s) does not match requested network (%s) at instance %d", currentNetwork, network, instance) return nil } var participants []uint64 diff --git a/itests/f3_test.go b/itests/f3_test.go index fb1af08cf5b..5d215949d61 100644 --- a/itests/f3_test.go +++ b/itests/f3_test.go @@ -229,14 +229,42 @@ func (e *testEnv) waitFor(f func(n *kit.TestFullNode) bool, timeout time.Duratio // and the second full-node is an observer that is not directly connected to // a miner. The last return value is the manifest sender for the network. func setup(t *testing.T, blocktime time.Duration) *testEnv { - manif := lf3.NewManifest(BaseNetworkName+"/1", DefaultFinality, DefaultBootstrapEpoch, blocktime, cid.Undef) - manif.Gpbft.Delta = 250 * time.Millisecond - manif.Gpbft.DeltaBackOffExponent = 1.3 - manif.Gpbft.RebroadcastBackoffBase = manif.Gpbft.Delta * 2 - manif.Gpbft.RebroadcastBackoffMax = manif.Gpbft.RebroadcastBackoffBase * 2 - manif.Gpbft.RebroadcastBackoffExponent = manif.Gpbft.DeltaBackOffExponent - - return setupWithStaticManifest(t, manif, false) + return setupWithStaticManifest(t, newTestManifest(blocktime), false) +} + +func newTestManifest(blocktime time.Duration) *manifest.Manifest { + return &manifest.Manifest{ + ProtocolVersion: manifest.VersionCapability, + BootstrapEpoch: DefaultBootstrapEpoch, + NetworkName: BaseNetworkName + "/1", + InitialPowerTable: cid.Undef, + CommitteeLookback: manifest.DefaultCommitteeLookback, + CatchUpAlignment: blocktime / 2, + Gpbft: manifest.GpbftConfig{ + // Use smaller time intervals for more responsive test progress/assertion. + Delta: 250 * time.Millisecond, + DeltaBackOffExponent: 1.3, + MaxLookaheadRounds: 5, + RebroadcastBackoffBase: 500 * time.Millisecond, + RebroadcastBackoffSpread: 0.1, + RebroadcastBackoffExponent: 1.3, + RebroadcastBackoffMax: 1 * time.Second, + }, + EC: manifest.EcConfig{ + Period: blocktime, + Finality: DefaultFinality, + DelayMultiplier: manifest.DefaultEcConfig.DelayMultiplier, + BaseDecisionBackoffTable: manifest.DefaultEcConfig.BaseDecisionBackoffTable, + HeadLookback: 0, + Finalize: true, + }, + CertificateExchange: manifest.CxConfig{ + ClientRequestTimeout: manifest.DefaultCxConfig.ClientRequestTimeout, + ServerRequestTimeout: manifest.DefaultCxConfig.ServerRequestTimeout, + MinimumPollInterval: blocktime, + MaximumPollInterval: 4 * blocktime, + }, + } } func setupWithStaticManifest(t *testing.T, manif *manifest.Manifest, testBootstrap bool) *testEnv { @@ -288,10 +316,7 @@ func setupWithStaticManifest(t *testing.T, manif *manifest.Manifest, testBootstr cancel() } - m, err := n1.F3GetManifest(ctx) - require.NoError(t, err) - - e := &testEnv{m: m, t: t, testCtx: ctx} + e := &testEnv{m: manif, t: t, testCtx: ctx} // in case we want to use more full-nodes in the future e.minerFullNodes = []*kit.TestFullNode{&n1, &n2, &n3} @@ -308,16 +333,9 @@ func setupWithStaticManifest(t *testing.T, manif *manifest.Manifest, testBootstr return e.ms.Run(ctx) }) - // Update initial manifest params to shorten the timeouts and backoff for - // testing, and assert it is consistently applied to all nodes. - e.m.Gpbft.Delta = 250 * time.Millisecond - e.m.Gpbft.DeltaBackOffExponent = 1.3 - e.m.Gpbft.RebroadcastBackoffBase = manif.Gpbft.Delta * 2 - e.m.Gpbft.RebroadcastBackoffMax = manif.Gpbft.RebroadcastBackoffBase * 2 - e.m.Gpbft.RebroadcastBackoffExponent = manif.Gpbft.DeltaBackOffExponent - e.ms.UpdateManifest(m) - e.waitTillManifestChange(m, 20*time.Second) - + // Assure manifest is picked up by all nodes. + e.ms.UpdateManifest(manif) + e.waitTillManifestChange(manif, 20*time.Second) return e } From ce08efd9fd7b3d34d3eb0804b5c68a0760344ef9 Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Wed, 16 Oct 2024 18:50:02 +0100 Subject: [PATCH 8/8] Remove the sanity check that all nodes use the same initial manifest --- itests/f3_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/itests/f3_test.go b/itests/f3_test.go index 5d215949d61..e3db29043c0 100644 --- a/itests/f3_test.go +++ b/itests/f3_test.go @@ -333,9 +333,6 @@ func setupWithStaticManifest(t *testing.T, manif *manifest.Manifest, testBootstr return e.ms.Run(ctx) }) - // Assure manifest is picked up by all nodes. - e.ms.UpdateManifest(manif) - e.waitTillManifestChange(manif, 20*time.Second) return e }