Add debug capability to F3 itests to print current progress

To aid debugging failing tests add option to print progress of all nodes
at every eventual assertion, disabled by default.

Author: masih

itests/f3_test.go

@@ -37,6 +37,7 @@ type testEnv struct {
 	m       *manifest.Manifest
 	t       *testing.T
 	testCtx context.Context
+	debug   bool
 }
 
 // Test that checks that F3 is enabled successfully,
@@ -100,7 +101,6 @@ func TestF3_PauseAndRebootstrap(t *testing.T) {
 	e.ms.UpdateManifest(&cpy)
 
 	e.waitTillManifestChange(&cpy, 20*time.Second)
-	e.waitTillF3Instance(0, 200*time.Second)
 	e.waitTillF3Rebootstrap(20 * time.Second)
 }
 
@@ -121,7 +121,7 @@ func TestF3_Bootstrap(t *testing.T) {
 	e.waitTillManifestChange(&dynamicManif, 20*time.Second)
 	e.waitTillF3Instance(2, 20*time.Second)
 	e.waitTillManifestChange(staticManif, 20*time.Second)
-	e.waitTillF3Instance(2, 20*time.Second)
+	e.waitTillF3Instance(2, 200*time.Second)
 
 	// Try to switch back, we should ignore the manifest update.
 	e.ms.UpdateManifest(&dynamicManif)
@@ -196,23 +196,26 @@ func (e *testEnv) waitFor(f func(n *kit.TestFullNode) bool, timeout time.Duratio
 	e.t.Helper()
 	require.Eventually(e.t, func() bool {
 		e.t.Helper()
-		for _, n := range e.minerFullNodes {
-			if !f(n) {
+		defer func() {
+			if e.debug {
 				var wg sync.WaitGroup
-				printProgress := func(n *kit.TestFullNode) {
+				printProgress := func(index int, n *kit.TestFullNode) {
 					defer wg.Done()
-					id, err := n.ID(e.testCtx)
-					require.NoError(e.t, err)
-
-					progress, err := n.F3GetProgress(e.testCtx)
-					require.NoError(e.t, err)
-					e.t.Logf("###### %s -> %v", id, progress)
+					if progress, err := n.F3GetProgress(e.testCtx); err != nil {
+						e.t.Logf("Node #%d progress: err: %v", index, err)
+					} else {
+						e.t.Logf("Node #%d progress: %v", index, progress)
+					}
 				}
-				for _, n := range e.minerFullNodes {
+				for i, n := range e.minerFullNodes {
 					wg.Add(1)
-					go printProgress(n)
+					go printProgress(i, n)
 				}
 				wg.Wait()
+			}
+		}()
+		for _, n := range e.minerFullNodes {
+			if !f(n) {
 				return false
 			}
 		}
@@ -227,12 +230,6 @@ func (e *testEnv) waitFor(f func(n *kit.TestFullNode) bool, timeout time.Duratio
 // a miner. The last return value is the manifest sender for the network.
 func setup(t *testing.T, blocktime time.Duration) *testEnv {
 	manif := lf3.NewManifest(BaseNetworkName+"/1", DefaultFinality, DefaultBootstrapEpoch, blocktime, cid.Undef)
-	manif.Gpbft.Delta = 250 * time.Millisecond
-	manif.Gpbft.DeltaBackOffExponent = 1.3
-	manif.Gpbft.RebroadcastBackoffBase = manif.Gpbft.Delta * 2
-	manif.Gpbft.RebroadcastBackoffMax = manif.Gpbft.RebroadcastBackoffBase * 2
-	manif.Gpbft.RebroadcastBackoffExponent = manif.Gpbft.DeltaBackOffExponent
-	manif.Gpbft.RebroadcastBackoffSpread = 0.2
 
 	return setupWithStaticManifest(t, manif, false)
 }
@@ -300,18 +297,6 @@ func setupWithStaticManifest(t *testing.T, manif *manifest.Manifest, testBootstr
 		require.NoError(t, err)
 	}
 
-	m.CertificateExchange.MinimumPollInterval = 200 * time.Millisecond
-	m.CertificateExchange.MaximumPollInterval = 1 * time.Second
-	m.Gpbft.Delta = 250 * time.Millisecond
-	m.Gpbft.DeltaBackOffExponent = 1.3
-	m.Gpbft.RebroadcastBackoffBase = m.Gpbft.Delta * 2
-	m.Gpbft.RebroadcastBackoffMax = m.Gpbft.RebroadcastBackoffBase * 2
-	m.Gpbft.RebroadcastBackoffExponent = m.Gpbft.DeltaBackOffExponent
-	m.Gpbft.RebroadcastBackoffSpread = 0.2
-
-	e.ms.UpdateManifest(m)
-	e.waitTillManifestChange(m, 30*time.Second)
-
 	errgrp.Go(func() error {
 		defer func() {
 			require.NoError(t, manifestServerHost.Close())