Main patch from tm-v0.34.16

.github/workflows/coverage.yml

@@ -79,7 +79,7 @@ jobs:
           path: crypto/vrf/internal/vrf/sodium
       - name: test & coverage report creation
         run: |
-          cat pkgs.txt.part.${{ matrix.part }} | xargs go test -mod=readonly -timeout 10m -race -coverprofile=${{ matrix.part }}profile.out -covermode=atomic -tags ${{ matrix.vrf }}
+          cat pkgs.txt.part.${{ matrix.part }} | xargs go test -mod=readonly -timeout 7m -race -coverprofile=${{ matrix.part }}profile.out -covermode=atomic -tags ${{ matrix.vrf }}
         if: env.GIT_DIFF
       - uses: actions/upload-artifact@v2
         with:

.github/workflows/e2e-manual.yml

@@ -0,0 +1,35 @@
+# Manually run randomly generated E2E testnets (as nightly).
+name: e2e-manual
+on:
+  workflow_dispatch:
+
+jobs:
+  e2e-nightly-test:
+    # Run parallel jobs for the listed testnet groups (must match the
+    # ./build/generator -g flag)
+    strategy:
+      fail-fast: false
+      matrix:
+        group: ['00', '01', '02', '03']
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+
+      - uses: actions/[email protected]
+
+      - name: Build
+        working-directory: test/e2e
+        # Run make jobs in parallel, since we can't run steps in parallel.
+        run: make -j2 docker generator runner tests
+
+      - name: Generate testnets
+        working-directory: test/e2e
+        # When changing -g, also change the matrix groups above
+        run: ./build/generator -g 4 -d networks/nightly/
+
+      - name: Run ${{ matrix.p2p }} p2p testnets
+        working-directory: test/e2e
+        run: ./run-multiple.sh networks/nightly/*-group${{ matrix.group }}-*.toml

Makefile

@@ -242,7 +242,7 @@ DESTINATION = ./index.html.md
 BRANCH := $(shell git branch --show-current)
 BRANCH_URI := $(shell git branch --show-current | sed 's/[\#]/%23/g')
 build-docs:
-	cd docs && \
+	@cd docs && \
 	npm install && \
 	VUEPRESS_BASE="/$(BRANCH_URI)/" npm run build && \
 	mkdir -p ~/output/$(BRANCH) && \

consensus/metrics.go

@@ -4,7 +4,7 @@ import (
 	"github.com/go-kit/kit/metrics"
 	"github.com/go-kit/kit/metrics/discard"
 
-	prometheus "github.com/go-kit/kit/metrics/prometheus"
+	"github.com/go-kit/kit/metrics/prometheus"
 	stdprometheus "github.com/prometheus/client_golang/prometheus"
 )
 
@@ -66,6 +66,22 @@ type Metrics struct {
 	// Number of blockparts transmitted by peer.
 	BlockParts metrics.Counter
 
+	// QuroumPrevoteMessageDelay is the interval in seconds between the proposal
+	// timestamp and the timestamp of the earliest prevote that achieved a quorum
+	// during the prevote step.
+	//
+	// To compute it, sum the voting power over each prevote received, in increasing
+	// order of timestamp. The timestamp of the first prevote to increase the sum to
+	// be above 2/3 of the total voting power of the network defines the endpoint
+	// the endpoint of the interval. Subtract the proposal timestamp from this endpoint
+	// to obtain the quorum delay.
+	QuorumPrevoteMessageDelay metrics.Gauge
+
+	// FullPrevoteMessageDelay is the interval in seconds between the proposal
+	// timestamp and the timestamp of the latest prevote in a round where 100%
+	// of the voting power on the network issued prevotes.
+	FullPrevoteMessageDelay metrics.Gauge
+
 	// ////////////////////////////////////
 	// Metrics for measuring performance
 	// ////////////////////////////////////
@@ -230,6 +246,20 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
 			Name:      "block_parts",
 			Help:      "Number of blockparts transmitted by peer.",
 		}, append(labels, "peer_id")).With(labelsAndValues...),
+		QuorumPrevoteMessageDelay: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "quorum_prevote_message_delay",
+			Help: "Difference in seconds between the proposal timestamp and the timestamp " +
+				"of the latest prevote that achieved a quorum in the prevote step.",
+		}, labels).With(labelsAndValues...),
+		FullPrevoteMessageDelay: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "full_prevote_message_delay",
+			Help: "Difference in seconds between the proposal timestamp and the timestamp " +
+				"of the latest prevote that achieved 100% of the voting power in the prevote step.",
+		}, labels).With(labelsAndValues...),
 		MissingProposal: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: MetricsSubsystem,
@@ -359,13 +389,15 @@ func NopMetrics() *Metrics {
 
 		BlockIntervalSeconds: discard.NewGauge(),
 
-		NumTxs:          discard.NewGauge(),
-		BlockSizeBytes:  discard.NewGauge(),
-		TotalTxs:        discard.NewGauge(),
-		CommittedHeight: discard.NewGauge(),
-		FastSyncing:     discard.NewGauge(),
-		StateSyncing:    discard.NewGauge(),
-		BlockParts:      discard.NewCounter(),
+		NumTxs:                    discard.NewGauge(),
+		BlockSizeBytes:            discard.NewGauge(),
+		TotalTxs:                  discard.NewGauge(),
+		CommittedHeight:           discard.NewGauge(),
+		FastSyncing:               discard.NewGauge(),
+		StateSyncing:              discard.NewGauge(),
+		BlockParts:                discard.NewCounter(),
+		QuorumPrevoteMessageDelay: discard.NewGauge(),
+		FullPrevoteMessageDelay:   discard.NewGauge(),
 
 		MissingProposal: discard.NewGauge(),
 		RoundFailures:   discard.NewHistogram(),

consensus/reactor.go

@@ -500,7 +500,6 @@ OUTER_LOOP:
 	for {
 		// Manage disconnects from self or peer.
 		if !peer.IsRunning() || !conR.IsRunning() {
-			logger.Info("Stopping gossipDataRoutine for peer")
 			return
 		}
 		rs := conR.conS.GetRoundState()
@@ -644,7 +643,6 @@ OUTER_LOOP:
 	for {
 		// Manage disconnects from self or peer.
 		if !peer.IsRunning() || !conR.IsRunning() {
-			logger.Info("Stopping gossipVotesRoutine for peer")
 			return
 		}
 		rs := conR.conS.GetRoundState()
@@ -771,13 +769,11 @@ func (conR *Reactor) gossipVotesForHeight(
 // NOTE: `queryMaj23Routine` has a simple crude design since it only comes
 // into play for liveness when there's a signature DDoS attack happening.
 func (conR *Reactor) queryMaj23Routine(peer p2p.Peer, ps *PeerState) {
-	logger := conR.Logger.With("peer", peer)
 
 OUTER_LOOP:
 	for {
 		// Manage disconnects from self or peer.
 		if !peer.IsRunning() || !conR.IsRunning() {
-			logger.Info("Stopping queryMaj23Routine for peer")
 			return
 		}
 

consensus/state.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"runtime/debug"
+	"sort"
 	"time"
 
 	"github.com/gogo/protobuf/proto"
@@ -1622,6 +1623,8 @@ func (cs *State) finalizeCommit(height int64) {
 		return
 	}
 
+	cs.calculatePrevoteMessageDelayMetrics()
+
 	blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority()
 	block, blockParts := cs.ProposalBlock, cs.ProposalBlockParts
 
@@ -2393,6 +2396,32 @@ func (cs *State) checkDoubleSigningRisk(height int64) error {
 	return nil
 }
 
+func (cs *State) calculatePrevoteMessageDelayMetrics() {
+	if cs.Proposal == nil {
+		return
+	}
+
+	ps := cs.Votes.Prevotes(cs.Round)
+	pl := ps.List()
+
+	sort.Slice(pl, func(i, j int) bool {
+		return pl[i].Timestamp.Before(pl[j].Timestamp)
+	})
+
+	var votingPowerSeen int64
+	for _, v := range pl {
+		_, voter := cs.Voters.GetByAddress(v.ValidatorAddress)
+		votingPowerSeen += voter.VotingPower
+		if votingPowerSeen >= cs.Voters.TotalVotingPower()*2/3+1 {
+			cs.metrics.QuorumPrevoteMessageDelay.Set(v.Timestamp.Sub(cs.Proposal.Timestamp).Seconds())
+			break
+		}
+	}
+	if ps.HasAll() {
+		cs.metrics.FullPrevoteMessageDelay.Set(pl[len(pl)-1].Timestamp.Sub(cs.Proposal.Timestamp).Seconds())
+	}
+}
+
 //---------------------------------------------------------
 
 func CompareHRS(h1 int64, r1 int32, s1 cstypes.RoundStepType, h2 int64, r2 int32, s2 cstypes.RoundStepType) int {

docs/package.json

@@ -4,7 +4,7 @@
   "description": "Ostracon Documentation",
   "main": "index.js",
   "dependencies": {
-    "vuepress-theme-cosmos": "^1.0.180"
+    "vuepress-theme-cosmos": "^1.0.183"
   },
   "devDependencies": {
     "watchpack": "^2.1.1"

p2p/pex/pex_reactor.go

@@ -397,15 +397,14 @@ func (r *Reactor) ReceiveAddrs(addrs []*p2p.NetAddress, src Peer) error {
 		// If this address came from a seed node, try to connect to it without
 		// waiting (#2093)
 		if srcIsSeed {
-			r.Logger.Info("Will dial address, which came from seed", "addr", netAddr, "seed", srcAddr)
 			go func(addr *p2p.NetAddress) {
 				err := r.dialPeer(addr)
 				if err != nil {
 					switch err.(type) {
 					case errMaxAttemptsToDial, errTooEarlyToDial, p2p.ErrCurrentlyDialingOrExistingAddress:
 						r.Logger.Debug(err.Error(), "addr", addr)
 					default:
-						r.Logger.Error(err.Error(), "addr", addr)
+						r.Logger.Debug(err.Error(), "addr", addr)
 					}
 				}
 			}(netAddr)
@@ -501,7 +500,6 @@ func (r *Reactor) ensurePeers() {
 		// TODO: consider moving some checks from toDial into here
 		// so we don't even consider dialing peers that we want to wait
 		// before dialling again, or have dialed too many times already
-		r.Logger.Info("Will dial address", "addr", try)
 		toDial[try.ID] = try
 	}
 
@@ -514,7 +512,7 @@ func (r *Reactor) ensurePeers() {
 				case errMaxAttemptsToDial, errTooEarlyToDial:
 					r.Logger.Debug(err.Error(), "addr", addr)
 				default:
-					r.Logger.Error(err.Error(), "addr", addr)
+					r.Logger.Debug(err.Error(), "addr", addr)
 				}
 			}
 		}(addr)
@@ -727,7 +725,7 @@ func (r *Reactor) crawlPeers(addrs []*p2p.NetAddress) {
 			case errMaxAttemptsToDial, errTooEarlyToDial, p2p.ErrCurrentlyDialingOrExistingAddress:
 				r.Logger.Debug(err.Error(), "addr", addr)
 			default:
-				r.Logger.Error(err.Error(), "addr", addr)
+				r.Logger.Debug(err.Error(), "addr", addr)
 			}
 			continue
 		}

state/rollback.go

@@ -41,7 +41,13 @@ func Rollback(bs BlockStore, ss Store) (int64, []byte, error) {
 	rollbackHeight := invalidState.LastBlockHeight - 1
 	rollbackBlock := bs.LoadBlockMeta(rollbackHeight)
 	if rollbackBlock == nil {
-		return -1, nil, fmt.Errorf("block at height %d not found", rollbackHeight)
+		return -1, nil, fmt.Errorf("block at RollbackHeight %d not found", rollbackHeight)
+	}
+	// We also need to retrieve the latest block because the app hash and last
+	// results hash is only agreed upon in the following block.
+	latestBlock := bs.LoadBlockMeta(invalidState.LastBlockHeight)
+	if latestBlock == nil {
+		return -1, nil, fmt.Errorf("block at LastBlockHeight %d not found", invalidState.LastBlockHeight)
 	}
 
 	_, prevVoterSet, _, _, err := ss.LoadVoters(rollbackHeight, nil)
@@ -99,8 +105,8 @@ func Rollback(bs BlockStore, ss Store) (int64, []byte, error) {
 		ConsensusParams:                  previousParams,
 		LastHeightConsensusParamsChanged: paramsChangeHeight,
 
-		LastResultsHash: rollbackBlock.Header.LastResultsHash,
-		AppHash:         rollbackBlock.Header.AppHash,
+		LastResultsHash: latestBlock.Header.LastResultsHash,
+		AppHash:         latestBlock.Header.AppHash,
 	}
 
 	// persist the new state. This overrides the invalid one. NOTE: this will also

state/rollback_test.go

@@ -7,6 +7,7 @@ import (
 	dbm "github.com/line/tm-db/v2/memdb"
 	"github.com/stretchr/testify/require"
 
+	"github.com/line/ostracon/crypto"
 	"github.com/line/ostracon/crypto/tmhash"
 	tmstate "github.com/line/ostracon/proto/ostracon/state"
 	tmversion "github.com/line/ostracon/proto/ostracon/version"
@@ -56,12 +57,22 @@ func TestRollback(t *testing.T) {
 		BlockID: initialState.LastBlockID,
 		Header: types.Header{
 			Height:          initialState.LastBlockHeight,
-			AppHash:         initialState.AppHash,
+			AppHash:         crypto.CRandBytes(tmhash.Size),
 			LastBlockID:     makeBlockIDRandom(),
 			LastResultsHash: initialState.LastResultsHash,
 		},
 	}
-	blockStore.On("LoadBlockMeta", initialState.LastBlockHeight).Return(block)
+	nextBlock := &types.BlockMeta{
+		BlockID: initialState.LastBlockID,
+		Header: types.Header{
+			Height:          nextState.LastBlockHeight,
+			AppHash:         initialState.AppHash,
+			LastBlockID:     block.BlockID,
+			LastResultsHash: nextState.LastResultsHash,
+		},
+	}
+	blockStore.On("LoadBlockMeta", height).Return(block)
+	blockStore.On("LoadBlockMeta", nextHeight).Return(nextBlock)
 	blockStore.On("Height").Return(nextHeight)
 
 	// rollback the state
@@ -91,11 +102,18 @@ func TestRollbackNoBlocks(t *testing.T) {
 	stateStore := setupStateStore(t, height)
 	blockStore := &mocks.BlockStore{}
 	blockStore.On("Height").Return(height)
-	blockStore.On("LoadBlockMeta", height-1).Return(nil)
+	blockStore.On("LoadBlockMeta", height-1).Once().Return(nil)
 
 	_, _, err := state.Rollback(blockStore, stateStore)
 	require.Error(t, err)
-	require.Contains(t, err.Error(), "block at height 99 not found")
+	require.Contains(t, err.Error(), "block at RollbackHeight 99 not found")
+
+	blockStore.On("LoadBlockMeta", height-1).Once().Return(&types.BlockMeta{})
+	blockStore.On("LoadBlockMeta", height).Return(nil)
+
+	_, _, err = state.Rollback(blockStore, stateStore)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "block at LastBlockHeight 100 not found")
 }
 
 func TestRollbackDifferentStateHeight(t *testing.T) {

statesync/reactor_test.go

@@ -257,7 +257,7 @@ func makeTestStateAndCommit(appHash string, height int64) (sm.State, *types.Comm
 		Version: tmstate.Version{
 			Consensus: tmversion.Consensus{
 				Block: version.BlockProtocol,
-				App:   0,
+				App:   9,
 			},
 
 			Software: version.OCCoreSemVer,

statesync/stateprovider.go

@@ -18,6 +18,7 @@ import (
 	rpchttp "github.com/line/ostracon/rpc/client/http"
 	sm "github.com/line/ostracon/state"
 	"github.com/line/ostracon/types"
+	"github.com/line/ostracon/version"
 	"github.com/line/tm-db/v2/memdb"
 )
 
@@ -171,6 +172,10 @@ func (s *lightClientStateProvider) State(ctx context.Context, height uint64) (sm
 		return sm.State{}, err
 	}
 
+	state.Version = tmstate.Version{
+		Consensus: currentLightBlock.Version,
+		Software:  version.OCCoreSemVer,
+	}
 	state.LastBlockHeight = lastLightBlock.Height
 	state.LastBlockTime = lastLightBlock.Time
 	state.LastBlockID = lastLightBlock.Commit.BlockID