From 72da44e2d451b98147f0496b271032a5cc3a3482 Mon Sep 17 00:00:00 2001 From: noot Date: Tue, 27 Jul 2021 17:26:22 -0400 Subject: [PATCH] comment out pause/catch up logic, store highest setID/round in db --- dot/core/interface.go | 1 - dot/network/service.go | 9 ++++- dot/network/state.go | 2 +- dot/network/sync.go | 6 +-- dot/state/block.go | 4 ++ dot/state/block_finalisation.go | 67 ++++++++++++++++++++++++++++++- dot/sync/interface.go | 2 +- dot/sync/mocks/FinalityGadget.go | 27 +++++++++++++ dot/sync/mocks/finality_gadget.go | 24 ----------- dot/sync/syncer.go | 8 +--- dot/sync/test_helpers.go | 4 +- lib/grandpa/errors.go | 9 +++-- lib/grandpa/grandpa.go | 16 ++++---- lib/grandpa/message_handler.go | 28 +++++++------ lib/grandpa/vote_message.go | 5 ++- 15 files changed, 142 insertions(+), 70 deletions(-) create mode 100644 dot/sync/mocks/FinalityGadget.go delete mode 100644 dot/sync/mocks/finality_gadget.go diff --git a/dot/core/interface.go b/dot/core/interface.go index a27db00227..c9760ae0ce 100644 --- a/dot/core/interface.go +++ b/dot/core/interface.go @@ -41,7 +41,6 @@ type BlockState interface { GetSlotForBlock(common.Hash) (uint64, error) GetFinalisedHeader(uint64, uint64) (*types.Header, error) GetFinalisedHash(uint64, uint64) (common.Hash, error) - SetFinalisedHash(common.Hash, uint64, uint64) error RegisterImportedChannel(ch chan<- *types.Block) (byte, error) UnregisterImportedChannel(id byte) RegisterFinalizedChannel(ch chan<- *types.FinalisationInfo) (byte, error) diff --git a/dot/network/service.go b/dot/network/service.go index fa89abf502..c85a682bff 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -343,7 +343,7 @@ func (s *Service) sentBlockIntervalTelemetry() { } bestHash := best.Hash() - finalized, err := s.blockState.GetFinalisedHeader(0, 0) //nolint + finalized, err := s.blockState.GetHighestFinalisedHeader() //nolint if err != nil { continue } @@ -519,16 +519,21 @@ func (s *Service) GossipMessage(msg NotificationsMessage) { func (s *Service) SendMessage(to peer.ID, msg NotificationsMessage) error { s.notificationsMu.Lock() defer s.notificationsMu.Unlock() + for msgID, prtl := range s.notificationsProtocols { - if msg.Type() != msgID || prtl == nil { + if msg.Type() != msgID { continue } + hs, err := prtl.getHandshake() if err != nil { return err } + s.sendData(to, hs, prtl, msg) + return nil } + return errors.New("message not supported by any notifications protocol") } diff --git a/dot/network/state.go b/dot/network/state.go index ea471ac354..e71fd1f229 100644 --- a/dot/network/state.go +++ b/dot/network/state.go @@ -29,7 +29,7 @@ type BlockState interface { BestBlockNumber() (*big.Int, error) GenesisHash() common.Hash HasBlockBody(common.Hash) (bool, error) - GetFinalisedHeader(round, setID uint64) (*types.Header, error) + GetHighestFinalisedHeader() (*types.Header, error) GetHashByNumber(num *big.Int) (common.Hash, error) } diff --git a/dot/network/sync.go b/dot/network/sync.go index 79fcaa90fd..9e93311952 100644 --- a/dot/network/sync.go +++ b/dot/network/sync.go @@ -351,7 +351,7 @@ func (q *syncQueue) benchmark() { goal := atomic.LoadInt64(&q.goal) if before.Number.Int64() >= goal { - finalised, err := q.s.blockState.GetFinalisedHeader(0, 0) //nolint + finalised, err := q.s.blockState.GetHighestFinalisedHeader() //nolint if err != nil { continue } @@ -767,7 +767,7 @@ func (q *syncQueue) handleBlockJustification(data []*types.BlockData) { } func (q *syncQueue) handleBlockData(data []*types.BlockData) { - finalised, err := q.s.blockState.GetFinalisedHeader(0, 0) + finalised, err := q.s.blockState.GetHighestFinalisedHeader() if err != nil { panic(err) // this should never happen } @@ -815,7 +815,7 @@ func (q *syncQueue) handleBlockDataFailure(idx int, err error, data []*types.Blo logger.Warn("failed to handle block data", "failed on block", q.currStart+int64(idx), "error", err) if errors.Is(err, chaindb.ErrKeyNotFound) || errors.Is(err, blocktree.ErrParentNotFound) { - finalised, err := q.s.blockState.GetFinalisedHeader(0, 0) + finalised, err := q.s.blockState.GetHighestFinalisedHeader() if err != nil { panic(err) } diff --git a/dot/state/block.go b/dot/state/block.go index 1d53e88f8a..cf7cc131ca 100644 --- a/dot/state/block.go +++ b/dot/state/block.go @@ -123,6 +123,10 @@ func NewBlockStateFromGenesis(db chaindb.Database, header *types.Header) (*Block bs.genesisHash = header.Hash() + if err := bs.db.Put(highestRoundAndSetIDKey, roundSetIDKey(0, 0)); err != nil { + return nil, err + } + // set the latest finalised head to the genesis header if err := bs.SetFinalisedHash(bs.genesisHash, 0, 0); err != nil { return nil, err diff --git a/dot/state/block_finalisation.go b/dot/state/block_finalisation.go index 1a674491cc..1e9d763a3a 100644 --- a/dot/state/block_finalisation.go +++ b/dot/state/block_finalisation.go @@ -17,6 +17,7 @@ package state import ( + "encoding/binary" "fmt" "math/big" @@ -24,6 +25,8 @@ import ( "github.com/ChainSafe/gossamer/lib/common" ) +var highestRoundAndSetIDKey = []byte("hrs") + // finalisedHashKey = FinalizedBlockHashKey + round + setID (LE encoded) func finalisedHashKey(round, setID uint64) []byte { return append(common.FinalizedBlockHashKey, roundSetIDKey(round, setID)...) @@ -69,8 +72,63 @@ func (bs *BlockState) GetFinalisedHash(round, setID uint64) (common.Hash, error) return common.NewHash(h), nil } +func roundSetIDNum(round, setID uint64) *big.Int { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, round) + buf2 := make([]byte, 8) + binary.BigEndian.PutUint64(buf2, setID) + return big.NewInt(0).SetBytes(append(buf, buf2...)) +} + +func (bs *BlockState) setHighestRoundAndSetID(round, setID uint64) error { + currRound, currSetID, err := bs.getHighestRoundAndSetID() + if err != nil { + return err + } + + if roundSetIDNum(round, setID).Cmp(roundSetIDNum(currRound, currSetID)) <= 0 { + return nil + } + + return bs.db.Put(highestRoundAndSetIDKey, roundSetIDKey(round, setID)) +} + +func (bs *BlockState) getHighestRoundAndSetID() (uint64, uint64, error) { + b, err := bs.db.Get(highestRoundAndSetIDKey) + if err != nil { + return 0, 0, err + } + + round := binary.LittleEndian.Uint64(b[:8]) + setID := binary.LittleEndian.Uint64(b[8:16]) + return round, setID, nil +} + +func (bs *BlockState) GetHighestFinalisedHash() (common.Hash, error) { + round, setID, err := bs.getHighestRoundAndSetID() + if err != nil { + return common.Hash{}, err + } + + return bs.GetFinalisedHash(round, setID) +} + +// GetHighestFinalisedHeader returns the most recently finalised block header +func (bs *BlockState) GetHighestFinalisedHeader() (*types.Header, error) { + h, err := bs.GetHighestFinalisedHash() + if err != nil { + return nil, err + } + + header, err := bs.GetHeader(h) + if err != nil { + return nil, err + } + + return header, nil +} + // SetFinalisedHash sets the latest finalised block header -// Note that using round=0 and setID=0 would refer to the latest finalised hash func (bs *BlockState) SetFinalisedHash(hash common.Hash, round, setID uint64) error { bs.Lock() defer bs.Unlock() @@ -114,7 +172,12 @@ func (bs *BlockState) SetFinalisedHash(hash common.Hash, round, setID uint64) er } bs.lastFinalised = hash - return bs.db.Put(finalisedHashKey(round, setID), hash[:]) + + if err := bs.db.Put(finalisedHashKey(round, setID), hash[:]); err != nil { + return err + } + + return bs.setHighestRoundAndSetID(round, setID) } func (bs *BlockState) setFirstSlotOnFinalisation() error { diff --git a/dot/sync/interface.go b/dot/sync/interface.go index 987dbc8648..f99fd4d86f 100644 --- a/dot/sync/interface.go +++ b/dot/sync/interface.go @@ -78,7 +78,7 @@ type Verifier interface { // FinalityGadget implements justification verification functionality type FinalityGadget interface { - VerifyBlockJustification([]byte) error + VerifyBlockJustification(common.Hash, []byte) error } // BlockImportHandler is the interface for the handler of newly imported blocks diff --git a/dot/sync/mocks/FinalityGadget.go b/dot/sync/mocks/FinalityGadget.go new file mode 100644 index 0000000000..6c79912932 --- /dev/null +++ b/dot/sync/mocks/FinalityGadget.go @@ -0,0 +1,27 @@ +// Code generated by mockery v2.8.0. DO NOT EDIT. + +package sync + +import ( + common "github.com/ChainSafe/gossamer/lib/common" + mock "github.com/stretchr/testify/mock" +) + +// FinalityGadget is an autogenerated mock type for the FinalityGadget type +type FinalityGadget struct { + mock.Mock +} + +// VerifyBlockJustification provides a mock function with given fields: _a0, _a1 +func (_m *FinalityGadget) VerifyBlockJustification(_a0 common.Hash, _a1 []byte) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(common.Hash, []byte) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/dot/sync/mocks/finality_gadget.go b/dot/sync/mocks/finality_gadget.go deleted file mode 100644 index 5386205968..0000000000 --- a/dot/sync/mocks/finality_gadget.go +++ /dev/null @@ -1,24 +0,0 @@ -// Code generated by mockery v2.8.0. DO NOT EDIT. - -package sync - -import mock "github.com/stretchr/testify/mock" - -// MockFinalityGadget is an autogenerated mock type for the FinalityGadget type -type MockFinalityGadget struct { - mock.Mock -} - -// VerifyBlockJustification provides a mock function with given fields: _a0 -func (_m *MockFinalityGadget) VerifyBlockJustification(_a0 []byte) error { - ret := _m.Called(_a0) - - var r0 error - if rf, ok := ret.Get(0).(func([]byte) error); ok { - r0 = rf(_a0) - } else { - r0 = ret.Error(0) - } - - return r0 -} diff --git a/dot/sync/syncer.go b/dot/sync/syncer.go index 7089099df3..5c4de5135a 100644 --- a/dot/sync/syncer.go +++ b/dot/sync/syncer.go @@ -366,18 +366,12 @@ func (s *Service) handleJustification(header *types.Header, justification []byte return } - err := s.finalityGadget.VerifyBlockJustification(justification) + err := s.finalityGadget.VerifyBlockJustification(header.Hash(), justification) if err != nil { logger.Warn("failed to verify block justification", "hash", header.Hash(), "number", header.Number, "error", err) return } - err = s.blockState.SetFinalisedHash(header.Hash(), 0, 0) - if err != nil { - logger.Error("failed to set finalised hash", "error", err) - return - } - err = s.blockState.SetJustification(header.Hash(), justification) if err != nil { logger.Error("failed tostore justification", "error", err) diff --git a/dot/sync/test_helpers.go b/dot/sync/test_helpers.go index 67dc43f47c..2a155a86ea 100644 --- a/dot/sync/test_helpers.go +++ b/dot/sync/test_helpers.go @@ -42,8 +42,8 @@ import ( ) // NewMockFinalityGadget create and return sync FinalityGadget interface mock -func NewMockFinalityGadget() *syncmocks.MockFinalityGadget { - m := new(syncmocks.MockFinalityGadget) +func NewMockFinalityGadget() *syncmocks.FinalityGadget { + m := new(syncmocks.FinalityGadget) // using []uint8 instead of []byte: https://github.com/stretchr/testify/pull/969 m.On("VerifyBlockJustification", mock.AnythingOfType("[]uint8")).Return(nil) return m diff --git a/lib/grandpa/errors.go b/lib/grandpa/errors.go index 3b7cab645c..ab92f892e9 100644 --- a/lib/grandpa/errors.go +++ b/lib/grandpa/errors.go @@ -18,10 +18,16 @@ package grandpa import ( "errors" + "fmt" "github.com/ChainSafe/gossamer/lib/blocktree" ) +// errRoundMismatch is returned when trying to validate a vote message that isn't for the current round +func errRoundMismatch(got, want uint64) error { + return fmt.Errorf("rounds do not match: got %d, want %d", got, want) +} + //nolint var ( ErrNilBlockState = errors.New("cannot have nil BlockState") @@ -39,9 +45,6 @@ var ( // ErrSetIDMismatch is returned when trying to validate a vote message with an invalid voter set ID, or when receiving a catch up message with a different set ID ErrSetIDMismatch = errors.New("set IDs do not match") - // ErrRoundMismatch is returned when trying to validate a vote message that isn't for the current round - ErrRoundMismatch = errors.New("rounds do not match") - // ErrEquivocation is returned when trying to validate a vote for that is equivocatory ErrEquivocation = errors.New("vote is equivocatory") diff --git a/lib/grandpa/grandpa.go b/lib/grandpa/grandpa.go index 09015da886..fc78808489 100644 --- a/lib/grandpa/grandpa.go +++ b/lib/grandpa/grandpa.go @@ -345,7 +345,8 @@ func (s *Service) initiate() error { } if err != nil { - return err + logger.Warn("play grandpa round error", "error", err) + continue } if s.ctx.Err() != nil { @@ -802,12 +803,7 @@ func (s *Service) finalise() error { return err } - if err = s.grandpaState.SetLatestRound(s.state.round); err != nil { - return err - } - - // set latest finalised head in db - return s.blockState.SetFinalisedHash(bfc.Hash, 0, 0) + return s.grandpaState.SetLatestRound(s.state.round) } // createJustification collects the signed precommits received for this round and turns them into @@ -1015,8 +1011,10 @@ func (s *Service) getPreVotedBlock() (Vote, error) { func (s *Service) getGrandpaGHOST() (Vote, error) { threshold := s.state.threshold() - var blocks map[common.Hash]uint32 - var err error + var ( + blocks map[common.Hash]uint32 + err error + ) for { blocks, err = s.getPossibleSelectedBlocks(prevote, threshold) diff --git a/lib/grandpa/message_handler.go b/lib/grandpa/message_handler.go index e44b6699f0..8d938fb701 100644 --- a/lib/grandpa/message_handler.go +++ b/lib/grandpa/message_handler.go @@ -142,17 +142,11 @@ func (h *MessageHandler) handleCommitMessage(msg *CommitMessage) (*ConsensusMess return nil, err } - if msg.Round >= h.grandpa.state.round { - // set latest finalised head in db - err = h.blockState.SetFinalisedHash(msg.Vote.Hash, 0, 0) - if err != nil { - return nil, err - } - } + return nil, nil // check if msg has same setID but is 2 or more rounds ahead of us, if so, return catch-up request to send if msg.Round > h.grandpa.state.round+1 && !h.grandpa.paused.Load().(bool) { // TODO: CommitMessage does not have setID, confirm this is correct - h.grandpa.paused.Store(true) + //h.grandpa.paused.Store(true) h.grandpa.state.round = msg.Round + 1 req := newCatchUpRequest(msg.Round, h.grandpa.state.setID) logger.Debug("sending catch-up request; paused service", "round", msg.Round) @@ -192,12 +186,13 @@ func (h *MessageHandler) handleCatchUpResponse(msg *catchUpResponse) error { } logger.Debug("received catch up response", "round", msg.Round, "setID", msg.SetID, "hash", msg.Hash) + return nil // if we aren't currently expecting a catch up response, return - if !h.grandpa.paused.Load().(bool) { - logger.Debug("not currently paused, ignoring catch up response") - return nil - } + // if !h.grandpa.paused.Load().(bool) { + // logger.Debug("not currently paused, ignoring catch up response") + // return nil + // } if msg.SetID != h.grandpa.state.setID { return ErrSetIDMismatch @@ -288,6 +283,7 @@ func (h *MessageHandler) verifyCommitMessageJustification(fm *CommitMessage) err isDescendant, err := h.blockState.IsDescendantOf(fm.Vote.Hash, just.Vote.Hash) if err != nil { logger.Warn("verifyCommitMessageJustification", "error", err) + continue } if isDescendant { @@ -400,7 +396,7 @@ func (h *MessageHandler) verifyJustification(just *SignedVote, round, setID uint } // VerifyBlockJustification verifies the finality justification for a block -func (s *Service) VerifyBlockJustification(justification []byte) error { +func (s *Service) VerifyBlockJustification(hash common.Hash, justification []byte) error { r := &bytes.Buffer{} _, _ = r.Write(justification) fj := new(Justification) @@ -465,6 +461,12 @@ func (s *Service) VerifyBlockJustification(justification []byte) error { } } + err = s.blockState.SetFinalisedHash(hash, fj.Round, setID) + if err != nil { + return err + } + + logger.Debug("set finalised block", "hash", hash, "round", fj.Round, "setID", setID) return nil } diff --git a/lib/grandpa/vote_message.go b/lib/grandpa/vote_message.go index 4c56c604cd..ccaf36f8e3 100644 --- a/lib/grandpa/vote_message.go +++ b/lib/grandpa/vote_message.go @@ -170,12 +170,13 @@ func (s *Service) validateMessage(from peer.ID, m *VoteMessage) (*Vote, error) { } if err = s.network.SendMessage(from, msg); err != nil { - return nil, err + logger.Warn("failed to send CommitMessage", "error", err) + //return nil, err } } // TODO: get justification if your round is lower, or just do catch-up? - return nil, ErrRoundMismatch + return nil, errRoundMismatch(m.Round, s.state.round) } // check for equivocation ie. multiple votes within one subround