From 0d0c3ae9928c8e079abfd65d1bbcf9ba11633e10 Mon Sep 17 00:00:00 2001 From: Zsolt Felfoldi Date: Sun, 13 Aug 2017 11:46:22 +0200 Subject: [PATCH] les: implement PPTProofsMsg --- les/handler.go | 100 ++++++++++++++++++++++++-- les/odr.go | 2 +- les/odr_requests.go | 172 +++++++++++++++++++++++++++++++++++++------- les/peer.go | 21 +++--- light/odr.go | 23 ++++-- 5 files changed, 276 insertions(+), 42 deletions(-) diff --git a/les/handler.go b/les/handler.go index 2238c5e3d33c..f987bcd70fdf 100644 --- a/les/handler.go +++ b/les/handler.go @@ -826,22 +826,90 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) return p.SendHeaderProofs(req.ReqID, bv, proofs) - case HeaderProofsMsg: + case GetPPTProofsMsg: + p.Log().Trace("Received PPT proof request") + // Decode the retrieval message + var req struct { + ReqID uint64 + Reqs []PPTReq + } + if err := msg.Decode(&req); err != nil { + return errResp(ErrDecode, "msg %v: %v", msg, err) + } + // Gather state data until the fetch or network limits is reached + var ( + auxBytes int + auxData [][]byte + ) + reqCnt := len(req.Reqs) + if reject(uint64(reqCnt), MaxPPTProofsFetch) { + return errResp(ErrRequestRejected, "") + } + + var ( + lastIdx uint64 + lastPPTId uint + root common.Hash + tr *trie.Trie + ) + + nodes := light.NewNodeSet() + + for _, req := range req.Reqs { + if nodes.DataSize()+auxBytes >= softResponseLimit { + break + } + if tr == nil || req.PPTId != lastPPTId || req.TrieIdx != lastIdx { + var prefix string + root, prefix = pm.getPPT(req.PPTId, req.TrieIdx) + if root != (common.Hash{}) { + if t, err := trie.New(root, ethdb.NewTable(pm.chainDb, prefix)); err == nil { + tr = t + } + } + lastPPTId = req.PPTId + lastIdx = req.TrieIdx + } + if req.AuxReq == PPTAuxRoot { + var data []byte + if root != (common.Hash{}) { + data = root[:] + } + auxData = append(auxData, data) + auxBytes += len(data) + } else { + if tr != nil { + tr.Prove(req.Key, req.FromLevel, nodes) + } + if req.AuxReq != 0 { + data := pm.getPPTAuxData(req) + auxData = append(auxData, data) + auxBytes += len(data) + } + } + } + proofs := nodes.NodeList() + bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) + pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) + return p.SendPPTProofs(req.ReqID, bv, PPTResps{Proofs: proofs, AuxData: auxData}) + + case PPTProofsMsg: if pm.odr == nil { return errResp(ErrUnexpectedResponse, "") } - p.Log().Trace("Received headers proof response") + p.Log().Trace("Received PPT proof response") var resp struct { ReqID, BV uint64 - Data []ChtResp + Data PPTResps } if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } + p.fcServer.GotReply(resp.ReqID, resp.BV) deliverMsg = &Msg{ - MsgType: MsgHeaderProofs, + MsgType: MsgPPTProofs, ReqID: resp.ReqID, Obj: resp.Data, } @@ -884,6 +952,30 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return nil } +// getPPT returns the post-processed trie root for the given trie ID and section index +func (pm *ProtocolManager) getPPT(id uint, idx uint64) (common.Hash, string) { + switch id { + case PPTChain: + return light.GetChtRoot(pm.chainDb, idx), light.ChtTablePrefix + case PPTBloomBits: + return light.GetBloomTrieRoot(pm.chainDb, idx), light.BloomTrieTablePrefix + } + return common.Hash{}, "" +} + +// getPPTAuxData returns requested auxiliary data for the given PPT request +func (pm *ProtocolManager) getPPTAuxData(req PPTReq) []byte { + if req.PPTId == PPTChain && req.AuxReq == PPTChainAuxHeader { + if len(req.Key) != 8 { + return nil + } + blockNum := binary.BigEndian.Uint64(req.Key) + hash := core.GetCanonicalHash(pm.chainDb, blockNum) + return core.GetHeaderRLP(pm.chainDb, hash, blockNum) + } + return nil +} + // NodeInfo retrieves some protocol metadata about the running host node. func (self *ProtocolManager) NodeInfo() *eth.EthNodeInfo { return ð.EthNodeInfo{ diff --git a/les/odr.go b/les/odr.go index 3f7584b48e12..3b79388a7253 100644 --- a/les/odr.go +++ b/les/odr.go @@ -52,7 +52,7 @@ const ( MsgCode MsgReceipts MsgProofs - MsgHeaderProofs + MsgPPTProofs ) // Msg encodes a LES message that delivers reply data for a request diff --git a/les/odr_requests.go b/les/odr_requests.go index 65cba43d3168..1e8a6b7c2c47 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -36,13 +36,14 @@ import ( var ( errInvalidMessageType = errors.New("invalid message type") - errMultipleEntries = errors.New("multiple response entries") + errInvalidEntryCount = errors.New("invalid number of response entries") errHeaderUnavailable = errors.New("header unavailable") errTxHashMismatch = errors.New("transaction hash mismatch") errUncleHashMismatch = errors.New("uncle hash mismatch") errReceiptHashMismatch = errors.New("receipt hash mismatch") errDataHashMismatch = errors.New("data hash mismatch") errCHTHashMismatch = errors.New("cht hash mismatch") + errCHTNumberMismatch = errors.New("cht number mismatch") errUselessNodes = errors.New("useless nodes in merkle proof nodeset") ) @@ -65,6 +66,8 @@ func LesRequest(req light.OdrRequest) LesOdrRequest { return (*CodeRequest)(r) case *light.ChtRequest: return (*ChtRequest)(r) + case *light.BloomRequest: + return (*BloomRequest)(r) default: return nil } @@ -102,7 +105,7 @@ func (r *BlockRequest) Validate(db ethdb.Database, msg *Msg) error { } bodies := msg.Obj.([]*types.Body) if len(bodies) != 1 { - return errMultipleEntries + return errInvalidEntryCount } body := bodies[0] @@ -158,7 +161,7 @@ func (r *ReceiptsRequest) Validate(db ethdb.Database, msg *Msg) error { } receipts := msg.Obj.([]types.Receipts) if len(receipts) != 1 { - return errMultipleEntries + return errInvalidEntryCount } receipt := receipts[0] @@ -187,7 +190,7 @@ type TrieRequest light.TrieRequest // GetCost returns the cost of the given ODR request according to the serving // peer's cost table (implementation of LesOdrRequest) func (r *TrieRequest) GetCost(peer *peer) uint64 { - return peer.GetRequestCost(GetProofsMsg, 1) + return peer.GetRequestCost(GetProofsV2Msg, 1) } // CanSend tells if a certain peer is suitable for serving the given request @@ -198,12 +201,12 @@ func (r *TrieRequest) CanSend(peer *peer) bool { // Request sends an ODR request to the LES network (implementation of LesOdrRequest) func (r *TrieRequest) Request(reqID uint64, peer *peer) error { peer.Log().Debug("Requesting trie proof", "root", r.Id.Root, "key", r.Key) - req := &ProofReq{ + req := ProofReq{ BHash: r.Id.BlockHash, AccKey: r.Id.AccKey, Key: r.Key, } - return peer.RequestProofs(reqID, r.GetCost(peer), []*ProofReq{req}) + return peer.RequestProofs(reqID, r.GetCost(peer), []ProofReq{req}) } // Valid processes an ODR request reply message from the LES network @@ -254,11 +257,11 @@ func (r *CodeRequest) CanSend(peer *peer) bool { // Request sends an ODR request to the LES network (implementation of LesOdrRequest) func (r *CodeRequest) Request(reqID uint64, peer *peer) error { peer.Log().Debug("Requesting code data", "hash", r.Hash) - req := &CodeReq{ + req := CodeReq{ BHash: r.Id.BlockHash, AccKey: r.Id.AccKey, } - return peer.RequestCode(reqID, r.GetCost(peer), []*CodeReq{req}) + return peer.RequestCode(reqID, r.GetCost(peer), []CodeReq{req}) } // Valid processes an ODR request reply message from the LES network @@ -273,7 +276,7 @@ func (r *CodeRequest) Validate(db ethdb.Database, msg *Msg) error { } reply := msg.Obj.([][]byte) if len(reply) != 1 { - return errMultipleEntries + return errInvalidEntryCount } data := reply[0] @@ -285,10 +288,32 @@ func (r *CodeRequest) Validate(db ethdb.Database, msg *Msg) error { return nil } +const ( + PPTChain = iota + PPTBloomBits + + PPTAuxRoot = 1 + PPTChainAuxHeader = 2 +) + +type PPTReq struct { + PPTId uint + TrieIdx uint64 + Key []byte + FromLevel, AuxReq uint +} + +type PPTResps struct { // describes all responses, not just a single one + Proofs light.NodeList + AuxData [][]byte +} + +// legacy LES/1 type ChtReq struct { ChtNum, BlockNum, FromLevel uint64 } +// legacy LES/1 type ChtResp struct { Header *types.Header Proof []rlp.RawValue @@ -300,7 +325,7 @@ type ChtRequest light.ChtRequest // GetCost returns the cost of the given ODR request according to the serving // peer's cost table (implementation of LesOdrRequest) func (r *ChtRequest) GetCost(peer *peer) uint64 { - return peer.GetRequestCost(GetHeaderProofsMsg, 1) + return peer.GetRequestCost(GetPPTProofsMsg, 1) } // CanSend tells if a certain peer is suitable for serving the given request @@ -308,17 +333,21 @@ func (r *ChtRequest) CanSend(peer *peer) bool { peer.lock.RLock() defer peer.lock.RUnlock() - return r.ChtNum <= (peer.headInfo.Number-light.ChtConfirmations)/light.ChtFrequency + return peer.headInfo.Number >= light.ChtConfirmations && r.ChtNum <= (peer.headInfo.Number-light.ChtConfirmations)/light.ChtFrequency } // Request sends an ODR request to the LES network (implementation of LesOdrRequest) func (r *ChtRequest) Request(reqID uint64, peer *peer) error { peer.Log().Debug("Requesting CHT", "cht", r.ChtNum, "block", r.BlockNum) - req := &ChtReq{ - ChtNum: r.ChtNum, - BlockNum: r.BlockNum, + var encNum [8]byte + binary.BigEndian.PutUint64(encNum[:], r.BlockNum) + req := PPTReq{ + PPTId: PPTChain, + TrieIdx: r.ChtNum, + Key: encNum[:], + AuxReq: PPTChainAuxHeader, } - return peer.RequestHeaderProofs(reqID, r.GetCost(peer), []*ChtReq{req}) + return peer.RequestPPTProofs(reqID, r.GetCost(peer), []PPTReq{req}) } // Valid processes an ODR request reply message from the LES network @@ -328,34 +357,127 @@ func (r *ChtRequest) Validate(db ethdb.Database, msg *Msg) error { log.Debug("Validating CHT", "cht", r.ChtNum, "block", r.BlockNum) // Ensure we have a correct message with a single proof element - if msg.MsgType != MsgHeaderProofs { + if msg.MsgType != MsgPPTProofs { return errInvalidMessageType } - proofs := msg.Obj.([]ChtResp) - if len(proofs) != 1 { - return errMultipleEntries + resp := msg.Obj.(PPTResps) + if len(resp.AuxData) != 1 { + return errInvalidEntryCount + } + pdb := resp.Proofs.NodeSet() + headerEnc := resp.AuxData[0] + if len(headerEnc) == 0 { + return errHeaderUnavailable + } + header := new(types.Header) + if err := rlp.DecodeBytes(headerEnc, header); err != nil { + return errHeaderUnavailable } - proof := proofs[0] // Verify the CHT var encNumber [8]byte binary.BigEndian.PutUint64(encNumber[:], r.BlockNum) - value, err, _ := trie.VerifyProof(r.ChtRoot, encNumber[:], light.NodeList(proof.Proof).NodeSet()) + cdb := pdb.ReadCache() + value, err, _ := trie.VerifyProof(r.ChtRoot, encNumber[:], cdb) if err != nil { - return err + return fmt.Errorf("merkle proof verification failed: %v", err) } + if pdb.KeyCount() != cdb.KeyCount() { + return errUselessNodes + } + var node light.ChtNode if err := rlp.DecodeBytes(value, &node); err != nil { return err } - if node.Hash != proof.Header.Hash() { + if node.Hash != header.Hash() { return errCHTHashMismatch } + if r.BlockNum != header.Number.Uint64() { + return errCHTNumberMismatch + } // Verifications passed, store and return - r.Header = proof.Header - r.Proof = proof.Proof + r.Header = header + r.Proof = pdb r.Td = node.Td return nil } + +type BloomReq struct { + BltNum, BitIdx, SectionIdx, FromLevel uint64 +} + +// ODR request type for requesting headers by Canonical Hash Trie, see LesOdrRequest interface +type BloomRequest light.BloomRequest + +// GetCost returns the cost of the given ODR request according to the serving +// peer's cost table (implementation of LesOdrRequest) +func (r *BloomRequest) GetCost(peer *peer) uint64 { + return peer.GetRequestCost(GetPPTProofsMsg, len(r.SectionIdxList)) +} + +// CanSend tells if a certain peer is suitable for serving the given request +func (r *BloomRequest) CanSend(peer *peer) bool { + peer.lock.RLock() + defer peer.lock.RUnlock() + + return peer.headInfo.Number >= light.BloomTrieConfirmations && r.BltNum <= (peer.headInfo.Number-light.BloomTrieConfirmations)/light.BloomTrieFrequency +} + +// Request sends an ODR request to the LES network (implementation of LesOdrRequest) +func (r *BloomRequest) Request(reqID uint64, peer *peer) error { + peer.Log().Debug("Requesting BloomBits", "blt", r.BltNum, "bitIdx", r.BitIdx, "sections", r.SectionIdxList) + reqs := make([]PPTReq, len(r.SectionIdxList)) + + var encNumber [10]byte + binary.BigEndian.PutUint16(encNumber[0:2], uint16(r.BitIdx)) + + for i, sectionIdx := range r.SectionIdxList { + binary.BigEndian.PutUint64(encNumber[2:10], sectionIdx) + reqs[i] = PPTReq{ + PPTId: PPTBloomBits, + TrieIdx: r.BltNum, + Key: common.CopyBytes(encNumber[:]), + } + } + return peer.RequestPPTProofs(reqID, r.GetCost(peer), reqs) +} + +// Valid processes an ODR request reply message from the LES network +// returns true and stores results in memory if the message was a valid reply +// to the request (implementation of LesOdrRequest) +func (r *BloomRequest) Validate(db ethdb.Database, msg *Msg) error { + log.Debug("Validating BloomBits", "blt", r.BltNum, "bitIdx", r.BitIdx, "sections", r.SectionIdxList) + + // Ensure we have a correct message with a single proof element + if msg.MsgType != MsgPPTProofs { + return errInvalidMessageType + } + resps := msg.Obj.(PPTResps) + proofs := resps.Proofs + pdb := proofs.NodeSet() + cdb := pdb.ReadCache() + + r.BloomBits = make([][]byte, len(r.SectionIdxList)) + + // Verify the proofs + var encNumber [10]byte + binary.BigEndian.PutUint16(encNumber[0:2], uint16(r.BitIdx)) + + for i, idx := range r.SectionIdxList { + binary.BigEndian.PutUint64(encNumber[2:10], idx) + value, err, _ := trie.VerifyProof(r.BltRoot, encNumber[:], cdb) + if err != nil { + return err + } + r.BloomBits[i] = value + } + + if pdb.KeyCount() != cdb.KeyCount() { + return errUselessNodes + } + r.Proofs = pdb + return nil +} diff --git a/les/peer.go b/les/peer.go index 1825f85a3a55..65080c543f4d 100644 --- a/les/peer.go +++ b/les/peer.go @@ -198,7 +198,7 @@ func (p *peer) SendReceiptsRLP(reqID, bv uint64, receipts []rlp.RawValue) error return sendResponse(p.rw, ReceiptsMsg, reqID, bv, receipts) } -// SendProofs sends a batch of merkle proofs, corresponding to the ones requested. +// SendProofs sends a batch of legacy LES/1 merkle proofs, corresponding to the ones requested. func (p *peer) SendProofs(reqID, bv uint64, proofs proofsData) error { return sendResponse(p.rw, ProofsMsg, reqID, bv, proofs) } @@ -208,11 +208,16 @@ func (p *peer) SendProofsV2(reqID, bv uint64, proofs light.NodeList) error { return sendResponse(p.rw, ProofsV2Msg, reqID, bv, proofs) } -// SendHeaderProofs sends a batch of header proofs, corresponding to the ones requested. +// SendHeaderProofs sends a batch of legacy LES/1 header proofs, corresponding to the ones requested. func (p *peer) SendHeaderProofs(reqID, bv uint64, proofs []ChtResp) error { return sendResponse(p.rw, HeaderProofsMsg, reqID, bv, proofs) } +// SendPPTProofs sends a batch of PPT proofs, corresponding to the ones requested. +func (p *peer) SendPPTProofs(reqID, bv uint64, resp PPTResps) error { + return sendResponse(p.rw, PPTProofsMsg, reqID, bv, resp) +} + // RequestHeadersByHash fetches a batch of blocks' headers corresponding to the // specified header query, based on the hash of an origin block. func (p *peer) RequestHeadersByHash(reqID, cost uint64, origin common.Hash, amount int, skip int, reverse bool) error { @@ -236,7 +241,7 @@ func (p *peer) RequestBodies(reqID, cost uint64, hashes []common.Hash) error { // RequestCode fetches a batch of arbitrary data from a node's known state // data, corresponding to the specified hashes. -func (p *peer) RequestCode(reqID, cost uint64, reqs []*CodeReq) error { +func (p *peer) RequestCode(reqID, cost uint64, reqs []CodeReq) error { p.Log().Debug("Fetching batch of codes", "count", len(reqs)) return sendRequest(p.rw, GetCodeMsg, reqID, cost, reqs) } @@ -248,15 +253,15 @@ func (p *peer) RequestReceipts(reqID, cost uint64, hashes []common.Hash) error { } // RequestProofs fetches a batch of merkle proofs from a remote node. -func (p *peer) RequestProofs(reqID, cost uint64, reqs []*ProofReq) error { +func (p *peer) RequestProofs(reqID, cost uint64, reqs []ProofReq) error { p.Log().Debug("Fetching batch of proofs", "count", len(reqs)) return sendRequest(p.rw, GetProofsV2Msg, reqID, cost, reqs) } -// RequestHeaderProofs fetches a batch of header merkle proofs from a remote node. -func (p *peer) RequestHeaderProofs(reqID, cost uint64, reqs []*ChtReq) error { - p.Log().Debug("Fetching batch of header proofs", "count", len(reqs)) - return sendRequest(p.rw, GetHeaderProofsMsg, reqID, cost, reqs) +// RequestPPTProofs fetches a batch of PPT merkle proofs from a remote node. +func (p *peer) RequestPPTProofs(reqID, cost uint64, reqs []PPTReq) error { + p.Log().Debug("Fetching batch of PPT proofs", "count", len(reqs)) + return sendRequest(p.rw, GetPPTProofsMsg, reqID, cost, reqs) } func (p *peer) SendTxs(reqID, cost uint64, txs types.Transactions) error { diff --git a/light/odr.go b/light/odr.go index caa00b3ab5d0..c2cfcbe83f34 100644 --- a/light/odr.go +++ b/light/odr.go @@ -26,7 +26,6 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/rlp" ) // NoOdr is the default context passed to an ODR capable function when the ODR @@ -126,14 +125,14 @@ func (req *ReceiptsRequest) StoreResult(db ethdb.Database) { core.WriteBlockReceipts(db, req.Hash, req.Number, req.Receipts) } -// TrieRequest is the ODR request type for state/storage trie entries +// ChtRequest is the ODR request type for state/storage trie entries type ChtRequest struct { OdrRequest ChtNum, BlockNum uint64 ChtRoot common.Hash Header *types.Header Td *big.Int - Proof []rlp.RawValue + Proof *NodeSet } // StoreResult stores the retrieved data in local database @@ -143,5 +142,21 @@ func (req *ChtRequest) StoreResult(db ethdb.Database) { hash, num := req.Header.Hash(), req.Header.Number.Uint64() core.WriteTd(db, hash, num, req.Td) core.WriteCanonicalHash(db, hash, num) - //storeProof(db, req.Proof) +} + +// BloomRequest is the ODR request type for retrieving bloom filters from a CHT structure +type BloomRequest struct { + OdrRequest + BltNum, BitIdx uint64 + SectionIdxList []uint64 + BltRoot common.Hash + BloomBits [][]byte + Proofs *NodeSet +} + +// StoreResult stores the retrieved data in local database +func (req *BloomRequest) StoreResult(db ethdb.Database) { + for i, sectionIdx := range req.SectionIdxList { + core.StoreBloomBits(db, req.BitIdx, sectionIdx, req.BloomBits[i]) + } }