fix(drand): StateGetBeaconEntry uses chain beacons for historical epochs (#12428)

* fix(drand): `StateGetBeaconEntry` uses chain beacons for historical epochs

Fixes: #12414

Previously StateGetBeaconEntry would always try and use a drand beacon to get
the appropriate round. But as drand has shut down old beacons and we've
removed client details from Lotus, it has stopped working for historical
beacons.
This fix restores historical beacon entries by using the on-chain lookup,
however it now follows the rules used by StateGetRandomnessFromBeacon and the
get_beacon_randomness syscall which has some quirks with null rounds prior to
nv14. See #12414 (comment)
for specifics.

StateGetBeaconEntry still blocks for future epochs and uses live drand beacon
clients to wait for and fetch rounds as they are available.

* fixup! fix(drand): `StateGetBeaconEntry` uses chain beacons for historical epochs

* fixup! fix(drand): `StateGetBeaconEntry` uses chain beacons for historical epochs

Author: rvagg

CHANGELOG.md

@@ -17,6 +17,7 @@ Add `EthGetBlockReceipts` RPC method to retrieve transaction receipts for a spec
 ## Improvements
 
 - Reduce size of embedded genesis CAR files by removing WASM actor blocks and compressing with zstd. This reduces the `lotus` binary size by approximately 10 MiB. ([filecoin-project/lotus#12439](https://github.com/filecoin-project/lotus/pull/12439))
+- Legacy/historical Drand lookups via `StateGetBeaconEntry` now work again for all historical epochs. `StateGetBeaconEntry` now uses the on-chain beacon entries and follows the same rules for historical Drand round matching as `StateGetRandomnessFromBeacon` and the `get_beacon_randomness` FVM syscall. Be aware that there will be some some variance in matching Filecoin epochs to Drand rounds where null Filecoin rounds are involved prior to network version 14. ([filecoin-project/lotus#12428](https://github.com/filecoin-project/lotus/pull/12428)).
 
 ## Bug Fixes
 

api/api_full.go

@@ -592,9 +592,10 @@ type FullNode interface {
 	// StateGetRandomnessDigestFromBeacon is used to sample the beacon for randomness.
 	StateGetRandomnessDigestFromBeacon(ctx context.Context, randEpoch abi.ChainEpoch, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
 
-	// StateGetBeaconEntry returns the beacon entry for the given filecoin epoch. If
-	// the entry has not yet been produced, the call will block until the entry
-	// becomes available
+	// StateGetBeaconEntry returns the beacon entry for the given filecoin epoch
+	// by using the recorded entries on the chain. If the entry for the requested
+	// epoch has not yet been produced, the call will block until the entry
+	// becomes available.
 	StateGetBeaconEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) //perm:read
 
 	// StateGetNetworkParams return current network params

build/openrpc/full.json

@@ -19266,7 +19266,7 @@
         {
             "name": "Filecoin.StateGetBeaconEntry",
             "description": "```go\nfunc (s *FullNodeStruct) StateGetBeaconEntry(p0 context.Context, p1 abi.ChainEpoch) (*types.BeaconEntry, error) {\n\tif s.Internal.StateGetBeaconEntry == nil {\n\t\treturn nil, ErrNotSupported\n\t}\n\treturn s.Internal.StateGetBeaconEntry(p0, p1)\n}\n```",
-            "summary": "StateGetBeaconEntry returns the beacon entry for the given filecoin epoch. If\nthe entry has not yet been produced, the call will block until the entry\nbecomes available\n",
+            "summary": "StateGetBeaconEntry returns the beacon entry for the given filecoin epoch\nby using the recorded entries on the chain. If the entry for the requested\nepoch has not yet been produced, the call will block until the entry\nbecomes available.\n",
             "paramStructure": "by-position",
             "params": [
                 {

chain/beacon/mock.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
+	"sync"
 	"time"
 
 	"golang.org/x/crypto/blake2b"
@@ -15,26 +16,54 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
-// mockBeacon assumes that filecoin rounds are 1:1 mapped with the beacon rounds
-type mockBeacon struct {
-	interval time.Duration
+// MockBeacon assumes that filecoin rounds are 1:1 mapped with the beacon rounds
+type MockBeacon struct {
+	interval     time.Duration
+	maxIndex     int
+	waitingEntry int
+	lk           sync.Mutex
+	cond         *sync.Cond
 }
 
-func (mb *mockBeacon) IsChained() bool {
+func (mb *MockBeacon) IsChained() bool {
 	return true
 }
 
 func NewMockBeacon(interval time.Duration) RandomBeacon {
-	mb := &mockBeacon{interval: interval}
-
+	mb := &MockBeacon{interval: interval, maxIndex: -1}
+	mb.cond = sync.NewCond(&mb.lk)
 	return mb
 }
 
-func (mb *mockBeacon) RoundTime() time.Duration {
+// SetMaxIndex sets the maximum index that the beacon will return, and optionally blocks until all
+// waiting requests are satisfied. If maxIndex is -1, the beacon will return entries indefinitely.
+func (mb *MockBeacon) SetMaxIndex(maxIndex int, blockTillNoneWaiting bool) {
+	mb.lk.Lock()
+	defer mb.lk.Unlock()
+	mb.maxIndex = maxIndex
+	mb.cond.Broadcast()
+	if !blockTillNoneWaiting {
+		return
+	}
+
+	for mb.waitingEntry > 0 {
+		mb.cond.Wait()
+	}
+}
+
+// WaitingOnEntryCount returns the number of requests that are currently waiting for an entry. Where
+// maxIndex has not been set, this will always return 0 as beacon entries are generated on demand.
+func (mb *MockBeacon) WaitingOnEntryCount() int {
+	mb.lk.Lock()
+	defer mb.lk.Unlock()
+	return mb.waitingEntry
+}
+
+func (mb *MockBeacon) RoundTime() time.Duration {
 	return mb.interval
 }
 
-func (mb *mockBeacon) entryForIndex(index uint64) types.BeaconEntry {
+func (mb *MockBeacon) entryForIndex(index uint64) types.BeaconEntry {
 	buf := make([]byte, 8)
 	binary.BigEndian.PutUint64(buf, index)
 	rval := blake2b.Sum256(buf)
@@ -44,14 +73,32 @@ func (mb *mockBeacon) entryForIndex(index uint64) types.BeaconEntry {
 	}
 }
 
-func (mb *mockBeacon) Entry(ctx context.Context, index uint64) <-chan Response {
-	e := mb.entryForIndex(index)
+func (mb *MockBeacon) Entry(ctx context.Context, index uint64) <-chan Response {
 	out := make(chan Response, 1)
-	out <- Response{Entry: e}
+
+	mb.lk.Lock()
+	defer mb.lk.Unlock()
+
+	if mb.maxIndex >= 0 && index > uint64(mb.maxIndex) {
+		mb.waitingEntry++
+		go func() {
+			mb.lk.Lock()
+			defer mb.lk.Unlock()
+			for index > uint64(mb.maxIndex) {
+				mb.cond.Wait()
+			}
+			out <- Response{Entry: mb.entryForIndex(index)}
+			mb.waitingEntry--
+			mb.cond.Broadcast()
+		}()
+	} else {
+		out <- Response{Entry: mb.entryForIndex(index)}
+	}
+
 	return out
 }
 
-func (mb *mockBeacon) VerifyEntry(from types.BeaconEntry, _prevEntrySig []byte) error {
+func (mb *MockBeacon) VerifyEntry(from types.BeaconEntry, _prevEntrySig []byte) error {
 	// TODO: cache this, especially for bls
 	oe := mb.entryForIndex(from.Round)
 	if !bytes.Equal(from.Data, oe.Data) {
@@ -60,9 +107,9 @@ func (mb *mockBeacon) VerifyEntry(from types.BeaconEntry, _prevEntrySig []byte)
 	return nil
 }
 
-func (mb *mockBeacon) MaxBeaconRoundForEpoch(nv network.Version, epoch abi.ChainEpoch) uint64 {
+func (mb *MockBeacon) MaxBeaconRoundForEpoch(nv network.Version, epoch abi.ChainEpoch) uint64 {
 	// offset for better testing
 	return uint64(epoch + 100)
 }
 
-var _ RandomBeacon = (*mockBeacon)(nil)
+var _ RandomBeacon = (*MockBeacon)(nil)

chain/gen/genesis/miners.go

@@ -647,6 +647,11 @@ func (fr *fakeRand) GetChainRandomness(ctx context.Context, randEpoch abi.ChainE
 	return *(*[32]byte)(out), nil
 }
 
+func (fr *fakeRand) GetBeaconEntry(ctx context.Context, randEpoch abi.ChainEpoch) (*types.BeaconEntry, error) {
+	r, _ := fr.GetChainRandomness(ctx, randEpoch)
+	return &types.BeaconEntry{Round: 10, Data: r[:]}, nil
+}
+
 func (fr *fakeRand) GetBeaconRandomness(ctx context.Context, randEpoch abi.ChainEpoch) ([32]byte, error) {
 	out := make([]byte, 32)
 	_, _ = rand.New(rand.NewSource(int64(randEpoch))).Read(out) //nolint

chain/rand/rand.go

@@ -111,6 +111,7 @@ type stateRand struct {
 
 type Rand interface {
 	GetChainRandomness(ctx context.Context, round abi.ChainEpoch) ([32]byte, error)
+	GetBeaconEntry(ctx context.Context, round abi.ChainEpoch) (*types.BeaconEntry, error)
 	GetBeaconRandomness(ctx context.Context, round abi.ChainEpoch) ([32]byte, error)
 }
 
@@ -124,48 +125,58 @@ func NewStateRand(cs *store.ChainStore, blks []cid.Cid, b beacon.Schedule, netwo
 }
 
 // network v0-12
-func (sr *stateRand) getBeaconRandomnessV1(ctx context.Context, round abi.ChainEpoch) ([32]byte, error) {
+func (sr *stateRand) getBeaconEntryV1(ctx context.Context, round abi.ChainEpoch) (*types.BeaconEntry, error) {
 	randTs, err := sr.GetBeaconRandomnessTipset(ctx, round, true)
 	if err != nil {
-		return [32]byte{}, err
-	}
-
-	be, err := sr.cs.GetLatestBeaconEntry(ctx, randTs)
-	if err != nil {
-		return [32]byte{}, err
+		return nil, err
 	}
-
-	return blake2b.Sum256(be.Data), nil
+	return sr.cs.GetLatestBeaconEntry(ctx, randTs)
 }
 
 // network v13
-func (sr *stateRand) getBeaconRandomnessV2(ctx context.Context, round abi.ChainEpoch) ([32]byte, error) {
+func (sr *stateRand) getBeaconEntryV2(ctx context.Context, round abi.ChainEpoch) (*types.BeaconEntry, error) {
 	randTs, err := sr.GetBeaconRandomnessTipset(ctx, round, false)
 	if err != nil {
-		return [32]byte{}, err
-	}
-
-	be, err := sr.cs.GetLatestBeaconEntry(ctx, randTs)
-	if err != nil {
-		return [32]byte{}, err
+		return nil, err
 	}
-
-	return blake2b.Sum256(be.Data), nil
+	return sr.cs.GetLatestBeaconEntry(ctx, randTs)
 }
 
 // network v14 and on
-func (sr *stateRand) getBeaconRandomnessV3(ctx context.Context, filecoinEpoch abi.ChainEpoch) ([32]byte, error) {
+func (sr *stateRand) getBeaconEntryV3(ctx context.Context, filecoinEpoch abi.ChainEpoch) (*types.BeaconEntry, error) {
 	if filecoinEpoch < 0 {
-		return sr.getBeaconRandomnessV2(ctx, filecoinEpoch)
+		return sr.getBeaconEntryV2(ctx, filecoinEpoch)
 	}
 
-	be, err := sr.extractBeaconEntryForEpoch(ctx, filecoinEpoch)
+	randTs, err := sr.GetBeaconRandomnessTipset(ctx, filecoinEpoch, false)
 	if err != nil {
-		log.Errorf("failed to get beacon entry as expected: %s", err)
-		return [32]byte{}, err
+		return nil, err
 	}
 
-	return blake2b.Sum256(be.Data), nil
+	nv := sr.networkVersionGetter(ctx, filecoinEpoch)
+
+	round := sr.beacon.BeaconForEpoch(filecoinEpoch).MaxBeaconRoundForEpoch(nv, filecoinEpoch)
+
+	// Search back for the beacon entry, in normal operation it should be in randTs but for devnets
+	// where the blocktime is faster than the beacon period we may need to search back a bit to find
+	// the beacon entry for the requested round.
+	for i := 0; i < 20; i++ {
+		cbe := randTs.Blocks()[0].BeaconEntries
+		for _, v := range cbe {
+			if v.Round == round {
+				return &v, nil
+			}
+		}
+
+		next, err := sr.cs.LoadTipSet(ctx, randTs.Parents())
+		if err != nil {
+			return nil, xerrors.Errorf("failed to load parents when searching back for beacon entry: %w", err)
+		}
+
+		randTs = next
+	}
+
+	return nil, xerrors.Errorf("didn't find beacon for round %d (epoch %d)", round, filecoinEpoch)
 }
 
 func (sr *stateRand) GetChainRandomness(ctx context.Context, filecoinEpoch abi.ChainEpoch) ([32]byte, error) {
@@ -178,15 +189,27 @@ func (sr *stateRand) GetChainRandomness(ctx context.Context, filecoinEpoch abi.C
 	return sr.getChainRandomness(ctx, filecoinEpoch, true)
 }
 
-func (sr *stateRand) GetBeaconRandomness(ctx context.Context, filecoinEpoch abi.ChainEpoch) ([32]byte, error) {
+func (sr *stateRand) GetBeaconEntry(ctx context.Context, filecoinEpoch abi.ChainEpoch) (*types.BeaconEntry, error) {
 	nv := sr.networkVersionGetter(ctx, filecoinEpoch)
 
 	if nv >= network.Version14 {
-		return sr.getBeaconRandomnessV3(ctx, filecoinEpoch)
+		be, err := sr.getBeaconEntryV3(ctx, filecoinEpoch)
+		if err != nil {
+			log.Errorf("failed to get beacon entry as expected: %s", err)
+		}
+		return be, err
 	} else if nv == network.Version13 {
-		return sr.getBeaconRandomnessV2(ctx, filecoinEpoch)
+		return sr.getBeaconEntryV2(ctx, filecoinEpoch)
+	}
+	return sr.getBeaconEntryV1(ctx, filecoinEpoch)
+}
+
+func (sr *stateRand) GetBeaconRandomness(ctx context.Context, filecoinEpoch abi.ChainEpoch) ([32]byte, error) {
+	be, err := sr.GetBeaconEntry(ctx, filecoinEpoch)
+	if err != nil {
+		return [32]byte{}, err
 	}
-	return sr.getBeaconRandomnessV1(ctx, filecoinEpoch)
+	return blake2b.Sum256(be.Data), nil
 }
 
 func (sr *stateRand) DrawChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, filecoinEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
@@ -218,32 +241,3 @@ func (sr *stateRand) DrawBeaconRandomness(ctx context.Context, pers crypto.Domai
 
 	return ret, nil
 }
-
-func (sr *stateRand) extractBeaconEntryForEpoch(ctx context.Context, filecoinEpoch abi.ChainEpoch) (*types.BeaconEntry, error) {
-	randTs, err := sr.GetBeaconRandomnessTipset(ctx, filecoinEpoch, false)
-	if err != nil {
-		return nil, err
-	}
-
-	nv := sr.networkVersionGetter(ctx, filecoinEpoch)
-
-	round := sr.beacon.BeaconForEpoch(filecoinEpoch).MaxBeaconRoundForEpoch(nv, filecoinEpoch)
-
-	for i := 0; i < 20; i++ {
-		cbe := randTs.Blocks()[0].BeaconEntries
-		for _, v := range cbe {
-			if v.Round == round {
-				return &v, nil
-			}
-		}
-
-		next, err := sr.cs.LoadTipSet(ctx, randTs.Parents())
-		if err != nil {
-			return nil, xerrors.Errorf("failed to load parents when searching back for beacon entry: %w", err)
-		}
-
-		randTs = next
-	}
-
-	return nil, xerrors.Errorf("didn't find beacon for round %d (epoch %d)", round, filecoinEpoch)
-}

chain/stmgr/stmgr.go

@@ -572,9 +572,17 @@ func (sm *StateManager) GetRandomnessDigestFromBeacon(ctx context.Context, randE
 	}
 
 	r := rand.NewStateRand(sm.ChainStore(), pts.Cids(), sm.beacon, sm.GetNetworkVersion)
-
 	return r.GetBeaconRandomness(ctx, randEpoch)
+}
 
+func (sm *StateManager) GetBeaconEntry(ctx context.Context, randEpoch abi.ChainEpoch, tsk types.TipSetKey) (*types.BeaconEntry, error) {
+	pts, err := sm.ChainStore().GetTipSetFromKey(ctx, tsk)
+	if err != nil {
+		return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
+	}
+
+	r := rand.NewStateRand(sm.ChainStore(), pts.Cids(), sm.beacon, sm.GetNetworkVersion)
+	return r.GetBeaconEntry(ctx, randEpoch)
 }
 
 func (sm *StateManager) GetRandomnessDigestFromTickets(ctx context.Context, randEpoch abi.ChainEpoch, tsk types.TipSetKey) ([32]byte, error) {
@@ -584,6 +592,5 @@ func (sm *StateManager) GetRandomnessDigestFromTickets(ctx context.Context, rand
 	}
 
 	r := rand.NewStateRand(sm.ChainStore(), pts.Cids(), sm.beacon, sm.GetNetworkVersion)
-
 	return r.GetChainRandomness(ctx, randEpoch)
 }

chain/store/store.go

@@ -1344,6 +1344,10 @@ func (cs *ChainStore) GetTipSetFromKey(ctx context.Context, tsk types.TipSetKey)
 
 func (cs *ChainStore) GetLatestBeaconEntry(ctx context.Context, ts *types.TipSet) (*types.BeaconEntry, error) {
 	cur := ts
+
+	// Search for a beacon entry, in normal operation one should be in the requested tipset, but for
+	// devnets where the blocktime is faster than the beacon period we may need to search back a bit
+	// to find a tipset with a beacon entry.
 	for i := 0; i < 20; i++ {
 		cbe := cur.Blocks()[0].BeaconEntries
 		if len(cbe) > 0 {

conformance/rand_fixed.go

@@ -6,6 +6,7 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 
 	"github.com/filecoin-project/lotus/chain/rand"
+	"github.com/filecoin-project/lotus/chain/types"
 )
 
 type fixedRand struct{}
@@ -22,6 +23,10 @@ func (r *fixedRand) GetChainRandomness(_ context.Context, _ abi.ChainEpoch) ([32
 	return *(*[32]byte)([]byte("i_am_random_____i_am_random_____")), nil
 }
 
+func (r *fixedRand) GetBeaconEntry(_ context.Context, _ abi.ChainEpoch) (*types.BeaconEntry, error) {
+	return &types.BeaconEntry{Round: 10, Data: []byte("i_am_random_____i_am_random_____")}, nil
+}
+
 func (r *fixedRand) GetBeaconRandomness(_ context.Context, _ abi.ChainEpoch) ([32]byte, error) {
 	return *(*[32]byte)([]byte("i_am_random_____i_am_random_____")), nil // 32 bytes.
 }