update to latest light client libp2p protocol

AllTests-mainnet.md

@@ -265,11 +265,11 @@ OK: 3/3 Fail: 0/3 Skip: 0/3
 ## Light client processor [Preset: mainnet]
 ```diff
 + Duplicate bootstrap [Preset: mainnet]                                                      OK
-+ Forced update [Preset: mainnet]                                                            OK
 + Invalid bootstrap [Preset: mainnet]                                                        OK
++ Missing bootstrap (finality update) [Preset: mainnet]                                      OK
 + Missing bootstrap (optimistic update) [Preset: mainnet]                                    OK
 + Missing bootstrap (update) [Preset: mainnet]                                               OK
-+ Standard sync [Preset: mainnet]                                                            OK
++ Sync [Preset: mainnet]                                                                     OK
 ```
 OK: 6/6 Fail: 0/6 Skip: 0/6
 ## ListKeys requests [Preset: mainnet]

ConsensusSpecPreset-mainnet.md

@@ -804,6 +804,11 @@ OK: 35/35 Fail: 0/35 Skip: 0/35
   All tests                                                                                  Skip
 ```
 OK: 0/1 Fail: 0/1 Skip: 1/1
+## EF - Altair - Sync protocol - Update ranking [Preset: mainnet]
+```diff
+  All tests                                                                                  Skip
+```
+OK: 0/1 Fail: 0/1 Skip: 1/1
 ## EF - Altair - Unittests - Sync protocol [Preset: mainnet]
 ```diff
 + process_light_client_update_finality_updated                                               OK
@@ -1215,4 +1220,4 @@ OK: 44/44 Fail: 0/44 Skip: 0/44
 OK: 27/27 Fail: 0/27 Skip: 0/27
 
 ---TOTAL---
-OK: 1035/1037 Fail: 0/1037 Skip: 2/1037
+OK: 1035/1038 Fail: 0/1038 Skip: 3/1038

ConsensusSpecPreset-minimal.md

@@ -845,6 +845,11 @@ OK: 35/35 Fail: 0/35 Skip: 0/35
   All tests                                                                                  Skip
 ```
 OK: 0/1 Fail: 0/1 Skip: 1/1
+## EF - Altair - Sync protocol - Update ranking [Preset: minimal]
+```diff
+  All tests                                                                                  Skip
+```
+OK: 0/1 Fail: 0/1 Skip: 1/1
 ## EF - Altair - Unittests - Sync protocol [Preset: minimal]
 ```diff
 + process_light_client_update_finality_updated                                               OK
@@ -1292,4 +1297,4 @@ OK: 48/48 Fail: 0/48 Skip: 0/48
 OK: 30/30 Fail: 0/30 Skip: 0/30
 
 ---TOTAL---
-OK: 1085/1106 Fail: 0/1106 Skip: 21/1106
+OK: 1085/1107 Fail: 0/1107 Skip: 22/1107

beacon_chain/consensus_object_pools/block_pools_types.nim

@@ -216,8 +216,10 @@ type
       ## On beacon chain reorganization
     onFinHappened*: OnFinalizedCallback
       ## On finalization callback
-    onOptimisticLightClientUpdate*: OnOptimisticLightClientUpdateCallback
-      ## On `OptimisticLightClientUpdate` updated callback
+    onLightClientFinalityUpdate*: OnLightClientFinalityUpdateCallback
+      ## On new `LightClientFinalityUpdate` callback
+    onLightClientOptimisticUpdate*: OnLightClientOptimisticUpdateCallback
+      ## On new `LightClientOptimisticUpdate` callback
 
     headSyncCommittees*: SyncCommitteeCache
       ## A cache of the sync committees, as they appear in the head state -

beacon_chain/consensus_object_pools/block_pools_types_light_client.nim

@@ -13,13 +13,16 @@
 import
   # Status libraries
   stew/bitops2,
+  chronos,
   # Beacon chain internals
   ../spec/datatypes/altair,
   ./block_dag
 
 type
-  OnOptimisticLightClientUpdateCallback* =
-    proc(data: OptimisticLightClientUpdate) {.gcsafe, raises: [Defect].}
+  OnLightClientFinalityUpdateCallback* =
+    proc(data: altair.LightClientFinalityUpdate) {.gcsafe, raises: [Defect].}
+  OnLightClientOptimisticUpdateCallback* =
+    proc(data: altair.LightClientOptimisticUpdate) {.gcsafe, raises: [Defect].}
 
   ImportLightClientData* {.pure.} = enum
     ## Controls which classes of light client data are imported.
@@ -30,18 +33,17 @@ type
     Full = "full"
       ## Import light client data for entire weak subjectivity period.
     OnDemand = "on-demand"
-      ## No precompute of historic data. Is slow and may miss validator duties.
+      ## Don't precompute historic data. Slow, may miss validator duties.
 
   CachedLightClientData* = object
     ## Cached data from historical non-finalized states to improve speed when
     ## creating future `LightClientUpdate` and `LightClientBootstrap` instances.
     current_sync_committee_branch*:
       array[log2trunc(altair.CURRENT_SYNC_COMMITTEE_INDEX), Eth2Digest]
-
     next_sync_committee_branch*:
       array[log2trunc(altair.NEXT_SYNC_COMMITTEE_INDEX), Eth2Digest]
 
-    finalized_bid*: BlockId
+    finalized_slot*: Slot
     finality_branch*:
       array[log2trunc(altair.FINALIZED_ROOT_INDEX), Eth2Digest]
 
@@ -55,40 +57,44 @@ type
     data*: Table[BlockId, CachedLightClientData]
       ## Cached data for creating future `LightClientUpdate` instances.
       ## Key is the block ID of which the post state was used to get the data.
-      ## Data is stored for the most recent 4 finalized checkpoints, as well as
-      ## for all non-finalized blocks.
+      ## Data stored for the finalized head block and all non-finalized blocks.
 
     bootstrap*: Table[Slot, CachedLightClientBootstrap]
       ## Cached data for creating future `LightClientBootstrap` instances.
       ## Key is the block slot of which the post state was used to get the data.
-      ## Data is stored for finalized epoch boundary blocks.
-
-    latestCheckpoints*: array[4, Checkpoint]
-      ## Keeps track of the latest four `finalized_checkpoint` references
-      ## leading to `finalizedHead`. Used to prune `data`.
-      ## Non-finalized states may only refer to these checkpoints.
+      ## Data stored for all finalized epoch boundary blocks.
 
-    lastCheckpointIndex*: int
-      ## Last index that was modified in `latestCheckpoints`.
-
-    bestUpdates*: Table[SyncCommitteePeriod, altair.LightClientUpdate]
+    best*: Table[SyncCommitteePeriod, altair.LightClientUpdate]
       ## Stores the `LightClientUpdate` with the most `sync_committee_bits` per
-      ## `SyncCommitteePeriod`. Updates with a finality proof have precedence.
+      ## `SyncCommitteePeriod`. Sync committee finality gives precedence.
 
-    pendingBestUpdates*:
+    pendingBest*:
       Table[(SyncCommitteePeriod, Eth2Digest), altair.LightClientUpdate]
-      ## Same as `bestUpdates`, but for `SyncCommitteePeriod` with
-      ## `next_sync_committee` that are not finalized. Key is `(period,
+      ## Same as `best`, but for `SyncCommitteePeriod` with not yet finalized
+      ## `next_sync_committee`. Key is `(attested_period,
       ## hash_tree_root(current_sync_committee | next_sync_committee)`.
 
-    latestUpdate*: altair.LightClientUpdate
-      ## Tracks the `LightClientUpdate` for the latest slot. This may be older
-      ## than head for empty slots or if not signed by sync committee.
-
-    optimisticUpdate*: OptimisticLightClientUpdate
-      ## Tracks the `OptimisticLightClientUpdate` for the latest slot. This may
-      ## be older than head for empty slots or if not signed by sync committee.
+    latest*: altair.LightClientFinalityUpdate
+      ## Tracks light client data for the latest slot that was signed by
+      ## at least `MIN_SYNC_COMMITTEE_PARTICIPANTS`. May be older than head.
 
     importTailSlot*: Slot
-      ## The earliest slot for which light client data is collected.
-      ## Only relevant for `ImportLightClientData.OnlyNew`.
+      ## The earliest slot for which light client data is imported.
+
+    latestForwardedFinalitySlot*: Slot
+      ## Latest finality update that was forwarded on libp2p gossip.
+      ## Tracks `finality_update.finalized_header.slot`.
+
+    latestForwardedOptimisticSlot*: Slot
+      ## Latest optimistic update that was forwarded on libp2p gossip.
+      ## Tracks `optimistic_update.attested_header.slot`.
+
+    latestBroadcastedSlot*: Slot
+      ## Latest slot for which updates were broadcasted on libp2p gossip.
+      ## Tracks `update.signature_slot`.
+
+    broadcastGossipFut*: Future[void]
+      ## Task to broadcast libp2p gossip. Started when a sync committee message
+      ## is sent. Tracked separately from `handleValidatorDuties` to catch the
+      ## case where `node.attachedValidators[].count == 0` at function start,
+      ## and then a sync committee message gets sent from a remote VC via REST.

beacon_chain/consensus_object_pools/blockchain_dag.nim

@@ -556,10 +556,10 @@ proc updateBeaconMetrics(
 import blockchain_dag_light_client
 
 export
-  blockchain_dag_light_client.getBestLightClientUpdateForPeriod,
-  blockchain_dag_light_client.getLatestLightClientUpdate,
-  blockchain_dag_light_client.getOptimisticLightClientUpdate,
-  blockchain_dag_light_client.getLightClientBootstrap
+  blockchain_dag_light_client.getLightClientBootstrap,
+  blockchain_dag_light_client.getLightClientUpdateForPeriod,
+  blockchain_dag_light_client.getLightClientFinalityUpdate,
+  blockchain_dag_light_client.getLightClientOptimisticUpdate
 
 proc getViableHead(cfg: RuntimeConfig, db: BeaconChainDB): Opt[BlockId] =
   # When the database has been written with a pre-fork version of the
@@ -659,9 +659,9 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB,
            validatorMonitor: ref ValidatorMonitor, updateFlags: UpdateFlags,
            eraPath = ".",
            onBlockCb: OnBlockCallback = nil, onHeadCb: OnHeadCallback = nil,
-           onReorgCb: OnReorgCallback = nil,
-           onFinCb: OnFinalizedCallback = nil,
-           onOptimisticLCUpdateCb: OnOptimisticLightClientUpdateCallback = nil,
+           onReorgCb: OnReorgCallback = nil, onFinCb: OnFinalizedCallback = nil,
+           onLCFinalityUpdateCb: OnLightClientFinalityUpdateCallback = nil,
+           onLCOptimisticUpdateCb: OnLightClientOptimisticUpdateCallback = nil,
            serveLightClientData = false,
            importLightClientData = ImportLightClientData.None): ChainDAGRef =
   # TODO move fork version sanity checking elsewhere?
@@ -712,7 +712,8 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB,
       onHeadChanged: onHeadCb,
       onReorgHappened: onReorgCb,
       onFinHappened: onFinCb,
-      onOptimisticLightClientUpdate: onOptimisticLCUpdateCb
+      onLightClientFinalityUpdate: onLCFinalityUpdateCb,
+      onLightClientOptimisticUpdate: onLCOptimisticUpdateCb
     )
     loadTick = Moment.now()
 
@@ -1636,6 +1637,7 @@ proc updateHead*(
       justified = shortLog(getStateField(
         dag.headState, current_justified_checkpoint)),
       finalized = shortLog(getStateField(dag.headState, finalized_checkpoint))
+    let oldFinalizedHead = dag.finalizedHead
 
     block:
       # Update `dag.finalizedBlocks` with all newly finalized blocks (those
@@ -1663,7 +1665,7 @@ proc updateHead*(
     dag.pruneBlocksDAG()
 
     # Update light client data
-    dag.processFinalizationForLightClient()
+    dag.processFinalizationForLightClient(oldFinalizedHead)
 
     # Send notification about new finalization point via callback.
     if not(isNil(dag.onFinHappened)):