From 99a5060d5ba034c416a699e0e54bfa834663a647 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sat, 13 Jul 2019 04:07:14 -0400 Subject: [PATCH 01/31] Minimal attestation simplification --- specs/core/1_shard-data-chains.md | 88 ++++++++++++++----------------- 1 file changed, 39 insertions(+), 49 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index fb4f353efb..b6acc538f4 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -13,11 +13,10 @@ - [Misc](#misc) - [Initial values](#initial-values) - [Time parameters](#time-parameters) - - [Signature domain types](#signature-domain-types) + - [Signature domains](#signature-domains) - [TODO PLACEHOLDER](#todo-placeholder) - [Data structures](#data-structures) - [`ShardBlockBody`](#shardblockbody) - - [`ShardAttestation`](#shardattestation) - [`ShardBlock`](#shardblock) - [`ShardBlockHeader`](#shardblockheader) - [Helper functions](#helper-functions) @@ -26,7 +25,6 @@ - [`get_persistent_committee`](#get_persistent_committee) - [`get_shard_proposer_index`](#get_shard_proposer_index) - [`get_shard_header`](#get_shard_header) - - [`verify_shard_attestation_signature`](#verify_shard_attestation_signature) - [`compute_crosslink_data_root`](#compute_crosslink_data_root) - [Object validity](#object-validity) - [Shard blocks](#shard-blocks) @@ -40,6 +38,14 @@ This document describes the shard data layer and the shard fork choice rule in Phase 1 of Ethereum 2.0. +## Custom types + +We define the following Python custom types for type hinting and readability: + +| Name | SSZ equivalent | Description | +| - | - | - | +| `ShardSlot` | `uint64` | a slot number in shard chain | + ## Configuration ### Misc @@ -48,6 +54,8 @@ This document describes the shard data layer and the shard fork choice rule in P | - | - | | `BYTES_PER_SHARD_BLOCK_BODY` | `2**14` (= 16,384) | | `MAX_SHARD_ATTESTIONS` | `2**4` (= 16) | +| `SHARD_SLOTS_PER_BEACON_SLOT` | `2**1` (= 2) | +| `SHARD_SLOT_COMMITTEE_SIZE` | `2**5` (= 32) | ### Initial values @@ -86,37 +94,26 @@ class ShardBlockBody(Container): data: Vector[Bytes[PLACEHOLDER], BYTES_PER_SHARD_BLOCK_BODY] ``` -### `ShardAttestation` - -```python -class ShardAttestation(Container): - class data(Container): - slot: Slot - shard: Shard - shard_block_root: Hash - aggregation_bits: Bitlist[PLACEHOLDER] - aggregate_signature: BLSSignature -``` - ### `ShardBlock` ```python class ShardBlock(Container): - slot: Slot + slot: ShardSlot shard: Shard beacon_chain_root: Hash parent_root: Hash data: ShardBlockBody state_root: Hash - attestations: List[ShardAttestation, PLACEHOLDER] - signature: BLSSignature + attester_bitfield: BitVector[SHARD_SLOT_COMMITTEE_SIZE] + attestation_signature: BLSSignature + proposer_signature: BLSSignature ``` ### `ShardBlockHeader` ```python class ShardBlockHeader(Container): - slot: Slot + slot: ShardSlot shard: Shard beacon_chain_root: Hash parent_root: Hash @@ -128,6 +125,13 @@ class ShardBlockHeader(Container): ## Helper functions +### `compute_epoch_of_shard_slot` + +```python +def compute_epoch_of_shard_slot(slot: ShardSlot) -> Epoch: + return Epoch(slot // SHARD_SLOTS_PER_BEACON_SLOT // SLOTS_PER_EPOCH) +``` + ### `get_period_committee` ```python @@ -161,11 +165,11 @@ def get_switchover_epoch(state: BeaconState, epoch: Epoch, index: ValidatorIndex ```python def get_persistent_committee(state: BeaconState, shard: Shard, - slot: Slot) -> Sequence[ValidatorIndex]: + slot: ShardSlot) -> Sequence[ValidatorIndex]: """ Return the persistent committee for the given ``shard`` at the given ``slot``. """ - epoch = compute_epoch_of_slot(slot) + epoch = compute_epoch_of_shard_slot(slot) earlier_start_epoch = Epoch(epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2) later_start_epoch = Epoch(epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD) @@ -193,7 +197,7 @@ def get_persistent_committee(state: BeaconState, ```python def get_shard_proposer_index(state: BeaconState, shard: Shard, - slot: Slot) -> Optional[ValidatorIndex]: + slot: ShardSlot) -> Optional[ValidatorIndex]: # Randomly shift persistent committee persistent_committee = list(get_persistent_committee(state, shard, slot)) seed = hash(state.current_shuffling_seed + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) @@ -225,27 +229,6 @@ def get_shard_header(block: ShardBlock) -> ShardBlockHeader: ) ``` -### `verify_shard_attestation_signature` - -```python -def verify_shard_attestation_signature(state: BeaconState, - attestation: ShardAttestation) -> None: - data = attestation.data - persistent_committee = get_persistent_committee(state, data.shard, data.slot) - pubkeys = [] - for i, index in enumerate(persistent_committee): - if attestation.aggregation_bits[i]: - validator = state.validators[index] - assert is_active_validator(validator, get_current_epoch(state)) - pubkeys.append(validator.pubkey) - assert bls_verify( - pubkey=bls_aggregate_pubkeys(pubkeys), - message_hash=data.shard_block_root, - signature=attestation.aggregate_signature, - domain=get_domain(state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_slot(data.slot)) - ) -``` - ### `compute_crosslink_data_root` ```python @@ -327,12 +310,19 @@ def is_valid_shard_block(beacon_blocks: Sequence[BeaconBlock], assert signing_root(beacon_blocks[parent_block.slot]) == parent_block.beacon_chain_root # Check attestations - assert len(candidate.attestations) <= MAX_SHARD_ATTESTIONS - for _, attestation in enumerate(candidate.attestations): - assert max(GENESIS_SHARD_SLOT, candidate.slot - SLOTS_PER_EPOCH) <= attestation.data.slot - assert attestation.data.slot <= candidate.slot - MIN_ATTESTATION_INCLUSION_DELAY - assert attestation.data.crosslink.shard == candidate.shard - verify_shard_attestation_signature(beacon_state, attestation) + persistent_committee = get_persistent_committee(beacon_state, block.shard, block.slot) + pubkeys = [] + for i, index in enumerate(persistent_committee): + if aggregation_bits[i]: + validator = state.validators[index] + assert is_active_validator(validator, get_current_epoch(state)) + pubkeys.append(validator.pubkey) + assert bls_verify( + pubkey=bls_aggregate_pubkeys(pubkeys), + message_hash=block.parent_root, + signature=aggregate_signature, + domain=get_domain(state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_slot(data.slot)) + ) # Check signature proposer_index = get_shard_proposer_index(beacon_state, candidate.shard, candidate.slot) From 14010f5a5d8886c3363e2f7387c1925dd8d454b6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Jul 2019 15:52:25 +0800 Subject: [PATCH 02/31] minor fix --- specs/core/1_shard-data-chains.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index b6acc538f4..f0ccfeac9a 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -9,17 +9,19 @@ - [Ethereum 2.0 Phase 1 -- Shard Data Chains](#ethereum-20-phase-1----shard-data-chains) - [Table of contents](#table-of-contents) - [Introduction](#introduction) + - [Custom types](#custom-types) - [Configuration](#configuration) - [Misc](#misc) - [Initial values](#initial-values) - [Time parameters](#time-parameters) - - [Signature domains](#signature-domains) + - [Signature domain types](#signature-domain-types) - [TODO PLACEHOLDER](#todo-placeholder) - [Data structures](#data-structures) - [`ShardBlockBody`](#shardblockbody) - [`ShardBlock`](#shardblock) - [`ShardBlockHeader`](#shardblockheader) - [Helper functions](#helper-functions) + - [`compute_epoch_of_shard_slot`](#compute_epoch_of_shard_slot) - [`get_period_committee`](#get_period_committee) - [`get_switchover_epoch`](#get_switchover_epoch) - [`get_persistent_committee`](#get_persistent_committee) @@ -104,7 +106,7 @@ class ShardBlock(Container): parent_root: Hash data: ShardBlockBody state_root: Hash - attester_bitfield: BitVector[SHARD_SLOT_COMMITTEE_SIZE] + attester_bitfield: Bitvector[SHARD_SLOT_COMMITTEE_SIZE] attestation_signature: BLSSignature proposer_signature: BLSSignature ``` @@ -119,7 +121,7 @@ class ShardBlockHeader(Container): parent_root: Hash body_root: Hash state_root: Hash - attestations: List[ShardAttestation, PLACEHOLDER] + attester_bitfield: Bitvector[SHARD_SLOT_COMMITTEE_SIZE] signature: BLSSignature ``` From 0cfca0153340503cc51f9feffacc12d8474f48d0 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Jul 2019 17:30:17 +0800 Subject: [PATCH 03/31] Make the tests pass --- specs/core/1_shard-data-chains.md | 61 +++++++++---------------------- 1 file changed, 17 insertions(+), 44 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index f0ccfeac9a..4dda406351 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -25,12 +25,11 @@ - [`get_period_committee`](#get_period_committee) - [`get_switchover_epoch`](#get_switchover_epoch) - [`get_persistent_committee`](#get_persistent_committee) - - [`get_shard_proposer_index`](#get_shard_proposer_index) + - [`get_shard_block_proposer_index`](#get_shard_block_proposer_index) - [`get_shard_header`](#get_shard_header) - [`compute_crosslink_data_root`](#compute_crosslink_data_root) - [Object validity](#object-validity) - [Shard blocks](#shard-blocks) - - [Shard attestations](#shard-attestations) - [Beacon attestations](#beacon-attestations) - [Shard fork choice rule](#shard-fork-choice-rule) @@ -122,6 +121,7 @@ class ShardBlockHeader(Container): body_root: Hash state_root: Hash attester_bitfield: Bitvector[SHARD_SLOT_COMMITTEE_SIZE] + attestation_signature: BLSSignature signature: BLSSignature ``` @@ -194,12 +194,12 @@ def get_persistent_committee(state: BeaconState, ))) ``` -### `get_shard_proposer_index` +### `get_shard_block_proposer_index` ```python -def get_shard_proposer_index(state: BeaconState, - shard: Shard, - slot: ShardSlot) -> Optional[ValidatorIndex]: +def get_shard_block_proposer_index(state: BeaconState, + shard: Shard, + slot: ShardSlot) -> Optional[ValidatorIndex]: # Randomly shift persistent committee persistent_committee = list(get_persistent_committee(state, shard, slot)) seed = hash(state.current_shuffling_seed + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) @@ -226,7 +226,7 @@ def get_shard_header(block: ShardBlock) -> ShardBlockHeader: parent_root=block.parent_root, body_root=hash_tree_root(block.body), state_root=block.state_root, - attestations=block.attestations, + attestation_signature=block.attestation_signature, signature=block.signature, ) ``` @@ -275,8 +275,8 @@ Let: - `candidate` be a candidate `ShardBlock` for which validity is to be determined by running `is_valid_shard_block` ```python -def is_valid_shard_block(beacon_blocks: Sequence[BeaconBlock], - beacon_state: BeaconState, +def is_valid_shard_block(beacon_state: BeaconState, + beacon_blocks: Sequence[BeaconBlock], valid_shard_blocks: Sequence[ShardBlock], candidate: ShardBlock) -> bool: # Check if block is already determined valid @@ -315,19 +315,19 @@ def is_valid_shard_block(beacon_blocks: Sequence[BeaconBlock], persistent_committee = get_persistent_committee(beacon_state, block.shard, block.slot) pubkeys = [] for i, index in enumerate(persistent_committee): - if aggregation_bits[i]: - validator = state.validators[index] - assert is_active_validator(validator, get_current_epoch(state)) + if block.attester_bitfield[i]: + validator = beacon_state.validators[index] + assert is_active_validator(validator, get_current_epoch(beacon_state)) pubkeys.append(validator.pubkey) assert bls_verify( pubkey=bls_aggregate_pubkeys(pubkeys), - message_hash=block.parent_root, - signature=aggregate_signature, - domain=get_domain(state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_slot(data.slot)) + message_hash=candidate.parent_root, + signature=candidate.attestation_signature, + domain=get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_slot(candidate.slot)) ) - # Check signature - proposer_index = get_shard_proposer_index(beacon_state, candidate.shard, candidate.slot) + # Check proposer + proposer_index = get_shard_block_proposer_index(beacon_state, candidate.shard, candidate.slot) assert proposer_index is not None assert bls_verify( pubkey=beacon_state.validators[proposer_index].pubkey, @@ -339,33 +339,6 @@ def is_valid_shard_block(beacon_blocks: Sequence[BeaconBlock], return True ``` -### Shard attestations - -Let: - -- `valid_shard_blocks` be the list of valid `ShardBlock` -- `beacon_state` be the canonical `BeaconState` -- `candidate` be a candidate `ShardAttestation` for which validity is to be determined by running `is_valid_shard_attestation` - -```python -def is_valid_shard_attestation(valid_shard_blocks: Sequence[ShardBlock], - beacon_state: BeaconState, - candidate: ShardAttestation) -> bool: - # Check shard block - shard_block = next( - (block for block in valid_shard_blocks if signing_root(block) == candidate.data.shard_block_root), - None, - ) - assert shard_block is not None - assert shard_block.slot == candidate.data.slot - assert shard_block.shard == candidate.data.shard - - # Check signature - verify_shard_attestation_signature(beacon_state, candidate) - - return True -``` - ### Beacon attestations Let: From f77df3f2008394a985b9cf2c1729ee1c5b7786e7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Jul 2019 18:24:45 +0800 Subject: [PATCH 04/31] Decrease `PLACEHOLDER`, Use `compute_epoch_of_shard_slot` --- specs/core/1_shard-data-chains.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 4dda406351..53f16136ee 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -84,7 +84,7 @@ The following types are defined, mapping into `DomainType` (little endian): | Name | Value | | - | - | -| `PLACEHOLDER` | `2**32` | +| `PLACEHOLDER` | `2**3` | ## Data structures @@ -323,7 +323,7 @@ def is_valid_shard_block(beacon_state: BeaconState, pubkey=bls_aggregate_pubkeys(pubkeys), message_hash=candidate.parent_root, signature=candidate.attestation_signature, - domain=get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_slot(candidate.slot)) + domain=get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(candidate.slot)) ) # Check proposer @@ -333,7 +333,7 @@ def is_valid_shard_block(beacon_state: BeaconState, pubkey=beacon_state.validators[proposer_index].pubkey, message_hash=signing_root(candidate), signature=candidate.signature, - domain=get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_slot(candidate.slot)), + domain=get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(candidate.slot)), ) return True From bcfff596a1b79a672f6406d473b503a7c0d27f54 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Jul 2019 19:08:21 +0800 Subject: [PATCH 05/31] Fix proposer signature name and use get_seed() to calculate current_shuffling_seed --- specs/core/1_shard-data-chains.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 53f16136ee..6839a3b143 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -107,7 +107,7 @@ class ShardBlock(Container): state_root: Hash attester_bitfield: Bitvector[SHARD_SLOT_COMMITTEE_SIZE] attestation_signature: BLSSignature - proposer_signature: BLSSignature + signature: BLSSignature ``` ### `ShardBlockHeader` @@ -202,7 +202,7 @@ def get_shard_block_proposer_index(state: BeaconState, slot: ShardSlot) -> Optional[ValidatorIndex]: # Randomly shift persistent committee persistent_committee = list(get_persistent_committee(state, shard, slot)) - seed = hash(state.current_shuffling_seed + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) + seed = hash(get_seed(state, get_current_epoch(state)) + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) random_index = bytes_to_int(seed[0:8]) % len(persistent_committee) persistent_committee = persistent_committee[random_index:] + persistent_committee[:random_index] From be1bc5221e48035df5c07bceb914a924d01d1901 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Jul 2019 19:16:24 +0800 Subject: [PATCH 06/31] Fix linter error --- specs/core/1_shard-data-chains.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 6839a3b143..6d3f4d0e99 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -202,7 +202,9 @@ def get_shard_block_proposer_index(state: BeaconState, slot: ShardSlot) -> Optional[ValidatorIndex]: # Randomly shift persistent committee persistent_committee = list(get_persistent_committee(state, shard, slot)) - seed = hash(get_seed(state, get_current_epoch(state)) + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) + seed = hash( + get_seed(state, get_current_epoch(state)) + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8) + ) random_index = bytes_to_int(seed[0:8]) % len(persistent_committee) persistent_committee = persistent_committee[random_index:] + persistent_committee[:random_index] From 7ca50313321129cf82a25dae6ca6584c5f111688 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Jul 2019 19:18:02 +0800 Subject: [PATCH 07/31] Add the WIP `test_is_valid_shard_block` --- .../eth2spec/test/helpers/phase1/__init__.py | 0 .../test/helpers/phase1/shard_block.py | 43 +++++++++++++++++++ .../shard_data_chain/test_shard_block.py | 26 +++++++++++ 3 files changed, 69 insertions(+) create mode 100644 test_libs/pyspec/eth2spec/test/helpers/phase1/__init__.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py create mode 100644 test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/__init__.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py new file mode 100644 index 0000000000..f67d7da87b --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py @@ -0,0 +1,43 @@ +from eth2spec.test.helpers.keys import privkeys +from eth2spec.utils.bls import ( + bls_sign, + only_with_bls, +) +from eth2spec.utils.ssz.ssz_impl import ( + signing_root, +) + + +@only_with_bls() +def sign_shard_block(spec, state, block, proposer_index=None): + if proposer_index is None: + proposer_index = spec.get_shard_block_proposer_index(state, block.shard, block.slot) + + privkey = privkeys[proposer_index] + + block.signature = bls_sign( + message_hash=signing_root(block), + privkey=privkey, + domain=spec.get_domain( + state, + spec.DOMAIN_SHARD_PROPOSER, + spec.compute_epoch_of_shard_slot(block.slot), + ) + ) + + +def build_empty_shard_block(spec, state, slot, shard, parent_root, signed=False): + if slot is None: + slot = state.slot + block = spec.ShardBlock( + slot=slot, + shard=shard, + beacon_chain_root=state.block_roots[state.slot // spec.SLOTS_PER_HISTORICAL_ROOT], + parent_root=parent_root, + signature=b'\x12' * 96, + ) + + if signed: + sign_shard_block(spec, state, block) + + return block diff --git a/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py b/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py new file mode 100644 index 0000000000..359350d39a --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py @@ -0,0 +1,26 @@ +from eth2spec.test.helpers.phase1.shard_block import ( + build_empty_shard_block, +) +from eth2spec.test.context import ( + with_all_phases_except, + spec_state_test, + always_bls, +) + + +@with_all_phases_except(['phase0']) +@always_bls +@spec_state_test +def test_is_valid_shard_block(spec, state): + block = build_empty_shard_block( + spec, + state, + slot=spec.Slot(spec.PERSISTENT_COMMITTEE_PERIOD * 100), + shard=spec.Shard(1), + parent_root=spec.Hash(), + signed=True, + ) + + # TODO: test `is_valid_shard_block` + + yield 'blocks', (block,) From ac29581b4439b8780c8ba877427c97cad1ff6a20 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Jul 2019 13:50:05 +0800 Subject: [PATCH 08/31] Add `get_shard_block_attester_committee` --- specs/core/1_shard-data-chains.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 6d3f4d0e99..1e23c99f29 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -26,6 +26,7 @@ - [`get_switchover_epoch`](#get_switchover_epoch) - [`get_persistent_committee`](#get_persistent_committee) - [`get_shard_block_proposer_index`](#get_shard_block_proposer_index) + - [`get_shard_block_attester_committee`](#get_shard_block_attester_committee) - [`get_shard_header`](#get_shard_header) - [`compute_crosslink_data_root`](#compute_crosslink_data_root) - [Object validity](#object-validity) @@ -217,6 +218,20 @@ def get_shard_block_proposer_index(state: BeaconState, return None ``` +### `get_shard_block_attester_committee` + +```python +def get_shard_block_attester_committee(state: BeaconState, + shard: Shard, + slot: ShardSlot) -> Sequence[Optional[ValidatorIndex]]: + persistent_committee = get_persistent_committee(state, shard, slot) + committee_size = min( + len(persistent_committee), + SHARD_SLOT_COMMITTEE_SIZE, + ) + return [get_shard_block_proposer_index(state, shard, ShardSlot(slot - i)) for i in range(committee_size)] +``` + ### `get_shard_header` ```python @@ -314,9 +329,9 @@ def is_valid_shard_block(beacon_state: BeaconState, assert signing_root(beacon_blocks[parent_block.slot]) == parent_block.beacon_chain_root # Check attestations - persistent_committee = get_persistent_committee(beacon_state, block.shard, block.slot) + attester_committee = get_shard_block_attester_committee(beacon_state, block.shard, block.slot) pubkeys = [] - for i, index in enumerate(persistent_committee): + for i, index in enumerate(attester_committee): if block.attester_bitfield[i]: validator = beacon_state.validators[index] assert is_active_validator(validator, get_current_epoch(beacon_state)) From 2b2b14324d0881947d1356f20fbd81f35196ca98 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Thu, 18 Jul 2019 05:10:31 -0400 Subject: [PATCH 09/31] Simplified committee selection --- specs/core/1_shard-data-chains.md | 62 +++++++++++++------------------ 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 1e23c99f29..97ecc9d151 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -57,7 +57,7 @@ We define the following Python custom types for type hinting and readability: | `BYTES_PER_SHARD_BLOCK_BODY` | `2**14` (= 16,384) | | `MAX_SHARD_ATTESTIONS` | `2**4` (= 16) | | `SHARD_SLOTS_PER_BEACON_SLOT` | `2**1` (= 2) | -| `SHARD_SLOT_COMMITTEE_SIZE` | `2**5` (= 32) | +| `MAX_PERSISTENT_COMMITTEE_SIZE` | `2**5` (= 128) | ### Initial values @@ -70,7 +70,8 @@ We define the following Python custom types for type hinting and readability: | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `CROSSLINK_LOOKBACK` | `2**0` (= 1) | epochs | 6.2 minutes | +| `CROSSLINK_LOOKBACK` | `2**0` (= 1) | epochs | 6.4 minutes | +| `PERSISTENT_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~27 hours | ### Signature domain types @@ -106,7 +107,7 @@ class ShardBlock(Container): parent_root: Hash data: ShardBlockBody state_root: Hash - attester_bitfield: Bitvector[SHARD_SLOT_COMMITTEE_SIZE] + attester_bitfield: Bitvector[MAX_PERSISTENT_COMMITTEE_SIZE * 2] attestation_signature: BLSSignature signature: BLSSignature ``` @@ -121,7 +122,7 @@ class ShardBlockHeader(Container): parent_root: Hash body_root: Hash state_root: Hash - attester_bitfield: Bitvector[SHARD_SLOT_COMMITTEE_SIZE] + attester_bitfield: Bitvector[MAX_PERSISTENT_COMMITTEE_SIZE * 2] attestation_signature: BLSSignature signature: BLSSignature ``` @@ -140,18 +141,27 @@ def compute_epoch_of_shard_slot(slot: ShardSlot) -> Epoch: ```python def get_period_committee(state: BeaconState, epoch: Epoch, - shard: Shard, - index: uint64, - count: uint64) -> Sequence[ValidatorIndex]: + shard: Shard) -> List[ValidatorIndex, MAX_PERSISTENT_COMMITTEE_SIZE]: """ Return committee for a period. Used to construct persistent committees. """ - return compute_committee( + seed = get_seed(state, epoch) + full_committee = compute_committee( indices=get_active_validator_indices(state, epoch), - seed=get_seed(state, epoch), + seed=seed, index=shard * count + index, count=SHARD_COUNT * count, ) + + def active_and_balance_filter(i: ValidatorIndex) -> bool: + if not is_active_validator(state.validators[i], epoch): + return False + active_threshold = MAX_EFFECTIVE_BALANCE * seed[i % 32] // MAX_RANDOM_BYTE + if state.validators[i].effective_balance < active_threshold: + return False + return True + + return [i for i in full_committee if active_and_balance_filter(i)][:MAX_PERSISTENT_COMMITTEE_SIZE] ``` ### `get_switchover_epoch` @@ -176,23 +186,15 @@ def get_persistent_committee(state: BeaconState, earlier_start_epoch = Epoch(epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2) later_start_epoch = Epoch(epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD) - committee_count = max( - len(get_active_validator_indices(state, earlier_start_epoch)) // - (SHARD_COUNT * TARGET_COMMITTEE_SIZE), - len(get_active_validator_indices(state, later_start_epoch)) // - (SHARD_COUNT * TARGET_COMMITTEE_SIZE), - ) + 1 - - index = slot % committee_count - earlier_committee = get_period_committee(state, earlier_start_epoch, shard, index, committee_count) - later_committee = get_period_committee(state, later_start_epoch, shard, index, committee_count) + earlier_committee = get_period_committee(state, earlier_start_epoch, shard) + later_committee = get_period_committee(state, later_start_epoch, shard) # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated - return sorted(list(set( + return sorted(set( [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(state, epoch, i)] + [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(state, epoch, i)] - ))) + )) ``` ### `get_shard_block_proposer_index` @@ -218,20 +220,6 @@ def get_shard_block_proposer_index(state: BeaconState, return None ``` -### `get_shard_block_attester_committee` - -```python -def get_shard_block_attester_committee(state: BeaconState, - shard: Shard, - slot: ShardSlot) -> Sequence[Optional[ValidatorIndex]]: - persistent_committee = get_persistent_committee(state, shard, slot) - committee_size = min( - len(persistent_committee), - SHARD_SLOT_COMMITTEE_SIZE, - ) - return [get_shard_block_proposer_index(state, shard, ShardSlot(slot - i)) for i in range(committee_size)] -``` - ### `get_shard_header` ```python @@ -329,13 +317,15 @@ def is_valid_shard_block(beacon_state: BeaconState, assert signing_root(beacon_blocks[parent_block.slot]) == parent_block.beacon_chain_root # Check attestations - attester_committee = get_shard_block_attester_committee(beacon_state, block.shard, block.slot) + attester_committee = get_persistent_committee(beacon_state, block.shard, block.slot) pubkeys = [] for i, index in enumerate(attester_committee): if block.attester_bitfield[i]: validator = beacon_state.validators[index] assert is_active_validator(validator, get_current_epoch(beacon_state)) pubkeys.append(validator.pubkey) + for i in range(len(attester_committee), MAX_PERSISTENT_COMMITTEE_SIZE * 2): + assert block.attester_bitfield[i] is False assert bls_verify( pubkey=bls_aggregate_pubkeys(pubkeys), message_hash=candidate.parent_root, From b5de66a6fe247f3b6875290bf797d5e0a1f6d5d6 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 19 Jul 2019 00:12:18 -0400 Subject: [PATCH 10/31] Added some helpers and simplified --- specs/core/1_shard-data-chains.md | 37 +++++++++++++------------------ 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 97ecc9d151..a7b541e903 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -22,6 +22,7 @@ - [`ShardBlockHeader`](#shardblockheader) - [Helper functions](#helper-functions) - [`compute_epoch_of_shard_slot`](#compute_epoch_of_shard_slot) + - [`get_period_start_block`](#get_period_start_block) - [`get_period_committee`](#get_period_committee) - [`get_switchover_epoch`](#get_switchover_epoch) - [`get_persistent_committee`](#get_persistent_committee) @@ -136,6 +137,13 @@ def compute_epoch_of_shard_slot(slot: ShardSlot) -> Epoch: return Epoch(slot // SHARD_SLOTS_PER_BEACON_SLOT // SLOTS_PER_EPOCH) ``` +### `get_period_start_block` + +```python +def get_period_start_block(epoch: Epoch, lookback:Epoch = 0): + return epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - lookback * PERSISTENT_COMMITTEE_PERIOD +``` + ### `get_period_committee` ```python @@ -168,7 +176,7 @@ def get_period_committee(state: BeaconState, ```python def get_switchover_epoch(state: BeaconState, epoch: Epoch, index: ValidatorIndex) -> int: - earlier_start_epoch = Epoch(epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2) + earlier_start_epoch = get_period_start_epoch(epoch, lookback=2) return (bytes_to_int(hash(get_seed(state, earlier_start_epoch) + int_to_bytes(index, length=3)[0:8])) % PERSISTENT_COMMITTEE_PERIOD) ``` @@ -183,11 +191,9 @@ def get_persistent_committee(state: BeaconState, Return the persistent committee for the given ``shard`` at the given ``slot``. """ epoch = compute_epoch_of_shard_slot(slot) - earlier_start_epoch = Epoch(epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2) - later_start_epoch = Epoch(epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD) - earlier_committee = get_period_committee(state, earlier_start_epoch, shard) - later_committee = get_period_committee(state, later_start_epoch, shard) + earlier_committee = get_period_committee(state, get_period_start_epoch(epoch, lookback=2), shard) + later_committee = get_period_committee(state, get_period_start_epoch(epoch, lookback=1), shard) # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated @@ -202,22 +208,11 @@ def get_persistent_committee(state: BeaconState, ```python def get_shard_block_proposer_index(state: BeaconState, shard: Shard, - slot: ShardSlot) -> Optional[ValidatorIndex]: + slot: Slot) -> Optional[ValidatorIndex]: # Randomly shift persistent committee persistent_committee = list(get_persistent_committee(state, shard, slot)) - seed = hash( - get_seed(state, get_current_epoch(state)) + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8) - ) - random_index = bytes_to_int(seed[0:8]) % len(persistent_committee) - persistent_committee = persistent_committee[random_index:] + persistent_committee[:random_index] - - # Search for an active proposer - for index in persistent_committee: - if is_active_validator(state.validators[index], get_current_epoch(state)): - return index - - # No block can be proposed if no validator is active - return None + seed = hash(state.current_shuffling_seed + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) + return persistent_committee[bytes_to_int(seed[0:8]) % len(persistent_committee)] ``` ### `get_shard_header` @@ -321,9 +316,7 @@ def is_valid_shard_block(beacon_state: BeaconState, pubkeys = [] for i, index in enumerate(attester_committee): if block.attester_bitfield[i]: - validator = beacon_state.validators[index] - assert is_active_validator(validator, get_current_epoch(beacon_state)) - pubkeys.append(validator.pubkey) + pubkeys.append(beacon_state.validators[index].pubkey) for i in range(len(attester_committee), MAX_PERSISTENT_COMMITTEE_SIZE * 2): assert block.attester_bitfield[i] is False assert bls_verify( From 8611f910d3e6454f77af8d0ea5efc876dc577995 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 19 Jul 2019 12:13:35 +0800 Subject: [PATCH 11/31] Update specs/core/1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index a7b541e903..74755f5dad 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -58,7 +58,7 @@ We define the following Python custom types for type hinting and readability: | `BYTES_PER_SHARD_BLOCK_BODY` | `2**14` (= 16,384) | | `MAX_SHARD_ATTESTIONS` | `2**4` (= 16) | | `SHARD_SLOTS_PER_BEACON_SLOT` | `2**1` (= 2) | -| `MAX_PERSISTENT_COMMITTEE_SIZE` | `2**5` (= 128) | +| `MAX_PERSISTENT_COMMITTEE_SIZE` | `2**7` (= 128) | ### Initial values From eb03f033f61de76284a07252aecad6ec870607eb Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 19 Jul 2019 00:15:13 -0400 Subject: [PATCH 12/31] Update 1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 74755f5dad..7ade451d88 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -22,7 +22,7 @@ - [`ShardBlockHeader`](#shardblockheader) - [Helper functions](#helper-functions) - [`compute_epoch_of_shard_slot`](#compute_epoch_of_shard_slot) - - [`get_period_start_block`](#get_period_start_block) + - [`get_period_start_epoch`](#get_period_start_epoch) - [`get_period_committee`](#get_period_committee) - [`get_switchover_epoch`](#get_switchover_epoch) - [`get_persistent_committee`](#get_persistent_committee) @@ -137,10 +137,10 @@ def compute_epoch_of_shard_slot(slot: ShardSlot) -> Epoch: return Epoch(slot // SHARD_SLOTS_PER_BEACON_SLOT // SLOTS_PER_EPOCH) ``` -### `get_period_start_block` +### `get_period_start_epoch` ```python -def get_period_start_block(epoch: Epoch, lookback:Epoch = 0): +def get_period_start_epoch(epoch: Epoch, lookback:Epoch = 0) -> Epoch: return epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - lookback * PERSISTENT_COMMITTEE_PERIOD ``` From 8f0ad70823ebe94e559f645d57d28dfa37bf7575 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 23 Jul 2019 18:51:14 -0400 Subject: [PATCH 13/31] Simplified switchover epochs, changed block structure, changed crosslink structure --- specs/core/1_shard-data-chains.md | 191 ++++++++++++++++-------------- 1 file changed, 103 insertions(+), 88 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 7ade451d88..c13d46a24d 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -17,18 +17,21 @@ - [Signature domain types](#signature-domain-types) - [TODO PLACEHOLDER](#todo-placeholder) - [Data structures](#data-structures) - - [`ShardBlockBody`](#shardblockbody) - - [`ShardBlock`](#shardblock) - [`ShardBlockHeader`](#shardblockheader) + - [`ShardBlock`](#shardblock) + - [`ShardBlockSignatures`](#shardblocksignatures) + - [`ShardBlockCore`](#shardblockcore) + - [`ExtendedShardBlockCore`](#extendedshardblockcore) - [Helper functions](#helper-functions) - [`compute_epoch_of_shard_slot`](#compute_epoch_of_shard_slot) - [`get_period_start_epoch`](#get_period_start_epoch) - [`get_period_committee`](#get_period_committee) - - [`get_switchover_epoch`](#get_switchover_epoch) - [`get_persistent_committee`](#get_persistent_committee) - [`get_shard_block_proposer_index`](#get_shard_block_proposer_index) - [`get_shard_block_attester_committee`](#get_shard_block_attester_committee) - [`get_shard_header`](#get_shard_header) + - [`pad`](#pad) + - [`flatten_shard_header`](#flatten_shard_header) - [`compute_crosslink_data_root`](#compute_crosslink_data_root) - [Object validity](#object-validity) - [Shard blocks](#shard-blocks) @@ -55,7 +58,8 @@ We define the following Python custom types for type hinting and readability: | Name | Value | | - | - | -| `BYTES_PER_SHARD_BLOCK_BODY` | `2**14` (= 16,384) | +| `SHARD_HEADER_SIZE` | `2**9` (= 512) | +| `SHARD_BLOCK_SIZE_LIMIT` | `2**16` (= 65,536) | | `MAX_SHARD_ATTESTIONS` | `2**4` (= 16) | | `SHARD_SLOTS_PER_BEACON_SLOT` | `2**1` (= 2) | | `MAX_PERSISTENT_COMMITTEE_SIZE` | `2**7` (= 128) | @@ -72,7 +76,7 @@ We define the following Python custom types for type hinting and readability: | Name | Value | Unit | Duration | | - | - | :-: | :-: | | `CROSSLINK_LOOKBACK` | `2**0` (= 1) | epochs | 6.4 minutes | -| `PERSISTENT_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~27 hours | +| `EPOCHS_PER_SHARD_PERIOD` | `2**8` (= 256) | epochs | ~27 hours | ### Signature domain types @@ -91,41 +95,54 @@ The following types are defined, mapping into `DomainType` (little endian): ## Data structures -### `ShardBlockBody` +_Note: the shard block header structure is carefully designed so that all of the values have the same depth in a hash tree implementation, so `hash_tree_root(SSZ_partial(x)) == hash_tree_root(x)` (using the "left-to-right leaves" scheme [here](https://github.com/ethereum/eth2.0-specs/issues/1303)), which allows shard block headers to look like an SSZ object when in the crosslink structure. This is done by balancing it so that 7 or 8 items are on the left side (the "core") and two 96-byte (ie. 3*2 = 6 chunk) items are on the right side. Change with care._ + +### `ShardBlockHeader` ```python -class ShardBlockBody(Container): - data: Vector[Bytes[PLACEHOLDER], BYTES_PER_SHARD_BLOCK_BODY] +class ShardBlockHeader(Container): + core: ShardBlockCore + signatures: ShardBlockSignatures ``` ### `ShardBlock` ```python class ShardBlock(Container): + core: ExtendedShardBlockCore + signatures: ShardBlockSignatures +``` + +### `ShardBlockSignatures` + +```python +class ShardBlockSignatures(Container): + attestation_signature: BLSSignature + proposer_signature: BLSSignature +``` + +### `ShardBlockCore` + +```python slot: ShardSlot - shard: Shard beacon_chain_root: Hash parent_root: Hash - data: ShardBlockBody + data_root: Hash state_root: Hash + total_bytes: uint64 attester_bitfield: Bitvector[MAX_PERSISTENT_COMMITTEE_SIZE * 2] - attestation_signature: BLSSignature - signature: BLSSignature ``` -### `ShardBlockHeader` +### `ExtendedShardBlockCore` ```python -class ShardBlockHeader(Container): slot: ShardSlot - shard: Shard beacon_chain_root: Hash parent_root: Hash - body_root: Hash + data_root: Bytes[SHARD_BLOCK_SIZE_LIMIT - SHARD_HEADER_SIZE] state_root: Hash + total_bytes: uint64 attester_bitfield: Bitvector[MAX_PERSISTENT_COMMITTEE_SIZE * 2] - attestation_signature: BLSSignature - signature: BLSSignature ``` ## Helper functions @@ -137,10 +154,10 @@ def compute_epoch_of_shard_slot(slot: ShardSlot) -> Epoch: return Epoch(slot // SHARD_SLOTS_PER_BEACON_SLOT // SLOTS_PER_EPOCH) ``` -### `get_period_start_epoch` +### `get_shard_period_start_epoch` ```python -def get_period_start_epoch(epoch: Epoch, lookback:Epoch = 0) -> Epoch: +def get_shard_period_start_epoch(epoch: Epoch, lookback:Epoch = 0) -> Epoch: return epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - lookback * PERSISTENT_COMMITTEE_PERIOD ``` @@ -172,15 +189,6 @@ def get_period_committee(state: BeaconState, return [i for i in full_committee if active_and_balance_filter(i)][:MAX_PERSISTENT_COMMITTEE_SIZE] ``` -### `get_switchover_epoch` - -```python -def get_switchover_epoch(state: BeaconState, epoch: Epoch, index: ValidatorIndex) -> int: - earlier_start_epoch = get_period_start_epoch(epoch, lookback=2) - return (bytes_to_int(hash(get_seed(state, earlier_start_epoch) + int_to_bytes(index, length=3)[0:8])) - % PERSISTENT_COMMITTEE_PERIOD) -``` - ### `get_persistent_committee` ```python @@ -198,8 +206,8 @@ def get_persistent_committee(state: BeaconState, # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated return sorted(set( - [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(state, epoch, i)] - + [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(state, epoch, i)] + [i for i in earlier_committee if epoch % EPOCHS_PER_SHARD_PERIOD < i % EPOCHS_PER_SHARD_PERIOD] + + [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= i % EPOCHS_PER_SHARD_PERIOD] )) ``` @@ -220,14 +228,45 @@ def get_shard_block_proposer_index(state: BeaconState, ```python def get_shard_header(block: ShardBlock) -> ShardBlockHeader: return ShardBlockHeader( - slot=block.slot, - shard=block.shard, - beacon_chain_root=block.beacon_chain_root, - parent_root=block.parent_root, - body_root=hash_tree_root(block.body), - state_root=block.state_root, - attestation_signature=block.attestation_signature, - signature=block.signature, + core=ShardBlockCore( + slot=block.core.slot, + beacon_chain_root=block.core.beacon_chain_root, + parent_root=block.core.parent_root, + data_root=hash_tree_root(block.core.data), + state_root=block.core.state_root, + total_bytes=block.core.total_bytes, + attester_bitfield=block.core.attester_bitfield + ), + signatures=block.signatures + ) +``` + +### `pad` + +```python +def pad(x: bytes, length: int) -> bytes: + assert len(x) <= length + return x + b'\x00' * (length - len(x)) +``` + +### `flatten_shard_header` + +```python +def flatten_shard_header(header: ShardBlockHeader) -> Bytes[SHARD_HEADER_SIZE]: + """ + Converts a shard block header into a flat object with the same hash tree root. Used + in the crosslink construction. + """ + return ( + pad(int_to_bytes8(header.core.slot), 32) + + header.core.beacon_chain_root + + header.core.parent_root + + header.core.data_root + + header.core.state_root + + pad(int_to_bytes8(header.core.total_bytes), 32) + + b'\x00' * 32 + + pad(header.signatures.attestation_signature, 128) + + pad(header.signatures.proposer_signature, 128) ) ``` @@ -235,32 +274,10 @@ def get_shard_header(block: ShardBlock) -> ShardBlockHeader: ```python def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Hash: - def is_power_of_two(value: uint64) -> bool: - return (value > 0) and (value & (value - 1) == 0) - - def pad_to_power_of_2(values: MutableSequence[bytes]) -> Sequence[bytes]: - while not is_power_of_two(len(values)): - values.append(b'\x00' * BYTES_PER_SHARD_BLOCK_BODY) - return values - - def hash_tree_root_of_bytes(data: bytes) -> Hash: - return hash_tree_root([data[i:i + 32] for i in range(0, len(data), 32)]) - - def zpad(data: bytes, length: uint64) -> bytes: - return data + b'\x00' * (length - len(data)) - - return hash( - # TODO untested code. - # Need to either pass a typed list to hash-tree-root, or merkleize_chunks(values, pad_to=2**x) - hash_tree_root(pad_to_power_of_2([ - hash_tree_root_of_bytes( - zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_BODY) - ) for block in blocks - ])) - + hash_tree_root(pad_to_power_of_2([ - hash_tree_root_of_bytes(block.body) for block in blocks - ])) - ) + header = b''.join([flatten_shard_header(get_shard_header(block)) for block in blocks]) + footer = b''.join([block.core.data for block in blocks]) + MAX_SIZE = SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK + return hash_tree_root(pad(header + footer, MAX_SIZE)) ``` ## Object validity @@ -271,12 +288,14 @@ Let: - `beacon_blocks` be the `BeaconBlock` list such that `beacon_blocks[slot]` is the canonical `BeaconBlock` at slot `slot` - `beacon_state` be the canonical `BeaconState` after processing `beacon_blocks[-1]` +- `shard` is the shard ID - `valid_shard_blocks` be the list of valid `ShardBlock`, recursively defined - `candidate` be a candidate `ShardBlock` for which validity is to be determined by running `is_valid_shard_block` ```python def is_valid_shard_block(beacon_state: BeaconState, beacon_blocks: Sequence[BeaconBlock], + shard: Shard, valid_shard_blocks: Sequence[ShardBlock], candidate: ShardBlock) -> bool: # Check if block is already determined valid @@ -285,55 +304,51 @@ def is_valid_shard_block(beacon_state: BeaconState, return True # Check slot number - assert candidate.slot >= PHASE_1_FORK_SLOT - - # Check shard number - assert candidate.shard <= SHARD_COUNT + assert candidate.core.slot >= PHASE_1_FORK_SLOT # Check beacon block - beacon_block = beacon_blocks[candidate.slot] - assert candidate.beacon_block_root == signing_root(beacon_block) - assert beacon_block.slot <= candidate.slot + beacon_block = beacon_blocks[candidate.core.slot - candidate.core.slot % (SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH)] + assert candidate.core.beacon_block_root == signing_root(beacon_block) + assert beacon_block.slot <= candidate.core.slot # Check state root - assert candidate.state_root == Hash() # [to be removed in phase 2] + assert candidate.core.state_root == Hash() # [to be removed in phase 2] # Check parent block - if candidate.slot == PHASE_1_FORK_SLOT: - assert candidate.parent_root == Hash() + if candidate.core.slot == PHASE_1_FORK_SLOT: + assert candidate.core.parent_root == Hash() else: parent_block = next( - (block for block in valid_shard_blocks if signing_root(block) == candidate.parent_root), + (block for block in valid_shard_blocks if hash_tree_root(block.core) == candidate.core.parent_root), None ) assert parent_block is not None - assert parent_block.shard == candidate.shard - assert parent_block.slot < candidate.slot - assert signing_root(beacon_blocks[parent_block.slot]) == parent_block.beacon_chain_root + assert parent_block.core.slot < candidate.core.slot + assert signing_root(beacon_blocks[parent_block.core.slot]) == parent_block.core.beacon_chain_root # Check attestations - attester_committee = get_persistent_committee(beacon_state, block.shard, block.slot) + attester_committee = get_persistent_committee(beacon_state, shard, block.core.slot) pubkeys = [] for i, index in enumerate(attester_committee): - if block.attester_bitfield[i]: + if block.core.attester_bitfield[i]: pubkeys.append(beacon_state.validators[index].pubkey) for i in range(len(attester_committee), MAX_PERSISTENT_COMMITTEE_SIZE * 2): assert block.attester_bitfield[i] is False assert bls_verify( pubkey=bls_aggregate_pubkeys(pubkeys), - message_hash=candidate.parent_root, - signature=candidate.attestation_signature, - domain=get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(candidate.slot)) + message_hash=candidate.core.parent_root, + signature=candidate.signatures.attestation_signature, + domain=get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(candidate.core.slot)) ) # Check proposer - proposer_index = get_shard_block_proposer_index(beacon_state, candidate.shard, candidate.slot) + proposer_index = get_shard_block_proposer_index(beacon_state, shard, candidate.core.slot) assert proposer_index is not None assert bls_verify( pubkey=beacon_state.validators[proposer_index].pubkey, - message_hash=signing_root(candidate), - signature=candidate.signature, - domain=get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(candidate.slot)), + message_hash=hash_tree_root(candidate.core), + signature=candidate.signatures.proposer_signature, + domain=get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(candidate.core.slot)), ) return True From 6812e46e3cf84ba1d760cf7182b3efba9bc5af7e Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 24 Jul 2019 09:29:26 -0400 Subject: [PATCH 14/31] Update 1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index c13d46a24d..e7367906b2 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -158,7 +158,7 @@ def compute_epoch_of_shard_slot(slot: ShardSlot) -> Epoch: ```python def get_shard_period_start_epoch(epoch: Epoch, lookback:Epoch = 0) -> Epoch: - return epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - lookback * PERSISTENT_COMMITTEE_PERIOD + return epoch - (epoch % EPOCHS_PER_SHARD_PERIOD) - lookback * EPOCHS_PER_SHARD_PERIOD ``` ### `get_period_committee` @@ -207,7 +207,7 @@ def get_persistent_committee(state: BeaconState, # later committee; return a sorted list of the union of the two, deduplicated return sorted(set( [i for i in earlier_committee if epoch % EPOCHS_PER_SHARD_PERIOD < i % EPOCHS_PER_SHARD_PERIOD] - + [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= i % EPOCHS_PER_SHARD_PERIOD] + + [i for i in later_committee if epoch % EPOCHS_PER_SHARD_PERIOD >= i % EPOCHS_PER_SHARD_PERIOD] )) ``` From 827b181192250039ebe18cf140cda24152fb0f92 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 24 Jul 2019 12:48:06 -0400 Subject: [PATCH 15/31] Moved balance dependency to proposer selection --- specs/core/1_shard-data-chains.md | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index e7367906b2..d68143e6c4 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -170,23 +170,14 @@ def get_period_committee(state: BeaconState, """ Return committee for a period. Used to construct persistent committees. """ - seed = get_seed(state, epoch) full_committee = compute_committee( indices=get_active_validator_indices(state, epoch), - seed=seed, + seed=get_seed(state, epoch), index=shard * count + index, count=SHARD_COUNT * count, ) - def active_and_balance_filter(i: ValidatorIndex) -> bool: - if not is_active_validator(state.validators[i], epoch): - return False - active_threshold = MAX_EFFECTIVE_BALANCE * seed[i % 32] // MAX_RANDOM_BYTE - if state.validators[i].effective_balance < active_threshold: - return False - return True - - return [i for i in full_committee if active_and_balance_filter(i)][:MAX_PERSISTENT_COMMITTEE_SIZE] + return [i for i in full_committee if is_active_validator(state.validators[i], epoch)][:MAX_PERSISTENT_COMMITTEE_SIZE] ``` ### `get_persistent_committee` @@ -220,7 +211,14 @@ def get_shard_block_proposer_index(state: BeaconState, # Randomly shift persistent committee persistent_committee = list(get_persistent_committee(state, shard, slot)) seed = hash(state.current_shuffling_seed + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) - return persistent_committee[bytes_to_int(seed[0:8]) % len(persistent_committee)] + i = 0 + while True: + candidate_index = persistent_committee[(slot + i) % len(persistent_committee)] + random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] + effective_balance = state.validators[candidate_index].effective_balance + if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: + return ValidatorIndex(candidate_index) + i += 1 ``` ### `get_shard_header` From 96b3cb9d6a169bc154a66a2cc5270299ca76c526 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 24 Jul 2019 15:13:19 -0400 Subject: [PATCH 16/31] Update specs/core/1_shard-data-chains.md Co-Authored-By: Danny Ryan --- specs/core/1_shard-data-chains.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index d68143e6c4..2a575edaf8 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -60,7 +60,6 @@ We define the following Python custom types for type hinting and readability: | - | - | | `SHARD_HEADER_SIZE` | `2**9` (= 512) | | `SHARD_BLOCK_SIZE_LIMIT` | `2**16` (= 65,536) | -| `MAX_SHARD_ATTESTIONS` | `2**4` (= 16) | | `SHARD_SLOTS_PER_BEACON_SLOT` | `2**1` (= 2) | | `MAX_PERSISTENT_COMMITTEE_SIZE` | `2**7` (= 128) | From d33ee7c47eaaad70a60df6423df4885d1bce8c86 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 24 Jul 2019 15:13:45 -0400 Subject: [PATCH 17/31] Update specs/core/1_shard-data-chains.md Co-Authored-By: Danny Ryan --- specs/core/1_shard-data-chains.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 2a575edaf8..3d489dd204 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -123,6 +123,7 @@ class ShardBlockSignatures(Container): ### `ShardBlockCore` ```python +class ShardBlockCore(Container): slot: ShardSlot beacon_chain_root: Hash parent_root: Hash From b859b2619a740977a4aa35238f6cf2eeade906ef Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 24 Jul 2019 15:16:00 -0400 Subject: [PATCH 18/31] Update specs/core/1_shard-data-chains.md Co-Authored-By: Danny Ryan --- specs/core/1_shard-data-chains.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 3d489dd204..6bb4221762 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -136,6 +136,7 @@ class ShardBlockCore(Container): ### `ExtendedShardBlockCore` ```python +class ExtendedShardBlockCore(Container): slot: ShardSlot beacon_chain_root: Hash parent_root: Hash From 82993994586e0a7365dd91c9c124b7d7ed8b5e47 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 24 Jul 2019 15:16:29 -0400 Subject: [PATCH 19/31] Update specs/core/1_shard-data-chains.md Co-Authored-By: Danny Ryan --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 6bb4221762..7f9a7c3250 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -158,7 +158,7 @@ def compute_epoch_of_shard_slot(slot: ShardSlot) -> Epoch: ### `get_shard_period_start_epoch` ```python -def get_shard_period_start_epoch(epoch: Epoch, lookback:Epoch = 0) -> Epoch: +def get_shard_period_start_epoch(epoch: Epoch, lookback: Epoch=0) -> Epoch: return epoch - (epoch % EPOCHS_PER_SHARD_PERIOD) - lookback * EPOCHS_PER_SHARD_PERIOD ``` From 2af058f782f3455b888a7154665f40dcec6fc0da Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 24 Jul 2019 15:16:58 -0400 Subject: [PATCH 20/31] Update specs/core/1_shard-data-chains.md Co-Authored-By: Danny Ryan --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 7f9a7c3250..a8949aaa1a 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -174,7 +174,7 @@ def get_period_committee(state: BeaconState, full_committee = compute_committee( indices=get_active_validator_indices(state, epoch), seed=get_seed(state, epoch), - index=shard * count + index, + index=shard, count=SHARD_COUNT * count, ) From 4951768a9b7fbe80f4d5e5e9242f4051d75a6db3 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 24 Jul 2019 15:17:23 -0400 Subject: [PATCH 21/31] Update specs/core/1_shard-data-chains.md Co-Authored-By: Danny Ryan --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index a8949aaa1a..ef0e793b50 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -175,7 +175,7 @@ def get_period_committee(state: BeaconState, indices=get_active_validator_indices(state, epoch), seed=get_seed(state, epoch), index=shard, - count=SHARD_COUNT * count, + count=SHARD_COUNT, ) return [i for i in full_committee if is_active_validator(state.validators[i], epoch)][:MAX_PERSISTENT_COMMITTEE_SIZE] From 7f2e7737b57157ae4cee88f5db76259cdc26c3f8 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 24 Jul 2019 15:18:10 -0400 Subject: [PATCH 22/31] Update specs/core/1_shard-data-chains.md Co-Authored-By: Danny Ryan --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index ef0e793b50..dae6975d07 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -178,7 +178,7 @@ def get_period_committee(state: BeaconState, count=SHARD_COUNT, ) - return [i for i in full_committee if is_active_validator(state.validators[i], epoch)][:MAX_PERSISTENT_COMMITTEE_SIZE] + return full_committee[:MAX_PERSISTENT_COMMITTEE_SIZE] ``` ### `get_persistent_committee` From cc80342f4218e0789f019222fb91d1de8594d314 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 24 Jul 2019 15:21:29 -0400 Subject: [PATCH 23/31] Update specs/core/1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index dae6975d07..0c29e887ba 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -218,7 +218,8 @@ def get_shard_block_proposer_index(state: BeaconState, random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] effective_balance = state.validators[candidate_index].effective_balance if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: - return ValidatorIndex(candidate_index) + if is_active_validator(state.validators[candidate_index], shard_slot_to_epoch(slot)): + return ValidatorIndex(candidate_index) i += 1 ``` From 8ba14089f4adc95463aca0f5414914e6b1ef0e14 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 24 Jul 2019 15:29:04 -0400 Subject: [PATCH 24/31] Fixed shard header flattening --- specs/core/1_shard-data-chains.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 0c29e887ba..e8446fc568 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -208,7 +208,7 @@ def get_persistent_committee(state: BeaconState, ```python def get_shard_block_proposer_index(state: BeaconState, shard: Shard, - slot: Slot) -> Optional[ValidatorIndex]: + slot: ShardSlot) -> Optional[ValidatorIndex]: # Randomly shift persistent committee persistent_committee = list(get_persistent_committee(state, shard, slot)) seed = hash(state.current_shuffling_seed + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) @@ -218,7 +218,7 @@ def get_shard_block_proposer_index(state: BeaconState, random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] effective_balance = state.validators[candidate_index].effective_balance if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: - if is_active_validator(state.validators[candidate_index], shard_slot_to_epoch(slot)): + if is_active_validator(state.validators[candidate_index], compute_epoch_of_shard_slot(slot)): return ValidatorIndex(candidate_index) i += 1 ``` @@ -251,12 +251,14 @@ def pad(x: bytes, length: int) -> bytes: ### `flatten_shard_header` -```python +```pythonpad_ def flatten_shard_header(header: ShardBlockHeader) -> Bytes[SHARD_HEADER_SIZE]: """ Converts a shard block header into a flat object with the same hash tree root. Used in the crosslink construction. """ + attester_bits = [header.core.attester_bitfield[i] if i < len(header.core.attester_bitfield) else 0 for i in range(256)] + attester_bytes = bytes([sum([attester_bits[i+j] << j for j in range(8)]) for i in range(0, 256, 8)]) return ( pad(int_to_bytes8(header.core.slot), 32) + header.core.beacon_chain_root + @@ -264,6 +266,7 @@ def flatten_shard_header(header: ShardBlockHeader) -> Bytes[SHARD_HEADER_SIZE]: header.core.data_root + header.core.state_root + pad(int_to_bytes8(header.core.total_bytes), 32) + + attester_bytes + b'\x00' * 32 + pad(header.signatures.attestation_signature, 128) + pad(header.signatures.proposer_signature, 128) From f523ddd27107e469f2bff756bfcdd774ebcc0cee Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 24 Jul 2019 15:38:45 -0400 Subject: [PATCH 25/31] Update specs/core/1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index e8446fc568..e3bc0ab3f9 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -280,7 +280,7 @@ def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Hash: header = b''.join([flatten_shard_header(get_shard_header(block)) for block in blocks]) footer = b''.join([block.core.data for block in blocks]) MAX_SIZE = SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK - return hash_tree_root(pad(header + footer, MAX_SIZE)) + return hash_tree_root(BytesN[MAX_SIZE](pad(header + footer, MAX_SIZE))) ``` ## Object validity From c7f8b11fccedf5125fae2782ea01063f8c126386 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 24 Jul 2019 18:55:24 -0400 Subject: [PATCH 26/31] Minor fixes --- specs/core/1_shard-data-chains.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index e3bc0ab3f9..2eececbe2c 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -24,6 +24,7 @@ - [`ExtendedShardBlockCore`](#extendedshardblockcore) - [Helper functions](#helper-functions) - [`compute_epoch_of_shard_slot`](#compute_epoch_of_shard_slot) + - [`compute_slot_of_shard_slot`](#compute_slot_of_shard_slot) - [`get_period_start_epoch`](#get_period_start_epoch) - [`get_period_committee`](#get_period_committee) - [`get_persistent_committee`](#get_persistent_committee) @@ -148,6 +149,13 @@ class ExtendedShardBlockCore(Container): ## Helper functions +### `compute_slot_of_shard_slot` + +```python +def compute_slot_of_shard_slot(slot: ShardSlot) -> Epoch: + return Epoch(slot // SHARD_SLOTS_PER_BEACON_SLOT) +``` + ### `compute_epoch_of_shard_slot` ```python @@ -307,10 +315,11 @@ def is_valid_shard_block(beacon_state: BeaconState, return True # Check slot number - assert candidate.core.slot >= PHASE_1_FORK_SLOT + assert compute_slot_of_shard_slot(candidate.core.slot) >= PHASE_1_FORK_SLOT # Check beacon block - beacon_block = beacon_blocks[candidate.core.slot - candidate.core.slot % (SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH)] + beacon_block_slot = compute_start_slot_of_epoch(compute_epoch_of_shard_slot(candidate.core.slot)) + beacon_block = beacon_blocks[beacon_block_slot] assert candidate.core.beacon_block_root == signing_root(beacon_block) assert beacon_block.slot <= candidate.core.slot @@ -318,16 +327,15 @@ def is_valid_shard_block(beacon_state: BeaconState, assert candidate.core.state_root == Hash() # [to be removed in phase 2] # Check parent block - if candidate.core.slot == PHASE_1_FORK_SLOT: - assert candidate.core.parent_root == Hash() - else: + if candidate.core.parent_root != Hash(): parent_block = next( (block for block in valid_shard_blocks if hash_tree_root(block.core) == candidate.core.parent_root), None ) assert parent_block is not None assert parent_block.core.slot < candidate.core.slot - assert signing_root(beacon_blocks[parent_block.core.slot]) == parent_block.core.beacon_chain_root + parent_beacon_block_slot = compute_start_slot_of_epoch(compute_epoch_of_shard_slot(parent_block.core.slot)) + assert signing_root(beacon_blocks[parent_beacon_block_slot]) == parent_block.core.beacon_chain_root # Check attestations attester_committee = get_persistent_committee(beacon_state, shard, block.core.slot) From 7df91f8b6b99520d598303cb6266f6aac077d5f4 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Thu, 25 Jul 2019 14:22:37 -0400 Subject: [PATCH 27/31] Update specs/core/1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 2eececbe2c..e2e9d3dbd5 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -141,7 +141,7 @@ class ExtendedShardBlockCore(Container): slot: ShardSlot beacon_chain_root: Hash parent_root: Hash - data_root: Bytes[SHARD_BLOCK_SIZE_LIMIT - SHARD_HEADER_SIZE] + data: Bytes[SHARD_BLOCK_SIZE_LIMIT - SHARD_HEADER_SIZE] state_root: Hash total_bytes: uint64 attester_bitfield: Bitvector[MAX_PERSISTENT_COMMITTEE_SIZE * 2] From e03534431725adca6f3736455199d57e2648b623 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Thu, 25 Jul 2019 14:24:11 -0400 Subject: [PATCH 28/31] Update specs/core/1_shard-data-chains.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index e2e9d3dbd5..d940269adb 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -259,7 +259,7 @@ def pad(x: bytes, length: int) -> bytes: ### `flatten_shard_header` -```pythonpad_ +```python def flatten_shard_header(header: ShardBlockHeader) -> Bytes[SHARD_HEADER_SIZE]: """ Converts a shard block header into a flat object with the same hash tree root. Used From dc77c0649f5523f91a9fea44a4cc1a6c797b33a1 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 25 Jul 2019 15:27:31 -0600 Subject: [PATCH 29/31] cleanup testing and lint --- scripts/build_spec.py | 3 +-- specs/core/1_shard-data-chains.md | 23 ++++++++++-------- .../test/helpers/phase1/shard_block.py | 24 +++++++++++-------- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 8b541ff50c..96866cc8a8 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -48,11 +48,10 @@ from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, - serialize, is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( - bit, boolean, Container, List, Vector, Bytes, uint64, + uint64, bit, boolean, Container, List, Vector, Bytes, BytesN, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index d940269adb..3223544d4a 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -25,7 +25,7 @@ - [Helper functions](#helper-functions) - [`compute_epoch_of_shard_slot`](#compute_epoch_of_shard_slot) - [`compute_slot_of_shard_slot`](#compute_slot_of_shard_slot) - - [`get_period_start_epoch`](#get_period_start_epoch) + - [`get_shard_period_start_epoch`](#get_shard_period_start_epoch) - [`get_period_committee`](#get_period_committee) - [`get_persistent_committee`](#get_persistent_committee) - [`get_shard_block_proposer_index`](#get_shard_block_proposer_index) @@ -166,8 +166,8 @@ def compute_epoch_of_shard_slot(slot: ShardSlot) -> Epoch: ### `get_shard_period_start_epoch` ```python -def get_shard_period_start_epoch(epoch: Epoch, lookback: Epoch=0) -> Epoch: - return epoch - (epoch % EPOCHS_PER_SHARD_PERIOD) - lookback * EPOCHS_PER_SHARD_PERIOD +def get_shard_period_start_epoch(epoch: Epoch, lookback: Epoch=Epoch(0)) -> Epoch: + return Epoch(epoch - (epoch % EPOCHS_PER_SHARD_PERIOD) - lookback * EPOCHS_PER_SHARD_PERIOD) ``` ### `get_period_committee` @@ -200,8 +200,8 @@ def get_persistent_committee(state: BeaconState, """ epoch = compute_epoch_of_shard_slot(slot) - earlier_committee = get_period_committee(state, get_period_start_epoch(epoch, lookback=2), shard) - later_committee = get_period_committee(state, get_period_start_epoch(epoch, lookback=1), shard) + earlier_committee = get_period_committee(state, get_shard_period_start_epoch(epoch, lookback=Epoch(2)), shard) + later_committee = get_period_committee(state, get_shard_period_start_epoch(epoch, lookback=Epoch(1)), shard) # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated @@ -219,7 +219,9 @@ def get_shard_block_proposer_index(state: BeaconState, slot: ShardSlot) -> Optional[ValidatorIndex]: # Randomly shift persistent committee persistent_committee = list(get_persistent_committee(state, shard, slot)) - seed = hash(state.current_shuffling_seed + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) + MAX_RANDOM_BYTE = 2**8 - 1 + current_epoch = get_current_epoch(state) + seed = hash(get_seed(state, current_epoch) + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) i = 0 while True: candidate_index = persistent_committee[(slot + i) % len(persistent_committee)] @@ -265,15 +267,16 @@ def flatten_shard_header(header: ShardBlockHeader) -> Bytes[SHARD_HEADER_SIZE]: Converts a shard block header into a flat object with the same hash tree root. Used in the crosslink construction. """ - attester_bits = [header.core.attester_bitfield[i] if i < len(header.core.attester_bitfield) else 0 for i in range(256)] - attester_bytes = bytes([sum([attester_bits[i+j] << j for j in range(8)]) for i in range(0, 256, 8)]) + committee_size = len(header.core.attester_bitfield) + attester_bits = [header.core.attester_bitfield[i] if i < committee_size else 0 for i in range(256)] + attester_bytes = bytes([sum([attester_bits[i + j] << j for j in range(8)]) for i in range(0, 256, 8)]) return ( - pad(int_to_bytes8(header.core.slot), 32) + + pad(int_to_bytes(header.core.slot, length=8), 32) + header.core.beacon_chain_root + header.core.parent_root + header.core.data_root + header.core.state_root + - pad(int_to_bytes8(header.core.total_bytes), 32) + + pad(int_to_bytes(header.core.total_bytes, length=8), 32) + attester_bytes + b'\x00' * 32 + pad(header.signatures.attestation_signature, 128) + diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py index f67d7da87b..4e19817276 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py @@ -9,19 +9,19 @@ @only_with_bls() -def sign_shard_block(spec, state, block, proposer_index=None): +def sign_shard_block(spec, state, block, shard, proposer_index=None): if proposer_index is None: - proposer_index = spec.get_shard_block_proposer_index(state, block.shard, block.slot) + proposer_index = spec.get_shard_block_proposer_index(state, shard, block.core.slot) privkey = privkeys[proposer_index] - block.signature = bls_sign( + block.signatures.proposer_signature = bls_sign( message_hash=signing_root(block), privkey=privkey, domain=spec.get_domain( state, spec.DOMAIN_SHARD_PROPOSER, - spec.compute_epoch_of_shard_slot(block.slot), + spec.compute_epoch_of_shard_slot(block.core.slot), ) ) @@ -30,14 +30,18 @@ def build_empty_shard_block(spec, state, slot, shard, parent_root, signed=False) if slot is None: slot = state.slot block = spec.ShardBlock( - slot=slot, - shard=shard, - beacon_chain_root=state.block_roots[state.slot // spec.SLOTS_PER_HISTORICAL_ROOT], - parent_root=parent_root, - signature=b'\x12' * 96, + core=spec.ExtendedShardBlockCore( + slot=slot, + beacon_chain_root=state.block_roots[state.slot % spec.SLOTS_PER_HISTORICAL_ROOT], + parent_root=parent_root, + ), + signatures=spec.ShardBlockSignatures( + attestation_signature=b'\x12' * 96, + proposer_signature=b'\x25' * 96, + ) ) if signed: - sign_shard_block(spec, state, block) + sign_shard_block(spec, state, block, shard) return block From ad30945b9b0cb63d00b0ea09fb0ccba9c6e35957 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 25 Jul 2019 15:44:10 -0600 Subject: [PATCH 30/31] return none if not active validators in persistent committee --- specs/core/1_shard-data-chains.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 3223544d4a..0ac6d5c1b4 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -219,8 +219,12 @@ def get_shard_block_proposer_index(state: BeaconState, slot: ShardSlot) -> Optional[ValidatorIndex]: # Randomly shift persistent committee persistent_committee = list(get_persistent_committee(state, shard, slot)) - MAX_RANDOM_BYTE = 2**8 - 1 current_epoch = get_current_epoch(state) + + if not any([i for i in persistent_committee if is_active_validator(state.validators[i], current_epoch)]): + return None + + MAX_RANDOM_BYTE = 2**8 - 1 seed = hash(get_seed(state, current_epoch) + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) i = 0 while True: @@ -228,8 +232,7 @@ def get_shard_block_proposer_index(state: BeaconState, random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] effective_balance = state.validators[candidate_index].effective_balance if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: - if is_active_validator(state.validators[candidate_index], compute_epoch_of_shard_slot(slot)): - return ValidatorIndex(candidate_index) + return ValidatorIndex(candidate_index) i += 1 ``` From 21f4e0334b60e217c6f1ac7948265448ab8665c0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 26 Jul 2019 15:43:02 -0600 Subject: [PATCH 31/31] only allow active validators as shard proposer --- specs/core/1_shard-data-chains.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 0ac6d5c1b4..65c2b12d83 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -221,14 +221,15 @@ def get_shard_block_proposer_index(state: BeaconState, persistent_committee = list(get_persistent_committee(state, shard, slot)) current_epoch = get_current_epoch(state) - if not any([i for i in persistent_committee if is_active_validator(state.validators[i], current_epoch)]): + active_indices = [i for i in persistent_committee if is_active_validator(state.validators[i], current_epoch)] + if not any(active_indices): return None MAX_RANDOM_BYTE = 2**8 - 1 seed = hash(get_seed(state, current_epoch) + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) i = 0 while True: - candidate_index = persistent_committee[(slot + i) % len(persistent_committee)] + candidate_index = active_indices[(slot + i) % len(active_indices)] random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] effective_balance = state.validators[candidate_index].effective_balance if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: