diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 8b541ff50c..1b4ffd541e 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -46,14 +46,15 @@ ) from eth2spec.utils.ssz.ssz_impl import ( + zpad, hash_tree_root, + is_empty, signing_root, serialize, - is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, uint64, - Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, + Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, BytesN, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 613b4c4c22..7d0c77bf9e 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -13,7 +13,7 @@ - [Misc](#misc) - [Initial values](#initial-values) - [Time parameters](#time-parameters) - - [Signature domains](#signature-domains) + - [Signature domain types](#signature-domain-types) - [TODO PLACEHOLDER](#todo-placeholder) - [Data structures](#data-structures) - [`ShardBlockBody`](#shardblockbody) @@ -27,6 +27,8 @@ - [`get_shard_proposer_index`](#get_shard_proposer_index) - [`get_shard_header`](#get_shard_header) - [`verify_shard_attestation_signature`](#verify_shard_attestation_signature) + - [`flatten_block`](#flatten_block) + - [`compute_crosslink_data`](#compute_crosslink_data) - [`compute_crosslink_data_root`](#compute_crosslink_data_root) - [Object validity](#object-validity) - [Shard blocks](#shard-blocks) @@ -46,6 +48,7 @@ This document describes the shard data layer and the shard fork choice rule in P | Name | Value | | - | - | +| `BYTES_PER_SHARD_BLOCK_HEADER` | `2**9` (= 512) | | `BYTES_PER_SHARD_BLOCK_BODY` | `2**14` (= 16,384) | | `MAX_SHARD_ATTESTIONS` | `2**4` (= 16) | @@ -76,7 +79,7 @@ The following types are defined, mapping into `DomainType` (little endian): | Name | Value | | - | - | -| `PLACEHOLDER` | `2**32` | +| `PLACEHOLDER` | `2**3` | ## Data structures @@ -107,7 +110,7 @@ class ShardBlock(Container): shard: Shard beacon_chain_root: Bytes32 parent_root: Bytes32 - data: ShardBlockBody + body: ShardBlockBody state_root: Bytes32 attestations: List[ShardAttestation, PLACEHOLDER] signature: BLSSignature @@ -247,36 +250,37 @@ def verify_shard_attestation_signature(state: BeaconState, ) ``` +### `flatten_block` + +```python +def flatten_block(block: ShardBlock) -> bytes: + return zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_HEADER) + serialize(block.body) +``` + +### `compute_crosslink_data` + +```python +def compute_crosslink_data(blocks: Sequence[ShardBlock]) -> bytes: + """ + Flattens a series of blocks. Designed to be equivalent to SSZ serializing a list of + flattened blocks with one empty block at the end as a termination marker. + """ + positions = [4 * len(blocks)] + bodies = [] + for block in blocks: + bodies.append(flatten_block(block)) + positions.append(positions[-1] + len(bodies[-1])) + return b''.join([int_to_bytes(pos, 4) for pos in positions]) + b''.join(bodies) +``` + ### `compute_crosslink_data_root` ```python def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: - def is_power_of_two(value: uint64) -> bool: - return (value > 0) and (value & (value - 1) == 0) - - def pad_to_power_of_2(values: MutableSequence[bytes]) -> Sequence[bytes]: - while not is_power_of_two(len(values)): - values.append(b'\x00' * BYTES_PER_SHARD_BLOCK_BODY) - return values - - def hash_tree_root_of_bytes(data: bytes) -> bytes: - return hash_tree_root([data[i:i + 32] for i in range(0, len(data), 32)]) - - def zpad(data: bytes, length: uint64) -> bytes: - return data + b'\x00' * (length - len(data)) - - return hash( - # TODO untested code. - # Need to either pass a typed list to hash-tree-root, or merkleize_chunks(values, pad_to=2**x) - hash_tree_root(pad_to_power_of_2([ - hash_tree_root_of_bytes( - zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_BODY) - ) for block in blocks - ])) - + hash_tree_root(pad_to_power_of_2([ - hash_tree_root_of_bytes(block.body) for block in blocks - ])) - ) + MAXLEN = ( + BYTES_PER_SHARD_BLOCK_HEADER + BYTES_PER_SHARD_BLOCK_BODY + ) * SHARD_SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK + return hash_tree_root(BytesN[MAXLEN](zpad(compute_crosslink_data(blocks), MAXLEN))) ``` ## Object validity diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index d5855a755f..148347acde 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,4 +1,6 @@ -from ..merkle_minimal import merkleize_chunks +from ..merkle_minimal import ( + merkleize_chunks, +) from ..hash_function import hash from .ssz_typing import ( SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bits, boolean, Container, List, Bytes, @@ -8,6 +10,7 @@ # SSZ Serialization # ----------------------------- +BYTES_PER_CHUNK = 32 BYTES_PER_LENGTH_OFFSET = 4 @@ -101,14 +104,17 @@ def pack(values: Series): return b''.join([serialize_basic(value) for value in values]) -def chunkify(bytez): - # pad `bytez` to nearest 32-byte multiple - bytez += b'\x00' * (-len(bytez) % 32) - return [bytez[i:i + 32] for i in range(0, len(bytez), 32)] +def zpad(bytez, bytes_per_chunk): + return bytez + b'\x00' * (-len(bytez) % bytes_per_chunk) + +def chunkify(bytez, bytes_per_chunk): + # pad `bytez` to nearest `bytes_per_chunk`-byte multiple + bytez = zpad(bytez, bytes_per_chunk) + return [bytez[i:i + bytes_per_chunk] for i in range(0, len(bytez), bytes_per_chunk)] def mix_in_length(root, length): - return hash(root + length.to_bytes(32, 'little')) + return hash(root + length.to_bytes(BYTES_PER_CHUNK, 'little')) def is_bottom_layer_kind(typ: SSZType): @@ -122,7 +128,7 @@ def item_length(typ: SSZType) -> int: if issubclass(typ, BasicValue): return typ.byte_len else: - return 32 + return BYTES_PER_CHUNK def chunk_count(typ: SSZType) -> int: @@ -131,7 +137,7 @@ def chunk_count(typ: SSZType) -> int: elif issubclass(typ, Bits): return (typ.length + 255) // 256 elif issubclass(typ, Elements): - return (typ.length * item_length(typ.elem_type) + 31) // 32 + return (typ.length * item_length(typ.elem_type) + BYTES_PER_CHUNK - 1) // BYTES_PER_CHUNK elif issubclass(typ, Container): return len(typ.get_fields()) else: @@ -141,11 +147,11 @@ def chunk_count(typ: SSZType) -> int: def hash_tree_root(obj: SSZValue): if isinstance(obj, Series): if is_bottom_layer_kind(obj.type()): - leaves = chunkify(pack(obj)) + leaves = chunkify(pack(obj), BYTES_PER_CHUNK) else: leaves = [hash_tree_root(value) for value in obj] elif isinstance(obj, BasicValue): - leaves = chunkify(serialize_basic(obj)) + leaves = chunkify(serialize_basic(obj), BYTES_PER_CHUNK) else: raise Exception(f"Type not supported: {type(obj)}") @@ -159,4 +165,4 @@ def signing_root(obj: Container): # ignore last field fields = [field for field in obj][:-1] leaves = [hash_tree_root(f) for f in fields] - return merkleize_chunks(chunkify(b''.join(leaves))) + return merkleize_chunks(chunkify(b''.join(leaves), BYTES_PER_CHUNK))