Alternative crosslink data construction, take 1

scripts/build_spec.py

@@ -46,14 +46,15 @@
 )
 
 from eth2spec.utils.ssz.ssz_impl import (
+    zpad,
     hash_tree_root,
+    is_empty,
     signing_root,
     serialize,
-    is_empty,
 )
 from eth2spec.utils.ssz.ssz_typing import (
     bit, boolean, Container, List, Vector, Bytes, uint64,
-    Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector,
+    Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, BytesN, Bitlist, Bitvector,
 )
 from eth2spec.utils.bls import (
     bls_aggregate_pubkeys,

specs/core/1_shard-data-chains.md

@@ -13,7 +13,7 @@
         - [Misc](#misc)
         - [Initial values](#initial-values)
         - [Time parameters](#time-parameters)
-        - [Signature domains](#signature-domains)
+        - [Signature domain types](#signature-domain-types)
         - [TODO PLACEHOLDER](#todo-placeholder)
     - [Data structures](#data-structures)
         - [`ShardBlockBody`](#shardblockbody)
@@ -27,6 +27,8 @@
         - [`get_shard_proposer_index`](#get_shard_proposer_index)
         - [`get_shard_header`](#get_shard_header)
         - [`verify_shard_attestation_signature`](#verify_shard_attestation_signature)
+        - [`flatten_block`](#flatten_block)
+        - [`compute_crosslink_data`](#compute_crosslink_data)
         - [`compute_crosslink_data_root`](#compute_crosslink_data_root)
     - [Object validity](#object-validity)
         - [Shard blocks](#shard-blocks)
@@ -46,6 +48,7 @@ This document describes the shard data layer and the shard fork choice rule in P
 
 | Name | Value |
 | - | - |
+| `BYTES_PER_SHARD_BLOCK_HEADER` | `2**9` (= 512) |
 | `BYTES_PER_SHARD_BLOCK_BODY` | `2**14` (= 16,384) |
 | `MAX_SHARD_ATTESTIONS` | `2**4` (= 16) |
 
@@ -76,7 +79,7 @@ The following types are defined, mapping into `DomainType` (little endian):
 
 | Name | Value |
 | - | - |
-| `PLACEHOLDER` | `2**32` |
+| `PLACEHOLDER` | `2**3` |
 
 ## Data structures
 
@@ -107,7 +110,7 @@ class ShardBlock(Container):
     shard: Shard
     beacon_chain_root: Bytes32
     parent_root: Bytes32
-    data: ShardBlockBody
+    body: ShardBlockBody
     state_root: Bytes32
     attestations: List[ShardAttestation, PLACEHOLDER]
     signature: BLSSignature
@@ -247,36 +250,37 @@ def verify_shard_attestation_signature(state: BeaconState,
     )
 ```
 
+### `flatten_block`
+
+```python
+def flatten_block(block: ShardBlock) -> Bytes:
+    return zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_HEADER) + serialize(block.body)
+```
+
+### `compute_crosslink_data`
+
+```python
+def compute_crosslink_data(blocks: Sequence[ShardBlock]) -> bytes:
+    """
+    Flattens a series of blocks. Designed to be equivalent to SSZ serializing a list of
+    flattened blocks with one empty block at the end as a termination marker.
+    """
+    positions = [4 * len(blocks)]
+    bodies = []
+    for block in blocks:
+        bodies.append(flatten_block(block))
+        positions.append(positions[-1] + len(bodies[-1]))
+    return b''.join([int_to_bytes(pos, 4) for pos in positions]) + b''.join(bodies)    
+```
+
 ### `compute_crosslink_data_root`
 
 ```python
 def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32:
-    def is_power_of_two(value: uint64) -> bool:
-        return (value > 0) and (value & (value - 1) == 0)
-
-    def pad_to_power_of_2(values: MutableSequence[bytes]) -> Sequence[bytes]:
-        while not is_power_of_two(len(values)):
-            values.append(b'\x00' * BYTES_PER_SHARD_BLOCK_BODY)
-        return values
-
-    def hash_tree_root_of_bytes(data: bytes) -> bytes:
-        return hash_tree_root([data[i:i + 32] for i in range(0, len(data), 32)])
-
-    def zpad(data: bytes, length: uint64) -> bytes:
-        return data + b'\x00' * (length - len(data))
-
-    return hash(
-        # TODO untested code.
-        #  Need to either pass a typed list to hash-tree-root, or merkleize_chunks(values, pad_to=2**x)
-        hash_tree_root(pad_to_power_of_2([
-            hash_tree_root_of_bytes(
-                zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_BODY)
-            ) for block in blocks
-        ]))
-        + hash_tree_root(pad_to_power_of_2([
-            hash_tree_root_of_bytes(block.body) for block in blocks
-        ]))
-    )
+    MAXLEN = (
+        BYTES_PER_SHARD_BLOCK_HEADER + BYTES_PER_SHARD_BLOCK_BODY
+    ) * SHARD_SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK
+    return hash_tree_root(BytesN[MAXLEN](zpad(compute_crosslink_data(blocks), MAXLEN)))
 ```
 
 ## Object validity

test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py

@@ -1,4 +1,6 @@
-from ..merkle_minimal import merkleize_chunks
+from ..merkle_minimal import (
+    merkleize_chunks,
+)
 from ..hash_function import hash
 from .ssz_typing import (
     SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bits, boolean, Container, List, Bytes,
@@ -8,6 +10,7 @@
 # SSZ Serialization
 # -----------------------------
 
+BYTES_PER_CHUNK = 32
 BYTES_PER_LENGTH_OFFSET = 4
 
 
@@ -101,14 +104,17 @@ def pack(values: Series):
     return b''.join([serialize_basic(value) for value in values])
 
 
-def chunkify(bytez):
-    # pad `bytez` to nearest 32-byte multiple
-    bytez += b'\x00' * (-len(bytez) % 32)
-    return [bytez[i:i + 32] for i in range(0, len(bytez), 32)]
+def zpad(bytez, bytes_per_chunk):
+    return bytez + b'\x00' * (-len(bytez) % bytes_per_chunk)
+
+def chunkify(bytez, bytes_per_chunk):
+    # pad `bytez` to nearest `bytes_per_chunk`-byte multiple
+    bytez = zpad(bytez, bytes_per_chunk)
+    return [bytez[i:i + bytes_per_chunk] for i in range(0, len(bytez), bytes_per_chunk)]
 
 
 def mix_in_length(root, length):
-    return hash(root + length.to_bytes(32, 'little'))
+    return hash(root + length.to_bytes(BYTES_PER_CHUNK, 'little'))
 
 
 def is_bottom_layer_kind(typ: SSZType):
@@ -122,7 +128,7 @@ def item_length(typ: SSZType) -> int:
     if issubclass(typ, BasicValue):
         return typ.byte_len
     else:
-        return 32
+        return BYTES_PER_CHUNK
 
 
 def chunk_count(typ: SSZType) -> int:
@@ -131,7 +137,7 @@ def chunk_count(typ: SSZType) -> int:
     elif issubclass(typ, Bits):
         return (typ.length + 255) // 256
     elif issubclass(typ, Elements):
-        return (typ.length * item_length(typ.elem_type) + 31) // 32
+        return (typ.length * item_length(typ.elem_type) + BYTES_PER_CHUNK - 1) // BYTES_PER_CHUNK
     elif issubclass(typ, Container):
         return len(typ.get_fields())
     else:
@@ -141,11 +147,11 @@ def chunk_count(typ: SSZType) -> int:
 def hash_tree_root(obj: SSZValue):
     if isinstance(obj, Series):
         if is_bottom_layer_kind(obj.type()):
-            leaves = chunkify(pack(obj))
+            leaves = chunkify(pack(obj), BYTES_PER_CHUNK)
         else:
             leaves = [hash_tree_root(value) for value in obj]
     elif isinstance(obj, BasicValue):
-        leaves = chunkify(serialize_basic(obj))
+        leaves = chunkify(serialize_basic(obj), BYTES_PER_CHUNK)
     else:
         raise Exception(f"Type not supported: {type(obj)}")
 
@@ -159,4 +165,4 @@ def signing_root(obj: Container):
     # ignore last field
     fields = [field for field in obj][:-1]
     leaves = [hash_tree_root(f) for f in fields]
-    return merkleize_chunks(chunkify(b''.join(leaves)))
+    return merkleize_chunks(chunkify(b''.join(leaves), BYTES_PER_CHUNK))