From f4de6217cb8f4313a535218a4cb4a6fb4911e045 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Mon, 8 Jul 2019 00:17:02 -0400 Subject: [PATCH 1/6] Alternative crosslink data construction, take 1 Properties: * Optimal data packing * Fixed depth * All data is encoded; makes fraud proofs not-too-difficult Weaknesses: * Uses SSZ serialization. * Positions of blocks dynamic, so accesses require an extra Merkle branch --- specs/core/1_shard-data-chains.md | 54 ++++++++++++++++--------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 613b4c4c22..fbfc276e04 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -27,6 +27,8 @@ - [`get_shard_proposer_index`](#get_shard_proposer_index) - [`get_shard_header`](#get_shard_header) - [`verify_shard_attestation_signature`](#verify_shard_attestation_signature) + - [`flatten_block`](#flatten_block) + - [`compute_crosslink_data`](#compute_crosslink_data) - [`compute_crosslink_data_root`](#compute_crosslink_data_root) - [Object validity](#object-validity) - [Shard blocks](#shard-blocks) @@ -46,6 +48,7 @@ This document describes the shard data layer and the shard fork choice rule in P | Name | Value | | - | - | +| `BYTES_PER_SHARD_BLOCK_HEADER` | `2**9` (= 512) | | `BYTES_PER_SHARD_BLOCK_BODY` | `2**14` (= 16,384) | | `MAX_SHARD_ATTESTIONS` | `2**4` (= 16) | @@ -247,36 +250,35 @@ def verify_shard_attestation_signature(state: BeaconState, ) ``` +### `flatten_block` + +```python +def flatten_block(block: ShardBlock) -> Bytes: + return zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_HEADER) + block.body +``` + +### `compute_crosslink_data` + +```python +def compute_crosslink_data(blocks: Sequence[ShardBlock]) -> Bytes32: + """ + Flattens a series of blocks. Designed to be equivalent to SSZ serializing a list of + flattened blocks with one empty block at the end as a termination marker. + """ + positions = [4 * len(blocks)] + bodies = [] + for block in blocks: + bodies.append(flatten_block(block)) + positions.append(positions[-1] + len(bodies[-1])) + return b''.join([int_to_bytes4(pos) for pos in positions]) + b''.join(bodies) +``` + ### `compute_crosslink_data_root` ```python def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: - def is_power_of_two(value: uint64) -> bool: - return (value > 0) and (value & (value - 1) == 0) - - def pad_to_power_of_2(values: MutableSequence[bytes]) -> Sequence[bytes]: - while not is_power_of_two(len(values)): - values.append(b'\x00' * BYTES_PER_SHARD_BLOCK_BODY) - return values - - def hash_tree_root_of_bytes(data: bytes) -> bytes: - return hash_tree_root([data[i:i + 32] for i in range(0, len(data), 32)]) - - def zpad(data: bytes, length: uint64) -> bytes: - return data + b'\x00' * (length - len(data)) - - return hash( - # TODO untested code. - # Need to either pass a typed list to hash-tree-root, or merkleize_chunks(values, pad_to=2**x) - hash_tree_root(pad_to_power_of_2([ - hash_tree_root_of_bytes( - zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_BODY) - ) for block in blocks - ])) - + hash_tree_root(pad_to_power_of_2([ - hash_tree_root_of_bytes(block.body) for block in blocks - ])) - ) + MAXLEN = (BYTES_PER_SHARD_BLOCK_HEADER + BYTES_PER_SHARD_BLOCK_BODY) * SHARD_SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK + return hash_tree_root(BytesN[MAXLEN](zpad(compute_crosslink_data(blocks), MAXLEN))) ``` ## Object validity From 457908dfcac3b12313cad5050bf6506560ad035a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Jul 2019 12:19:26 +0800 Subject: [PATCH 2/6] Minor fix --- scripts/build_spec.py | 5 ++-- specs/core/1_shard-data-chains.md | 12 +++++---- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 25 +++++++++++-------- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 8b541ff50c..7f99c774d7 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -46,14 +46,15 @@ ) from eth2spec.utils.ssz.ssz_impl import ( + chunkify, hash_tree_root, + is_empty, signing_root, serialize, - is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, uint64, - Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, + Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, BytesN, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index fbfc276e04..f5b9346cc4 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -13,7 +13,7 @@ - [Misc](#misc) - [Initial values](#initial-values) - [Time parameters](#time-parameters) - - [Signature domains](#signature-domains) + - [Signature domain types](#signature-domain-types) - [TODO PLACEHOLDER](#todo-placeholder) - [Data structures](#data-structures) - [`ShardBlockBody`](#shardblockbody) @@ -254,7 +254,7 @@ def verify_shard_attestation_signature(state: BeaconState, ```python def flatten_block(block: ShardBlock) -> Bytes: - return zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_HEADER) + block.body + return chunkify(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_HEADER) + block.body ``` ### `compute_crosslink_data` @@ -270,15 +270,17 @@ def compute_crosslink_data(blocks: Sequence[ShardBlock]) -> Bytes32: for block in blocks: bodies.append(flatten_block(block)) positions.append(positions[-1] + len(bodies[-1])) - return b''.join([int_to_bytes4(pos) for pos in positions]) + b''.join(bodies) + return b''.join([int_to_bytes(pos, 4) for pos in positions]) + b''.join(bodies) ``` ### `compute_crosslink_data_root` ```python def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: - MAXLEN = (BYTES_PER_SHARD_BLOCK_HEADER + BYTES_PER_SHARD_BLOCK_BODY) * SHARD_SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK - return hash_tree_root(BytesN[MAXLEN](zpad(compute_crosslink_data(blocks), MAXLEN))) + MAXLEN = ( + BYTES_PER_SHARD_BLOCK_HEADER + BYTES_PER_SHARD_BLOCK_BODY + ) * SHARD_SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK + return hash_tree_root(BytesN[MAXLEN](chunkify(compute_crosslink_data(blocks), MAXLEN))) ``` ## Object validity diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index d5855a755f..f113f501f0 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,4 +1,6 @@ -from ..merkle_minimal import merkleize_chunks +from ..merkle_minimal import ( + merkleize_chunks, +) from ..hash_function import hash from .ssz_typing import ( SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bits, boolean, Container, List, Bytes, @@ -8,6 +10,7 @@ # SSZ Serialization # ----------------------------- +BYTES_PER_CHUNK = 32 BYTES_PER_LENGTH_OFFSET = 4 @@ -101,14 +104,14 @@ def pack(values: Series): return b''.join([serialize_basic(value) for value in values]) -def chunkify(bytez): - # pad `bytez` to nearest 32-byte multiple - bytez += b'\x00' * (-len(bytez) % 32) - return [bytez[i:i + 32] for i in range(0, len(bytez), 32)] +def chunkify(bytez, bytes_per_chunk): + # pad `bytez` to nearest `bytes_per_chunk`-byte multiple + bytez += b'\x00' * (-len(bytez) % bytes_per_chunk) + return [bytez[i:i + bytes_per_chunk] for i in range(0, len(bytez), bytes_per_chunk)] def mix_in_length(root, length): - return hash(root + length.to_bytes(32, 'little')) + return hash(root + length.to_bytes(BYTES_PER_CHUNK, 'little')) def is_bottom_layer_kind(typ: SSZType): @@ -122,7 +125,7 @@ def item_length(typ: SSZType) -> int: if issubclass(typ, BasicValue): return typ.byte_len else: - return 32 + return BYTES_PER_CHUNK def chunk_count(typ: SSZType) -> int: @@ -131,7 +134,7 @@ def chunk_count(typ: SSZType) -> int: elif issubclass(typ, Bits): return (typ.length + 255) // 256 elif issubclass(typ, Elements): - return (typ.length * item_length(typ.elem_type) + 31) // 32 + return (typ.length * item_length(typ.elem_type) + BYTES_PER_CHUNK - 1) // BYTES_PER_CHUNK elif issubclass(typ, Container): return len(typ.get_fields()) else: @@ -141,11 +144,11 @@ def chunk_count(typ: SSZType) -> int: def hash_tree_root(obj: SSZValue): if isinstance(obj, Series): if is_bottom_layer_kind(obj.type()): - leaves = chunkify(pack(obj)) + leaves = chunkify(pack(obj), BYTES_PER_CHUNK) else: leaves = [hash_tree_root(value) for value in obj] elif isinstance(obj, BasicValue): - leaves = chunkify(serialize_basic(obj)) + leaves = chunkify(serialize_basic(obj), BYTES_PER_CHUNK) else: raise Exception(f"Type not supported: {type(obj)}") @@ -159,4 +162,4 @@ def signing_root(obj: Container): # ignore last field fields = [field for field in obj][:-1] leaves = [hash_tree_root(f) for f in fields] - return merkleize_chunks(chunkify(b''.join(leaves))) + return merkleize_chunks(chunkify(b''.join(leaves), BYTES_PER_CHUNK)) From dcd0dc5297cbe769ccc69b1d976745bd6a999c9c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Jul 2019 14:17:24 +0800 Subject: [PATCH 3/6] Rework zpad --- specs/core/1_shard-data-chains.md | 4 ++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index f5b9346cc4..4f6774b016 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -254,7 +254,7 @@ def verify_shard_attestation_signature(state: BeaconState, ```python def flatten_block(block: ShardBlock) -> Bytes: - return chunkify(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_HEADER) + block.body + return zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_HEADER) + block.body ``` ### `compute_crosslink_data` @@ -280,7 +280,7 @@ def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: MAXLEN = ( BYTES_PER_SHARD_BLOCK_HEADER + BYTES_PER_SHARD_BLOCK_BODY ) * SHARD_SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK - return hash_tree_root(BytesN[MAXLEN](chunkify(compute_crosslink_data(blocks), MAXLEN))) + return hash_tree_root(BytesN[MAXLEN](zpad(compute_crosslink_data(blocks), MAXLEN))) ``` ## Object validity diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index f113f501f0..148347acde 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -104,9 +104,12 @@ def pack(values: Series): return b''.join([serialize_basic(value) for value in values]) +def zpad(bytez, bytes_per_chunk): + return bytez + b'\x00' * (-len(bytez) % bytes_per_chunk) + def chunkify(bytez, bytes_per_chunk): # pad `bytez` to nearest `bytes_per_chunk`-byte multiple - bytez += b'\x00' * (-len(bytez) % bytes_per_chunk) + bytez = zpad(bytez, bytes_per_chunk) return [bytez[i:i + bytes_per_chunk] for i in range(0, len(bytez), bytes_per_chunk)] From 467d7b45400e20a6be185e9714e93d267591c668 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Jul 2019 14:57:19 +0800 Subject: [PATCH 4/6] minor fix --- scripts/build_spec.py | 2 +- specs/core/1_shard-data-chains.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 7f99c774d7..1b4ffd541e 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -46,7 +46,7 @@ ) from eth2spec.utils.ssz.ssz_impl import ( - chunkify, + zpad, hash_tree_root, is_empty, signing_root, diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 4f6774b016..a9a68bedf1 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -79,7 +79,7 @@ The following types are defined, mapping into `DomainType` (little endian): | Name | Value | | - | - | -| `PLACEHOLDER` | `2**32` | +| `PLACEHOLDER` | `2**3` | ## Data structures @@ -110,7 +110,7 @@ class ShardBlock(Container): shard: Shard beacon_chain_root: Bytes32 parent_root: Bytes32 - data: ShardBlockBody + body: ShardBlockBody state_root: Bytes32 attestations: List[ShardAttestation, PLACEHOLDER] signature: BLSSignature @@ -254,7 +254,7 @@ def verify_shard_attestation_signature(state: BeaconState, ```python def flatten_block(block: ShardBlock) -> Bytes: - return zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_HEADER) + block.body + return zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_HEADER) + serialize(block.body) ``` ### `compute_crosslink_data` From 2af45b44e89afbbbe12f621695caebbbc8eaf8b5 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Mon, 15 Jul 2019 16:37:58 +0800 Subject: [PATCH 5/6] Update specs/core/1_shard-data-chains.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index a9a68bedf1..a999fad963 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -260,7 +260,7 @@ def flatten_block(block: ShardBlock) -> Bytes: ### `compute_crosslink_data` ```python -def compute_crosslink_data(blocks: Sequence[ShardBlock]) -> Bytes32: +def compute_crosslink_data(blocks: Sequence[ShardBlock]) -> bytes: """ Flattens a series of blocks. Designed to be equivalent to SSZ serializing a list of flattened blocks with one empty block at the end as a termination marker. From 102473a9488de6a6bb356626aaf146c4a0b9cebd Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 28 Jul 2019 08:04:14 -0400 Subject: [PATCH 6/6] Update specs/core/1_shard-data-chains.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index a999fad963..7d0c77bf9e 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -253,7 +253,7 @@ def verify_shard_attestation_signature(state: BeaconState, ### `flatten_block` ```python -def flatten_block(block: ShardBlock) -> Bytes: +def flatten_block(block: ShardBlock) -> bytes: return zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_HEADER) + serialize(block.body) ```