Merge pull request #1077 from ethereum/ssz-impl-rework

SSZ implementation for exec. spec - Support for Python 3 typing.

.circleci/config.yml

@@ -60,13 +60,13 @@ jobs:
       - restore_cache:
           key: v1-specs-repo-{{ .Branch }}-{{ .Revision }}
       - restore_cached_venv:
-          venv_name: v1-pyspec-03
+          venv_name: v2-pyspec
           reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}'
       - run:
           name: Install pyspec requirements
-          command: make install_test && make install_lint
+          command: make install_test
       - save_cached_venv:
-          venv_name: v1-pyspec-03
+          venv_name: v2-pyspec
           reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}'
           venv_path: ./test_libs/pyspec/venv
   test:
@@ -77,7 +77,7 @@ jobs:
       - restore_cache:
           key: v1-specs-repo-{{ .Branch }}-{{ .Revision }}
       - restore_cached_venv:
-          venv_name: v1-pyspec-03
+          venv_name: v2-pyspec
           reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}'
       - run:
           name: Run py-tests
@@ -92,7 +92,7 @@ jobs:
       - restore_cache:
           key: v1-specs-repo-{{ .Branch }}-{{ .Revision }}
       - restore_cached_venv:
-          venv_name: v1-pyspec-03
+          venv_name: v2-pyspec
           reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}'
       - run:
           name: Run linter

Makefile

@@ -39,12 +39,9 @@ test: $(PY_SPEC_ALL_TARGETS)
 citest: $(PY_SPEC_ALL_TARGETS)
 	cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml .
 
-install_lint:
-	cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.5.0
-
 lint: $(PY_SPEC_ALL_TARGETS)
 	cd $(PY_SPEC_DIR); . venv/bin/activate; \
-	flake8 --max-line-length=120 ./eth2spec;
+	flake8  --ignore=E252,W504 --max-line-length=120 ./eth2spec;
 
 # "make pyspec" to create the pyspec for all phases.
 pyspec: $(PY_SPEC_ALL_TARGETS)

scripts/phase0/build_spec.py

@@ -13,31 +13,33 @@ def build_phase0_spec(sourcefile, outfile):
     NewType,
     Tuple,
 )
-from eth2spec.utils.minimal_ssz import (
-    SSZType,
+from eth2spec.utils.ssz.ssz_impl import (
     hash_tree_root,
     signing_root,
 )
+from eth2spec.utils.ssz.ssz_typing import (
+    # unused: uint8, uint16, uint32, uint128, uint256,
+    uint64, Container, Vector, BytesN
+)
 from eth2spec.utils.hash_function import hash
 from eth2spec.utils.bls import (
     bls_aggregate_pubkeys,
     bls_verify,
     bls_verify_multiple,
 )
 
-
-# stub, will get overwritten by real var
-SLOTS_PER_EPOCH = 64
-
+# Note: 'int' type defaults to being interpreted as a uint64 by SSZ implementation.
 Slot = NewType('Slot', int)  # uint64
 Epoch = NewType('Epoch', int)  # uint64
 Shard = NewType('Shard', int)  # uint64
 ValidatorIndex = NewType('ValidatorIndex', int)  # uint64
 Gwei = NewType('Gwei', int)  # uint64
-Bytes32 = NewType('Bytes32', bytes)  # bytes32
-BLSPubkey = NewType('BLSPubkey', bytes)  # bytes48
-BLSSignature = NewType('BLSSignature', bytes)  # bytes96
-Store = None
+
+Bytes4 = BytesN[4]
+Bytes32 = BytesN[32]
+Bytes48 = BytesN[48]
+Bytes96 = BytesN[96]
+
 """)
 
     code_lines += function_puller.get_spec(sourcefile)

scripts/phase0/function_puller.py

@@ -1,14 +1,15 @@
-import sys
 from typing import List
 
 
 def get_spec(file_name: str) -> List[str]:
     code_lines = []
     pulling_from = None
     current_name = None
+    # list of current type definition being parsed, or None otherwise
     current_typedef = None
+    # list of (name, definition lines list) tuples.
     type_defs = []
-    for linenum, line in enumerate(open(sys.argv[1]).readlines()):
+    for linenum, line in enumerate(open(file_name).readlines()):
         line = line.rstrip()
         if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`':
             current_name = line[line[:-1].rfind('`') + 1: -1]
@@ -19,26 +20,25 @@ def get_spec(file_name: str) -> List[str]:
             if pulling_from is None:
                 pulling_from = linenum
             else:
+                pulling_from = None
                 if current_typedef is not None:
-                    assert code_lines[-1] == '}'
-                    code_lines[-1] = '})'
-                    current_typedef[-1] = '})'
                     type_defs.append((current_name, current_typedef))
-                pulling_from = None
-                current_typedef = None
+                    current_typedef = None
         else:
-            if pulling_from == linenum and line == '{':
-                code_lines.append('%s = SSZType({' % current_name)
-                current_typedef = ['global_vars["%s"] = SSZType({' % current_name]
-            elif pulling_from is not None:
+            if pulling_from is not None:
                 # Add some whitespace between functions
-                if line[:3] == 'def':
+                if line[:3] == 'def' or line[:5] == 'class':
                     code_lines.append('')
                     code_lines.append('')
-                code_lines.append(line)
-                # Remember type def lines
+                # Check for SSZ type definitions
+                if len(line) > 18 and line[:6] == 'class ' and line[-12:] == '(Container):':
+                    name = line[6:-12]
+                    # Check consistency with markdown header
+                    assert name == current_name
+                    current_typedef = []
                 if current_typedef is not None:
                     current_typedef.append(line)
+                code_lines.append(line)
             elif pulling_from is None and len(line) > 0 and line[0] == '|':
                 row = line[1:].split('|')
                 if len(row) >= 2:
@@ -56,20 +56,26 @@ def get_spec(file_name: str) -> List[str]:
                         code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890')))
     # Build type-def re-initialization
     code_lines.append('\n')
+    code_lines.append('ssz_types = [\n')
+    for (ssz_type_name, _) in type_defs:
+        code_lines.append(f'    {ssz_type_name},')
+    code_lines.append(']')
+    code_lines.append('\n')
     code_lines.append('def init_SSZ_types():')
     code_lines.append('    global_vars = globals()')
     for ssz_type_name, ssz_type in type_defs:
         code_lines.append('')
         for type_line in ssz_type:
             if len(type_line) > 0:
                 code_lines.append('    ' + type_line)
-    code_lines.append('\n')
-    code_lines.append('ssz_types = [\n')
     for (ssz_type_name, _) in type_defs:
-        code_lines.append(f'    "{ssz_type_name}",\n')
-    code_lines.append(']')
+        code_lines.append(f'    global_vars["{ssz_type_name}"] = {ssz_type_name}')
+    code_lines.append('    global_vars["ssz_types"] = [')
+    for (ssz_type_name, _) in type_defs:
+        code_lines.append(f'        "{ssz_type_name}",')
+    code_lines.append('    ]')
     code_lines.append('\n')
-    code_lines.append('def get_ssz_type_by_name(name: str) -> SSZType:')
+    code_lines.append('def get_ssz_type_by_name(name: str) -> Container:')
     code_lines.append('    return globals()[name]')
     code_lines.append('')
     return code_lines