Update test vec generation and make it dynamic

This commit updates the validator shuffling generator and tries to make
the test validator sets more life-like.

constants.py:
* Add FAR_FUTURE_EPOCH

core_helpers.py:
* Update the helpers needed for validator shuffling

Remove enums.py: status flags are not used since 2018-12-28

tgen_shuffling.py:
* Use a randomized probabilistic approach for generating configurably
diverse validator sets; pre-FAR_FUTURE_EPOCH exit epochs truncated for
readability
* Change the YAML layout to include validator data needed for
present-day shuffling

yaml_objects.py:
* Change ValidatorRecord name to Validator (as in spec); change fields
for up-to-date shuffling
* Remove ShardCommittee (not used for shuffling at all as of today)

test_vector_shuffling.yml:
* Regenerate

eth2_testgen/shuffling/constants.py

@@ -1,4 +1,4 @@
-
-SHARD_COUNT = 2**10  # 1024
 EPOCH_LENGTH = 2**6  # 64 slots, 6.4 minutes
+FAR_FUTURE_EPOCH = 2**64 - 1  # uint64 max
+SHARD_COUNT = 2**10  # 1024
 TARGET_COMMITTEE_SIZE = 2**8  # 256 validators

eth2_testgen/shuffling/core_helpers.py

@@ -8,26 +8,29 @@
 from eth_typing import Hash32
 
 from constants import EPOCH_LENGTH, SHARD_COUNT, TARGET_COMMITTEE_SIZE
-from enums import ValidatorStatusCode
 from utils import hash
-from yaml_objects import ShardCommittee, ValidatorRecord
+from yaml_objects import Validator
 
+EpochNumber = int
+ValidatorIndex = int
+Bytes32 = bytes
 
-def is_active_validator(validator: ValidatorRecord) -> bool:
+
+def is_active_validator(validator: Validator, epoch: EpochNumber) -> bool:
     """
     Checks if ``validator`` is active.
     """
-    return validator.status in [ValidatorStatusCode.ACTIVE, ValidatorStatusCode.ACTIVE_PENDING_EXIT]
+    return validator.activation_epoch <= epoch < validator.exit_epoch
 
 
-def get_active_validator_indices(validators: [ValidatorRecord]) -> List[int]:
+def get_active_validator_indices(validators: List[Validator], epoch: EpochNumber) -> List[ValidatorIndex]:
     """
     Gets indices of active validators from ``validators``.
     """
-    return [i for i, v in enumerate(validators) if is_active_validator(v)]
+    return [i for i, v in enumerate(validators) if is_active_validator(v, epoch)]
 
 
-def shuffle(values: List[Any], seed: Hash32) -> List[Any]:
+def shuffle(values: List[Any], seed: Bytes32) -> List[Any]:
     """
     Returns the shuffled ``values`` with ``seed`` as entropy.
     """
@@ -78,57 +81,51 @@ def shuffle(values: List[Any], seed: Hash32) -> List[Any]:
     return output
 
 
-def split(values: List[Any], split_count: int) -> List[Any]:
+def split(values: List[Any], split_count: int) -> List[List[Any]]:
     """
     Splits ``values`` into ``split_count`` pieces.
     """
     list_length = len(values)
     return [
-        values[
-            (list_length * i // split_count): (list_length * (i + 1) // split_count)
-        ]
+        values[(list_length * i // split_count)
+                : (list_length * (i + 1) // split_count)]
         for i in range(split_count)
     ]
 
 
-def get_new_shuffling(seed: Hash32,
-                      validators: List[ValidatorRecord],
-                      crosslinking_start_shard: int) -> List[List[ShardCommittee]]:
-    """
-    Shuffles ``validators`` into shard committees using ``seed`` as entropy.
-    """
-    active_validator_indices = get_active_validator_indices(validators)
-
-    committees_per_slot = max(
+def get_epoch_committee_count(active_validator_count: int) -> int:
+    return max(
         1,
         min(
             SHARD_COUNT // EPOCH_LENGTH,
-            len(active_validator_indices) // EPOCH_LENGTH // TARGET_COMMITTEE_SIZE,
+            active_validator_count // EPOCH_LENGTH // TARGET_COMMITTEE_SIZE,
         )
-    )
+    ) * EPOCH_LENGTH
 
-    # Shuffle with seed
-    shuffled_active_validator_indices = shuffle(active_validator_indices, seed)
 
-    # Split the shuffled list into epoch_length pieces
-    validators_per_slot = split(
-        shuffled_active_validator_indices, EPOCH_LENGTH)
+def xor(a: bytes, b: bytes) -> bytes:
+    return bytes(i ^ j for (i, j) in zip(a, b))
 
-    output = []
-    for slot, slot_indices in enumerate(validators_per_slot):
-        # Split the shuffled list into committees_per_slot pieces
-        shard_indices = split(slot_indices, committees_per_slot)
 
-        shard_id_start = crosslinking_start_shard + slot * committees_per_slot
+def int_to_bytes32(x) -> bytes:
+    return x.to_bytes(32, 'big')
 
-        shard_committees = [
-            ShardCommittee(
-                shard=(shard_id_start + shard_position) % SHARD_COUNT,
-                committee=indices,
-                total_validator_count=len(active_validator_indices),
-            )
-            for shard_position, indices in enumerate(shard_indices)
-        ]
-        output.append(shard_committees)
 
-    return output
+def get_shuffling(seed: Bytes32, validators: List[Validator], epoch: EpochNumber) -> List[List[ValidatorIndex]]:
+    """
+    Shuffles ``validators`` into crosslink committees seeded by ``seed`` and ``epoch``.
+    Returns a list of ``committees_per_epoch`` committees where each
+    committee is itself a list of validator indices.
+    """
+
+    active_validator_indices = get_active_validator_indices(validators, epoch)
+
+    committees_per_epoch = get_epoch_committee_count(
+        len(active_validator_indices))
+
+    # Shuffle
+    seed = xor(seed, int_to_bytes32(epoch))
+    shuffled_active_validator_indices = shuffle(active_validator_indices, seed)
+
+    # Split the shuffled list into committees_per_epoch pieces
+    return split(shuffled_active_validator_indices, committees_per_epoch)

eth2_testgen/shuffling/tgen_shuffling.py

@@ -4,10 +4,9 @@
 
 import yaml
 
-from constants import SHARD_COUNT
-from core_helpers import get_new_shuffling
-from enums import ValidatorStatusCode
-from yaml_objects import ShardCommittee, ValidatorRecord
+from constants import FAR_FUTURE_EPOCH
+from core_helpers import get_shuffling
+from yaml_objects import Validator
 
 
 def noop(self, *args, **kw):
@@ -17,20 +16,16 @@ def noop(self, *args, **kw):
 
 yaml.emitter.Emitter.process_tag = noop
 
+# Cap epoch randomization for human-readability
+MAX_RAND_EPOCH = 1000
 
-def yaml_ValidatorStatusCode(dumper, data):
-    # Try to deal with enums - otherwise for "ValidatorStatus.Active" you get [1], instead of 1
-    return dumper.represent_data(data.value)
-
-
-yaml.add_representer(ValidatorStatusCode, yaml_ValidatorStatusCode)
 
 if __name__ == '__main__':
 
     # Order not preserved - https://github.com/yaml/pyyaml/issues/110
     metadata = {
         'title': 'Shuffling Algorithm Tests',
-        'summary': 'Test vectors for shuffling a list based upon a seed using `shuffle`',
+        'summary': 'Test vectors for validator shuffling. Note: only relevant validator fields are defined.',
         'test_suite': 'shuffle',
         'fork': 'tchaikovsky',
         'version': 1.0
@@ -39,24 +34,53 @@ def yaml_ValidatorStatusCode(dumper, data):
     # Config
     random.seed(int("0xEF00BEAC", 16))
     num_cases = 10
-    list_val_state = list(ValidatorStatusCode)
     test_cases = []
 
     for case in range(num_cases):
         seedhash = bytes(random.randint(0, 255) for byte in range(32))
-        num_val = random.randint(128, 512)
-        validators = [
-            ValidatorRecord(
-                status=random.choice(list_val_state),
-                original_index=num_val)
-            for num_val in range(num_val)
-        ]
+        idx_max = random.randint(128, 512)
+
+        validators = []
+        activation_sum = 0
+        exit_sum = 0
+        for idx in range(idx_max):
+            # ~10% chance v is not activated. These don't partake in epoch avg
+            if not random.randint(0, 9):
+                v = Validator(activation_epoch=FAR_FUTURE_EPOCH,
+                              exit_epoch=FAR_FUTURE_EPOCH,
+                              original_index=idx)
+                validators.append(v)
+                continue
+
+            # ~90% chance v is activated
+            v = Validator(
+                activation_epoch=random.randint(0, MAX_RAND_EPOCH),
+                original_index=idx)
+
+            # ~72% chance v didn't leave at any point
+            if random.randint(0, 4):
+                v.exit_epoch = FAR_FUTURE_EPOCH
+            # ~18% chance v leaves at some point
+            else:
+                v.exit_epoch = v.activation_epoch + \
+                    random.randint(0, MAX_RAND_EPOCH - v.activation_epoch)
+
+            validators.append(v)
+            activation_sum += v.activation_epoch
+            exit_sum += v.exit_epoch
+
+        activation_avg = activation_sum // idx_max
+        exit_avg = exit_sum // idx_max
+
+        # The middle of the typical activated validator epoch span
+        epoch = (activation_avg + exit_avg) // 2
+
         input_ = {
-            'validators_status': [v.status.value for v in validators],
-            'crosslinking_start_shard': random.randint(0, SHARD_COUNT)
+            'validators': validators,
+            'epoch': epoch
         }
-        output = get_new_shuffling(
-            seedhash, validators, input_['crosslinking_start_shard'])
+        output = get_shuffling(
+            seedhash, validators, input_['epoch'])
 
         test_cases.append({
             'seed': '0x' + seedhash.hex(), 'input': input_, 'output': output

eth2_testgen/shuffling/yaml_objects.py

@@ -3,33 +3,15 @@
 import yaml
 
 
-class ValidatorRecord(yaml.YAMLObject):
+class Validator(yaml.YAMLObject):
+    """ 
+    A validator stub containing only the fields relevant for get_shuffling()
+    """
     fields = {
-        # Status code
-        'status': 'ValidatorStatusCode',
+        'activation_epoch': 'uint64',
+        'exit_epoch': 'uint64',
         # Extra index field to ease testing/debugging
-        'original_index': 'uint64'
-    }
-
-    def __init__(self, **kwargs):
-        for k in self.fields.keys():
-            setattr(self, k, kwargs.get(k))
-
-    def __setattr__(self, name: str, value: Any) -> None:
-        super().__setattr__(name, value)
-
-    def __getattribute__(self, name: str) -> Any:
-        return super().__getattribute__(name)
-
-
-class ShardCommittee(yaml.YAMLObject):
-    fields = {
-        # Shard number
-        'shard': 'uint64',
-        # Validator indices
-        'committee': ['uint24'],
-        # Total validator count (for proofs of custody)
-        'total_validator_count': 'uint64',
+        'original_index': 'uint64',
     }
 
     def __init__(self, **kwargs):