Prettify script

.gitignore

@@ -0,0 +1,4 @@
+.pytest_cache/
+.python-version
+__pycache__/
+test.py

LICENSE

@@ -0,0 +1,124 @@
+Creative Commons Legal Code
+
+CC0 1.0 Universal
+
+    CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+    LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+    ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+    INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+    REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+    PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+    THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+    HEREUNDER.
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator
+and subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for
+the purpose of contributing to a commons of creative, cultural and
+scientific works ("Commons") that the public can reliably and without fear
+of later claims of infringement build upon, modify, incorporate in other
+works, reuse and redistribute as freely as possible in any form whatsoever
+and for any purposes, including without limitation commercial purposes.
+These owners may contribute to the Commons to promote the ideal of a free
+culture and the further production of creative, cultural and scientific
+works, or to gain reputation or greater distribution for their Work in
+part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any
+expectation of additional consideration or compensation, the person
+associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+is an owner of Copyright and Related Rights in the Work, voluntarily
+elects to apply CC0 to the Work and publicly distribute the Work under its
+terms, with knowledge of his or her Copyright and Related Rights in the
+Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not
+limited to, the following:
+
+  i. the right to reproduce, adapt, distribute, perform, display,
+     communicate, and translate a Work;
+ ii. moral rights retained by the original author(s) and/or performer(s);
+iii. publicity and privacy rights pertaining to a person's image or
+     likeness depicted in a Work;
+ iv. rights protecting against unfair competition in regards to a Work,
+     subject to the limitations in paragraph 4(a), below;
+  v. rights protecting the extraction, dissemination, use and reuse of data
+     in a Work;
+ vi. database rights (such as those arising under Directive 96/9/EC of the
+     European Parliament and of the Council of 11 March 1996 on the legal
+     protection of databases, and under any national implementation
+     thereof, including any amended or successor version of such
+     directive); and
+vii. other similar, equivalent or corresponding rights throughout the
+     world based on applicable law or treaty, and any national
+     implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention
+of, applicable law, Affirmer hereby overtly, fully, permanently,
+irrevocably and unconditionally waives, abandons, and surrenders all of
+Affirmer's Copyright and Related Rights and associated claims and causes
+of action, whether now known or unknown (including existing as well as
+future claims and causes of action), in the Work (i) in all territories
+worldwide, (ii) for the maximum duration provided by applicable law or
+treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional
+purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+member of the public at large and to the detriment of Affirmer's heirs and
+successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public
+as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason
+be judged legally invalid or ineffective under applicable law, then the
+Waiver shall be preserved to the maximum extent permitted taking into
+account Affirmer's express Statement of Purpose. In addition, to the
+extent the Waiver is so judged Affirmer hereby grants to each affected
+person a royalty-free, non transferable, non sublicensable, non exclusive,
+irrevocable and unconditional license to exercise Affirmer's Copyright and
+Related Rights in the Work (i) in all territories worldwide, (ii) for the
+maximum duration provided by applicable law or treaty (including future
+time extensions), (iii) in any current or future medium and for any number
+of copies, and (iv) for any purpose whatsoever, including without
+limitation commercial, advertising or promotional purposes (the
+"License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any
+reason be judged legally invalid or ineffective under applicable law, such
+partial invalidity or ineffectiveness shall not invalidate the remainder
+of the License, and in such case Affirmer hereby affirms that he or she
+will not (i) exercise any of his or her remaining Copyright and Related
+Rights in the Work or (ii) assert any associated claims and causes of
+action with respect to the Work, in either case contrary to Affirmer's
+express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+    surrendered, licensed or otherwise affected by this document.
+ b. Affirmer offers the Work as-is and makes no representations or
+    warranties of any kind concerning the Work, express, implied,
+    statutory or otherwise, including without limitation warranties of
+    title, merchantability, fitness for a particular purpose, non
+    infringement, or the absence of latent or other defects, accuracy, or
+    the present or absence of errors, whether or not discoverable, all to
+    the greatest extent permissible under applicable law.
+ c. Affirmer disclaims responsibility for clearing rights of other persons
+    that may apply to the Work or any use thereof, including without
+    limitation any person's Copyright and Related Rights in the Work.
+    Further, Affirmer disclaims responsibility for obtaining any necessary
+    consents, permissions or other rights required for any use of the
+    Work.
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+    party to this document and has no duty or obligation with respect to
+    this CC0 or use of the Work.
+
+For more information, please see
+<http://creativecommons.org/publicdomain/zero/1.0/>

Makefile

@@ -0,0 +1,2 @@
+all:
+	python eth2_testgen/shuffling/tgen_shuffling.py test_vectors/test_vector_shuffling.yml

README.md

@@ -1,5 +1,25 @@
-# eth2.0-tests
+# Eth 2.0 test vectors generators
 
 [![Join the chat at https://gitter.im/eth2-0-tests/Lobby](https://badges.gitter.im/eth2-0-tests/Lobby.svg)](https://gitter.im/eth2-0-tests/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
+Test vectors generators based on the specifications at https://github.com/ethereum/eth2.0-specs.
+
 This repo serves as the common tests across eth2.0 implementations. See [YAML testing format](https://github.com/ethereum/eth2.0-specs/blob/master/specs/test-format.md).
+
+## Getting Started
+
+```
+make
+```
+
+## Implementation
+
+As much as possible the generators copy-paste the specifications. If an example implementation
+is not available in the specifications, the generator will be implemented using one of the
+Ethereum Foundation library instead.
+
+
+## License
+
+Similar to Eth 2.0 specifications, all code and generated test vectors
+are public domain under [CC0](https://creativecommons.org/publicdomain/zero/1.0/)

eth2_testgen/shuffling/README.md

@@ -0,0 +1,54 @@
+# Shuffling Test Generator
+
+```
+2018 Status Research & Development GmbH
+Copyright and related rights waived via [CC0](https://creativecommons.org/publicdomain/zero/1.0/).
+
+This work uses public domain work under CC0 from the Ethereum Foundation
+https://github.com/ethereum/eth2.0-specs
+```
+
+
+This file implements a test vectors generator for the shuffling algorithm described in the Ethereum
+[specs](https://github.com/ethereum/eth2.0-specs/blob/2983e68f0305551083fac7fcf9330c1fc9da3411/specs/core/0_beacon-chain.md#get_new_shuffling)
+
+
+
+Reference picture: 
+
+![](https://vitalik.ca/files/ShuffleAndAssign.png)
+
+and [description](https://github.com/ethereum/py-evm/blob/f2d0d5d187400ba46a6b8f5b1f1c9997dc7fbb5a/eth/beacon/helpers.py#L272-L344) from Py-EVM
+
+```
+validators:
+    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+After shuffling:
+    [6, 0, 2, 12, 14, 8, 10, 4, 9, 1, 5, 13, 15, 7, 3, 11]
+Split by slot:
+    [
+        [6, 0, 2, 12, 14], [8, 10, 4, 9, 1], [5, 13, 15, 7, 3, 11]
+    ]
+Split by shard:
+    [
+        [6, 0], [2, 12, 14], [8, 10], [4, 9, 1], [5, 13, 15] ,[7, 3, 11]
+    ]
+Fill to output:
+    [
+        # slot 0
+        [
+            ShardAndCommittee(shard_id=0, committee=[6, 0]),
+            ShardAndCommittee(shard_id=1, committee=[2, 12, 14]),
+        ],
+        # slot 1
+        [
+            ShardAndCommittee(shard_id=2, committee=[8, 10]),
+            ShardAndCommittee(shard_id=3, committee=[4, 9, 1]),
+        ],
+        # slot 2
+        [
+            ShardAndCommittee(shard_id=4, committee=[5, 13, 15]),
+            ShardAndCommittee(shard_id=5, committee=[7, 3, 11]),
+        ],
+    ]
+```

eth2_testgen/shuffling/constants.py

@@ -0,0 +1,4 @@
+
+SHARD_COUNT = 2**10  # 1024
+EPOCH_LENGTH = 2**6  # 64 slots, 6.4 minutes
+TARGET_COMMITTEE_SIZE = 2**8  # 256 validators

eth2_testgen/shuffling/core_helpers.py

@@ -0,0 +1,134 @@
+"""
+copy-pasted from specs
+"""
+
+from typing import Any, Dict, List
+
+import yaml
+from eth_typing import Hash32
+
+from constants import EPOCH_LENGTH, SHARD_COUNT, TARGET_COMMITTEE_SIZE
+from enums import ValidatorStatusCode
+from utils import hash
+from yaml_objects import ShardCommittee, ValidatorRecord
+
+
+def is_active_validator(validator: ValidatorRecord) -> bool:
+    """
+    Checks if ``validator`` is active.
+    """
+    return validator.status in [ValidatorStatusCode.ACTIVE, ValidatorStatusCode.ACTIVE_PENDING_EXIT]
+
+
+def get_active_validator_indices(validators: [ValidatorRecord]) -> List[int]:
+    """
+    Gets indices of active validators from ``validators``.
+    """
+    return [i for i, v in enumerate(validators) if is_active_validator(v)]
+
+
+def shuffle(values: List[Any], seed: Hash32) -> List[Any]:
+    """
+    Returns the shuffled ``values`` with ``seed`` as entropy.
+    """
+    values_count = len(values)
+
+    # Entropy is consumed from the seed in 3-byte (24 bit) chunks.
+    rand_bytes = 3
+    # The highest possible result of the RNG.
+    rand_max = 2 ** (rand_bytes * 8) - 1
+
+    # The range of the RNG places an upper-bound on the size of the list that
+    # may be shuffled. It is a logic error to supply an oversized list.
+    assert values_count < rand_max
+
+    output = [x for x in values]
+    source = seed
+    index = 0
+    while index < values_count - 1:
+        # Re-hash the `source` to obtain a new pattern of bytes.
+        source = hash(source)
+        # Iterate through the `source` bytes in 3-byte chunks.
+        for position in range(0, 32 - (32 % rand_bytes), rand_bytes):
+            # Determine the number of indices remaining in `values` and exit
+            # once the last index is reached.
+            remaining = values_count - index
+            if remaining == 1:
+                break
+
+            # Read 3-bytes of `source` as a 24-bit big-endian integer.
+            sample_from_source = int.from_bytes(
+                source[position:position + rand_bytes], 'big')
+
+            # Sample values greater than or equal to `sample_max` will cause
+            # modulo bias when mapped into the `remaining` range.
+            sample_max = rand_max - rand_max % remaining
+
+            # Perform a swap if the consumed entropy will not cause modulo bias.
+            if sample_from_source < sample_max:
+                # Select a replacement index for the current index.
+                replacement_position = (sample_from_source % remaining) + index
+                # Swap the current index with the replacement index.
+                output[index], output[replacement_position] = output[replacement_position], output[index]
+                index += 1
+            else:
+                # The sample causes modulo bias. A new sample should be read.
+                pass
+
+    return output
+
+
+def split(values: List[Any], split_count: int) -> List[Any]:
+    """
+    Splits ``values`` into ``split_count`` pieces.
+    """
+    list_length = len(values)
+    return [
+        values[
+            (list_length * i // split_count): (list_length * (i + 1) // split_count)
+        ]
+        for i in range(split_count)
+    ]
+
+
+def get_new_shuffling(seed: Hash32,
+                      validators: List[ValidatorRecord],
+                      crosslinking_start_shard: int) -> List[List[ShardCommittee]]:
+    """
+    Shuffles ``validators`` into shard committees using ``seed`` as entropy.
+    """
+    active_validator_indices = get_active_validator_indices(validators)
+
+    committees_per_slot = max(
+        1,
+        min(
+            SHARD_COUNT // EPOCH_LENGTH,
+            len(active_validator_indices) // EPOCH_LENGTH // TARGET_COMMITTEE_SIZE,
+        )
+    )
+
+    # Shuffle with seed
+    shuffled_active_validator_indices = shuffle(active_validator_indices, seed)
+
+    # Split the shuffled list into epoch_length pieces
+    validators_per_slot = split(
+        shuffled_active_validator_indices, EPOCH_LENGTH)
+
+    output = []
+    for slot, slot_indices in enumerate(validators_per_slot):
+        # Split the shuffled list into committees_per_slot pieces
+        shard_indices = split(slot_indices, committees_per_slot)
+
+        shard_id_start = crosslinking_start_shard + slot * committees_per_slot
+
+        shard_committees = [
+            ShardCommittee(
+                shard=(shard_id_start + shard_position) % SHARD_COUNT,
+                committee=indices,
+                total_validator_count=len(active_validator_indices),
+            )
+            for shard_position, indices in enumerate(shard_indices)
+        ]
+        output.append(shard_committees)
+
+    return output

eth2_testgen/shuffling/enums.py

@@ -0,0 +1,9 @@
+from enum import IntEnum
+
+
+class ValidatorStatusCode(IntEnum):
+    PENDING_ACTIVATION = 0
+    ACTIVE = 1
+    ACTIVE_PENDING_EXIT = 2
+    EXITED_WITHOUT_PENALTY = 3
+    EXITED_WITH_PENALTY = 4