Update slashing protection interchange to v5 (#1816)

## Proposed Changes

Update the slashing protection interchange format to v5 in preparation for finalisation as part of an EIP.

Also, add some more tests and update the commit hash for https://github.com/eth2-clients/slashing-protection-interchange-tests to include the new generated tests.

book/src/slashing-protection.md

@@ -60,11 +60,12 @@ Examples where it is **ineffective** are:
 
 ## Import and Export
 
-Lighthouse supports v4 of the slashing protection interchange format described
+Lighthouse supports v5 of the slashing protection interchange format described
 [here][interchange-spec]. An interchange file is a record of all blocks and attestations
 signing by a set of validator keys – basically a portable slashing protection database!
 
-You can import a `.json` interchange file from another client using this command:
+With your validator client stopped, you can import a `.json` interchange file from another client
+using this command:
 
 ```bash
 lighthouse account validator slashing-protection import <my_interchange.json>
@@ -85,6 +86,8 @@ You can export Lighthouse's database for use with another client with this comma
 lighthouse account validator slashing-protection export <lighthouse_interchange.json>
 ```
 
+The validator client needs to be stopped in order to export.
+
 [interchange-spec]: https://hackmd.io/@sproul/Bk0Y0qdGD
 
 ## Troubleshooting

validator_client/slashing_protection/Makefile

@@ -1,4 +1,4 @@
-TESTS_TAG := ac393b815b356c95569c028c215232b512df583d
+TESTS_TAG := 359085be9da6e5e19644977aa45947bcec5d99de
 GENERATE_DIR := generated-tests
 OUTPUT_DIR := interchange-tests
 TARBALL := $(OUTPUT_DIR)-$(TESTS_TAG).tar.gz

validator_client/slashing_protection/src/bin/test_generator.rs

@@ -1,6 +1,5 @@
 use slashing_protection::interchange::{
-    CompleteInterchangeData, Interchange, InterchangeFormat, InterchangeMetadata,
-    SignedAttestation, SignedBlock,
+    Interchange, InterchangeData, InterchangeMetadata, SignedAttestation, SignedBlock,
 };
 use slashing_protection::interchange_test::TestCase;
 use slashing_protection::test_utils::{pubkey, DEFAULT_GENESIS_VALIDATORS_ROOT};
@@ -11,31 +10,54 @@ use types::{Epoch, Hash256, Slot};
 
 fn metadata(genesis_validators_root: Hash256) -> InterchangeMetadata {
     InterchangeMetadata {
-        interchange_format: InterchangeFormat::Complete,
         interchange_format_version: SUPPORTED_INTERCHANGE_FORMAT_VERSION,
         genesis_validators_root,
     }
 }
 
-#[allow(clippy::type_complexity)]
-fn interchange(data: Vec<(usize, Vec<u64>, Vec<(u64, u64)>)>) -> Interchange {
+type TestPubkey = usize;
+type TestBlocks = Vec<u64>;
+type TestBlocksWithRoots = Vec<(u64, Option<u64>)>;
+type TestAttestations = Vec<(u64, u64)>;
+type TestAttestationsWithRoots = Vec<(u64, u64, Option<u64>)>;
+
+fn interchange(data: Vec<(TestPubkey, TestBlocks, TestAttestations)>) -> Interchange {
+    let data = data
+        .into_iter()
+        .map(|(pk, blocks, attestations)| {
+            (
+                pk,
+                blocks.into_iter().map(|slot| (slot, None)).collect(),
+                attestations
+                    .into_iter()
+                    .map(|(source, target)| (source, target, None))
+                    .collect(),
+            )
+        })
+        .collect();
+    interchange_with_signing_roots(data)
+}
+
+fn interchange_with_signing_roots(
+    data: Vec<(TestPubkey, TestBlocksWithRoots, TestAttestationsWithRoots)>,
+) -> Interchange {
     let data = data
         .into_iter()
-        .map(|(pk, blocks, attestations)| CompleteInterchangeData {
+        .map(|(pk, blocks, attestations)| InterchangeData {
             pubkey: pubkey(pk),
             signed_blocks: blocks
                 .into_iter()
-                .map(|slot| SignedBlock {
+                .map(|(slot, signing_root)| SignedBlock {
                     slot: Slot::new(slot),
-                    signing_root: None,
+                    signing_root: signing_root.map(Hash256::from_low_u64_be),
                 })
                 .collect(),
             signed_attestations: attestations
                 .into_iter()
-                .map(|(source, target)| SignedAttestation {
+                .map(|(source, target, signing_root)| SignedAttestation {
                     source_epoch: Epoch::new(source),
                     target_epoch: Epoch::new(target),
-                    signing_root: None,
+                    signing_root: signing_root.map(Hash256::from_low_u64_be),
                 })
                 .collect(),
         })
@@ -110,11 +132,56 @@ fn main() {
             (0, 11, 12, true),
             (0, 20, 25, true),
         ]),
+        TestCase::new(
+            "single_validator_single_block_and_attestation_signing_root",
+            interchange_with_signing_roots(vec![(0, vec![(19, Some(1))], vec![(0, 1, Some(2))])]),
+        ),
+        TestCase::new(
+            "multiple_validators_multiple_blocks_and_attestations",
+            interchange(vec![
+                (
+                    0,
+                    vec![10, 15, 20],
+                    vec![(0, 1), (0, 2), (1, 3), (2, 4), (4, 5)],
+                ),
+                (
+                    1,
+                    vec![3, 4, 100],
+                    vec![(0, 0), (0, 1), (1, 2), (2, 5), (5, 6)],
+                ),
+                (2, vec![10, 15, 20], vec![(1, 2), (1, 3), (2, 4)]),
+            ]),
+        )
+        .with_blocks(vec![
+            (0, 9, false),
+            (0, 10, false),
+            (0, 21, true),
+            (0, 11, true),
+            (1, 2, false),
+            (1, 3, false),
+            (1, 0, false),
+            (1, 101, true),
+            (2, 9, false),
+            (2, 10, false),
+            (2, 22, true),
+        ])
+        .with_attestations(vec![
+            (0, 0, 5, false),
+            (0, 3, 6, false),
+            (0, 4, 6, true),
+            (0, 5, 7, true),
+            (0, 6, 8, true),
+            (1, 1, 7, false),
+            (1, 1, 4, true),
+            (1, 5, 7, true),
+            (2, 0, 0, false),
+            (2, 0, 1, false),
+            (2, 2, 5, true),
+        ]),
         TestCase::new("wrong_genesis_validators_root", interchange(vec![]))
             .gvr(Hash256::from_low_u64_be(1))
             .should_fail(),
     ];
-    // TODO: multi-validator test
 
     let args = std::env::args().collect::<Vec<_>>();
     let output_dir = Path::new(&args[1]);

validator_client/slashing_protection/src/interchange.rs

@@ -3,24 +3,17 @@ use std::collections::HashSet;
 use std::iter::FromIterator;
 use types::{Epoch, Hash256, PublicKey, Slot};
 
-#[derive(Debug, Clone, Copy, PartialEq, Deserialize, Serialize)]
-#[serde(rename_all = "lowercase")]
-pub enum InterchangeFormat {
-    Complete,
-}
-
 #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
 #[serde(deny_unknown_fields)]
 pub struct InterchangeMetadata {
-    pub interchange_format: InterchangeFormat,
     #[serde(with = "serde_utils::quoted_u64::require_quotes")]
     pub interchange_format_version: u64,
     pub genesis_validators_root: Hash256,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize)]
 #[serde(deny_unknown_fields)]
-pub struct CompleteInterchangeData {
+pub struct InterchangeData {
     pub pubkey: PublicKey,
     pub signed_blocks: Vec<SignedBlock>,
     pub signed_attestations: Vec<SignedAttestation>,
@@ -49,7 +42,7 @@ pub struct SignedAttestation {
 #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
 pub struct Interchange {
     pub metadata: InterchangeMetadata,
-    pub data: Vec<CompleteInterchangeData>,
+    pub data: Vec<InterchangeData>,
 }
 
 impl Interchange {

validator_client/slashing_protection/src/slashing_database.rs

@@ -1,6 +1,6 @@
 use crate::interchange::{
-    CompleteInterchangeData, Interchange, InterchangeFormat, InterchangeMetadata,
-    SignedAttestation as InterchangeAttestation, SignedBlock as InterchangeBlock,
+    Interchange, InterchangeData, InterchangeMetadata, SignedAttestation as InterchangeAttestation,
+    SignedBlock as InterchangeBlock,
 };
 use crate::signed_attestation::InvalidAttestation;
 use crate::signed_block::InvalidBlock;
@@ -25,7 +25,7 @@ pub const CONNECTION_TIMEOUT: Duration = Duration::from_secs(5);
 pub const CONNECTION_TIMEOUT: Duration = Duration::from_millis(100);
 
 /// Supported version of the interchange format.
-pub const SUPPORTED_INTERCHANGE_FORMAT_VERSION: u64 = 4;
+pub const SUPPORTED_INTERCHANGE_FORMAT_VERSION: u64 = 5;
 
 #[derive(Debug, Clone)]
 pub struct SlashingDatabase {
@@ -673,15 +673,14 @@ impl SlashingDatabase {
         .collect::<Result<_, InterchangeError>>()?;
 
         let metadata = InterchangeMetadata {
-            interchange_format: InterchangeFormat::Complete,
             interchange_format_version: SUPPORTED_INTERCHANGE_FORMAT_VERSION,
             genesis_validators_root,
         };
 
         let data = data
             .into_iter()
             .map(|(pubkey, (signed_blocks, signed_attestations))| {
-                Ok(CompleteInterchangeData {
+                Ok(InterchangeData {
                     pubkey: pubkey.parse().map_err(InterchangeError::InvalidPubkey)?,
                     signed_blocks,
                     signed_attestations,