Squashed commit of #2489

commit e22b271 Author: Paul Hauner <[email protected]> Date: Wed Aug 4 10:55:15 2021 +1000 Separate name_ident and name_str commit d7e8408 Author: Paul Hauner <[email protected]> Date: Tue Aug 3 10:58:54 2021 +1000 Naming consistency fix commit cd73518 Author: Paul Hauner <[email protected]> Date: Tue Aug 3 10:44:26 2021 +1000 Refactor for a single source of truth commit 83280d6 Author: Paul Hauner <[email protected]> Date: Mon Aug 2 18:27:51 2021 +1000 Add test for name consistency commit 99be2ea Author: Paul Hauner <[email protected]> Date: Mon Aug 2 15:55:40 2021 +1000 Switch define_nets to lower case commit 777b419 Author: Paul Hauner <[email protected]> Date: Mon Aug 2 15:52:41 2021 +1000 Add instructional comments commit 33dda79 Author: Paul Hauner <[email protected]> Date: Mon Aug 2 15:36:14 2021 +1000 Use hardcoded list in CLI commit 892efe4 Author: Paul Hauner <[email protected]> Date: Mon Aug 2 15:34:58 2021 +1000 Add paste to eth2_config mod commit 03d6e49 Author: Paul Hauner <[email protected]> Date: Mon Aug 2 15:23:06 2021 +1000 Add hardcoded net names commit d3dadc8 Author: Paul Hauner <[email protected]> Date: Mon Aug 2 15:10:01 2021 +1000 Use paste in eth2_network_config commit 5053f53 Author: Paul Hauner <[email protected]> Date: Mon Aug 2 15:03:15 2021 +1000 Add define_nets!
sigp · Aug 4, 2021 · 47d0dff · 47d0dff
1 parent 187425c
commit 47d0dff
Show file tree

Hide file tree

Showing 6 changed files with 195 additions and 68 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/common/eth2_config/Cargo.toml b/common/eth2_config/Cargo.toml
@@ -8,3 +8,4 @@ edition = "2018"
 serde = "1.0.116"
 serde_derive = "1.0.116"
 types = { path = "../../consensus/types" }
+paste = "1.0.5"
diff --git a/common/eth2_config/src/lib.rs b/common/eth2_config/src/lib.rs
@@ -1,7 +1,16 @@
+//! This crate primarily exists to serve the `common/eth2_network_configs` crate, by providing the
+//! canonical list of built-in-networks and some tooling to help include those configurations in the
+//! `lighthouse` binary.
+//!
+//! It also provides some additional structs which are useful to other components of `lighthouse`
+//! (e.g., `Eth2Config`).
+
 use std::env;
 use std::path::PathBuf;
 use types::{ChainSpec, EthSpecId};
 
+pub use paste::paste;
+
 // A macro is used to define this constant so it can be used with `include_bytes!`.
 #[macro_export]
 macro_rules! predefined_networks_dir {
@@ -73,40 +82,154 @@ impl<'a> Eth2NetArchiveAndDirectory<'a> {
     }
 }
 
-macro_rules! define_net {
-    ($title: ident, $macro_title: tt, $name: tt, $genesis_is_known: tt) => {
-        #[macro_use]
-        pub mod $title {
-            use super::*;
-
-            pub const ETH2_NET_DIR: Eth2NetArchiveAndDirectory = Eth2NetArchiveAndDirectory {
-                name: $name,
-                unique_id: $name,
-                genesis_is_known: $genesis_is_known,
-            };
-
-            // A wrapper around `std::include_bytes` which includes a file from a specific network
-            // directory. Used by upstream crates to import files at compile time.
-            #[macro_export]
-            macro_rules! $macro_title {
-                ($base_dir: tt, $filename: tt) => {
-                    include_bytes!(concat!(
-                        $base_dir,
-                        "/",
-                        predefined_networks_dir!(),
-                        "/",
-                        $name,
-                        "/",
-                        $filename
-                    ))
+/// Indicates that the `genesis.ssz.zip` file is present on the filesystem. This means that the
+/// deposit ceremony has concluded and the final genesis `BeaconState` is known.
+const GENESIS_STATE_IS_KNOWN: bool = true;
+
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub struct HardcodedNet {
+    pub name: &'static str,
+    pub genesis_is_known: bool,
+    pub config: &'static [u8],
+    pub deploy_block: &'static [u8],
+    pub boot_enr: &'static [u8],
+    pub genesis_state_bytes: &'static [u8],
+}
+
+/// Defines an `Eth2NetArchiveAndDirectory` for some network.
+///
+/// It also defines a `include_<title>_bytes!` macro which provides a wrapper around
+/// `std::include_bytes`, allowing the inclusion of bytes from the specific testnet directory.
+macro_rules! define_archive {
+    ($name_ident: ident, $name_str: tt, $genesis_is_known: ident) => {
+        paste! {
+            #[macro_use]
+            pub mod $name_ident {
+                use super::*;
+
+                pub const ETH2_NET_DIR: Eth2NetArchiveAndDirectory = Eth2NetArchiveAndDirectory {
+                    name: $name_str,
+                    unique_id: $name_str,
+                    genesis_is_known: $genesis_is_known,
                 };
+
+                /// A wrapper around `std::include_bytes` which includes a file from a specific network
+                /// directory. Used by upstream crates to import files at compile time.
+                #[macro_export]
+                macro_rules! [<include_ $name_ident _file>] {
+                    ($this_crate: ident, $base_dir: tt, $filename: tt) => {
+                        include_bytes!(concat!(
+                            $base_dir,
+                            "/",
+                            $this_crate::predefined_networks_dir!(),
+                            "/",
+                            $name_str,
+                            "/",
+                            $filename
+                        ))
+                    };
+                }
             }
         }
     };
 }
 
-define_net!(pyrmont, include_pyrmont_file, "pyrmont", true);
+/// Creates a `HardcodedNet` definition for some network.
+#[macro_export]
+macro_rules! define_net {
+    ($this_crate: tt, $mod: ident, $include_file: tt) => {{
+        use $this_crate::$mod::ETH2_NET_DIR;
+
+        $this_crate::HardcodedNet {
+            name: ETH2_NET_DIR.name,
+            genesis_is_known: ETH2_NET_DIR.genesis_is_known,
+            config: $this_crate::$include_file!($this_crate, "../", "config.yaml"),
+            deploy_block: $this_crate::$include_file!($this_crate, "../", "deploy_block.txt"),
+            boot_enr: $this_crate::$include_file!($this_crate, "../", "boot_enr.yaml"),
+            genesis_state_bytes: $this_crate::$include_file!($this_crate, "../", "genesis.ssz"),
+        }
+    }};
+}
+
+/// Calls `define_net` on a list of networks, and then defines two more lists:
+///
+/// - `HARDCODED_NETS`: a list of all the networks defined by this macro.
+/// - `HARDCODED_NET_NAMES`: a list of the *names* of the networks defined by this macro.
+#[macro_export]
+macro_rules! define_nets {
+    ($this_crate: ident, $($name_ident: ident, $name_str: tt,)+) => {
+        $this_crate::paste! {
+            $(
+            const [<$name_ident:upper>]: $this_crate::HardcodedNet = $this_crate::define_net!($this_crate, $name_ident, [<include_ $name_ident _file>]);
+            )+
+            const HARDCODED_NETS: &[$this_crate::HardcodedNet] = &[$([<$name_ident:upper>],)+];
+            pub const HARDCODED_NET_NAMES: &[&'static str] = &[$($name_str,)+];
+        }
+    };
+}
+
+/// The canonical macro for defining built-in network configurations.
+///
+/// This macro will provide:
+///
+/// - An `Eth2NetArchiveAndDirectory` for each network.
+/// - `ETH2_NET_DIRS`: a list of all the above `Eth2NetArchiveAndDirectory`.
+/// - The `instantiate_hardcoded_nets` macro (see its documentation).
+///
+/// ## Design Justification
+///
+/// Ultimately, this macro serves as a single list of all the networks. The reason it is structured
+/// in such a complex web-of-macros way is because two requirements of built-in (hard-coded) networks:
+///
+/// 1. We must use `std::include_bytes!` to "bake" arbitrary bytes (genesis states, etc) into the binary.
+/// 2. We must use a `build.rs` script to decompress the genesis state from a zip file, before we
+///    can include those bytes.
+///
+/// Because of these two constraints, we must first define all of the networks and the paths to
+/// their files in this crate. Then, we must use another crate (`eth2_network_configs`) to run a
+/// `build.rs` which will unzip the genesis states. Then, that `eth2_network_configs` crate can
+/// perform the final step of using `std::include_bytes` to bake the files (bytes) into the binary.
+macro_rules! define_hardcoded_nets {
+    ($(($name_ident: ident, $name_str: tt, $genesis_is_known: ident)),+) => {
+        paste! {
+            $(
+            define_archive!($name_ident, $name_str, $genesis_is_known);
+            )+
+
+            pub const ETH2_NET_DIRS: &[Eth2NetArchiveAndDirectory<'static>] = &[$($name_ident::ETH2_NET_DIR,)+];
 
-define_net!(mainnet, include_mainnet_file, "mainnet", true);
+            /// This macro is designed to be called by an external crate. When called, it will
+            /// define in that external crate:
+            ///
+            /// - A `HardcodedNet` for each network.
+            /// - `HARDCODED_NETS`: a list of all the above `HardcodedNet`.
+            /// - `HARDCODED_NET_NAMES`: a list of all the names of the above `HardcodedNet` (as `&str`).
+            #[macro_export]
+            macro_rules! instantiate_hardcoded_nets {
+                ($this_crate: ident) => {
+                    $this_crate::define_nets!($this_crate, $($name_ident, $name_str,)+);
+                }
+            }
+        }
+    };
+}
 
-define_net!(prater, include_prater_file, "prater", true);
+// Add a new "built-in" network by adding it to the list below.
+//
+// The last entry must not end with a comma, otherwise compilation will fail.
+//
+// This is the canonical place for defining the built-in network configurations that are present in
+// the `common/eth2_network_config/built_in_network_configs` directory.
+//
+// Each net is defined as a three-tuple:
+//
+// 0. The name of the testnet as an "ident" (i.e. something that can be a Rust variable name).
+// 1. The human-friendly name of the testnet (i.e. usually with "-" instead of "_").
+// 2. A bool indicating if the genesis state is known and present as a `genesis.ssz.zip`.
+//
+// The directory containing the testnet files should match the human-friendly name (element 1).
+define_hardcoded_nets!(
+    (mainnet, "mainnet", GENESIS_STATE_IS_KNOWN),
+    (pyrmont, "pyrmont", GENESIS_STATE_IS_KNOWN),
+    (prater, "prater", GENESIS_STATE_IS_KNOWN)
+);
diff --git a/common/eth2_network_config/build.rs b/common/eth2_network_config/build.rs
@@ -1,15 +1,9 @@
 //! Extracts zipped genesis states on first run.
-use eth2_config::{mainnet, prater, pyrmont, Eth2NetArchiveAndDirectory, GENESIS_FILE_NAME};
+use eth2_config::{Eth2NetArchiveAndDirectory, ETH2_NET_DIRS, GENESIS_FILE_NAME};
 use std::fs::File;
 use std::io;
 use zip::ZipArchive;
 
-const ETH2_NET_DIRS: &[Eth2NetArchiveAndDirectory<'static>] = &[
-    mainnet::ETH2_NET_DIR,
-    pyrmont::ETH2_NET_DIR,
-    prater::ETH2_NET_DIR,
-];
-
 fn main() {
     for network in ETH2_NET_DIRS {
         match uncompress_state(network) {

diff --git a/common/eth2_network_config/src/lib.rs b/common/eth2_network_config/src/lib.rs
@@ -1,6 +1,18 @@
-use eth2_config::{predefined_networks_dir, *};
+//! Provides the `Eth2NetworkConfig` struct which defines the configuration of an eth2 network or
+//! test-network (aka "testnet").
+//!
+//! Whilst the `Eth2NetworkConfig` struct can be used to read a specification from a directory at
+//! runtime, this crate also includes some pre-defined network configurations "built-in" to the
+//! binary itself (the most notable of these being the "mainnet" configuration). When a network is
+//! "built-in", the  genesis state and configuration files is included in the final binary via the
+//! `std::include_bytes` macro. This provides convenience to the user, the binary is self-sufficient
+//! and does not require the configuration to be read from the filesystem at runtime.
+//!
+//! To add a new built-in testnet, add it to the `define_hardcoded_nets` invocation in the `eth2_config`
+//! crate.
 
 use enr::{CombinedKey, Enr};
+use eth2_config::{instantiate_hardcoded_nets, HardcodedNet};
 use std::fs::{create_dir_all, File};
 use std::io::{Read, Write};
 use std::path::PathBuf;
@@ -11,36 +23,13 @@ pub const BOOT_ENR_FILE: &str = "boot_enr.yaml";
 pub const GENESIS_STATE_FILE: &str = "genesis.ssz";
 pub const BASE_CONFIG_FILE: &str = "config.yaml";
 
-#[derive(Copy, Clone, Debug, PartialEq)]
-pub struct HardcodedNet {
-    pub name: &'static str,
-    pub genesis_is_known: bool,
-    pub config: &'static [u8],
-    pub deploy_block: &'static [u8],
-    pub boot_enr: &'static [u8],
-    pub genesis_state_bytes: &'static [u8],
-}
-
-macro_rules! define_net {
-    ($mod: ident, $include_file: tt) => {{
-        use eth2_config::$mod::ETH2_NET_DIR;
-
-        HardcodedNet {
-            name: ETH2_NET_DIR.name,
-            genesis_is_known: ETH2_NET_DIR.genesis_is_known,
-            config: $include_file!("../", "config.yaml"),
-            deploy_block: $include_file!("../", "deploy_block.txt"),
-            boot_enr: $include_file!("../", "boot_enr.yaml"),
-            genesis_state_bytes: $include_file!("../", "genesis.ssz"),
-        }
-    }};
-}
-
-const PYRMONT: HardcodedNet = define_net!(pyrmont, include_pyrmont_file);
-const MAINNET: HardcodedNet = define_net!(mainnet, include_mainnet_file);
-const PRATER: HardcodedNet = define_net!(prater, include_prater_file);
+// Creates definitions for:
+//
+// - Each of the `HardcodedNet` values (e.g., `MAINNET`, `PYRMONT`, etc).
+// - `HARDCODED_NETS: &[HardcodedNet]`
+// - `HARDCODED_NET_NAMES: &[&'static str]`
+instantiate_hardcoded_nets!(eth2_config);
 
-const HARDCODED_NETS: &[HardcodedNet] = &[PYRMONT, MAINNET, PRATER];
 pub const DEFAULT_HARDCODED_NETWORK: &str = "mainnet";
 
 /// Specifies an Eth2 network.
@@ -241,6 +230,19 @@ mod tests {
 
     type E = MainnetEthSpec;
 
+    #[test]
+    fn default_network_exists() {
+        assert!(HARDCODED_NET_NAMES.contains(&DEFAULT_HARDCODED_NETWORK));
+    }
+
+    #[test]
+    fn hardcoded_testnet_names() {
+        assert_eq!(HARDCODED_NET_NAMES.len(), HARDCODED_NETS.len());
+        for (name, net) in HARDCODED_NET_NAMES.iter().zip(HARDCODED_NETS.iter()) {
+            assert_eq!(name, &net.name);
+        }
+    }
+
     #[test]
     fn mainnet_config_eq_chain_spec() {
         let config = Eth2NetworkConfig::from_hardcoded_net(&MAINNET).unwrap();

diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs
@@ -8,7 +8,7 @@ use clap_utils::flags::DISABLE_MALLOC_TUNING_FLAG;
 use env_logger::{Builder, Env};
 use environment::EnvironmentBuilder;
 use eth2_hashing::have_sha_extensions;
-use eth2_network_config::{Eth2NetworkConfig, DEFAULT_HARDCODED_NETWORK};
+use eth2_network_config::{Eth2NetworkConfig, DEFAULT_HARDCODED_NETWORK, HARDCODED_NET_NAMES};
 use lighthouse_version::VERSION;
 use malloc_utils::configure_memory_allocator;
 use slog::{crit, info, warn};
@@ -127,7 +127,7 @@ fn main() {
                 .long("network")
                 .value_name("network")
                 .help("Name of the Eth2 chain Lighthouse will sync and follow.")
-                .possible_values(&["pyrmont", "mainnet", "prater"])
+                .possible_values(HARDCODED_NET_NAMES)
                 .conflicts_with("testnet-dir")
                 .takes_value(true)
                 .global(true)