diff --git a/Cargo.lock b/Cargo.lock index 22c3cf62d5d..8044a3799fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1848,6 +1848,7 @@ dependencies = [ name = "eth2_config" version = "0.2.0" dependencies = [ + "paste", "serde", "serde_derive", "types", @@ -4639,6 +4640,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "paste" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" + [[package]] name = "pbkdf2" version = "0.4.0" diff --git a/common/eth2_config/Cargo.toml b/common/eth2_config/Cargo.toml index 0cb3f1dfbc1..fe8708cd223 100644 --- a/common/eth2_config/Cargo.toml +++ b/common/eth2_config/Cargo.toml @@ -8,3 +8,4 @@ edition = "2018" serde = "1.0.116" serde_derive = "1.0.116" types = { path = "../../consensus/types" } +paste = "1.0.5" diff --git a/common/eth2_config/src/lib.rs b/common/eth2_config/src/lib.rs index c30dab5df7a..01b7eb73b03 100644 --- a/common/eth2_config/src/lib.rs +++ b/common/eth2_config/src/lib.rs @@ -1,7 +1,16 @@ +//! This crate primarily exists to serve the `common/eth2_network_configs` crate, by providing the +//! canonical list of built-in-networks and some tooling to help include those configurations in the +//! `lighthouse` binary. +//! +//! It also provides some additional structs which are useful to other components of `lighthouse` +//! (e.g., `Eth2Config`). + use std::env; use std::path::PathBuf; use types::{ChainSpec, EthSpecId}; +pub use paste::paste; + // A macro is used to define this constant so it can be used with `include_bytes!`. #[macro_export] macro_rules! predefined_networks_dir { @@ -73,40 +82,154 @@ impl<'a> Eth2NetArchiveAndDirectory<'a> { } } -macro_rules! define_net { - ($title: ident, $macro_title: tt, $name: tt, $genesis_is_known: tt) => { - #[macro_use] - pub mod $title { - use super::*; - - pub const ETH2_NET_DIR: Eth2NetArchiveAndDirectory = Eth2NetArchiveAndDirectory { - name: $name, - unique_id: $name, - genesis_is_known: $genesis_is_known, - }; - - // A wrapper around `std::include_bytes` which includes a file from a specific network - // directory. Used by upstream crates to import files at compile time. - #[macro_export] - macro_rules! $macro_title { - ($base_dir: tt, $filename: tt) => { - include_bytes!(concat!( - $base_dir, - "/", - predefined_networks_dir!(), - "/", - $name, - "/", - $filename - )) +/// Indicates that the `genesis.ssz.zip` file is present on the filesystem. This means that the +/// deposit ceremony has concluded and the final genesis `BeaconState` is known. +const GENESIS_STATE_IS_KNOWN: bool = true; + +#[derive(Copy, Clone, Debug, PartialEq)] +pub struct HardcodedNet { + pub name: &'static str, + pub genesis_is_known: bool, + pub config: &'static [u8], + pub deploy_block: &'static [u8], + pub boot_enr: &'static [u8], + pub genesis_state_bytes: &'static [u8], +} + +/// Defines an `Eth2NetArchiveAndDirectory` for some network. +/// +/// It also defines a `include__bytes!` macro which provides a wrapper around +/// `std::include_bytes`, allowing the inclusion of bytes from the specific testnet directory. +macro_rules! define_archive { + ($name_ident: ident, $name_str: tt, $genesis_is_known: ident) => { + paste! { + #[macro_use] + pub mod $name_ident { + use super::*; + + pub const ETH2_NET_DIR: Eth2NetArchiveAndDirectory = Eth2NetArchiveAndDirectory { + name: $name_str, + unique_id: $name_str, + genesis_is_known: $genesis_is_known, }; + + /// A wrapper around `std::include_bytes` which includes a file from a specific network + /// directory. Used by upstream crates to import files at compile time. + #[macro_export] + macro_rules! [<include_ $name_ident _file>] { + ($this_crate: ident, $base_dir: tt, $filename: tt) => { + include_bytes!(concat!( + $base_dir, + "/", + $this_crate::predefined_networks_dir!(), + "/", + $name_str, + "/", + $filename + )) + }; + } } } }; } -define_net!(pyrmont, include_pyrmont_file, "pyrmont", true); +/// Creates a `HardcodedNet` definition for some network. +#[macro_export] +macro_rules! define_net { + ($this_crate: tt, $mod: ident, $include_file: tt) => {{ + use $this_crate::$mod::ETH2_NET_DIR; + + $this_crate::HardcodedNet { + name: ETH2_NET_DIR.name, + genesis_is_known: ETH2_NET_DIR.genesis_is_known, + config: $this_crate::$include_file!($this_crate, "../", "config.yaml"), + deploy_block: $this_crate::$include_file!($this_crate, "../", "deploy_block.txt"), + boot_enr: $this_crate::$include_file!($this_crate, "../", "boot_enr.yaml"), + genesis_state_bytes: $this_crate::$include_file!($this_crate, "../", "genesis.ssz"), + } + }}; +} + +/// Calls `define_net` on a list of networks, and then defines two more lists: +/// +/// - `HARDCODED_NETS`: a list of all the networks defined by this macro. +/// - `HARDCODED_NET_NAMES`: a list of the *names* of the networks defined by this macro. +#[macro_export] +macro_rules! define_nets { + ($this_crate: ident, $($name_ident: ident, $name_str: tt,)+) => { + $this_crate::paste! { + $( + const [<$name_ident:upper>]: $this_crate::HardcodedNet = $this_crate::define_net!($this_crate, $name_ident, [<include_ $name_ident _file>]); + )+ + const HARDCODED_NETS: &[$this_crate::HardcodedNet] = &[$([<$name_ident:upper>],)+]; + pub const HARDCODED_NET_NAMES: &[&'static str] = &[$($name_str,)+]; + } + }; +} + +/// The canonical macro for defining built-in network configurations. +/// +/// This macro will provide: +/// +/// - An `Eth2NetArchiveAndDirectory` for each network. +/// - `ETH2_NET_DIRS`: a list of all the above `Eth2NetArchiveAndDirectory`. +/// - The `instantiate_hardcoded_nets` macro (see its documentation). +/// +/// ## Design Justification +/// +/// Ultimately, this macro serves as a single list of all the networks. The reason it is structured +/// in such a complex web-of-macros way is because two requirements of built-in (hard-coded) networks: +/// +/// 1. We must use `std::include_bytes!` to "bake" arbitrary bytes (genesis states, etc) into the binary. +/// 2. We must use a `build.rs` script to decompress the genesis state from a zip file, before we +/// can include those bytes. +/// +/// Because of these two constraints, we must first define all of the networks and the paths to +/// their files in this crate. Then, we must use another crate (`eth2_network_configs`) to run a +/// `build.rs` which will unzip the genesis states. Then, that `eth2_network_configs` crate can +/// perform the final step of using `std::include_bytes` to bake the files (bytes) into the binary. +macro_rules! define_hardcoded_nets { + ($(($name_ident: ident, $name_str: tt, $genesis_is_known: ident)),+) => { + paste! { + $( + define_archive!($name_ident, $name_str, $genesis_is_known); + )+ + + pub const ETH2_NET_DIRS: &[Eth2NetArchiveAndDirectory<'static>] = &[$($name_ident::ETH2_NET_DIR,)+]; -define_net!(mainnet, include_mainnet_file, "mainnet", true); + /// This macro is designed to be called by an external crate. When called, it will + /// define in that external crate: + /// + /// - A `HardcodedNet` for each network. + /// - `HARDCODED_NETS`: a list of all the above `HardcodedNet`. + /// - `HARDCODED_NET_NAMES`: a list of all the names of the above `HardcodedNet` (as `&str`). + #[macro_export] + macro_rules! instantiate_hardcoded_nets { + ($this_crate: ident) => { + $this_crate::define_nets!($this_crate, $($name_ident, $name_str,)+); + } + } + } + }; +} -define_net!(prater, include_prater_file, "prater", true); +// Add a new "built-in" network by adding it to the list below. +// +// The last entry must not end with a comma, otherwise compilation will fail. +// +// This is the canonical place for defining the built-in network configurations that are present in +// the `common/eth2_network_config/built_in_network_configs` directory. +// +// Each net is defined as a three-tuple: +// +// 0. The name of the testnet as an "ident" (i.e. something that can be a Rust variable name). +// 1. The human-friendly name of the testnet (i.e. usually with "-" instead of "_"). +// 2. A bool indicating if the genesis state is known and present as a `genesis.ssz.zip`. +// +// The directory containing the testnet files should match the human-friendly name (element 1). +define_hardcoded_nets!( + (mainnet, "mainnet", GENESIS_STATE_IS_KNOWN), + (pyrmont, "pyrmont", GENESIS_STATE_IS_KNOWN), + (prater, "prater", GENESIS_STATE_IS_KNOWN) +); diff --git a/common/eth2_network_config/build.rs b/common/eth2_network_config/build.rs index d84dbde4d88..fa45fafa4e7 100644 --- a/common/eth2_network_config/build.rs +++ b/common/eth2_network_config/build.rs @@ -1,15 +1,9 @@ //! Extracts zipped genesis states on first run. -use eth2_config::{mainnet, prater, pyrmont, Eth2NetArchiveAndDirectory, GENESIS_FILE_NAME}; +use eth2_config::{Eth2NetArchiveAndDirectory, ETH2_NET_DIRS, GENESIS_FILE_NAME}; use std::fs::File; use std::io; use zip::ZipArchive; -const ETH2_NET_DIRS: &[Eth2NetArchiveAndDirectory<'static>] = &[ - mainnet::ETH2_NET_DIR, - pyrmont::ETH2_NET_DIR, - prater::ETH2_NET_DIR, -]; - fn main() { for network in ETH2_NET_DIRS { match uncompress_state(network) { diff --git a/common/eth2_network_config/src/lib.rs b/common/eth2_network_config/src/lib.rs index 4f38905931c..1fa7a52321a 100644 --- a/common/eth2_network_config/src/lib.rs +++ b/common/eth2_network_config/src/lib.rs @@ -1,6 +1,18 @@ -use eth2_config::{predefined_networks_dir, *}; +//! Provides the `Eth2NetworkConfig` struct which defines the configuration of an eth2 network or +//! test-network (aka "testnet"). +//! +//! Whilst the `Eth2NetworkConfig` struct can be used to read a specification from a directory at +//! runtime, this crate also includes some pre-defined network configurations "built-in" to the +//! binary itself (the most notable of these being the "mainnet" configuration). When a network is +//! "built-in", the genesis state and configuration files is included in the final binary via the +//! `std::include_bytes` macro. This provides convenience to the user, the binary is self-sufficient +//! and does not require the configuration to be read from the filesystem at runtime. +//! +//! To add a new built-in testnet, add it to the `define_hardcoded_nets` invocation in the `eth2_config` +//! crate. use enr::{CombinedKey, Enr}; +use eth2_config::{instantiate_hardcoded_nets, HardcodedNet}; use std::fs::{create_dir_all, File}; use std::io::{Read, Write}; use std::path::PathBuf; @@ -11,36 +23,13 @@ pub const BOOT_ENR_FILE: &str = "boot_enr.yaml"; pub const GENESIS_STATE_FILE: &str = "genesis.ssz"; pub const BASE_CONFIG_FILE: &str = "config.yaml"; -#[derive(Copy, Clone, Debug, PartialEq)] -pub struct HardcodedNet { - pub name: &'static str, - pub genesis_is_known: bool, - pub config: &'static [u8], - pub deploy_block: &'static [u8], - pub boot_enr: &'static [u8], - pub genesis_state_bytes: &'static [u8], -} - -macro_rules! define_net { - ($mod: ident, $include_file: tt) => {{ - use eth2_config::$mod::ETH2_NET_DIR; - - HardcodedNet { - name: ETH2_NET_DIR.name, - genesis_is_known: ETH2_NET_DIR.genesis_is_known, - config: $include_file!("../", "config.yaml"), - deploy_block: $include_file!("../", "deploy_block.txt"), - boot_enr: $include_file!("../", "boot_enr.yaml"), - genesis_state_bytes: $include_file!("../", "genesis.ssz"), - } - }}; -} - -const PYRMONT: HardcodedNet = define_net!(pyrmont, include_pyrmont_file); -const MAINNET: HardcodedNet = define_net!(mainnet, include_mainnet_file); -const PRATER: HardcodedNet = define_net!(prater, include_prater_file); +// Creates definitions for: +// +// - Each of the `HardcodedNet` values (e.g., `MAINNET`, `PYRMONT`, etc). +// - `HARDCODED_NETS: &[HardcodedNet]` +// - `HARDCODED_NET_NAMES: &[&'static str]` +instantiate_hardcoded_nets!(eth2_config); -const HARDCODED_NETS: &[HardcodedNet] = &[PYRMONT, MAINNET, PRATER]; pub const DEFAULT_HARDCODED_NETWORK: &str = "mainnet"; /// Specifies an Eth2 network. @@ -241,6 +230,19 @@ mod tests { type E = MainnetEthSpec; + #[test] + fn default_network_exists() { + assert!(HARDCODED_NET_NAMES.contains(&DEFAULT_HARDCODED_NETWORK)); + } + + #[test] + fn hardcoded_testnet_names() { + assert_eq!(HARDCODED_NET_NAMES.len(), HARDCODED_NETS.len()); + for (name, net) in HARDCODED_NET_NAMES.iter().zip(HARDCODED_NETS.iter()) { + assert_eq!(name, &net.name); + } + } + #[test] fn mainnet_config_eq_chain_spec() { let config = Eth2NetworkConfig::from_hardcoded_net(&MAINNET).unwrap(); diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index 547e8b8c863..ccfe121efe9 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -8,7 +8,7 @@ use clap_utils::flags::DISABLE_MALLOC_TUNING_FLAG; use env_logger::{Builder, Env}; use environment::EnvironmentBuilder; use eth2_hashing::have_sha_extensions; -use eth2_network_config::{Eth2NetworkConfig, DEFAULT_HARDCODED_NETWORK}; +use eth2_network_config::{Eth2NetworkConfig, DEFAULT_HARDCODED_NETWORK, HARDCODED_NET_NAMES}; use lighthouse_version::VERSION; use malloc_utils::configure_memory_allocator; use slog::{crit, info, warn}; @@ -127,7 +127,7 @@ fn main() { .long("network") .value_name("network") .help("Name of the Eth2 chain Lighthouse will sync and follow.") - .possible_values(&["pyrmont", "mainnet", "prater"]) + .possible_values(HARDCODED_NET_NAMES) .conflicts_with("testnet-dir") .takes_value(true) .global(true)