[Remote signer] Fold signer into Lighthouse repository (#1852)

The remote signer relies on the `types` and `crypto/bls` crates from Lighthouse. Moreover, a number of tests of the remote signer consumption of LH leverages this very signer, making any important update a potential dependency nightmare.

Co-authored-by: Paul Hauner <[email protected]>

Cargo.toml

@@ -63,10 +63,15 @@ members = [
     "lighthouse",
     "lighthouse/environment",
 
-    "testing/simulator",
+    "remote_signer",
+    "remote_signer/backend",
+    "remote_signer/client",
+
     "testing/ef_tests",
     "testing/eth1_test_rig",
     "testing/node_test_rig",
+    "testing/remote_signer_test",
+    "testing/simulator",
     "testing/state_transition_vectors",
 
     "validator_client",

Makefile

@@ -74,7 +74,6 @@ build-release-tarballs:
 	$(MAKE) build-aarch64-portable
 	$(call tarball_release_binary,$(BUILD_PATH_AARCH64),$(AARCH64_TAG),"-portable")
 
-
 # Runs the full workspace tests in **release**, without downloading any additional
 # test vectors.
 test-release:

lighthouse/Cargo.toml

@@ -36,6 +36,7 @@ eth2_testnet_config = { path = "../common/eth2_testnet_config" }
 directory = { path = "../common/directory" }
 lighthouse_version = { path = "../common/lighthouse_version" }
 account_utils = { path = "../common/account_utils" }
+remote_signer = { "path" = "../remote_signer" }
 
 [dev-dependencies]
 tempfile = "3.1.0"

lighthouse/environment/src/lib.rs

@@ -15,7 +15,7 @@ use futures::channel::{
 };
 use futures::{future, StreamExt};
 
-use slog::{info, o, Drain, Level, Logger};
+use slog::{error, info, o, Drain, Level, Logger};
 use sloggers::{null::NullLoggerBuilder, Build};
 use std::cell::RefCell;
 use std::ffi::OsStr;
@@ -395,9 +395,16 @@ impl<E: EthSpec> Environment<E> {
         // setup for handling a Ctrl-C
         let (ctrlc_send, ctrlc_oneshot) = oneshot::channel();
         let ctrlc_send_c = RefCell::new(Some(ctrlc_send));
+        let log = self.log.clone();
         ctrlc::set_handler(move || {
             if let Some(ctrlc_send) = ctrlc_send_c.try_borrow_mut().unwrap().take() {
-                ctrlc_send.send(()).expect("Error sending ctrl-c message");
+                if let Err(e) = ctrlc_send.send(()) {
+                    error!(
+                        log,
+                        "Error sending ctrl-c message";
+                        "error" => e
+                    );
+                }
             }
         })
         .map_err(|e| format!("Could not set ctrlc handler: {:?}", e))?;

lighthouse/src/main.rs

@@ -125,6 +125,7 @@ fn main() {
         .subcommand(boot_node::cli_app())
         .subcommand(validator_client::cli_app())
         .subcommand(account_manager::cli_app())
+        .subcommand(remote_signer::cli_app())
         .get_matches();
 
     // Debugging output for libp2p and external crates.
@@ -292,7 +293,7 @@ fn run<E: EthSpec>(
                         .shutdown_sender()
                         .try_send("Failed to start beacon node");
                 }
-            })
+            });
         }
         ("validator_client", Some(matches)) => {
             let context = environment.core_context();
@@ -316,7 +317,17 @@ fn run<E: EthSpec>(
                         .shutdown_sender()
                         .try_send("Failed to start validator client");
                 }
-            })
+            });
+        }
+        ("remote_signer", Some(matches)) => {
+            if let Err(e) = remote_signer::run(&mut environment, matches) {
+                crit!(log, "Failed to start remote signer"; "reason" => e);
+                let _ = environment
+                    .core_context()
+                    .executor
+                    .shutdown_sender()
+                    .try_send("Failed to start remote signer");
+            }
         }
         _ => {
             crit!(log, "No subcommand supplied. See --help .");

remote_signer/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "remote_signer"
+version = "0.2.0"
+authors = ["Sigma Prime <[email protected]>"]
+edition = "2018"
+
+[features]
+# Compiles the BLS crypto code so that the binary is portable across machines.
+portable = ["bls/supranational-portable"]
+# Uses the slower Milagro BLS library, which is written in native Rust.
+milagro = ["bls/milagro"]
+
+[dev-dependencies]
+client_backend = { path = "./backend", package = "remote_signer_backend" }
+helpers = { path = "../testing/remote_signer_test", package = "remote_signer_test" }
+
+[dependencies]
+bls = { path = "../crypto/bls" }
+clap = "2.33.3"
+client = { path = "./client", package = "remote_signer_client" }
+environment = { path = "../lighthouse/environment" }
+serde_json = "1.0.58"
+slog = { version = "2.5.2", features = ["max_level_trace"] }
+types = { path = "../consensus/types"}

remote_signer/README.md

@@ -0,0 +1,137 @@
+# Remote BLS Signer
+
+## Overview
+
+Simple HTTP BLS signer service.
+
+This service is designed to be consumed by Ethereum 2.0 clients, looking for a more secure avenue to store their BLS12-381 secret keys, while running their validators in more permisive and/or scalable environments.
+
+One goal of this package is to be standard compliant. There is a [current draft for an Ethereum Improvement Proposal (EIP)](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-3030.md) in progress. Please refer to the [wishlist](#wishlist--roadmap) in this very document for a list of advanced features.
+
+## API
+
+### Standard
+
+### `GET /upcheck`
+
+_**Responses**_
+
+Success | <br>
+--- | ---
+Code | `200`
+Content | `{"status": "OK"}`
+
+---
+
+### `GET /keys`
+
+Returns the identifiers of the keys available to the signer.
+
+_**Responses**_
+
+Success | <br>
+--- | ---
+Code | `200`
+Content | `{"keys": "[identifier]"}`
+
+---
+
+### `POST /sign/:identifier`
+
+URL Parameter | <br>
+--- | ---
+`:identifier` | `public_key_hex_string_without_0x`
+
+_**Request**_
+
+JSON Body | <br> | <br>
+--- | --- | ---
+`bls_domain` | **Required** | The BLS Signature domain.<br>As defined in the [specification](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/phase0/beacon-chain.md#domain-types), in lowercase, omitting the `domain` prefix.<br>Supporting `beacon_proposer`, `beacon_attester`, and `randao`.
+`data` | **Required** | The data to be signed.<br>As defined in the specifications for [block](https://github.com/ethereum/eth2.0-APIs/blob/master/types/block.yaml), [attestation](https://github.com/ethereum/eth2.0-APIs/blob/master/types/attestation.yaml), and [epoch](https://github.com/ethereum/eth2.0-APIs/blob/master/types/misc.yaml).
+`fork` | **Required** | A `Fork` object containing previous and current versions.<br>As defined in the [specification](https://github.com/ethereum/eth2.0-APIs/blob/master/types/misc.yaml)
+`genesis_validators_root` | **Required** | A `Hash256` for domain separation and chain versioning.
+<br> | Optional | Any other field will be ignored by the signer
+
+_**Responses**_
+
+Success | <br>
+--- | ---
+Code |  `200`
+Content | `{"signature": "<signature_hex_string>"}`
+
+_or_
+
+Error | <br>
+--- | ---
+Code |  `400`
+Content | `{"error": "<Bad Request Error Message>"}`
+
+_or_
+
+Error | <br>
+--- | ---
+Code |  `404`
+Content | `{"error": "Key not found: <identifier>"}`
+
+## Build instructions
+
+1. [Get Rust](https://www.rust-lang.org/learn/get-started).
+2. Go to the root directory of this repository.
+3. Execute `make`
+4. The binary `lighthouse` will most likely be found in `./target/release`.
+5. Run it as `lighthouse remote_signer` or `lighthouse rs`.
+
+## Running the signer
+
+### Storing the secret keys as raw files
+
+* Steps to store a secret key
+  * Choose an empty directory, as the backend will parse every file looking for keys.
+  * Create a file named after the **hex representation of the public key without 0x**.
+  * Write the **hex representation of the secret key without 0x**.
+  * Store the file in your chosen directory.
+  * Use this directory as a command line parameter (`--storage-raw-dir`)
+
+### Command line flags
+
+```
+USAGE:
+    remote_signer [OPTIONS]
+
+FLAGS:
+    -h, --help       Prints help information
+    -V, --version    Prints version information
+
+OPTIONS:
+        --debug-level <LEVEL>         The verbosity level for emitting logs. [default: info]  [possible values:
+                                      info, debug, trace, warn, error, crit]
+        --listen-address <ADDRESS>    The address to listen for TCP connections. [default: 0.0.0.0]
+        --log-format <FORMAT>         Specifies the format used for logging. [possible values: JSON]
+        --logfile <FILE>              File path where output will be written.
+        --port <PORT>                 The TCP port to listen on. [default: 9000]
+        --spec <TITLE>                Specifies the default eth2 spec type. [default: mainnet]  [possible values:
+                                      mainnet, minimal, interop]
+        --storage-raw-dir <DIR>       Data directory for secret keys in raw files.
+```
+
+## Roadmap
+
+- [X] EIP standard compliant
+- [ ] Metrics
+- [ ] Benchmarking & Profiling
+- [ ] Release management
+- [ ] Architecture builds
+- [ ] Support EIP-2335, BLS12-381 keystore
+- [ ] Support storage in AWS Cloud HSM
+- [ ] Route with the `warp` library
+- [ ] Filter by the `message` field
+  - [ ] Middleware REST API
+  - [ ] Built-in middleware
+  - [ ] Flag to enforce the `message` field and compare it to the signing root
+- [ ] TLS/SSL support for requests
+- [ ] Authentication by HTTP Header support
+- [ ] Confidential computing support (e.g. Intel SGX)
+
+## LICENSE
+
+* Apache 2.0.

remote_signer/backend/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "remote_signer_backend"
+version = "0.2.0"
+authors = ["Herman Junge <[email protected]>"]
+edition = "2018"
+
+[dev-dependencies]
+helpers = { path = "../../testing/remote_signer_test", package = "remote_signer_test"  }
+sloggers = "1.0.1"
+tempdir = "0.3.7"
+
+[dependencies]
+bls = { path = "../../crypto/bls" }
+clap = "2.33.3"
+hex = "0.4.2"
+lazy_static = "1.4.0"
+regex = "1.3.9"
+slog = "2.5.2"
+types = { path = "../../consensus/types" }
+zeroize = { version = "1.1.1", features = ["zeroize_derive"] }