chain: Instrument provider initialization

Author: lutter

chain/ethereum/src/adapter.rs

@@ -727,10 +727,32 @@ impl EthereumBlockFilter {
     }
 }
 
+pub enum ProviderStatus {
+    Working,
+    VersionFail,
+    GenesisFail,
+    VersionTimeout,
+    GenesisTimeout,
+}
+
+impl From<ProviderStatus> for f64 {
+    fn from(state: ProviderStatus) -> Self {
+        match state {
+            ProviderStatus::Working => 0.0,
+            ProviderStatus::VersionFail => 1.0,
+            ProviderStatus::GenesisFail => 2.0,
+            ProviderStatus::VersionTimeout => 3.0,
+            ProviderStatus::GenesisTimeout => 4.0,
+        }
+    }
+}
+
+const STATUS_HELP: &str = "0 = ok, 1 = net_version failed, 2 = get genesis failed, 3 = net_version timeout, 4 = get genesis timeout";
 #[derive(Clone)]
 pub struct ProviderEthRpcMetrics {
     request_duration: Box<HistogramVec>,
     errors: Box<CounterVec>,
+    status: Box<GaugeVec>,
 }
 
 impl ProviderEthRpcMetrics {
@@ -750,9 +772,18 @@ impl ProviderEthRpcMetrics {
                 vec![String::from("method"), String::from("provider")],
             )
             .unwrap();
+        let status_help = format!("Whether the provider has failed ({STATUS_HELP})");
+        let status = registry
+            .new_gauge_vec(
+                "eth_rpc_status",
+                &status_help,
+                vec![String::from("provider")],
+            )
+            .unwrap();
         Self {
             request_duration,
             errors,
+            status,
         }
     }
 
@@ -765,6 +796,12 @@ impl ProviderEthRpcMetrics {
     pub fn add_error(&self, method: &str, provider: &str) {
         self.errors.with_label_values(&[method, provider]).inc();
     }
+
+    pub fn set_status(&self, status: ProviderStatus, provider: &str) {
+        self.status
+            .with_label_values(&[provider])
+            .set(status.into());
+    }
 }
 
 #[derive(Clone)]

chain/ethereum/src/ethereum_adapter.rs

@@ -42,6 +42,7 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Instant;
 
+use crate::adapter::ProviderStatus;
 use crate::chain::BlockFinality;
 use crate::{
     adapter::{
@@ -820,29 +821,56 @@ impl EthereumAdapterTrait for EthereumAdapter {
         let logger = self.logger.clone();
 
         let web3 = self.web3.clone();
+        let metrics = self.metrics.clone();
+        let provider = self.provider().to_string();
         let net_version_future = retry("net_version RPC call", &logger)
             .no_limit()
             .timeout_secs(20)
             .run(move || {
                 let web3 = web3.cheap_clone();
-                async move { web3.net().version().await.map_err(Into::into) }
+                let metrics = metrics.cheap_clone();
+                let provider = provider.clone();
+                async move {
+                    web3.net().version().await.map_err(|e| {
+                        metrics.set_status(ProviderStatus::VersionFail, &provider);
+                        e.into()
+                    })
+                }
+            })
+            .map_err(|e| {
+                self.metrics
+                    .set_status(ProviderStatus::VersionTimeout, self.provider());
+                e
             })
             .boxed();
 
         let web3 = self.web3.clone();
+        let metrics = self.metrics.clone();
+        let provider = self.provider().to_string();
         let gen_block_hash_future = retry("eth_getBlockByNumber(0, false) RPC call", &logger)
             .no_limit()
             .timeout_secs(30)
             .run(move || {
                 let web3 = web3.cheap_clone();
+                let metrics = metrics.cheap_clone();
+                let provider = provider.clone();
                 async move {
                     web3.eth()
                         .block(BlockId::Number(Web3BlockNumber::Number(0.into())))
-                        .await?
+                        .await
+                        .map_err(|e| {
+                            metrics.set_status(ProviderStatus::GenesisFail, &provider);
+                            e
+                        })?
                         .map(|gen_block| gen_block.hash.map(BlockHash::from))
                         .flatten()
                         .ok_or_else(|| anyhow!("Ethereum node could not find genesis block"))
                 }
+            })
+            .map_err(|e| {
+                self.metrics
+                    .set_status(ProviderStatus::GenesisTimeout, self.provider());
+                e
             });
 
         let (net_version, genesis_block_hash) =
@@ -858,6 +886,8 @@ impl EthereumAdapterTrait for EthereumAdapter {
             genesis_block_hash,
         };
 
+        self.metrics
+            .set_status(ProviderStatus::Working, self.provider());
         Ok(ident)
     }