feat: add new lazy loading parameters and improve documentation (#3063)

node/cli-opt/src/lib.rs

@@ -115,8 +115,10 @@ pub struct RpcConfig {
 
 #[derive(Clone)]
 pub struct LazyLoadingConfig {
-	pub state_rpc: String,
+	pub state_rpc: url::Url,
 	pub from_block: Option<H256>,
 	pub state_overrides_path: Option<PathBuf>,
 	pub runtime_override: Option<PathBuf>,
+	pub delay_between_requests: u32,
+	pub max_retries_per_request: u32,
 }

node/cli/Cargo.toml

@@ -9,6 +9,7 @@ clap = { workspace = true, features = ["derive"] }
 clap-num = { workspace = true }
 log = { workspace = true }
 parity-scale-codec = { workspace = true }
+url = { workspace = true }
 
 # Moonbeam
 moonbeam-cli-opt = { workspace = true }

node/cli/src/cli.rs

@@ -25,13 +25,25 @@ use moonbeam_service::chain_spec;
 use sc_cli::{Error as CliError, SubstrateCli};
 use std::path::PathBuf;
 use std::time::Duration;
+use url::Url;
 
 #[cfg(feature = "lazy-loading")]
 fn parse_block_hash(s: &str) -> Result<sp_core::H256, String> {
 	use std::str::FromStr;
 	sp_core::H256::from_str(s).map_err(|err| err.to_string())
 }
 
+fn validate_url(arg: &str) -> Result<Url, String> {
+	let url = Url::parse(arg).map_err(|e| e.to_string())?;
+
+	let scheme = url.scheme();
+	if scheme == "http" || scheme == "https" {
+		Ok(url)
+	} else {
+		Err(format!("'{}' URL scheme not supported.", url.scheme()))
+	}
+}
+
 /// Sub-commands supported by the collator.
 #[derive(Debug, clap::Subcommand)]
 pub enum Subcommand {
@@ -147,21 +159,69 @@ pub struct RunCmd {
 	#[clap(long)]
 	pub experimental_block_import_strategy: bool,
 
+	/// Specifies the URL used to fetch chain data via RPC.
+	///
+	/// The URL should point to the RPC endpoint of the chain being forked.
+	/// Ensure that the RPC has sufficient rate limits to handle the expected load.
 	#[cfg(feature = "lazy-loading")]
 	#[clap(long)]
-	pub fork_chain_from_rpc: Option<String>,
-
+	#[arg(
+		long,
+		value_parser = validate_url,
+		alias = "fork-chain-from-rpc"
+	)]
+	pub lazy_loading_remote_rpc: Option<Url>,
+
+	/// Optional parameter to specify the block hash for lazy loading.
+	///
+	/// This parameter allows the user to specify a block hash from which to start loading data.
+	///
+	/// If not provided, the latest block will be used.
 	#[cfg(feature = "lazy-loading")]
-	#[arg(long, value_name = "BLOCK", value_parser = parse_block_hash)]
-	pub block: Option<sp_core::H256>,
+	#[arg(
+		long,
+		value_name = "BLOCK",
+		value_parser = parse_block_hash,
+		alias = "block"
+	)]
+	pub lazy_loading_block: Option<sp_core::H256>,
+
+	/// Optional parameter to specify state overrides during lazy loading.
+	///
+	/// This parameter allows the user to provide a path to a file containing state overrides.
+	/// The file can contain any custom state modifications that should be applied.
+	#[cfg(feature = "lazy-loading")]
+	#[clap(
+		long,
+		value_name = "PATH",
+		value_parser,
+		alias = "fork-state-overrides"
+	)]
+	pub lazy_loading_state_overrides: Option<PathBuf>,
+
+	/// Optional parameter to specify a runtime override when starting the lazy loading.
+	///
+	/// If not provided, it will fetch the runtime from the block being forked.
+	#[cfg(feature = "lazy-loading")]
+	#[clap(long, value_name = "PATH", value_parser, alias = "runtime-override")]
+	pub lazy_loading_runtime_override: Option<PathBuf>,
 
+	/// The delay (in milliseconds) between RPC requests when using lazy loading.
+	///
+	/// This parameter controls the amount of time (in milliseconds) to wait between consecutive
+	/// RPC requests. This can help manage request rate and avoid overwhelming the server.
+	///
+	/// The default value is 100 milliseconds.
 	#[cfg(feature = "lazy-loading")]
-	#[clap(long, value_name = "PATH", value_parser)]
-	pub fork_state_overrides: Option<PathBuf>,
+	#[clap(long, default_value = "100")]
+	pub lazy_loading_delay_between_requests: u32,
 
+	/// The maximum number of retries for an RPC request when using lazy loading.
+	///
+	/// The default value is 10 retries.
 	#[cfg(feature = "lazy-loading")]
-	#[clap(long, value_name = "PATH", value_parser)]
-	pub runtime_override: Option<PathBuf>,
+	#[clap(long, default_value = "10")]
+	pub lazy_loading_max_retries_per_request: u32,
 
 	/// When blocks should be sealed in the dev service.
 	///

node/cli/src/command.rs

@@ -849,16 +849,18 @@ pub fn run() -> Result<()> {
 					};
 				}
 				#[cfg(feature = "lazy-loading")]
-				if let Some(fork_chain_from_rpc) = cli.run.fork_chain_from_rpc {
+				if let Some(lazy_loading_remote_rpc) = cli.run.lazy_loading_remote_rpc {
 					let author_id = Some(chain_spec::get_from_seed::<nimbus_primitives::NimbusId>(
 						"Alice",
 					));
 
 					let lazy_loading_config = moonbeam_cli_opt::LazyLoadingConfig {
-						state_rpc: fork_chain_from_rpc,
-						from_block: cli.run.block,
-						state_overrides_path: cli.run.fork_state_overrides,
-						runtime_override: cli.run.runtime_override,
+						state_rpc: lazy_loading_remote_rpc,
+						from_block: cli.run.lazy_loading_block,
+						state_overrides_path: cli.run.lazy_loading_state_overrides,
+						runtime_override: cli.run.lazy_loading_runtime_override,
+						delay_between_requests: cli.run.lazy_loading_delay_between_requests,
+						max_retries_per_request: cli.run.lazy_loading_max_retries_per_request,
 					};
 
 					let spec_builder =

node/service/src/lazy_loading/backend.rs

@@ -1455,16 +1455,16 @@ pub fn check_genesis_storage(storage: &Storage) -> sp_blockchain::Result<()> {
 #[derive(Debug, Clone)]
 pub struct RPC {
 	http_client: HttpClient,
-	delay_between_requests_ms: u64,
-	max_retries_per_request: usize,
+	delay_between_requests_ms: u32,
+	max_retries_per_request: u32,
 	counter: Arc<ReadWriteLock<u64>>,
 }
 
 impl RPC {
 	pub fn new(
 		http_client: HttpClient,
-		delay_between_requests_ms: u64,
-		max_retries_per_request: usize,
+		delay_between_requests_ms: u32,
+		max_retries_per_request: u32,
 	) -> Self {
 		Self {
 			http_client,
@@ -1654,7 +1654,8 @@ impl RPC {
 
 		tokio::task::block_in_place(move || {
 			Handle::current().block_on(async move {
-				let delay_between_requests = Duration::from_millis(self.delay_between_requests_ms);
+				let delay_between_requests =
+					Duration::from_millis(self.delay_between_requests_ms.into());
 
 				let start = std::time::Instant::now();
 				self.counter.write().add_assign(1);
@@ -1669,8 +1670,8 @@ impl RPC {
 
 				// Retry request in case of failure
 				// The maximum number of retries is specified by `self.max_retries_per_request`
-				let retry_strategy =
-					FixedInterval::new(delay_between_requests).take(self.max_retries_per_request);
+				let retry_strategy = FixedInterval::new(delay_between_requests)
+					.take(self.max_retries_per_request as usize);
 				let result = Retry::spawn(retry_strategy, f).await;
 
 				log::debug!(
@@ -1696,20 +1697,22 @@ where
 	Block: BlockT + DeserializeOwned,
 	Block::Hash: From<H256>,
 {
-	let uri: String = lazy_loading_config.state_rpc.clone().into();
-
 	let http_client = jsonrpsee::http_client::HttpClientBuilder::default()
 		.max_request_size(u32::MAX)
 		.max_response_size(u32::MAX)
 		.request_timeout(Duration::from_secs(10))
-		.build(uri)
+		.build(lazy_loading_config.state_rpc.clone())
 		.map_err(|e| {
 			sp_blockchain::Error::Backend(
 				format!("failed to build http client: {:?}", e).to_string(),
 			)
 		})?;
 
-	let rpc = RPC::new(http_client, 100, 10);
+	let rpc = RPC::new(
+		http_client,
+		lazy_loading_config.delay_between_requests,
+		lazy_loading_config.max_retries_per_request,
+	);
 	let block_hash = lazy_loading_config
 		.from_block
 		.map(|block| Into::<Block::Hash>::into(block));

test/moonwall.config.json

@@ -368,6 +368,8 @@
             "binPath": "../target/release/moonbeam",
             "newRpcBehaviour": true,
             "options": [
+              "--lazy-loading-delay-between-requests=10",
+              "--lazy-loading-max-retries-per-request=5",
               "--ethapi=txpool",
               "--no-hardware-benchmarks",
               "--no-telemetry",