feat: support multiple bootnodes & static peer ids (#81)

Co-authored-by: Barnabas Busa <[email protected]>
kurtosis-tech · Aug 3, 2023 · a71204c · a71204c
1 parent 6db2e07
commit a71204c
Show file tree

Hide file tree

Showing 8 changed files with 95 additions and 42 deletions.
diff --git a/package_io/constants.star b/package_io/constants.star
@@ -23,3 +23,7 @@ GLOBAL_CLIENT_LOG_LEVEL = struct(
 )
 
 VALIDATING_REWARDS_ACCOUNT = "0x878705ba3f8Bc32FCf7F4CAa1A35E72AF65CF766"
+
+# TODO - fix this and not have this at all after lodestar generates proper ENRs
+# the current one is invalid as the IP field is missing on decode
+ENR_TO_SKIP = "skip"
diff --git a/src/cl/cl_client_context.star b/src/cl/cl_client_context.star
@@ -5,13 +5,17 @@ def new_cl_client_context(
 	http_port_num,
 	cl_nodes_metrics_info,
 	beacon_service_name,
-	validator_service_name = ""):
+	validator_service_name = "",
+	multiaddr = "",
+	peer_id = ""):
 	return struct(
 		client_name = client_name,
 		enr = enr,
 		ip_addr = ip_addr,
 		http_port_num = http_port_num,
 		cl_nodes_metrics_info = cl_nodes_metrics_info,
 		beacon_service_name = beacon_service_name,
-		validator_service_name = validator_service_name
+		validator_service_name = validator_service_name,
+		multiaddr = multiaddr,
+		peer_id = peer_id,
 	)
diff --git a/src/cl/lighthouse/lighthouse_launcher.star b/src/cl/lighthouse/lighthouse_launcher.star
@@ -79,7 +79,7 @@ def launch(
 	image,
 	participant_log_level,
 	global_log_level,
-	bootnode_context,
+	bootnode_contexts,
 	el_client_context,
 	node_keystore_files,
 	bn_min_cpu,
@@ -112,7 +112,7 @@ def launch(
 	beacon_config = get_beacon_config(
 		launcher.genesis_data,
 		image,
-		bootnode_context,
+		bootnode_contexts,
 		el_client_context,
 		log_level,
 		bn_min_cpu,
@@ -149,10 +149,15 @@ def launch(
 		endpoint = "/eth/v1/node/identity",
 		port_id = BEACON_HTTP_PORT_ID,
 		extract = {
-			"enr": ".data.enr"
+			"enr": ".data.enr",
+			"multiaddr": ".data.discovery_addresses[0]",
+			"peer_id": ".data.peer_id"
 		}
 	)
-	beacon_node_enr = plan.request(recipe = beacon_node_identity_recipe, service_name = beacon_node_service_name)["extract.enr"]
+	response = plan.request(recipe = beacon_node_identity_recipe, service_name = beacon_node_service_name)
+	beacon_node_enr = response["extract.enr"]
+	beacon_multiaddr = response["extract.multiaddr"]
+	beacon_peer_id = response["extract.peer_id"]
 
 	beacon_metrics_port = beacon_service.ports[BEACON_METRICS_PORT_ID]
 	beacon_metrics_url = "{0}:{1}".format(beacon_service.ip_address, beacon_metrics_port.number)
@@ -171,14 +176,16 @@ def launch(
 		BEACON_HTTP_PORT_NUM,
 		nodes_metrics_info,
 		beacon_node_service_name,
-		validator_node_service_name
+		validator_node_service_name,
+		beacon_multiaddr,
+		beacon_peer_id
 	)
 
 
 def get_beacon_config(
 	genesis_data,
 	image,
-	boot_cl_client_ctx,
+	boot_cl_client_ctxs,
 	el_client_ctx,
 	log_level,
 	bn_min_cpu,
@@ -239,8 +246,9 @@ def get_beacon_config(
 		# ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^
 	]
 
-	if boot_cl_client_ctx != None:
-		cmd.append("--boot-nodes="+boot_cl_client_ctx.enr)
+	if boot_cl_client_ctxs != None:
+		cmd.append("--boot-nodes="+",".join([ctx.enr for ctx in boot_cl_client_ctxs if ctx.enr != package_io.ENR_TO_SKIP]))
+		cmd.append("--trusted-peers="+",".join([ctx.peer_id for ctx in boot_cl_client_ctxs]))
 
 	if len(extra_params) > 0:
 		# this is a repeated<proto type>, we convert it into Starlark

diff --git a/src/cl/lodestar/lodestar_launcher.star b/src/cl/lodestar/lodestar_launcher.star
@@ -69,7 +69,7 @@ def launch(
 	image,
 	participant_log_level,
 	global_log_level,
-	bootnode_context,
+	bootnode_contexts,
 	el_client_context,
 	node_keystore_files,
 	bn_min_cpu,
@@ -103,7 +103,7 @@ def launch(
 	beacon_config = get_beacon_config(
 		launcher.cl_genesis_data,
 		image,
-		bootnode_context,
+		bootnode_contexts,
 		el_client_context,
 		log_level,
 		bn_min_cpu,
@@ -143,10 +143,16 @@ def launch(
 		endpoint = "/eth/v1/node/identity",
 		port_id = HTTP_PORT_ID,
 		extract = {
-			"enr": ".data.enr"
+			"enr": ".data.enr",
+			"multiaddr": ".data.p2p_addresses[-1]",
+			"peer_id": ".data.peer_id"
 		}
 	)
-	beacon_node_enr = plan.request(recipe = beacon_node_identity_recipe, service_name = beacon_node_service_name)["extract.enr"]
+	response = plan.request(recipe = beacon_node_identity_recipe, service_name = beacon_node_service_name)
+	# TODO fix this after lodestar generates valid ENR with valid IP
+	beacon_node_enr = package_io.ENR_TO_SKIP
+	beacon_multiaddr = response["extract.multiaddr"]
+	beacon_peer_id = response["extract.peer_id"]
 
 	beacon_metrics_port = beacon_service.ports[METRICS_PORT_ID]
 	beacon_metrics_url = "{0}:{1}".format(beacon_service.ip_address, beacon_metrics_port.number)
@@ -161,14 +167,16 @@ def launch(
 		HTTP_PORT_NUM,
 		nodes_metrics_info,
 		beacon_node_service_name,
-		validator_node_service_name
+		validator_node_service_name,
+		beacon_multiaddr,
+		beacon_peer_id
 	)
 
 
 def get_beacon_config(
 	genesis_data,
 	image,
-	boot_cl_client_ctx,
+	bootnode_contexts,
 	el_client_ctx,
 	log_level,
 	bn_min_cpu,
@@ -208,6 +216,7 @@ def get_beacon_config(
 		"--rest.address=0.0.0.0",
 		"--rest.namespace=*",
 		"--rest.port={0}".format(HTTP_PORT_NUM),
+		"--nat=" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
 		"--enr.ip=" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
 		"--enr.tcp={0}".format(DISCOVERY_PORT_NUM),
 		"--enr.udp={0}".format(DISCOVERY_PORT_NUM),
@@ -221,8 +230,8 @@ def get_beacon_config(
 		# ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^
 	]
 
-	if boot_cl_client_ctx != None :
-		cmd.append("--bootnodes="+boot_cl_client_ctx.enr)
+	if bootnode_contexts != None :
+		cmd.append("--bootnodes="+",".join([ctx.enr for ctx in bootnode_contexts if ctx.enr != package_io.ENR_TO_SKIP]))
 
 	if len(extra_params) > 0:
 		# this is a repeated<proto type>, we convert it into Starlark

diff --git a/src/cl/nimbus/nimbus_launcher.star b/src/cl/nimbus/nimbus_launcher.star
@@ -73,7 +73,7 @@ def launch(
 	image,
 	participant_log_level,
 	global_log_level,
-	bootnode_context,
+	bootnode_contexts,
 	el_client_context,
 	node_keystore_files,
 	bn_min_cpu,
@@ -105,7 +105,7 @@ def launch(
 	config = get_config(
 		launcher.cl_genesis_data,
 		image,
-		bootnode_context,
+		bootnode_contexts,
 		el_client_context,
 		log_level,
 		node_keystore_files,
@@ -122,10 +122,15 @@ def launch(
 		endpoint = "/eth/v1/node/identity",
 		port_id = HTTP_PORT_ID,
 		extract = {
-			"enr": ".data.enr"
+			"enr": ".data.enr",
+			"multiaddr": ".data.discovery_addresses[0]",
+			"peer_id": ".data.peer_id"
 		}
 	)
-	node_enr = plan.request(recipe = cl_node_identity_recipe, service_name = service_name)["extract.enr"]
+	response = plan.request(recipe = cl_node_identity_recipe, service_name = service_name)
+	node_enr = response["extract.enr"]
+	multiaddr = response["extract.multiaddr"]
+	peer_id = response["extract.peer_id"]
 
 	metrics_port = nimbus_service.ports[METRICS_PORT_ID]
 	metrics_url = "{0}:{1}".format(nimbus_service.ip_address, metrics_port.number)
@@ -141,13 +146,15 @@ def launch(
 		HTTP_PORT_NUM,
 		nodes_metrics_info,
 		service_name,
+		multiaddr = multiaddr,
+		peer_id = peer_id,
 	)
 
 
 def get_config(
 	genesis_data,
 	image,
-	bootnode_context,
+	bootnode_contexts,
 	el_client_ctx,
 	log_level,
 	node_keystore_files,
@@ -228,12 +235,15 @@ def get_config(
 		"--metrics-port={0}".format(METRICS_PORT_NUM),
 		# ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^
 	]
-	if bootnode_context == None:
+	if bootnode_contexts == None:
 		# Copied from https://github.com/status-im/nimbus-eth2/blob/67ab477a27e358d605e99bffeb67f98d18218eca/scripts/launch_local_testnet.sh#L417
 		# See explanation there
 		cmd.append("--subscribe-all-subnets")
 	else:
-		cmd.append("--bootstrap-node="+bootnode_context.enr)
+		for ctx in bootnode_contexts:
+			if ctx.enr != package_io.ENR_TO_SKIP:
+				cmd.append("--bootstrap-node="+ctx.enr)
+			cmd.append("--direct-peer="+ctx.multiaddr)
 
 	if len(extra_params) > 0:
 		cmd.extend([param for param in extra_params])

diff --git a/src/cl/prysm/prysm_launcher.star b/src/cl/prysm/prysm_launcher.star
@@ -3,7 +3,6 @@ input_parser = import_module("github.com/kurtosis-tech/eth-network-package/packa
 cl_client_context = import_module("github.com/kurtosis-tech/eth-network-package/src/cl/cl_client_context.star")
 cl_node_metrics = import_module("github.com/kurtosis-tech/eth-network-package/src/cl/cl_node_metrics_info.star")
 cl_node_ready_conditions = import_module("github.com/kurtosis-tech/eth-network-package/src/cl/cl_node_ready_conditions.star")
-
 package_io = import_module("github.com/kurtosis-tech/eth-network-package/package_io/constants.star")
 
 IMAGE_SEPARATOR_DELIMITER	= ","
@@ -83,7 +82,7 @@ def launch(
 	images,
 	participant_log_level,
 	global_log_level,
-	bootnode_context,
+	bootnode_contexts,
 	el_client_context,
 	node_keystore_files,
 	bn_min_cpu,
@@ -127,7 +126,7 @@ def launch(
 	beacon_config = get_beacon_config(
 		launcher.genesis_data,
 		beacon_image,
-		bootnode_context,
+		bootnode_contexts,
 		el_client_context,
 		log_level,
 		bn_min_cpu,
@@ -169,10 +168,15 @@ def launch(
 		endpoint = "/eth/v1/node/identity",
 		port_id = HTTP_PORT_ID,
 		extract = {
-			"enr": ".data.enr"
+			"enr": ".data.enr",
+			"multiaddr": ".data.discovery_addresses[0]",
+			"peer_id": ".data.peer_id"
 		}
 	)
-	beacon_node_enr = plan.request(recipe = beacon_node_identity_recipe, service_name = beacon_node_service_name)["extract.enr"]
+	response = plan.request(recipe = beacon_node_identity_recipe, service_name = beacon_node_service_name)
+	beacon_node_enr = response["extract.enr"]
+	beacon_multiaddr = response["extract.multiaddr"]
+	beacon_peer_id = response["extract.peer_id"]
 
 	beacon_metrics_port = beacon_service.ports[BEACON_MONITORING_PORT_ID]
 	beacon_metrics_url = "{0}:{1}".format(beacon_service.ip_address, beacon_metrics_port.number)
@@ -192,14 +196,16 @@ def launch(
 		HTTP_PORT_NUM,
 		nodes_metrics_info,
 		beacon_node_service_name,
-		validator_node_service_name
+		validator_node_service_name,
+		beacon_multiaddr,
+		beacon_peer_id,
 	)
 
 
 def get_beacon_config(
 		genesis_data,
 		beacon_image,
-		bootnode_context,
+		bootnode_contexts,
 		el_client_context,
 		log_level,
 		bn_min_cpu,
@@ -235,6 +241,7 @@ def get_beacon_config(
 		"--p2p-udp-port={0}".format(DISCOVERY_UDP_PORT_NUM),
 		"--min-sync-peers={0}".format(MIN_PEERS),
 		"--verbosity=" + log_level,
+		"--suggested-fee-recipient=" + package_io.VALIDATING_REWARDS_ACCOUNT,
 		# Set per Pari's recommendation to reduce noise
 		"--subscribe-all-subnets=true",
 		"--jwt-secret={0}".format(jwt_secret_filepath),
@@ -245,8 +252,12 @@ def get_beacon_config(
 		# ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^
 	]
 
-	if bootnode_context != None:
-		cmd.append("--bootstrap-node="+bootnode_context.enr)
+	if bootnode_contexts != None:
+		for ctx in bootnode_contexts:
+			cmd.append("--peer="+ctx.multiaddr)
+			if ctx.enr != package_io.ENR_TO_SKIP:
+				cmd.append("--bootstrap-node="+ctx.enr)
+		cmd.append("--p2p-static-id=true")
 
 	if len(extra_params) > 0:
 		# we do the for loop as otherwise its a proto repeated array

diff --git a/src/cl/teku/teku_launcher.star b/src/cl/teku/teku_launcher.star
@@ -124,11 +124,15 @@ def launch(
 		endpoint = "/eth/v1/node/identity",
 		port_id = HTTP_PORT_ID,
 		extract = {
-			"enr": ".data.enr"
+			"enr": ".data.enr",
+			"multiaddr": ".data.discovery_addresses[0]",
+			"peer_id": ".data.peer_id"
 		}
 	)
-	node_enr = plan.request(recipe = node_identity_recipe, service_name = service_name)["extract.enr"]
-
+	response = plan.request(recipe = node_identity_recipe, service_name = service_name)
+	node_enr = response["extract.enr"]
+	multiaddr = response["extract.multiaddr"]
+	peer_id = response["extract.peer_id"]
 
 	teku_metrics_port = teku_service.ports[METRICS_PORT_ID]
 	teku_metrics_url = "{0}:{1}".format(teku_service.ip_address, teku_metrics_port.number)
@@ -142,14 +146,16 @@ def launch(
 		teku_service.ip_address,
 		HTTP_PORT_NUM,
 		nodes_metrics_info,
-		service_name
+		service_name,
+		multiaddr = multiaddr,
+		peer_id = peer_id,
 	)
 
 
 def get_config(
 	genesis_data,
 	image,
-	boot_cl_client_ctx,
+	bootnode_contexts,
 	el_client_ctx,
 	log_level,
 	node_keystore_files,
@@ -223,8 +229,9 @@ def get_config(
 		# ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^
 	]
 
-	if boot_cl_client_ctx != None:
-		cmd.append("--p2p-discovery-bootnodes="+boot_cl_client_ctx.enr)
+	if bootnode_contexts != None:
+		cmd.append("--p2p-discovery-bootnodes="+",".join([ctx.enr for ctx in bootnode_contexts if ctx.enr != package_io.ENR_TO_SKIP]))
+		cmd.append("--p2p-static-peers="+",".join([ctx.multiaddr for ctx in bootnode_contexts]))
 
 	if len(extra_params) > 0:
 		# we do the list comprehension as the default extra_params is a proto repeated string

diff --git a/src/participant_network.star b/src/participant_network.star
@@ -198,7 +198,7 @@ def launch_participant_network(plan, participants, network_params, global_log_le
 				participant.validator_extra_params,
 			)
 		else:
-			boot_cl_client_ctx = all_cl_client_contexts[0]
+			boot_cl_client_ctx = all_cl_client_contexts
 			cl_client_context = launch_method(
 				plan,
 				cl_launcher,