diff --git a/common/primitives/core/src/teebag/types.rs b/common/primitives/core/src/teebag/types.rs index bf8b02edf5..ec65014c3b 100644 --- a/common/primitives/core/src/teebag/types.rs +++ b/common/primitives/core/src/teebag/types.rs @@ -72,6 +72,7 @@ pub enum WorkerType { #[default] Identity, BitAcross, + OmniExecutor, } #[derive(Encode, Decode, Clone, Copy, Default, PartialEq, Eq, RuntimeDebug, TypeInfo)] diff --git a/tee-worker/omni-executor/Makefile b/tee-worker/omni-executor/Makefile index 5c91358fef..cb27222bc3 100644 --- a/tee-worker/omni-executor/Makefile +++ b/tee-worker/omni-executor/Makefile @@ -23,7 +23,7 @@ endif # of the DEBUG setting is to control Gramine's loglevel. -include $(SELF_EXE).d # See also: .cargo/config.toml $(SELF_EXE): Cargo.toml - cargo build --release + cargo build --release --features=gramine-quote omni-executor.manifest: omni-executor.manifest.template gramine-manifest \ @@ -73,6 +73,6 @@ start-local: stop-local: docker compose down -.PHONY: get-omni-pallet-metadata -get-omni-pallet-metadata: - subxt metadata --url http://localhost:9944 --allow-insecure --pallets OmniAccount > parentchain/artifacts/rococo-omni-account.scale +.PHONY: get-pallet-metadata +get-pallet-metadata: + subxt metadata --url http://localhost:9944 --allow-insecure --pallets OmniAccount,Teebag > parentchain/artifacts/rococo-omni-account.scale diff --git a/tee-worker/omni-executor/docker-compose.yml b/tee-worker/omni-executor/docker-compose.yml index b3798d98ed..98c7367c10 100644 --- a/tee-worker/omni-executor/docker-compose.yml +++ b/tee-worker/omni-executor/docker-compose.yml @@ -7,6 +7,7 @@ services: - ethereum-node - litentry-node command: ["executor-worker", "ws://litentry-node:9944", "http://ethereum-node:8545"] + restart: always ethereum-node: image: ghcr.io/foundry-rs/foundry command: diff --git a/tee-worker/omni-executor/omni-executor.manifest.template b/tee-worker/omni-executor/omni-executor.manifest.template index 526945bbdc..19638e4dbf 100644 --- a/tee-worker/omni-executor/omni-executor.manifest.template +++ b/tee-worker/omni-executor/omni-executor.manifest.template @@ -44,6 +44,8 @@ sgx.trusted_files = [ # https://gramine.readthedocs.io/en/stable/manifest-syntax.html#number-of-threads sgx.max_threads = {{ '1' if env.get('EDMM', '0') == '1' else '64' }} +sgx.remote_attestation = "dcap" + # for easy demo setup loader.insecure__use_cmdline_argv = true loader.insecure__use_host_env = true \ No newline at end of file diff --git a/tee-worker/omni-executor/parentchain/artifacts/rococo-omni-account.scale b/tee-worker/omni-executor/parentchain/artifacts/rococo-omni-account.scale index 2c9a40ede9..5f18bed23f 100644 Binary files a/tee-worker/omni-executor/parentchain/artifacts/rococo-omni-account.scale and b/tee-worker/omni-executor/parentchain/artifacts/rococo-omni-account.scale differ diff --git a/tee-worker/omni-executor/parentchain/listener/Cargo.toml b/tee-worker/omni-executor/parentchain/listener/Cargo.toml index ce990f2bb4..1981ef632a 100644 --- a/tee-worker/omni-executor/parentchain/listener/Cargo.toml +++ b/tee-worker/omni-executor/parentchain/listener/Cargo.toml @@ -21,3 +21,6 @@ env_logger = { workspace = true } [lints] workspace = true + +[features] +gramine-quote = [] diff --git a/tee-worker/omni-executor/parentchain/listener/src/event_handler.rs b/tee-worker/omni-executor/parentchain/listener/src/event_handler.rs index e1f443752b..230dee4a61 100644 --- a/tee-worker/omni-executor/parentchain/listener/src/event_handler.rs +++ b/tee-worker/omni-executor/parentchain/listener/src/event_handler.rs @@ -17,6 +17,7 @@ use crate::metadata::{MetadataProvider, SubxtMetadataProvider}; use crate::primitives::BlockEvent; use crate::rpc_client::{SubstrateRpcClient, SubstrateRpcClientFactory}; +use crate::transaction_signer::TransactionSigner; use async_trait::async_trait; use executor_core::event_handler::Error::RecoverableError; use executor_core::event_handler::{Error, EventHandler}; @@ -24,19 +25,17 @@ use executor_core::intent_executor::IntentExecutor; use executor_core::key_store::KeyStore; use executor_core::primitives::Intent; use log::error; -use parity_scale_codec::Decode; use std::marker::PhantomData; -use std::sync::RwLock; -use subxt::config::DefaultExtrinsicParamsBuilder; +use std::sync::Arc; use subxt::ext::scale_decode; use subxt::ext::scale_decode::DecodeAsFields; -use subxt::ext::subxt_core::tx; use subxt::{Config, Metadata}; use subxt_core::config::DefaultExtrinsicParams; use subxt_core::utils::{AccountId32, MultiAddress, MultiSignature}; use subxt_signer::sr25519::SecretKeyBytes; pub struct IntentEventHandler< + ChainConfig: Config, MetadataT, MetadataProviderT: MetadataProvider, EthereumIntentExecutorT: IntentExecutor, @@ -44,15 +43,24 @@ pub struct IntentEventHandler< RpcClient: SubstrateRpcClient, RpcClientFactory: SubstrateRpcClientFactory, > { - metadata_provider: MetadataProviderT, + metadata_provider: Arc, ethereum_intent_executor: EthereumIntentExecutorT, - key_store: KeyStoreT, rpc_client_factory: RpcClientFactory, - nonce: RwLock, + transaction_signer: Arc< + TransactionSigner< + KeyStoreT, + RpcClient, + RpcClientFactory, + ChainConfig, + MetadataT, + MetadataProviderT, + >, + >, phantom_data: PhantomData<(MetadataT, RpcClient)>, } impl< + ChainConfig: Config, MetadataT, MetadataProviderT: MetadataProvider, EthereumIntentExecutorT: IntentExecutor, @@ -61,6 +69,7 @@ impl< RpcClientFactory: SubstrateRpcClientFactory, > IntentEventHandler< + ChainConfig, MetadataT, MetadataProviderT, EthereumIntentExecutorT, @@ -70,17 +79,25 @@ impl< > { pub fn new( - metadata_provider: MetadataProviderT, + metadata_provider: Arc, ethereum_intent_executor: EthereumIntentExecutorT, - key_store: KeyStoreT, rpc_client_factory: RpcClientFactory, + transaction_signer: Arc< + TransactionSigner< + KeyStoreT, + RpcClient, + RpcClientFactory, + ChainConfig, + MetadataT, + MetadataProviderT, + >, + >, ) -> Self { Self { metadata_provider, ethereum_intent_executor, - key_store, rpc_client_factory, - nonce: RwLock::new(0), + transaction_signer, phantom_data: Default::default(), } } @@ -100,6 +117,7 @@ impl< RpcClientFactory: SubstrateRpcClientFactory + Send + Sync, > EventHandler for IntentEventHandler< + ChainConfig, Metadata, SubxtMetadataProvider, EthereumIntentExecutorT, @@ -119,7 +137,7 @@ impl< log::debug!("Got IntentRequested event: {:?}", event.id); - let metadata = self.metadata_provider.get(event.id.block_num).await; + let metadata = self.metadata_provider.get(Some(event.id.block_num)).await; let pallet = metadata.pallet_by_name(&event.pallet_name).ok_or_else(move || { log::error!( @@ -205,51 +223,13 @@ impl< execution_result, ); - let secret_key_bytes = self - .key_store - .read() - .map_err(|e| { - error!("Could not unseal key: {:?}", e); - }) - .unwrap(); - let signer = subxt_signer::sr25519::Keypair::from_secret_key(secret_key_bytes) - .map_err(|e| { - error!("Could not create secret key: {:?}", e); - }) - .unwrap(); - let mut client = self.rpc_client_factory.new_client().await.map_err(|e| { error!("Could not create RPC client: {:?}", e); RecoverableError })?; - let runtime_version = client.runtime_version().await.map_err(|e| { - error!("Could not get runtime version: {:?}", e); - RecoverableError - })?; - let genesis_hash = client.get_genesis_hash().await.map_err(|e| { - error!("Could not get genesis hash: {:?}", e); - RecoverableError - })?; - let nonce = *self.nonce.read().map_err(|e| { - error!("Could not read nonce: {:?}", e); - RecoverableError - })?; - let params = DefaultExtrinsicParamsBuilder::::new().nonce(nonce).build(); - *self.nonce.write().map_err(|e| { - error!("Could not write nonce: {:?}", e); - RecoverableError - })? = nonce + 1; - let state = tx::ClientState:: { - metadata: { metadata }, - genesis_hash: ChainConfig::Hash::decode(&mut genesis_hash.as_slice()).unwrap(), - runtime_version: tx::RuntimeVersion { - spec_version: runtime_version.spec_version, - transaction_version: runtime_version.transaction_version, - }, - }; - let signed_call = tx::create_signed(&call, &state, &signer, params).unwrap(); - client.submit_tx(signed_call.encoded()).await.map_err(|e| { + let signed_call = self.transaction_signer.sign(call).await; + client.submit_tx(&signed_call).await.map_err(|e| { error!("Error while submitting tx: {:?}", e); RecoverableError })?; diff --git a/tee-worker/omni-executor/parentchain/listener/src/fetcher.rs b/tee-worker/omni-executor/parentchain/listener/src/fetcher.rs index 7d3dfa800f..160dce2347 100644 --- a/tee-worker/omni-executor/parentchain/listener/src/fetcher.rs +++ b/tee-worker/omni-executor/parentchain/listener/src/fetcher.rs @@ -20,20 +20,21 @@ use crate::rpc_client::SubstrateRpcClientFactory; use async_trait::async_trait; use executor_core::fetcher::{EventsFetcher, LastFinalizedBlockNumFetcher}; use log::error; +use std::sync::Arc; /// Used for fetching data from parentchain pub struct Fetcher< RpcClient: SubstrateRpcClient, RpcClientFactory: SubstrateRpcClientFactory, > { - client_factory: RpcClientFactory, + client_factory: Arc, client: Option, } impl> Fetcher { - pub fn new(client_factory: RpcClientFactory) -> Self { + pub fn new(client_factory: Arc) -> Self { Self { client: None, client_factory } } diff --git a/tee-worker/omni-executor/parentchain/listener/src/lib.rs b/tee-worker/omni-executor/parentchain/listener/src/lib.rs index 9f8fab0384..5a0d1ea9d0 100644 --- a/tee-worker/omni-executor/parentchain/listener/src/lib.rs +++ b/tee-worker/omni-executor/parentchain/listener/src/lib.rs @@ -21,22 +21,29 @@ mod listener; mod metadata; mod primitives; mod rpc_client; +mod transaction_signer; use crate::event_handler::IntentEventHandler; use crate::fetcher::Fetcher; use crate::key_store::SubstrateKeyStore; use crate::listener::ParentchainListener; use crate::metadata::SubxtMetadataProvider; +use crate::rpc_client::SubstrateRpcClient; use crate::rpc_client::{SubxtClient, SubxtClientFactory}; +use crate::transaction_signer::TransactionSigner; use executor_core::intent_executor::IntentExecutor; use executor_core::key_store::KeyStore; use executor_core::listener::Listener; use executor_core::sync_checkpoint_repository::FileCheckpointRepository; +use litentry_rococo::runtime_types::core_primitives::teebag; use log::{error, info}; use scale_encode::EncodeAsType; +use std::sync::Arc; use subxt::config::signed_extensions; use subxt::Config; use subxt_core::utils::AccountId32; +use subxt_core::Metadata; +use subxt_signer::sr25519::Keypair; use tokio::runtime::Handle; use tokio::sync::oneshot::Receiver; @@ -92,14 +99,16 @@ pub async fn create_listener, (), > { - let client_factory: SubxtClientFactory = SubxtClientFactory::new(ws_rpc_endpoint); + let client_factory: Arc> = + Arc::new(SubxtClientFactory::new(ws_rpc_endpoint)); - let fetcher = Fetcher::new(client_factory); + let fetcher = Fetcher::new(client_factory.clone()); let last_processed_log_repository = FileCheckpointRepository::new("data/parentchain_last_log.bin"); - let metadata_provider = SubxtMetadataProvider::new(SubxtClientFactory::new(ws_rpc_endpoint)); - let key_store = SubstrateKeyStore::new("/data/parentchain_key.bin".to_string()); + let metadata_provider = + Arc::new(SubxtMetadataProvider::new(SubxtClientFactory::new(ws_rpc_endpoint))); + let key_store = Arc::new(SubstrateKeyStore::new("/data/parentchain_key.bin".to_string())); let secret_key_bytes = key_store .read() .map_err(|e| { @@ -114,11 +123,19 @@ pub async fn create_listener>, + signer: Keypair, + transaction_signer: &Arc< + TransactionSigner< + SubstrateKeyStore, + SubxtClient, + SubxtClientFactory, + CustomConfig, + Metadata, + SubxtMetadataProvider, + >, + >, +) -> Result<(), ()> { + let mut quote = vec![]; + let mut attestation_type = + litentry_rococo::teebag::calls::types::register_enclave::AttestationType::Dcap( + teebag::types::DcapProvider::Intel, + ); + + #[cfg(feature = "gramine-quote")] + { + use std::fs; + use std::fs::File; + use std::io::Write; + let mut f = File::create("/dev/attestation/user_report_data").unwrap(); + let content = signer.public_key().0; + f.write_all(&content).unwrap(); + + quote = fs::read("/dev/attestation/quote").unwrap(); + info!("Attestation quote {:?}", quote); + } + #[cfg(not(feature = "gramine-quote"))] + { + attestation_type = + litentry_rococo::teebag::calls::types::register_enclave::AttestationType::Ignore; + } + + let registration_call = litentry_rococo::tx().teebag().register_enclave( + litentry_rococo::teebag::calls::types::register_enclave::WorkerType::OmniExecutor, + litentry_rococo::teebag::calls::types::register_enclave::WorkerMode::OffChainWorker, + quote, + vec![], + None, + None, + attestation_type, + ); + + let mut client = client_factory.new_client_until_connected().await; + let signed_call = transaction_signer.sign(registration_call).await; + client.submit_tx(&signed_call).await.map_err(|e| { + error!("Error while submitting tx: {:?}", e); + })?; + Ok(()) +} diff --git a/tee-worker/omni-executor/parentchain/listener/src/listener.rs b/tee-worker/omni-executor/parentchain/listener/src/listener.rs index dabd440fa0..f98f87ae52 100644 --- a/tee-worker/omni-executor/parentchain/listener/src/listener.rs +++ b/tee-worker/omni-executor/parentchain/listener/src/listener.rs @@ -38,6 +38,7 @@ pub type ParentchainListener< IntentEventId, BlockEvent, IntentEventHandler< + ChainConfig, Metadata, SubxtMetadataProvider, EthereumIntentExecutor, diff --git a/tee-worker/omni-executor/parentchain/listener/src/metadata.rs b/tee-worker/omni-executor/parentchain/listener/src/metadata.rs index c6e930ab14..0f1b202210 100644 --- a/tee-worker/omni-executor/parentchain/listener/src/metadata.rs +++ b/tee-worker/omni-executor/parentchain/listener/src/metadata.rs @@ -21,7 +21,7 @@ use subxt::{Config, Metadata}; #[async_trait] pub trait MetadataProvider { - async fn get(&self, block_num: u64) -> M; + async fn get(&self, block_num: Option) -> M; } pub struct SubxtMetadataProvider { @@ -36,9 +36,9 @@ impl SubxtMetadataProvider { #[async_trait] impl MetadataProvider for SubxtMetadataProvider { - async fn get(&self, block_num: u64) -> Metadata { + async fn get(&self, block_num: Option) -> Metadata { let mut client = self.client_factory.new_client().await.unwrap(); - let raw_metadata = client.get_raw_metadata(Some(block_num)).await.unwrap(); + let raw_metadata = client.get_raw_metadata(block_num).await.unwrap(); Metadata::decode(&mut raw_metadata.as_slice()).unwrap() } diff --git a/tee-worker/omni-executor/parentchain/listener/src/rpc_client.rs b/tee-worker/omni-executor/parentchain/listener/src/rpc_client.rs index 672aa0a53a..bc43904964 100644 --- a/tee-worker/omni-executor/parentchain/listener/src/rpc_client.rs +++ b/tee-worker/omni-executor/parentchain/listener/src/rpc_client.rs @@ -20,6 +20,8 @@ use log::error; use parity_scale_codec::Encode; use std::marker::PhantomData; use std::ops::Deref; +use std::thread; +use std::time::Duration; use subxt::backend::legacy::LegacyRpcMethods; use subxt::backend::BlockRef; use subxt::config::Header; @@ -157,6 +159,24 @@ impl SubxtClientFactory { pub fn new(url: &str) -> Self { Self { url: url.to_string(), _phantom: PhantomData } } + pub async fn new_client_until_connected(&self) -> SubxtClient { + let mut client: Option> = None; + loop { + if client.is_some() { + break; + } + match self.new_client().await { + Ok(c) => { + client = Some(c); + }, + Err(e) => { + error!("Error creating client: {:?}", e); + }, + }; + thread::sleep(Duration::from_secs(1)) + } + client.unwrap() + } } #[async_trait] diff --git a/tee-worker/omni-executor/parentchain/listener/src/transaction_signer.rs b/tee-worker/omni-executor/parentchain/listener/src/transaction_signer.rs new file mode 100644 index 0000000000..d29bd53cd8 --- /dev/null +++ b/tee-worker/omni-executor/parentchain/listener/src/transaction_signer.rs @@ -0,0 +1,118 @@ +use crate::metadata::{MetadataProvider, SubxtMetadataProvider}; +use crate::rpc_client::{SubstrateRpcClient, SubstrateRpcClientFactory}; +use executor_core::event_handler::Error::RecoverableError; +use executor_core::key_store::KeyStore; +use log::error; +use parity_scale_codec::Decode; +use std::marker::PhantomData; +use std::sync::{Arc, RwLock}; +use subxt_core::config::{DefaultExtrinsicParams, DefaultExtrinsicParamsBuilder}; +use subxt_core::tx::payload::Payload; +use subxt_core::utils::{AccountId32, MultiAddress, MultiSignature}; +use subxt_core::{tx, Config, Metadata}; +use subxt_signer::sr25519::SecretKeyBytes; + +pub struct TransactionSigner< + KeyStoreT, + RpcClient: SubstrateRpcClient, + RpcClientFactory: SubstrateRpcClientFactory, + ChainConfig, + MetadataT, + MetadataProviderT: MetadataProvider, +> { + metadata_provider: Arc, + rpc_client_factory: Arc, + key_store: Arc, + nonce: RwLock, + phantom_data: PhantomData<(RpcClient, ChainConfig, MetadataT)>, +} + +impl< + KeyStoreT: KeyStore, + RpcClient: SubstrateRpcClient, + RpcClientFactory: SubstrateRpcClientFactory, + ChainConfig: Config< + ExtrinsicParams = DefaultExtrinsicParams, + AccountId = AccountId32, + Address = MultiAddress, + Signature = MultiSignature, + >, + > + TransactionSigner< + KeyStoreT, + RpcClient, + RpcClientFactory, + ChainConfig, + Metadata, + SubxtMetadataProvider, + > +{ + pub fn new( + metadata_provider: Arc>, + rpc_client_factory: Arc, + key_store: Arc, + ) -> Self { + Self { + metadata_provider, + rpc_client_factory, + key_store, + //todo: read nonce from chain + nonce: RwLock::new(0), + phantom_data: PhantomData, + } + } + + pub async fn sign(&self, call: Call) -> Vec { + let secret_key_bytes = self + .key_store + .read() + .map_err(|e| { + error!("Could not unseal key: {:?}", e); + }) + .unwrap(); + + let signer = subxt_signer::sr25519::Keypair::from_secret_key(secret_key_bytes) + .map_err(|e| { + error!("Could not create secret key: {:?}", e); + }) + .unwrap(); + let mut client = self.rpc_client_factory.new_client().await.unwrap(); + let runtime_version = client.runtime_version().await.unwrap(); + + let genesis_hash = client.get_genesis_hash().await.unwrap(); + + let nonce = *self + .nonce + .read() + .map_err(|e| { + error!("Could not read nonce: {:?}", e); + RecoverableError + }) + .unwrap(); + + *self + .nonce + .write() + .map_err(|e| { + error!("Could not write nonce: {:?}", e); + RecoverableError + }) + .unwrap() = nonce + 1; + + // we should get latest metadata + let metadata = self.metadata_provider.get(None).await; + + let state = tx::ClientState:: { + metadata: { metadata }, + genesis_hash: ChainConfig::Hash::decode(&mut genesis_hash.as_slice()).unwrap(), + runtime_version: tx::RuntimeVersion { + spec_version: runtime_version.spec_version, + transaction_version: runtime_version.transaction_version, + }, + }; + let params = DefaultExtrinsicParamsBuilder::::new().nonce(nonce).build(); + let signed_call = tx::create_signed(&call, &state, &signer, params).unwrap(); + + signed_call.encoded().to_vec() + } +}