Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Benchmark request-vc cli command #3016

Merged
merged 6 commits into from
Aug 28, 2024
Merged

Benchmark request-vc cli command #3016

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e2de7fd
bench request-vc
kziemianek Aug 21, 2024
cb43c82
Merge branch 'dev' into p-1004-benchmark-command-to-measure-request_v…
kziemianek Aug 26, 2024
f6c8f9d
Merge branch 'dev' into p-1004-benchmark-command-to-measure-request_v…
Kailai-Wang Aug 27, 2024
343ad7b
Merge branch 'dev' into p-1004-benchmark-command-to-measure-request_v…
kziemianek Aug 28, 2024
707872c
review suggestions
kziemianek Aug 28, 2024
04cdf9a
Merge branch 'dev' into p-1004-benchmark-command-to-measure-request_v…
Kailai-Wang Aug 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
380 changes: 2 additions & 378 deletions tee-worker/cli/src/benchmark/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,378 +1,2 @@
/*
Copyright 2021 Integritee AG and Supercomputing Systems AG

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

*/

use crate::{
command_utils::get_worker_api_direct,
get_layer_two_nonce,
trusted_cli::TrustedCli,
trusted_command_utils::{get_identifiers, get_keystore_path, get_pair_from_str},
trusted_operation::{get_json_request, get_state, wait_until},
Cli, CliResult, CliResultOk, SR25519_KEY_TYPE,
};
use codec::Decode;
use hdrhistogram::Histogram;
use ita_stf::{
Getter, Index, PublicGetter, TrustedCall, TrustedCallSigned, TrustedGetter, STF_TX_FEE,
};
use itc_rpc_client::direct_client::{DirectApi, DirectClient};
use itp_stf_primitives::{
traits::TrustedCallSigning,
types::{KeyPair, TrustedOperation},
};
use itp_types::{
Balance, ShardIdentifier, TrustedOperationStatus,
TrustedOperationStatus::{InSidechainBlock, Submitted},
};
use log::*;
use rand::Rng;
use rayon::prelude::*;
use sgx_crypto_helper::rsa3072::Rsa3072PubKey;
use sp_application_crypto::sr25519;
use sp_core::{sr25519 as sr25519_core, Pair};
use sp_keystore::Keystore;
use std::{
boxed::Box,
string::ToString,
sync::mpsc::{channel, Receiver},
thread, time,
time::Instant,
vec::Vec,
};
use substrate_client_keystore::LocalKeystore;

// Needs to be above the existential deposit minimum, otherwise an account will not
// be created and the state is not increased.
const EXISTENTIAL_DEPOSIT: Balance = 1000;

#[derive(Parser)]
pub struct BenchmarkCommand {
/// The number of clients (=threads) to be used in the benchmark
#[clap(default_value_t = 10)]
number_clients: u32,

/// The number of iterations to execute for each client
#[clap(default_value_t = 30)]
number_iterations: u128,

/// Adds a random wait before each transaction. This is the lower bound for the interval in ms.
#[clap(default_value_t = 0)]
random_wait_before_transaction_min_ms: u32,

/// Adds a random wait before each transaction. This is the upper bound for the interval in ms.
#[clap(default_value_t = 0)]
random_wait_before_transaction_max_ms: u32,

/// Whether to wait for "InSidechainBlock" confirmation for each transaction
#[clap(short, long)]
wait_for_confirmation: bool,

/// Account to be used for initial funding of generated accounts used in benchmark
#[clap(default_value_t = String::from("//Alice"))]
funding_account: String,
}

struct BenchmarkClient {
account: sr25519_core::Pair,
current_balance: u128,
client_api: DirectClient,
receiver: Receiver<String>,
}

impl BenchmarkClient {
fn new(
account: sr25519_core::Pair,
initial_balance: u128,
initial_request: String,
cli: &Cli,
) -> Self {
debug!("get direct api");
let client_api = get_worker_api_direct(cli);

debug!("setup sender and receiver");
let (sender, receiver) = channel();
client_api.watch(initial_request, sender);
BenchmarkClient { account, current_balance: initial_balance, client_api, receiver }
}
}

/// Stores timing information about a specific transaction
struct BenchmarkTransaction {
started: Instant,
submitted: Instant,
confirmed: Option<Instant>,
}

impl BenchmarkCommand {
pub(crate) fn run(&self, cli: &Cli, trusted_args: &TrustedCli) -> CliResult {
let random_wait_before_transaction_ms: (u32, u32) = (
self.random_wait_before_transaction_min_ms,
self.random_wait_before_transaction_max_ms,
);
let store = LocalKeystore::open(get_keystore_path(trusted_args, cli), None).unwrap();
let funding_account_keys = get_pair_from_str(trusted_args, &self.funding_account, cli);

let (mrenclave, shard) = get_identifiers(trusted_args, cli);

// Get shielding pubkey.
let worker_api_direct = get_worker_api_direct(cli);
let shielding_pubkey: Rsa3072PubKey = match worker_api_direct.get_rsa_pubkey() {
Ok(key) => key,
Err(err_msg) => panic!("{}", err_msg.to_string()),
};

let nonce_start = get_layer_two_nonce!(funding_account_keys, cli, trusted_args);
println!("Nonce for account {}: {}", self.funding_account, nonce_start);

let mut accounts = Vec::new();
let initial_balance = (self.number_iterations + 1) * (STF_TX_FEE + EXISTENTIAL_DEPOSIT);
// Setup new accounts and initialize them with money from Alice.
for i in 0..self.number_clients {
let nonce = i + nonce_start;
println!("Initializing account {} with initial amount {:?}", i, initial_balance);

// Create new account to use.
let a = LocalKeystore::sr25519_generate_new(&store, SR25519_KEY_TYPE, None).unwrap();
let account = get_pair_from_str(trusted_args, a.to_string().as_str(), cli);

// Transfer amount from Alice to new account.
let top: TrustedOperation<TrustedCallSigned, Getter> = TrustedCall::balance_transfer(
funding_account_keys.public().into(),
account.public().into(),
initial_balance,
)
.sign(
&KeyPair::Sr25519(Box::new(funding_account_keys.clone())),
nonce,
&mrenclave,
&shard,
)
.into_trusted_operation(trusted_args.direct);

// For the last account we wait for confirmation in order to ensure all accounts were setup correctly
let wait_for_confirmation = i == self.number_clients - 1;
let account_funding_request = get_json_request(shard, &top, shielding_pubkey);

let client =
BenchmarkClient::new(account, initial_balance, account_funding_request, cli);
let _result = wait_for_top_confirmation(wait_for_confirmation, &client);
accounts.push(client);
}

rayon::ThreadPoolBuilder::new()
.num_threads(self.number_clients as usize)
.build_global()
.unwrap();

let overall_start = Instant::now();

// Run actual benchmark logic, in parallel, for each account initialized above.
let outputs: Vec<Vec<BenchmarkTransaction>> = accounts
.into_par_iter()
.map(move |mut client| {
let mut output: Vec<BenchmarkTransaction> = Vec::new();

for i in 0..self.number_iterations {
println!("Iteration: {}", i);

if random_wait_before_transaction_ms.1 > 0 {
random_wait(random_wait_before_transaction_ms);
}

// Create new account.
let account_keys = LocalKeystore::sr25519_generate_new(&store, SR25519_KEY_TYPE, None).unwrap();

let new_account =
get_pair_from_str(trusted_args, account_keys.to_string().as_str(), cli);

println!(" Transfer amount: {}", EXISTENTIAL_DEPOSIT);
println!(" From: {:?}", client.account.public());
println!(" To: {:?}", new_account.public());

// Get nonce of account.
let nonce = get_nonce(client.account.clone(), shard, &client.client_api);

// Transfer money from client account to new account.
let top: TrustedOperation<TrustedCallSigned, Getter> = TrustedCall::balance_transfer(
client.account.public().into(),
new_account.public().into(),
EXISTENTIAL_DEPOSIT,
)
.sign(&KeyPair::Sr25519(Box::new(client.account.clone())), nonce, &mrenclave, &shard)
.into_trusted_operation(trusted_args.direct);

let last_iteration = i == self.number_iterations - 1;
let jsonrpc_call = get_json_request(shard, &top, shielding_pubkey);
client.client_api.send(&jsonrpc_call).unwrap();
let result = wait_for_top_confirmation(
self.wait_for_confirmation || last_iteration,
&client,
);

client.current_balance -= EXISTENTIAL_DEPOSIT;

let balance = get_balance(client.account.clone(), shard, &client.client_api);
println!("Balance: {}", balance.unwrap_or_default());
assert_eq!(client.current_balance, balance.unwrap_or_default());

output.push(result);

// FIXME: We probably should re-fund the account in this case.
if client.current_balance <= EXISTENTIAL_DEPOSIT + STF_TX_FEE {
error!("Account {:?} does not have enough balance anymore. Finishing benchmark early", client.account.public());
break;
}
}

client.client_api.close().unwrap();

output
})
.collect();

println!(
"Finished benchmark with {} clients and {} transactions in {} ms",
self.number_clients,
self.number_iterations,
overall_start.elapsed().as_millis()
);

print_benchmark_statistic(outputs, self.wait_for_confirmation);

Ok(CliResultOk::None)
}
}

fn get_balance(
account: sr25519::Pair,
shard: ShardIdentifier,
direct_client: &DirectClient,
) -> Option<u128> {
let getter = Getter::trusted(
TrustedGetter::free_balance(account.public().into())
.sign(&KeyPair::Sr25519(Box::new(account.clone()))),
);

let getter_start_timer = Instant::now();
let getter_result = direct_client.get_state(shard, &getter);
let getter_execution_time = getter_start_timer.elapsed().as_millis();

let balance = decode_balance(getter_result);
info!("Balance getter execution took {} ms", getter_execution_time,);
debug!("Retrieved {:?} Balance for {:?}", balance.unwrap_or_default(), account.public());
balance
}

fn get_nonce(
account: sr25519::Pair,
shard: ShardIdentifier,
direct_client: &DirectClient,
) -> Index {
let getter = Getter::public(PublicGetter::nonce(account.public().into()));

let getter_start_timer = Instant::now();
let nonce = get_state::<Index>(direct_client, shard, &getter).ok().unwrap_or_default();
let getter_execution_time = getter_start_timer.elapsed().as_millis();
info!("Nonce getter execution took {} ms", getter_execution_time,);
debug!("Retrieved {:?} nonce for {:?}", nonce, account.public());
nonce
}

fn print_benchmark_statistic(outputs: Vec<Vec<BenchmarkTransaction>>, wait_for_confirmation: bool) {
let mut hist = Histogram::<u64>::new(1).unwrap();
for output in outputs {
for t in output {
let benchmarked_timestamp =
if wait_for_confirmation { t.confirmed } else { Some(t.submitted) };
if let Some(confirmed) = benchmarked_timestamp {
hist += confirmed.duration_since(t.started).as_millis() as u64;
} else {
println!("Missing measurement data");
}
}
}

for i in (5..=100).step_by(5) {
let text = format!(
"{} percent are done within {} ms",
i,
hist.value_at_quantile(i as f64 / 100.0)
);
println!("{}", text);
}
}

fn random_wait(random_wait_before_transaction_ms: (u32, u32)) {
let mut rng = rand::thread_rng();
let sleep_time = time::Duration::from_millis(
rng.gen_range(random_wait_before_transaction_ms.0..=random_wait_before_transaction_ms.1)
.into(),
);
println!("Sleep for: {}ms", sleep_time.as_millis());
thread::sleep(sleep_time);
}

fn wait_for_top_confirmation(
wait_for_sidechain_block: bool,
client: &BenchmarkClient,
) -> BenchmarkTransaction {
let started = Instant::now();

let submitted = wait_until(&client.receiver, is_submitted);

let confirmed = if wait_for_sidechain_block {
// We wait for the transaction hash that actually matches the submitted hash
loop {
let transaction_information = wait_until(&client.receiver, is_sidechain_block);
if let Some((hash, _)) = transaction_information {
if hash == submitted.unwrap().0 {
break transaction_information
}
}
}
} else {
None
};
if let (Some(s), Some(c)) = (submitted, confirmed) {
// Assert the two hashes are identical
assert_eq!(s.0, c.0);
}

BenchmarkTransaction {
started,
submitted: submitted.unwrap().1,
confirmed: confirmed.map(|v| v.1),
}
}

fn is_submitted(s: TrustedOperationStatus) -> bool {
matches!(s, Submitted)
}

fn is_sidechain_block(s: TrustedOperationStatus) -> bool {
matches!(s, InSidechainBlock(_))
}

fn decode_balance(maybe_encoded_balance: Option<Vec<u8>>) -> Option<Balance> {
maybe_encoded_balance.and_then(|encoded_balance| {
if let Ok(vd) = Balance::decode(&mut encoded_balance.as_slice()) {
Some(vd)
} else {
warn!("Could not decode balance. maybe hasn't been set? {:x?}", encoded_balance);
None
}
})
}
pub mod request_vc;
pub mod stf;
Loading
Loading