From 41c40a81287863ae6342728e24cfe3303ec28493 Mon Sep 17 00:00:00 2001 From: Lou-Kamades <128186011+Lou-Kamades@users.noreply.github.com> Date: Thu, 14 Mar 2024 11:49:34 -0500 Subject: [PATCH] Use CLI instead of binaries for configuring benchmarks (#357) refactor: use CLI for benchmarks --- Cargo.lock | 54 ++------------- README.md | 2 +- bench/Cargo.toml | 19 +----- bench/bench-config.toml | 14 ---- bench/src/benches/api_load.rs | 24 +++---- bench/src/benches/confirmation_rate.rs | 34 ++++----- bench/src/benches/confirmation_slot.rs | 38 +++++------ bench/src/{ => benches}/mod.rs | 8 +-- bench/src/config.rs | 41 ----------- bench/src/lib.rs | 3 +- bench/src/main.rs | 95 ++++++++++++++++++++++++++ bench/src/tx_size.rs | 2 +- 12 files changed, 151 insertions(+), 183 deletions(-) delete mode 100644 bench/bench-config.toml rename bench/src/{ => benches}/mod.rs (72%) delete mode 100644 bench/src/config.rs create mode 100644 bench/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 2933c613..034d061e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -572,6 +572,7 @@ version = "0.2.4" dependencies = [ "anyhow", "bincode", + "clap 4.4.18", "dashmap 5.5.3", "futures", "itertools 0.10.5", @@ -586,7 +587,6 @@ dependencies = [ "solana-transaction-status", "spl-memo 4.0.0", "tokio", - "toml 0.8.10", "tracing", "tracing-subscriber", ] @@ -3215,7 +3215,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" dependencies = [ - "toml 0.5.11", + "toml", ] [[package]] @@ -3907,15 +3907,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_spanned" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" -dependencies = [ - "serde", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -5859,26 +5850,11 @@ dependencies = [ "serde", ] -[[package]] -name = "toml" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a9aad4a3066010876e8dcf5a8a06e70a558751117a145c6ce2b82c2e2054290" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.22.6", -] - [[package]] name = "toml_datetime" version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" -dependencies = [ - "serde", -] [[package]] name = "toml_edit" @@ -5888,7 +5864,7 @@ checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ "indexmap 2.2.2", "toml_datetime", - "winnow 0.5.37", + "winnow", ] [[package]] @@ -5899,20 +5875,7 @@ checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ "indexmap 2.2.2", "toml_datetime", - "winnow 0.5.37", -] - -[[package]] -name = "toml_edit" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1b5fd4128cc8d3e0cb74d4ed9a9cc7c7284becd4df68f5f940e1ad123606f6" -dependencies = [ - "indexmap 2.2.2", - "serde", - "serde_spanned", - "toml_datetime", - "winnow 0.6.5", + "winnow", ] [[package]] @@ -6589,15 +6552,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "winnow" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dffa400e67ed5a4dd237983829e66475f0a4a26938c4b04c21baede6262215b8" -dependencies = [ - "memchr", -] - [[package]] name = "winreg" version = "0.50.0" diff --git a/README.md b/README.md index 35cab130..b5458d10 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ $ cargo test *bench* ```bash $ cd bench -$ cargo run --bin confirmation_slot ./bench-config.toml +$ RUST_LOG=info cargo run -- --help ``` Find a new file named `metrics.csv` in the project root. diff --git a/bench/Cargo.toml b/bench/Cargo.toml index 6a755373..10eecdc7 100644 --- a/bench/Cargo.toml +++ b/bench/Cargo.toml @@ -3,24 +3,8 @@ name = "bench" version = "0.2.4" edition = "2021" -[lib] -name = "bench_lib" -path = "src/lib.rs" - -[[bin]] -name = "api_load" -path = "src/benches/api_load.rs" - -[[bin]] -name = "confirmation_slot" -path = "src/benches/confirmation_slot.rs" - -[[bin]] -name = "confirmation_rate" -path = "src/benches/confirmation_rate.rs" - - [dependencies] +clap = { workspace = true } solana-sdk = { workspace = true } solana-rpc-client = { workspace = true } solana-transaction-status = { workspace = true } @@ -39,7 +23,6 @@ dashmap = { workspace = true } bincode = { workspace = true } itertools = "0.10.5" spl-memo = "4.0.0" -toml = "0.8.10" [dev-dependencies] bincode = { workspace = true } diff --git a/bench/bench-config.toml b/bench/bench-config.toml deleted file mode 100644 index 110e547e..00000000 --- a/bench/bench-config.toml +++ /dev/null @@ -1,14 +0,0 @@ -payer_path = "/path/to/id.json" -lite_rpc_url = "http://127.0.0.1:8899" -rpc_url = "https://api.testnet.rpcpool.com/secret" - -[api_load] -time_ms = 3000 - -[confirmation_rate] -num_txns = 20 -num_runs = 5 -tx_size = "Small" - -[confirmation_slot] -tx_size = "Small" # Small or Large diff --git a/bench/src/benches/api_load.rs b/bench/src/benches/api_load.rs index 5b014fe4..bc244c55 100644 --- a/bench/src/benches/api_load.rs +++ b/bench/src/benches/api_load.rs @@ -1,26 +1,25 @@ -use bench_lib::{config::BenchConfig, create_memo_tx_small}; use log::{info, warn}; -use std::sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, +use std::{ + path::Path, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, }; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_sdk::signature::{read_keypair_file, Keypair, Signer}; -// TC3 measure how much load the API endpoint can take -#[tokio::main] -async fn main() -> anyhow::Result<()> { - tracing_subscriber::fmt::init(); +use crate::create_memo_tx_small; +// TC3 measure how much load the API endpoint can take +pub async fn api_load(payer_path: &Path, rpc_url: String, time_ms: u64) -> anyhow::Result<()> { warn!("THIS IS WORK IN PROGRESS"); - let config = BenchConfig::load().unwrap(); - - let rpc = Arc::new(RpcClient::new(config.lite_rpc_url.clone())); + let rpc = Arc::new(RpcClient::new(rpc_url)); info!("RPC: {}", rpc.as_ref().url()); - let payer: Arc = Arc::new(read_keypair_file(&config.payer_path).unwrap()); + let payer: Arc = Arc::new(read_keypair_file(payer_path).unwrap()); info!("Payer: {}", payer.pubkey().to_string()); let mut txs = 0; @@ -28,7 +27,6 @@ async fn main() -> anyhow::Result<()> { let success = Arc::new(AtomicUsize::new(0)); let hash = rpc.get_latest_blockhash().await?; - let time_ms = config.api_load.time_ms; let time = tokio::time::Instant::now(); while time.elapsed().as_millis() < time_ms.into() { diff --git a/bench/src/benches/confirmation_rate.rs b/bench/src/benches/confirmation_rate.rs index ff1bc8bd..48ced54b 100644 --- a/bench/src/benches/confirmation_rate.rs +++ b/bench/src/benches/confirmation_rate.rs @@ -1,13 +1,13 @@ +use crate::tx_size::TxSize; +use crate::{create_rng, generate_txs}; use anyhow::{bail, Error}; -use bench_lib::config::{BenchConfig, ConfirmationRateConfig}; -use bench_lib::tx_size::TxSize; -use bench_lib::{create_rng, generate_txs}; use futures::future::join_all; use futures::TryFutureExt; use itertools::Itertools; use log::{debug, info, trace, warn}; use std::collections::{HashMap, HashSet}; use std::iter::zip; +use std::path::Path; use std::sync::Arc; use std::time::Duration; @@ -31,29 +31,25 @@ pub struct RpcStat { } /// TC2 send multiple runs of num_txns, measure the confirmation rate -#[tokio::main] -async fn main() -> anyhow::Result<()> { - tracing_subscriber::fmt::init(); - +pub async fn confirmation_rate( + payer_path: &Path, + rpc_url: String, + tx_size: TxSize, + txns_per_round: usize, + num_rounds: usize, +) -> anyhow::Result<()> { warn!("THIS IS WORK IN PROGRESS"); - let config = BenchConfig::load().unwrap(); - let ConfirmationRateConfig { - tx_size, - num_txns, - num_runs, - } = config.confirmation_rate; - - let rpc = Arc::new(RpcClient::new(config.lite_rpc_url.clone())); + let rpc = Arc::new(RpcClient::new(rpc_url)); info!("RPC: {}", rpc.as_ref().url()); - let payer: Arc = Arc::new(read_keypair_file(&config.payer_path).unwrap()); + let payer: Arc = Arc::new(read_keypair_file(payer_path).unwrap()); info!("Payer: {}", payer.pubkey().to_string()); - let mut rpc_results = Vec::with_capacity(num_runs); + let mut rpc_results = Vec::with_capacity(num_rounds); - for _ in 0..num_runs { - let stat: RpcStat = send_bulk_txs_and_wait(&rpc, &payer, num_txns, tx_size).await?; + for _ in 0..num_rounds { + let stat: RpcStat = send_bulk_txs_and_wait(&rpc, &payer, txns_per_round, tx_size).await?; rpc_results.push(stat); } diff --git a/bench/src/benches/confirmation_slot.rs b/bench/src/benches/confirmation_slot.rs index 4e7ff7e1..dd298b99 100644 --- a/bench/src/benches/confirmation_slot.rs +++ b/bench/src/benches/confirmation_slot.rs @@ -1,7 +1,8 @@ +use std::path::Path; + +use crate::tx_size::TxSize; +use crate::{create_memo_tx, create_rng, send_and_confirm_transactions, Rng8}; use anyhow::Context; -use bench_lib::config::BenchConfig; -use bench_lib::tx_size::TxSize; -use bench_lib::{create_memo_tx, create_rng, send_and_confirm_transactions, Rng8}; use log::{info, warn}; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_sdk::signature::{read_keypair_file, Signer}; @@ -9,31 +10,30 @@ use solana_sdk::transaction::Transaction; use solana_sdk::{commitment_config::CommitmentConfig, signature::Keypair}; /// TC1 send 2 txs (one via LiteRPC, one via Solana RPC) and compare confirmation slot (=slot distance) -#[tokio::main] -async fn main() -> anyhow::Result<()> { - tracing_subscriber::fmt::init(); - +pub async fn confirmation_slot( + payer_path: &Path, + rpc_a_url: String, + rpc_b_url: String, + tx_size: TxSize, +) -> anyhow::Result<()> { warn!("THIS IS WORK IN PROGRESS"); - let config = BenchConfig::load().unwrap(); - let tx_size = config.confirmation_slot.tx_size; + let rpc_a = RpcClient::new(rpc_a_url); + info!("RPC A: {}", rpc_a.url()); - let lite_rpc = RpcClient::new(config.lite_rpc_url.clone()); - info!("Lite RPC: {}", lite_rpc.url()); - - let rpc = RpcClient::new(config.rpc_url.clone()); - info!("RPC: {}", rpc.url()); + let rpc_b = RpcClient::new(rpc_b_url); + info!("RPC B: {}", rpc_b.url()); let mut rng = create_rng(None); - let payer = read_keypair_file(&config.payer_path).expect("payer file"); + let payer = read_keypair_file(payer_path).expect("payer file"); info!("Payer: {}", payer.pubkey().to_string()); - let rpc_tx = create_tx(&rpc, &payer, &mut rng, tx_size).await?; - let lite_rpc_tx = create_tx(&lite_rpc, &payer, &mut rng, tx_size).await?; + let rpc_a_tx = create_tx(&rpc_a, &payer, &mut rng, tx_size).await?; + let rpc_b_tx = create_tx(&rpc_b, &payer, &mut rng, tx_size).await?; let (rpc_slot, lite_rpc_slot) = tokio::join!( - send_transaction_and_get_slot(&rpc, rpc_tx), - send_transaction_and_get_slot(&lite_rpc, lite_rpc_tx) + send_transaction_and_get_slot(&rpc_a, rpc_a_tx), + send_transaction_and_get_slot(&rpc_b, rpc_b_tx) ); info!("rpc_slot: {}", rpc_slot?); diff --git a/bench/src/mod.rs b/bench/src/benches/mod.rs similarity index 72% rename from bench/src/mod.rs rename to bench/src/benches/mod.rs index bbc30a85..361d0851 100644 --- a/bench/src/mod.rs +++ b/bench/src/benches/mod.rs @@ -1,7 +1,3 @@ -pub mod confirmation_slot; -pub mod confirmation_rate; pub mod api_load; -// pub mod tx_broadcast; - - - +pub mod confirmation_rate; +pub mod confirmation_slot; diff --git a/bench/src/config.rs b/bench/src/config.rs deleted file mode 100644 index 8d1076fa..00000000 --- a/bench/src/config.rs +++ /dev/null @@ -1,41 +0,0 @@ -use crate::tx_size::TxSize; -use serde::Deserialize; -use std::fs::File; -use std::io::Read; - -#[derive(Clone, Debug, Deserialize)] -pub struct BenchConfig { - pub payer_path: String, - pub lite_rpc_url: String, - pub rpc_url: String, - pub api_load: ApiLoadConfig, - pub confirmation_rate: ConfirmationRateConfig, - pub confirmation_slot: ConfirmationSlotConfig, -} - -#[derive(Clone, Debug, Deserialize)] -pub struct ApiLoadConfig { - pub time_ms: u64, -} - -#[derive(Clone, Debug, Deserialize)] -pub struct ConfirmationRateConfig { - pub num_txns: usize, - pub num_runs: usize, - pub tx_size: TxSize, -} - -#[derive(Clone, Debug, Deserialize)] -pub struct ConfirmationSlotConfig { - pub tx_size: TxSize, -} - -impl BenchConfig { - pub fn load() -> anyhow::Result { - let args: Vec = std::env::args().collect(); - let mut file = File::open(&args[1])?; - let mut contents = String::new(); - file.read_to_string(&mut contents).unwrap(); - Ok(toml::from_str(&contents)?) - } -} diff --git a/bench/src/lib.rs b/bench/src/lib.rs index 91fd213e..6eff3b61 100644 --- a/bench/src/lib.rs +++ b/bench/src/lib.rs @@ -20,7 +20,8 @@ use solana_transaction_status::TransactionStatus; use std::{str::FromStr, time::Duration}; use tokio::time::Instant; -pub mod config; +pub mod benches; +pub mod metrics; pub mod tx_size; //TODO: use CLAP diff --git a/bench/src/main.rs b/bench/src/main.rs new file mode 100644 index 00000000..94a655e1 --- /dev/null +++ b/bench/src/main.rs @@ -0,0 +1,95 @@ +use std::path::PathBuf; + +use bench::{ + benches::{ + api_load::api_load, confirmation_rate::confirmation_rate, + confirmation_slot::confirmation_slot, + }, + tx_size::TxSize, +}; +use clap::{Parser, Subcommand}; + +#[derive(Parser, Debug)] +#[clap(version, about)] + +struct Arguments { + #[clap(subcommand)] + subcommand: SubCommand, +} + +#[derive(Subcommand, Debug)] +enum SubCommand { + ApiLoad { + #[clap(short, long)] + payer_path: PathBuf, + #[clap(short, long)] + rpc_url: String, + #[clap(short, long)] + time_ms: u64, + }, + ConfirmationRate { + #[clap(short, long)] + payer_path: PathBuf, + #[clap(short, long)] + rpc_url: String, + #[clap(short, long)] + size_tx: TxSize, + #[clap(short, long)] + txns_per_round: usize, + #[clap(short, long)] + num_rounds: usize, + }, + ConfirmationSlot { + #[clap(short, long)] + payer_path: PathBuf, + #[clap(short, long)] + #[arg(short = 'a')] + rpc_a: String, + #[clap(short, long)] + #[arg(short = 'b')] + rpc_b: String, + #[clap(short, long)] + size_tx: TxSize, + }, +} + +pub fn initialize_logger() { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_thread_ids(true) + .with_line_number(true) + .init(); +} + +#[tokio::main(flavor = "multi_thread", worker_threads = 16)] +async fn main() { + let args = Arguments::parse(); + initialize_logger(); + + match args.subcommand { + SubCommand::ApiLoad { + payer_path, + rpc_url, + time_ms, + } => { + api_load(&payer_path, rpc_url, time_ms).await.unwrap(); + } + SubCommand::ConfirmationRate { + payer_path, + rpc_url, + size_tx, + txns_per_round, + num_rounds, + } => confirmation_rate(&payer_path, rpc_url, size_tx, txns_per_round, num_rounds) + .await + .unwrap(), + SubCommand::ConfirmationSlot { + payer_path, + rpc_a, + rpc_b, + size_tx, + } => confirmation_slot(&payer_path, rpc_a, rpc_b, size_tx) + .await + .unwrap(), + } +} diff --git a/bench/src/tx_size.rs b/bench/src/tx_size.rs index 23aee3b8..f6bc899d 100644 --- a/bench/src/tx_size.rs +++ b/bench/src/tx_size.rs @@ -3,7 +3,7 @@ use serde::Deserialize; // see https://spl.solana.com/memo for sizing of transactions // As of v1.5.1, an unsigned instruction can support single-byte UTF-8 of up to 566 bytes. // An instruction with a simple memo of 32 bytes can support up to 12 signers. -#[derive(Debug, Clone, Copy, Deserialize)] +#[derive(Debug, Clone, Copy, Deserialize, clap::ValueEnum)] pub enum TxSize { // 179 bytes, 5237 CUs Small,