From 288a3bdcd96ac5d3457b845a976091e8f9d85cdd Mon Sep 17 00:00:00 2001 From: Pankaj Garg Date: Tue, 11 Jun 2019 18:47:35 -0700 Subject: [PATCH] Provision bench client accounts in genesis block (#4648) * fixes to script * shellcheck * address review comments --- Cargo.lock | 3 + bench-tps/Cargo.toml | 3 + bench-tps/src/cli.rs | 31 +++++++++ bench-tps/src/main.rs | 59 ++++++++++++++--- genesis/src/main.rs | 129 +++++++++++++++++++++++++++++++++--- net/net.sh | 9 ++- net/remote/remote-client.sh | 4 ++ net/remote/remote-node.sh | 24 ++++++- 8 files changed, 239 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b666270069..a97cebd548 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2287,7 +2287,10 @@ dependencies = [ "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_yaml 0.8.9 (registry+https://github.com/rust-lang/crates.io-index)", "solana 0.16.0", "solana-client 0.16.0", "solana-drone 0.16.0", diff --git a/bench-tps/Cargo.toml b/bench-tps/Cargo.toml index 03720311c3..a0be9d930f 100644 --- a/bench-tps/Cargo.toml +++ b/bench-tps/Cargo.toml @@ -11,7 +11,10 @@ homepage = "https://solana.com/" clap = "2.33.0" log = "0.4.6" rayon = "1.0.3" +serde = "1.0.92" +serde_derive = "1.0.92" serde_json = "1.0.39" +serde_yaml = "0.8.9" solana = { path = "../core", version = "0.16.0" } solana-client = { path = "../client", version = "0.16.0" } solana-drone = { path = "../drone", version = "0.16.0" } diff --git a/bench-tps/src/cli.rs b/bench-tps/src/cli.rs index 166cb8cb1d..ab12c171be 100644 --- a/bench-tps/src/cli.rs +++ b/bench-tps/src/cli.rs @@ -17,6 +17,9 @@ pub struct Config { pub tx_count: usize, pub thread_batch_sleep_ms: usize, pub sustained: bool, + pub client_ids_and_stake_file: String, + pub write_to_client_file: bool, + pub read_from_client_file: bool, } impl Default for Config { @@ -31,6 +34,9 @@ impl Default for Config { tx_count: 500_000, thread_batch_sleep_ms: 0, sustained: false, + client_ids_and_stake_file: String::new(), + write_to_client_file: false, + read_from_client_file: false, } } } @@ -106,6 +112,20 @@ pub fn build_args<'a, 'b>() -> App<'a, 'b> { .takes_value(true) .help("Per-thread-per-iteration sleep in ms"), ) + .arg( + Arg::with_name("write-client-keys") + .long("write-client-keys") + .value_name("FILENAME") + .takes_value(true) + .help("Generate client keys and stakes and write the list to YAML file"), + ) + .arg( + Arg::with_name("read-client-keys") + .long("read-client-keys") + .value_name("FILENAME") + .takes_value(true) + .help("Read client keys and stakes from the YAML file"), + ) } /// Parses a clap `ArgMatches` structure into a `Config` @@ -163,5 +183,16 @@ pub fn extract_args<'a>(matches: &ArgMatches<'a>) -> Config { args.sustained = matches.is_present("sustained"); + if let Some(s) = matches.value_of("write-client-keys") { + args.write_to_client_file = true; + args.client_ids_and_stake_file = s.to_string(); + } + + if let Some(s) = matches.value_of("read-client-keys") { + assert!(!args.write_to_client_file); + args.read_from_client_file = true; + args.client_ids_and_stake_file = s.to_string(); + } + args } diff --git a/bench-tps/src/main.rs b/bench-tps/src/main.rs index faecf13617..a50dd0ebb4 100644 --- a/bench-tps/src/main.rs +++ b/bench-tps/src/main.rs @@ -1,8 +1,15 @@ mod bench; mod cli; -use crate::bench::{do_bench_tps, generate_and_fund_keypairs, Config, NUM_LAMPORTS_PER_ACCOUNT}; +use crate::bench::{ + do_bench_tps, generate_and_fund_keypairs, generate_keypairs, Config, NUM_LAMPORTS_PER_ACCOUNT, +}; use solana::gossip_service::{discover_cluster, get_multi_client}; +use solana_sdk::signature::Keypair; +use std::collections::HashMap; +use std::fs::File; +use std::io::prelude::*; +use std::path::Path; use std::process::exit; fn main() { @@ -22,8 +29,28 @@ fn main() { tx_count, thread_batch_sleep_ms, sustained, + client_ids_and_stake_file, + write_to_client_file, + read_from_client_file, } = cli_config; + if write_to_client_file { + let keypairs = generate_keypairs(&id, tx_count as u64 * 2); + let mut accounts = HashMap::new(); + keypairs.iter().for_each(|keypair| { + accounts.insert( + serde_json::to_string(&keypair.to_bytes().to_vec()).unwrap(), + NUM_LAMPORTS_PER_ACCOUNT as u64, + ); + }); + + let serialized = serde_yaml::to_string(&accounts).unwrap(); + let path = Path::new(&client_ids_and_stake_file); + let mut file = File::create(path).unwrap(); + file.write_all(&serialized.into_bytes()).unwrap(); + return; + } + println!("Connecting to the cluster"); let (nodes, _replicators) = discover_cluster(&entrypoint_addr, num_nodes).unwrap_or_else(|err| { @@ -41,13 +68,29 @@ fn main() { exit(1); } - let (keypairs, keypair_balance) = generate_and_fund_keypairs( - &client, - Some(drone_addr), - &id, - tx_count, - NUM_LAMPORTS_PER_ACCOUNT, - ); + let (keypairs, keypair_balance) = if read_from_client_file { + let path = Path::new(&client_ids_and_stake_file); + let file = File::open(path).unwrap(); + + let accounts: HashMap = serde_yaml::from_reader(file).unwrap(); + let mut keypairs = vec![]; + let mut last_balance = 0; + + accounts.into_iter().for_each(|(keypair, balance)| { + let bytes: Vec = serde_json::from_str(keypair.as_str()).unwrap(); + keypairs.push(Keypair::from_bytes(&bytes).unwrap()); + last_balance = balance; + }); + (keypairs, last_balance) + } else { + generate_and_fund_keypairs( + &client, + Some(drone_addr), + &id, + tx_count, + NUM_LAMPORTS_PER_ACCOUNT, + ) + }; let config = Config { id, diff --git a/genesis/src/main.rs b/genesis/src/main.rs index 0d8d67ccc5..cc29a5bbd8 100644 --- a/genesis/src/main.rs +++ b/genesis/src/main.rs @@ -24,7 +24,7 @@ use solana_sdk::genesis_block::GenesisBlock; use solana_sdk::hash::{hash, Hash}; use solana_sdk::poh_config::PohConfig; use solana_sdk::pubkey::Pubkey; -use solana_sdk::signature::{read_keypair, KeypairUtil}; +use solana_sdk::signature::{read_keypair, Keypair, KeypairUtil}; use solana_sdk::system_program; use solana_sdk::timing; use solana_stake_api::stake_state; @@ -39,18 +39,36 @@ use std::time::{Duration, Instant}; pub const BOOTSTRAP_LEADER_LAMPORTS: u64 = 42; -pub fn append_primordial_accounts(file: &str, genesis_block: &mut GenesisBlock) -> io::Result<()> { +pub enum AccountFileFormat { + Pubkey, + Keypair, +} + +pub fn append_primordial_accounts( + file: &str, + file_format: AccountFileFormat, + genesis_block: &mut GenesisBlock, +) -> io::Result<()> { let accounts_file = File::open(file.to_string())?; let primordial_accounts: HashMap = serde_yaml::from_reader(accounts_file) .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?; - primordial_accounts.into_iter().for_each(|primordial| { - genesis_block.accounts.push(( - Pubkey::from_str(primordial.0.as_str()).unwrap(), - Account::new(primordial.1, 0, &system_program::id()), - )) - }); + primordial_accounts + .into_iter() + .for_each(|(account, balance)| { + let pubkey = match file_format { + AccountFileFormat::Pubkey => Pubkey::from_str(account.as_str()).unwrap(), + AccountFileFormat::Keypair => { + let bytes: Vec = serde_json::from_str(account.as_str()).unwrap(); + Keypair::from_bytes(&bytes).unwrap().pubkey() + } + }; + + genesis_block + .accounts + .push((pubkey, Account::new(balance, 0, &system_program::id()))) + }); Ok(()) } @@ -219,6 +237,13 @@ fn main() -> Result<(), Box> { .takes_value(true) .help("The location of pubkey for primordial accounts and balance"), ) + .arg( + Arg::with_name("primordial_keypairs_file") + .long("primordial-keypairs-file") + .value_name("FILENAME") + .takes_value(true) + .help("The location of keypairs for primordial accounts and balance"), + ) .get_matches(); let bootstrap_leader_keypair_file = matches.value_of("bootstrap_leader_keypair_file").unwrap(); @@ -290,7 +315,11 @@ fn main() -> Result<(), Box> { ); if let Some(file) = matches.value_of("primordial_accounts_file") { - append_primordial_accounts(file, &mut genesis_block)?; + append_primordial_accounts(file, AccountFileFormat::Pubkey, &mut genesis_block)?; + } + + if let Some(file) = matches.value_of("primordial_keypairs_file") { + append_primordial_accounts(file, AccountFileFormat::Keypair, &mut genesis_block)?; } genesis_block.fee_calculator.target_lamports_per_signature = @@ -368,7 +397,12 @@ mod tests { let mut genesis_block = GenesisBlock::new(&[], &[]); // Test invalid file returns error - assert!(append_primordial_accounts("unknownfile", &mut genesis_block).is_err()); + assert!(append_primordial_accounts( + "unknownfile", + AccountFileFormat::Pubkey, + &mut genesis_block + ) + .is_err()); let mut primordial_accounts = HashMap::new(); primordial_accounts.insert(Pubkey::new_rand().to_string(), 2 as u64); @@ -383,6 +417,7 @@ mod tests { // Test valid file returns ok assert!(append_primordial_accounts( "test_append_primordial_accounts_to_genesis.yml", + AccountFileFormat::Pubkey, &mut genesis_block ) .is_ok()); @@ -413,6 +448,7 @@ mod tests { assert!(append_primordial_accounts( "test_append_primordial_accounts_to_genesis.yml", + AccountFileFormat::Pubkey, &mut genesis_block ) .is_ok()); @@ -444,5 +480,78 @@ mod tests { .lamports, ); }); + + // Test accounts from keypairs can be appended + let account_keypairs: Vec<_> = (0..3).map(|_| Keypair::new()).collect(); + let mut primordial_accounts2 = HashMap::new(); + primordial_accounts2.insert( + serde_json::to_string(&account_keypairs[0].to_bytes().to_vec()).unwrap(), + 20 as u64, + ); + primordial_accounts2.insert( + serde_json::to_string(&account_keypairs[1].to_bytes().to_vec()).unwrap(), + 15 as u64, + ); + primordial_accounts2.insert( + serde_json::to_string(&account_keypairs[2].to_bytes().to_vec()).unwrap(), + 30 as u64, + ); + + let serialized = serde_yaml::to_string(&primordial_accounts2).unwrap(); + let path = Path::new("test_append_primordial_accounts_to_genesis.yml"); + let mut file = File::create(path).unwrap(); + file.write_all(&serialized.into_bytes()).unwrap(); + + assert!(append_primordial_accounts( + "test_append_primordial_accounts_to_genesis.yml", + AccountFileFormat::Keypair, + &mut genesis_block + ) + .is_ok()); + + remove_file(path).unwrap(); + + // Test total number of accounts is correct + assert_eq!( + genesis_block.accounts.len(), + primordial_accounts.len() + primordial_accounts1.len() + primordial_accounts2.len() + ); + + // Test old accounts are still there + (0..primordial_accounts.len()).for_each(|i| { + assert_eq!( + primordial_accounts[&genesis_block.accounts[i].0.to_string()], + genesis_block.accounts[i].1.lamports, + ); + }); + + // Test new account data matches + (0..primordial_accounts1.len()).for_each(|i| { + assert_eq!( + primordial_accounts1[&genesis_block.accounts[primordial_accounts.len() + i] + .0 + .to_string()], + genesis_block.accounts[primordial_accounts.len() + i] + .1 + .lamports, + ); + }); + + let offset = primordial_accounts.len() + primordial_accounts1.len(); + // Test account data for keypairs matches + account_keypairs.iter().for_each(|keypair| { + let mut i = 0; + (offset..(offset + account_keypairs.len())).for_each(|n| { + if keypair.pubkey() == genesis_block.accounts[n].0 { + i = n; + } + }); + + assert_ne!(i, 0); + assert_eq!( + primordial_accounts2[&serde_json::to_string(&keypair.to_bytes().to_vec()).unwrap()], + genesis_block.accounts[i].1.lamports, + ); + }); } } diff --git a/net/net.sh b/net/net.sh index a09002f625..68853ae7e9 100755 --- a/net/net.sh +++ b/net/net.sh @@ -340,6 +340,8 @@ startBootstrapLeader() { \"$externalPrimordialAccountsFile\" \ \"$stakeNodesInGenesisBlock\" \ $nodeIndex \ + $numBenchTpsClients \"$benchTpsExtraArgs\" \ + $numBenchExchangeClients \"$benchExchangeExtraArgs\" \ \"$genesisOptions\" \ " ) >> "$logFile" 2>&1 || { @@ -383,6 +385,7 @@ startNode() { startClient() { declare ipAddress=$1 declare clientToRun="$2" + declare clientIndex="$3" declare logFile="$netLogDir/client-$clientToRun-$ipAddress.log" echo "--- Starting client: $ipAddress - $clientToRun" echo "start log: $logFile" @@ -391,7 +394,7 @@ startClient() { startCommon "$ipAddress" ssh "${sshOptions[@]}" -f "$ipAddress" \ "./solana/net/remote/remote-client.sh $deployMethod $entrypointIp \ - $clientToRun \"$RUST_LOG\" \"$benchTpsExtraArgs\" \"$benchExchangeExtraArgs\"" + $clientToRun \"$RUST_LOG\" \"$benchTpsExtraArgs\" \"$benchExchangeExtraArgs\" $clientIndex" ) >> "$logFile" 2>&1 || { cat "$logFile" echo "^^^ +++" @@ -562,9 +565,9 @@ start() { SECONDS=0 for ((i=0; i < "$numClients" && i < "$numClientsRequested"; i++)) do if [[ $i -lt "$numBenchTpsClients" ]]; then - startClient "${clientIpList[$i]}" "solana-bench-tps" + startClient "${clientIpList[$i]}" "solana-bench-tps" "$i" else - startClient "${clientIpList[$i]}" "solana-bench-exchange" + startClient "${clientIpList[$i]}" "solana-bench-exchange" "$i" fi done clientDeployTime=$SECONDS diff --git a/net/remote/remote-client.sh b/net/remote/remote-client.sh index ecf6b9ac7e..8ef78f905e 100755 --- a/net/remote/remote-client.sh +++ b/net/remote/remote-client.sh @@ -11,6 +11,7 @@ clientToRun="$3" RUST_LOG="$4" benchTpsExtraArgs="$5" benchExchangeExtraArgs="$6" +clientIndex="$7" export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info missing() { @@ -54,6 +55,8 @@ scripts/net-stats.sh > net-stats.log 2>&1 & case $clientToRun in solana-bench-tps) + net/scripts/rsync-retry.sh -vPrc \ + "$entrypointIp":~/solana/solana-client-accounts/bench-tps"$clientIndex".yml ./client-accounts.yml clientCommand="\ solana-bench-tps \ --entrypoint $entrypointIp:8001 \ @@ -62,6 +65,7 @@ solana-bench-tps) --sustained \ --threads $threadCount \ $benchTpsExtraArgs \ + --read-client-keys ./client-accounts.yml \ " ;; solana-bench-exchange) diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index c556a3109b..1d9612cc26 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -14,7 +14,11 @@ failOnValidatorBootupFailure="$7" externalPrimordialAccountsFile="$8" stakeNodesInGenesisBlock="$9" nodeIndex="${10}" -genesisOptions="${11}" +numBenchTpsClients="${11}" +benchTpsExtraArgs="${12}" +numBenchExchangeClients="${13}" +benchExchangeExtraArgs="${14}" +genesisOptions="${15}" set +x export RUST_LOG @@ -90,9 +94,25 @@ local|tar) echo "${pubkey}: $stakeNodesInGenesisBlock" >> ./solana-node-stakes/fullnode-stakes.yml done fi + rm -rf ./solana-client-accounts + mkdir ./solana-client-accounts + for i in $(seq 0 $((numBenchTpsClients-1))); do + # shellcheck disable=SC2086 # Do not want to quote $benchTpsExtraArgs + solana-bench-tps --write-client-keys ./solana-client-accounts/bench-tps"$i".yml $benchTpsExtraArgs + # Skip first line, as it contains header + tail -n +2 -q ./solana-client-accounts/bench-tps"$i".yml >> ./solana-client-accounts/client-accounts.yml + echo "" >> ./solana-client-accounts/client-accounts.yml + done + for i in $(seq "$numBenchTpsClients" "$numBenchExchangeClients"); do + # shellcheck disable=SC2086 # Do not want to quote $benchExchangeExtraArgs + echo $benchExchangeExtraArgs +# solana-bench-exchange -w ./solana-client-accounts/bench-exchange"$i".yml $benchExchangeExtraArgs +# tail -n +2 -q ./solana-client-accounts/bench-exchange"$i".yml >> ./solana-client-accounts/client-accounts.yml + done [[ -z $externalPrimordialAccountsFile ]] || cat "$externalPrimordialAccountsFile" >> ./solana-node-stakes/fullnode-stakes.yml if [ -f ./solana-node-stakes/fullnode-stakes.yml ]; then - genesisOptions+=" --primordial-accounts-file ./solana-node-stakes/fullnode-stakes.yml" + genesisOptions+=" --primordial-accounts-file ./solana-node-stakes/fullnode-stakes.yml \ + --primordial-keypairs-file ./solana-client-accounts/client-accounts.yml" fi if [[ $skipSetup != true ]]; then args=(