blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Create bench exchange accounts in genesis block (#4655) 

* fix script

* review comments
This commit is contained in:
Pankaj Garg 2019-06-13 11:51:35 -07:00 committed by GitHub
parent 8abf22f34b
commit e3f895d7d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 169 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -2256,6 +2256,7 @@ dependencies = [
"serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_yaml 0.8.9 (registry+https://github.com/rust-lang/crates.io-index)",
"solana 0.16.0",
"solana-client 0.16.0",
"solana-drone 0.16.0",

3
bench-exchange/Cargo.toml
View File

@ -21,7 +21,8 @@ rand = "0.6.5"
rayon = "1.0.3"
serde = "1.0.92"
serde_derive = "1.0.92"
serde_json = "1.0.38"
serde_json = "1.0.39"
serde_yaml = "0.8.9"
# solana-runtime = { path = "../solana/runtime"}
solana = { path = "../core", version = "0.16.0" }
solana-client = { path = "../client", version = "0.16.0" }

108
bench-exchange/src/bench.rs
View File

@ -20,9 +20,12 @@ use solana_sdk::system_instruction;
use solana_sdk::timing::{duration_as_ms, duration_as_s};
use solana_sdk::transaction::Transaction;
use std::cmp;
use std::collections::VecDeque;
use std::collections::{HashMap, VecDeque};
use std::fs::File;
use std::io::prelude::*;
use std::mem;
use std::net::SocketAddr;
use std::path::Path;
use std::process::exit;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::mpsc::{channel, Receiver, Sender};
@ -48,6 +51,8 @@ pub struct Config {
pub batch_size: usize,
pub chunk_size: usize,
pub account_groups: usize,
pub client_ids_and_stake_file: String,
pub read_from_client_file: bool,
}
impl Default for Config {
@ -61,10 +66,38 @@ impl Default for Config {
batch_size: 10,
chunk_size: 10,
account_groups: 100,
client_ids_and_stake_file: String::new(),
read_from_client_file: false,
}
}
}
pub fn create_client_accounts_file(
client_ids_and_stake_file: &str,
batch_size: usize,
account_groups: usize,
fund_amount: u64,
) {
let accounts_in_groups = batch_size * account_groups;
const NUM_KEYPAIR_GROUPS: u64 = 2;
let total_keys = accounts_in_groups as u64 * NUM_KEYPAIR_GROUPS;
let keypairs = generate_keypairs(total_keys);
let mut accounts = HashMap::new();
keypairs.iter().for_each(|keypair| {
accounts.insert(
serde_json::to_string(&keypair.to_bytes().to_vec()).unwrap(),
fund_amount,
);
});
let serialized = serde_yaml::to_string(&accounts).unwrap();
let path = Path::new(&client_ids_and_stake_file);
let mut file = File::create(path).unwrap();
file.write_all(&serialized.into_bytes()).unwrap();
}
pub fn do_bench_exchange<T>(clients: Vec<T>, config: Config)
where
T: 'static + Client + Send + Sync,
@ -78,6 +111,8 @@ where
batch_size,
chunk_size,
account_groups,
client_ids_and_stake_file,
read_from_client_file,
} = config;
info!(
@ -92,35 +127,55 @@ where
);
let accounts_in_groups = batch_size * account_groups;
let exit_signal = Arc::new(AtomicBool::new(false));
const NUM_KEYPAIR_GROUPS: u64 = 2;
let total_keys = accounts_in_groups as u64 * NUM_KEYPAIR_GROUPS;
let mut signer_keypairs = if read_from_client_file {
let path = Path::new(&client_ids_and_stake_file);
let file = File::open(path).unwrap();
let accounts: HashMap<String, u64> = serde_yaml::from_reader(file).unwrap();
accounts
.into_iter()
.map(|(keypair, _)| {
let bytes: Vec<u8> = serde_json::from_str(keypair.as_str()).unwrap();
Keypair::from_bytes(&bytes).unwrap()
})
.collect()
} else {
info!("Generating {:?} signer keys", total_keys);
generate_keypairs(total_keys)
};
let trader_signers: Vec<_> = signer_keypairs
.drain(0..accounts_in_groups)
.map(Arc::new)
.collect();
let swapper_signers: Vec<_> = signer_keypairs
.drain(0..accounts_in_groups)
.map(Arc::new)
.collect();
let clients: Vec<_> = clients.into_iter().map(Arc::new).collect();
let client = clients[0].as_ref();
const NUM_KEYPAIR_GROUPS: u64 = 4;
let total_keys = accounts_in_groups as u64 * NUM_KEYPAIR_GROUPS;
info!("Generating {:?} keys", total_keys);
let mut keypairs = generate_keypairs(total_keys);
let trader_signers: Vec<_> = keypairs
.drain(0..accounts_in_groups)
.map(Arc::new)
.collect();
let swapper_signers: Vec<_> = keypairs
.drain(0..accounts_in_groups)
.map(Arc::new)
.collect();
let src_pubkeys: Vec<_> = keypairs
.drain(0..accounts_in_groups)
.map(|keypair| keypair.pubkey())
.collect();
let profit_pubkeys: Vec<_> = keypairs
.drain(0..accounts_in_groups)
.map(|keypair| keypair.pubkey())
.collect();
if !read_from_client_file {
info!("Fund trader accounts");
fund_keys(client, &identity, &trader_signers, fund_amount);
info!("Fund swapper accounts");
fund_keys(client, &identity, &swapper_signers, fund_amount);
}
info!("Fund trader accounts");
fund_keys(client, &identity, &trader_signers, fund_amount);
info!("Fund swapper accounts");
fund_keys(client, &identity, &swapper_signers, fund_amount);
info!("Generating {:?} account keys", total_keys);
let mut account_keypairs = generate_keypairs(total_keys);
let src_pubkeys: Vec<_> = account_keypairs
.drain(0..accounts_in_groups)
.map(|keypair| keypair.pubkey())
.collect();
let profit_pubkeys: Vec<_> = account_keypairs
.drain(0..accounts_in_groups)
.map(|keypair| keypair.pubkey())
.collect();
info!("Create {:?} source token accounts", src_pubkeys.len());
create_token_accounts(client, &trader_signers, &src_pubkeys);
@ -136,6 +191,7 @@ where
transfer_delay
);
let exit_signal = Arc::new(AtomicBool::new(false));
let shared_txs: SharedTransactions = Arc::new(RwLock::new(VecDeque::new()));
let total_txs_sent_count = Arc::new(AtomicUsize::new(0));
let s_threads: Vec<_> = (0..threads)

30
bench-exchange/src/cli.rs
View File

@ -18,6 +18,9 @@ pub struct Config {
pub batch_size: usize,
pub chunk_size: usize,
pub account_groups: usize,
pub client_ids_and_stake_file: String,
pub write_to_client_file: bool,
pub read_from_client_file: bool,
}
impl Default for Config {
@ -34,6 +37,9 @@ impl Default for Config {
batch_size: 100,
chunk_size: 100,
account_groups: 100,
client_ids_and_stake_file: String::new(),
write_to_client_file: false,
read_from_client_file: false,
}
}
}
@ -141,6 +147,20 @@ pub fn build_args<'a, 'b>() -> App<'a, 'b> {
.default_value("10")
.help("Number of account groups to cycle for each batch"),
)
.arg(
Arg::with_name("write-client-keys")
.long("write-client-keys")
.value_name("FILENAME")
.takes_value(true)
.help("Generate client keys and stakes and write the list to YAML file"),
)
.arg(
Arg::with_name("read-client-keys")
.long("read-client-keys")
.value_name("FILENAME")
.takes_value(true)
.help("Read client keys and stakes from the YAML file"),
)
}
pub fn extract_args<'a>(matches: &ArgMatches<'a>) -> Config {
@ -184,5 +204,15 @@ pub fn extract_args<'a>(matches: &ArgMatches<'a>) -> Config {
args.account_groups = value_t!(matches.value_of("account-groups"), usize)
.expect("Failed to parse account-groups");
if let Some(s) = matches.value_of("write-client-keys") {
args.write_to_client_file = true;
args.client_ids_and_stake_file = s.to_string();
}
if let Some(s) = matches.value_of("read-client-keys") {
assert!(!args.write_to_client_file);
args.read_from_client_file = true;
args.client_ids_and_stake_file = s.to_string();
}
args
}

67
bench-exchange/src/main.rs
View File

@ -6,7 +6,7 @@ pub mod order_book;
#[macro_use]
extern crate solana_exchange_program;
use crate::bench::{airdrop_lamports, do_bench_exchange, Config};
use crate::bench::{airdrop_lamports, create_client_accounts_file, do_bench_exchange, Config};
use log::*;
use solana::gossip_service::{discover_cluster, get_multi_client};
use solana_sdk::signature::KeypairUtil;
@ -30,33 +30,12 @@ fn main() {
batch_size,
chunk_size,
account_groups,
client_ids_and_stake_file,
write_to_client_file,
read_from_client_file,
..
} = cli_config;
info!("Connecting to the cluster");
let (nodes, _replicators) =
discover_cluster(&entrypoint_addr, num_nodes).unwrap_or_else(|_| {
panic!("Failed to discover nodes");
});
let (client, num_clients) = get_multi_client(&nodes);
info!("{} nodes found", num_clients);
if num_clients < num_nodes {
panic!("Error: Insufficient nodes discovered");
}
info!("Funding keypair: {}", identity.pubkey());
let accounts_in_groups = batch_size * account_groups;
const NUM_SIGNERS: u64 = 2;
airdrop_lamports(
&client,
&drone_addr,
&identity,
fund_amount * (accounts_in_groups + 1) as u64 * NUM_SIGNERS,
);
let config = Config {
identity,
threads,
@ -66,7 +45,43 @@ fn main() {
batch_size,
chunk_size,
account_groups,
client_ids_and_stake_file,
read_from_client_file,
};
do_bench_exchange(vec![client], config);
if write_to_client_file {
create_client_accounts_file(
&config.client_ids_and_stake_file,
config.batch_size,
config.account_groups,
config.fund_amount,
);
} else {
info!("Connecting to the cluster");
let (nodes, _replicators) =
discover_cluster(&entrypoint_addr, num_nodes).unwrap_or_else(|_| {
panic!("Failed to discover nodes");
});
let (client, num_clients) = get_multi_client(&nodes);
info!("{} nodes found", num_clients);
if num_clients < num_nodes {
panic!("Error: Insufficient nodes discovered");
}
if !read_from_client_file {
info!("Funding keypair: {}", config.identity.pubkey());
let accounts_in_groups = batch_size * account_groups;
const NUM_SIGNERS: u64 = 2;
airdrop_lamports(
&client,
&drone_addr,
&config.identity,
fund_amount * (accounts_in_groups + 1) as u64 * NUM_SIGNERS,
);
}
do_bench_exchange(vec![client], config);
}
}

2
net/net.sh
View File

@ -567,7 +567,7 @@ start() {
if [[ $i -lt "$numBenchTpsClients" ]]; then
startClient "${clientIpList[$i]}" "solana-bench-tps" "$i"
else
startClient "${clientIpList[$i]}" "solana-bench-exchange" "$i"
startClient "${clientIpList[$i]}" "solana-bench-exchange" $((i-numBenchTpsClients))
fi
done
clientDeployTime=$SECONDS

3
net/remote/remote-client.sh
View File

@ -70,6 +70,8 @@ solana-bench-tps)
;;
solana-bench-exchange)
solana-keygen new -f -o bench.keypair
net/scripts/rsync-retry.sh -vPrc \
"$entrypointIp":~/solana/solana-client-accounts/bench-exchange"$clientIndex".yml ./client-accounts.yml
clientCommand="\
solana-bench-exchange \
--entrypoint $entrypointIp:8001 \
@ -80,6 +82,7 @@ solana-bench-exchange)
--duration 7500 \
--identity bench.keypair \
$benchExchangeExtraArgs \
--read-client-keys ./client-accounts.yml \
"
;;
*)

15
net/remote/remote-node.sh
View File

@ -103,16 +103,19 @@ local|tar)
tail -n +2 -q ./solana-client-accounts/bench-tps"$i".yml >> ./solana-client-accounts/client-accounts.yml
echo "" >> ./solana-client-accounts/client-accounts.yml
done
for i in $(seq "$numBenchTpsClients" "$numBenchExchangeClients"); do
for i in $(seq 0 $((numBenchExchangeClients-1))); do
# shellcheck disable=SC2086 # Do not want to quote $benchExchangeExtraArgs
echo $benchExchangeExtraArgs
# solana-bench-exchange -w ./solana-client-accounts/bench-exchange"$i".yml $benchExchangeExtraArgs
# tail -n +2 -q ./solana-client-accounts/bench-exchange"$i".yml >> ./solana-client-accounts/client-accounts.yml
solana-bench-exchange --batch-size 1000 --fund-amount 20000 \
--write-client-keys ./solana-client-accounts/bench-exchange"$i".yml $benchExchangeExtraArgs
tail -n +2 -q ./solana-client-accounts/bench-exchange"$i".yml >> ./solana-client-accounts/client-accounts.yml
echo "" >> ./solana-client-accounts/client-accounts.yml
done
[[ -z $externalPrimordialAccountsFile ]] || cat "$externalPrimordialAccountsFile" >> ./solana-node-stakes/fullnode-stakes.yml
if [ -f ./solana-node-stakes/fullnode-stakes.yml ]; then
genesisOptions+=" --primordial-accounts-file ./solana-node-stakes/fullnode-stakes.yml \
--primordial-keypairs-file ./solana-client-accounts/client-accounts.yml"
genesisOptions+=" --primordial-accounts-file ./solana-node-stakes/fullnode-stakes.yml"
fi
if [ -f ./solana-client-accounts/client-accounts.yml ]; then
genesisOptions+=" --primordial-keypairs-file ./solana-client-accounts/client-accounts.yml"
fi
if [[ $skipSetup != true ]]; then
args=(