blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Facility to provision primordial accounts for fullnodes in genesis block (#4631) 

* updated usage

* shellcheck

* support replicators

* disable airdrops if primordial accounts are used

* review comments
This commit is contained in:
Pankaj Garg 2019-06-10 19:42:49 -07:00 committed by GitHub
parent b4d4edb645
commit 9259d342ac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 105 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
genesis/src/main.rs
View File

@ -13,7 +13,6 @@ extern crate solana_config_program;
extern crate solana_exchange_program;
use clap::{crate_description, crate_name, crate_version, value_t_or_exit, App, Arg};
use serde_derive::{Deserialize, Serialize};
use solana::blocktree::create_new_ledger;
use solana_sdk::account::Account;
use solana_sdk::fee_calculator::FeeCalculator;
@ -27,29 +26,25 @@ use solana_sdk::timing;
use solana_stake_api::stake_state;
use solana_storage_program::genesis_block_util::GenesisBlockUtil;
use solana_vote_api::vote_state;
use std::collections::HashMap;
use std::error;
use std::fs::File;
use std::io;
use std::str::FromStr;
use std::time::{Duration, Instant};
pub const BOOTSTRAP_LEADER_LAMPORTS: u64 = 42;
#[derive(Serialize, Deserialize, Default, Debug, PartialEq)]
pub struct PrimordialAccount {
pub pubkey: Pubkey,
pub lamports: u64,
}
pub fn append_primordial_accounts(file: &str, genesis_block: &mut GenesisBlock) -> io::Result<()> {
let accounts_file = File::open(file.to_string())?;
let primordial_accounts: Vec<PrimordialAccount> = serde_yaml::from_reader(accounts_file)
let primordial_accounts: HashMap<String, u64> = serde_yaml::from_reader(accounts_file)
.map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
primordial_accounts.iter().for_each(|primordial| {
primordial_accounts.into_iter().for_each(|primordial| {
genesis_block.accounts.push((
primordial.pubkey,
Account::new(primordial.lamports, 0, &system_program::id()),
Pubkey::from_str(primordial.0.as_str()).unwrap(),
Account::new(primordial.1, 0, &system_program::id()),
))
});
@ -314,6 +309,7 @@ mod tests {
use hashbrown::HashSet;
use solana_sdk::genesis_block::GenesisBlock;
use solana_sdk::pubkey::Pubkey;
use std::collections::HashMap;
use std::fs::remove_file;
use std::io::Write;
use std::path::Path;
@ -343,20 +339,10 @@ mod tests {
// Test invalid file returns error
assert!(append_primordial_accounts("unknownfile", &mut genesis_block).is_err());
let primordial_accounts = [
PrimordialAccount {
pubkey: Pubkey::new_rand(),
lamports: 2,
},
PrimordialAccount {
pubkey: Pubkey::new_rand(),
lamports: 1,
},
PrimordialAccount {
pubkey: Pubkey::new_rand(),
lamports: 3,
},
];
let mut primordial_accounts = HashMap::new();
primordial_accounts.insert(Pubkey::new_rand().to_string(), 2 as u64);
primordial_accounts.insert(Pubkey::new_rand().to_string(), 1 as u64);
primordial_accounts.insert(Pubkey::new_rand().to_string(), 3 as u64);
let serialized = serde_yaml::to_string(&primordial_accounts).unwrap();
let path = Path::new("test_append_primordial_accounts_to_genesis.yml");
@ -377,28 +363,17 @@ mod tests {
// Test account data matches
(0..primordial_accounts.len()).for_each(|i| {
assert_eq!(genesis_block.accounts[i].0, primordial_accounts[i].pubkey);
assert_eq!(
primordial_accounts[&genesis_block.accounts[i].0.to_string()],
genesis_block.accounts[i].1.lamports,
primordial_accounts[i].lamports
);
});
// Test more accounts can be appended
let primordial_accounts1 = [
PrimordialAccount {
pubkey: Pubkey::new_rand(),
lamports: 6,
},
PrimordialAccount {
pubkey: Pubkey::new_rand(),
lamports: 5,
},
PrimordialAccount {
pubkey: Pubkey::new_rand(),
lamports: 10,
},
];
let mut primordial_accounts1 = HashMap::new();
primordial_accounts1.insert(Pubkey::new_rand().to_string(), 6 as u64);
primordial_accounts1.insert(Pubkey::new_rand().to_string(), 5 as u64);
primordial_accounts1.insert(Pubkey::new_rand().to_string(), 10 as u64);
let serialized = serde_yaml::to_string(&primordial_accounts1).unwrap();
let path = Path::new("test_append_primordial_accounts_to_genesis.yml");
@ -421,24 +396,21 @@ mod tests {
// Test old accounts are still there
(0..primordial_accounts.len()).for_each(|i| {
assert_eq!(genesis_block.accounts[i].0, primordial_accounts[i].pubkey);
assert_eq!(
primordial_accounts[&genesis_block.accounts[i].0.to_string()],
genesis_block.accounts[i].1.lamports,
primordial_accounts[i].lamports
);
});
// Test new account data matches
(0..primordial_accounts1.len()).for_each(|i| {
assert_eq!(
genesis_block.accounts[primordial_accounts.len() + i].0,
primordial_accounts1[i].pubkey
);
assert_eq!(
primordial_accounts1[&genesis_block.accounts[primordial_accounts.len() + i]
.0
.to_string()],
genesis_block.accounts[primordial_accounts.len() + i]
.1
.lamports,
primordial_accounts1[i].lamports
);
});
}

23
multinode-demo/fullnode.sh
View File

@ -29,6 +29,7 @@ Start a validator or a replicator
--no-voting - start node without vote signer
--rpc-port port - custom RPC port for this node
--no-restart - do not restart the node if it exits
--no-airdrop - The genesis block has an account for the node. Airdrops are not required.
EOF
exit 1
@ -97,9 +98,14 @@ setup_validator_accounts() {
if [[ -f $configured_flag ]]; then
echo "Vote and stake accounts have already been configured"
else
# Fund the node with enough tokens to fund its Vote, Staking, and Storage accounts
declare fees=100 # TODO: No hardcoded transaction fees, fetch the current cluster fees
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" airdrop $((node_lamports+stake_lamports+fees)) || return $?
if ((airdrops_enabled)); then
# Fund the node with enough tokens to fund its Vote, Staking, and Storage accounts
declare fees=100 # TODO: No hardcoded transaction fees, fetch the current cluster fees
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" airdrop $((node_lamports+stake_lamports+fees)) || return $?
else
echo "current account balance is "
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" balance || return $?
fi
# Fund the vote account from the node, with the node as the node_pubkey
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" \
@ -149,7 +155,12 @@ setup_replicator_account() {
if [[ -f $configured_flag ]]; then
echo "Replicator account has already been configured"
else
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" airdrop "$node_lamports" || return $?
if ((airdrops_enabled)); then
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" airdrop "$node_lamports" || return $?
else
echo "current account balance is "
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" balance || return $?
fi
# Setup replicator storage account
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" \
@ -179,6 +190,7 @@ poll_for_new_genesis_block=0
label=
identity_keypair_path=
no_restart=0
airdrops_enabled=1
positional_args=()
while [[ -n $1 ]]; do
@ -233,6 +245,9 @@ while [[ -n $1 ]]; do
elif [[ $1 = --gossip-port ]]; then
args+=("$1" "$2")
shift 2
elif [[ $1 = --no-airdrop ]]; then
airdrops_enabled=0
shift
elif [[ $1 = -h ]]; then
fullnode_usage "$@"
else

34
net/net.sh
View File

@ -55,6 +55,12 @@ Operate a configured testnet
- Override the default --hashes-per-tick for the cluster
-n NUM_FULL_NODES - Number of fullnodes to apply command to.
-x Accounts and Stakes for external nodes
- A YML file with a list of account pubkeys and corresponding stakes
for external nodes
-s Num lamports per node in genesis block
- Create account keypairs for internal nodes and assign these many lamports
sanity/start/update-specific options:
-F - Discard validator nodes that didn't bootup successfully
-o noLedgerVerify - Skip ledger verification
@ -89,6 +95,8 @@ benchExchangeExtraArgs=
failOnValidatorBootupFailure=true
genesisOptions=
numFullnodesRequested=
externalPrimordialAccountsFile=
stakeNodesInGenesisBlock=
command=$1
[[ -n $command ]] || usage
@ -112,7 +120,7 @@ while [[ -n $1 ]]; do
fi
done
while getopts "h?T:t:o:f:rD:c:Fn:" opt "${shortArgs[@]}"; do
while getopts "h?T:t:o:f:rD:c:Fn:x:s:" opt "${shortArgs[@]}"; do
case $opt in
h | \?)
usage
@ -191,6 +199,12 @@ while getopts "h?T:t:o:f:rD:c:Fn:" opt "${shortArgs[@]}"; do
F)
failOnValidatorBootupFailure=false
;;
x)
externalPrimordialAccountsFile=$OPTARG
;;
s)
stakeNodesInGenesisBlock=$OPTARG
;;
*)
usage "Error: unhandled option: $opt"
;;
@ -291,6 +305,7 @@ startCommon() {
startBootstrapLeader() {
declare ipAddress=$1
declare logFile="$2"
declare nodeIndex="$3"
echo "--- Starting bootstrap leader: $ipAddress"
echo "start log: $logFile"
@ -299,6 +314,8 @@ startBootstrapLeader() {
(
set -x
startCommon "$ipAddress" || exit 1
[[ -z "$externalPrimordialAccountsFile" ]] || rsync -vPrc -e "ssh ${sshOptions[*]}" "$externalPrimordialAccountsFile" \
"$ipAddress:~/solana/config/external-primodial-accounts.yml"
case $deployMethod in
tar)
rsync -vPrc -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/solana-release/bin/* "$ipAddress:~/.cargo/bin/"
@ -316,10 +333,13 @@ startBootstrapLeader() {
$deployMethod \
bootstrap-leader \
$entrypointIp \
$((${#fullnodeIpList[@]} + ${#blockstreamerIpList[@]})) \
$((${#fullnodeIpList[@]} + ${#blockstreamerIpList[@]} + ${#replicatorIpList[@]})) \
\"$RUST_LOG\" \
$skipSetup \
$failOnValidatorBootupFailure \
\"$externalPrimordialAccountsFile\" \
\"$stakeNodesInGenesisBlock\" \
$nodeIndex \
\"$genesisOptions\" \
"
) >> "$logFile" 2>&1 || {
@ -332,6 +352,7 @@ startBootstrapLeader() {
startNode() {
declare ipAddress=$1
declare nodeType=$2
declare nodeIndex="$3"
declare logFile="$netLogDir/fullnode-$ipAddress.log"
echo "--- Starting $nodeType: $ipAddress"
@ -344,10 +365,13 @@ startNode() {
$deployMethod \
$nodeType \
$entrypointIp \
$((${#fullnodeIpList[@]} + ${#blockstreamerIpList[@]})) \
$((${#fullnodeIpList[@]} + ${#blockstreamerIpList[@]} + ${#replicatorIpList[@]})) \
\"$RUST_LOG\" \
$skipSetup \
$failOnValidatorBootupFailure \
\"$externalPrimordialAccountsFile\" \
\"$stakeNodesInGenesisBlock\" \
$nodeIndex \
\"$genesisOptions\" \
"
) >> "$logFile" 2>&1 &
@ -492,7 +516,7 @@ start() {
if $bootstrapLeader; then
SECONDS=0
declare bootstrapNodeDeployTime=
startBootstrapLeader "$ipAddress" "$netLogDir/bootstrap-leader-$ipAddress.log"
startBootstrapLeader "$ipAddress" "$netLogDir/bootstrap-leader-$ipAddress.log" $loopCount
bootstrapNodeDeployTime=$SECONDS
$metricsWriteDatapoint "testnet-deploy net-bootnode-leader-started=1"
@ -500,7 +524,7 @@ start() {
SECONDS=0
pids=()
else
startNode "$ipAddress" $nodeType
startNode "$ipAddress" $nodeType $loopCount
# Stagger additional node start time. If too many nodes start simultaneously
# the bootstrap node gets more rsync requests from the additional nodes than

38
net/remote/remote-node.sh
View File

@ -11,7 +11,10 @@ numNodes="$4"
RUST_LOG="$5"
skipSetup="$6"
failOnValidatorBootupFailure="$7"
genesisOptions="$8"
externalPrimordialAccountsFile="$8"
stakeNodesInGenesisBlock="$9"
nodeIndex="${10}"
genesisOptions="${11}"
set +x
export RUST_LOG
@ -77,6 +80,20 @@ local|tar)
export SOLANA_CUDA=1
fi
set -x
rm -rf ./solana-node-keys
rm -rf ./solana-node-stakes
mkdir ./solana-node-stakes
if [[ -n $stakeNodesInGenesisBlock ]]; then
for i in $(seq 0 "$numNodes"); do
solana-keygen new -o ./solana-node-keys/"$i"
pubkey="$(solana-keygen pubkey ./solana-node-keys/"$i")"
echo "${pubkey}: $stakeNodesInGenesisBlock" >> ./solana-node-stakes/fullnode-stakes.yml
done
fi
[[ -z $externalPrimordialAccountsFile ]] || cat "$externalPrimordialAccountsFile" >> ./solana-node-stakes/fullnode-stakes.yml
if [ -f ./solana-node-stakes/fullnode-stakes.yml ]; then
genesisOptions+=" --primordial-accounts-file ./solana-node-stakes/fullnode-stakes.yml"
fi
if [[ $skipSetup != true ]]; then
args=(
--bootstrap-leader-stake-lamports "$stake"
@ -92,11 +109,17 @@ local|tar)
--gossip-port "$entrypointIp":8001
)
if [[ -n $stakeNodesInGenesisBlock ]]; then
args+=(--no-airdrop)
fi
nohup ./multinode-demo/validator.sh --bootstrap-leader "${args[@]}" > fullnode.log 2>&1 &
sleep 1
;;
validator|blockstreamer)
net/scripts/rsync-retry.sh -vPrc "$entrypointIp":~/.cargo/bin/ ~/.cargo/bin/
rm -f ~/solana/fullnode-identity.json
[[ -z $stakeNodesInGenesisBlock ]] || net/scripts/rsync-retry.sh -vPrc \
"$entrypointIp":~/solana/solana-node-keys/"$nodeIndex" ~/solana/fullnode-identity.json
if [[ -e /dev/nvidia0 && -x ~/.cargo/bin/solana-validator-cuda ]]; then
echo Selecting solana-validator-cuda
@ -119,6 +142,14 @@ local|tar)
args+=(--enable-rpc-exit)
fi
if [[ -f ~/solana/fullnode-identity.json ]]; then
args+=(--identity ~/solana/fullnode-identity.json)
fi
if [[ -n $stakeNodesInGenesisBlock ]]; then
args+=(--no-airdrop)
fi
set -x
if [[ $skipSetup != true ]]; then
./multinode-demo/clear-config.sh
@ -157,6 +188,11 @@ local|tar)
args=(
"$entrypointIp":~/solana "$entrypointIp:8001"
)
if [[ -n $stakeNodesInGenesisBlock ]]; then
args+=(--no-airdrop)
fi
if [[ $skipSetup != true ]]; then
./multinode-demo/clear-config.sh
fi