diff --git a/ci/testnet-deploy.sh b/ci/testnet-deploy.sh index e9aea0aa3..74d4fd3cf 100755 --- a/ci/testnet-deploy.sh +++ b/ci/testnet-deploy.sh @@ -13,6 +13,7 @@ snapChannel=edge tarChannelOrTag=edge delete=false enableGpu=false +leaderRotation=true useTarReleaseChannel=false usage() { @@ -42,6 +43,7 @@ Deploys a CD testnet -P - Use public network IP addresses (default: $publicNetwork) -G - Enable GPU, and set count/type of GPUs to use (e.g n1-standard-16 --accelerator count=4,type=nvidia-tesla-k80) -g - Enable GPU (default: $enableGpu) + -b - Disable leader rotation -a [address] - Set the bootstrap fullnode's external IP address to this GCE address -d - Delete the network @@ -59,7 +61,7 @@ zone=$3 [[ -n $zone ]] || usage "Zone not specified" shift 3 -while getopts "h?p:Pn:c:s:t:gG:a:d" opt; do +while getopts "h?p:Pn:c:s:t:gG:a:db" opt; do case $opt in h | \?) usage @@ -94,6 +96,9 @@ while getopts "h?p:Pn:c:s:t:gG:a:d" opt; do ;; esac ;; + b) + leaderRotation=false + ;; g) enableGpu=true ;; @@ -130,6 +135,10 @@ if $enableGpu; then fi fi +if ! $leaderRotation; then + create_args+=(-b) +fi + if $publicNetwork; then create_args+=(-P) fi diff --git a/multinode-demo/bootstrap-leader.sh b/multinode-demo/bootstrap-leader.sh index 44c9d750e..fd62dfd78 100755 --- a/multinode-demo/bootstrap-leader.sh +++ b/multinode-demo/bootstrap-leader.sh @@ -29,11 +29,28 @@ else program="$solana_fullnode" fi +maybe_no_leader_rotation= +if [[ $1 = --no-leader-rotation ]]; then + maybe_no_leader_rotation="--no-leader-rotation" + shift +fi + +if [[ -n $1 ]]; then + echo "Unknown argument: $1" + exit 1 +fi + +if [[ -d $SNAP ]]; then + if [[ $(snapctl get leader-rotation) = false ]]; then + maybe_no_leader_rotation="--no-leader-rotation" + fi +fi + tune_networking trap 'kill "$pid" && wait "$pid"' INT TERM $program \ - --no-leader-rotation \ + $maybe_no_leader_rotation \ --identity "$SOLANA_CONFIG_DIR"/bootstrap-leader.json \ --ledger "$SOLANA_CONFIG_DIR"/bootstrap-leader-ledger \ --rpc 8899 \ diff --git a/multinode-demo/fullnode.sh b/multinode-demo/fullnode.sh index 862aa4e3f..1807d8352 100755 --- a/multinode-demo/fullnode.sh +++ b/multinode-demo/fullnode.sh @@ -20,11 +20,12 @@ usage() { echo "$*" echo fi - echo "usage: $0 [-x] [rsync network path to bootstrap leader configuration] [network entry point]" + echo "usage: $0 [-x] [--no-leader-rotation] [rsync network path to bootstrap leader configuration] [network entry point]" echo echo " Start a full node on the specified network" echo echo " -x: runs a new, dynamically-configured full node" + echo " --no-leader-rotation: disable leader rotation" echo exit 1 } @@ -33,13 +34,25 @@ if [[ $1 = -h ]]; then usage fi -if [[ $1 == -x ]]; then +if [[ $1 = -x ]]; then self_setup=1 shift else self_setup=0 fi +maybe_no_leader_rotation= +if [[ $1 = --no-leader-rotation ]]; then + maybe_no_leader_rotation="--no-leader-rotation" + shift +fi + +if [[ -d $SNAP ]]; then + if [[ $(snapctl get leader-rotation) = false ]]; then + maybe_no_leader_rotation="--no-leader-rotation" + fi +fi + if [[ -n $3 ]]; then usage fi @@ -175,7 +188,7 @@ $solana_wallet \ trap 'kill "$pid" && wait "$pid"' INT TERM $program \ - --no-leader-rotation \ + $maybe_no_leader_rotation \ --identity "$fullnode_json_path" \ --network "$leader_address" \ --ledger "$ledger_config_dir"/ledger \ diff --git a/net/common.sh b/net/common.sh index 156f3509c..128f8c6ab 100644 --- a/net/common.sh +++ b/net/common.sh @@ -29,6 +29,7 @@ fullnodeIpList=() fullnodeIpListPrivate=() clientIpList=() clientIpListPrivate=() +leaderRotation= buildSshOptions() { sshOptions=( @@ -50,6 +51,7 @@ loadConfigFile() { [[ -n "$publicNetwork" ]] || usage "Config file invalid, publicNetwork unspecified: $configFile" [[ -n "$netBasename" ]] || usage "Config file invalid, netBasename unspecified: $configFile" [[ -n $sshPrivateKey ]] || usage "Config file invalid, sshPrivateKey unspecified: $configFile" + [[ -n $leaderRotation ]] || usage "Config file invalid, leaderRotation unspecified: $configFile" [[ ${#fullnodeIpList[@]} -gt 0 ]] || usage "Config file invalid, fullnodeIpList unspecified: $configFile" [[ ${#fullnodeIpListPrivate[@]} -gt 0 ]] || usage "Config file invalid, fullnodeIpListPrivate unspecified: $configFile" diff --git a/net/gce.sh b/net/gce.sh index 649b4ccb3..e5b876329 100755 --- a/net/gce.sh +++ b/net/gce.sh @@ -43,6 +43,7 @@ clientBootDiskSizeInGb=75 publicNetwork=false enableGpu=false bootstrapLeaderAddress= +leaderRotation=true usage() { exitcode=0 @@ -76,6 +77,7 @@ Manage testnet instances For EC2, [address] is the "allocation ID" of the desired Elastic IP. -d [disk-type] - Specify a boot disk type (default None) Use pd-ssd to get ssd on GCE. + -b - Disable leader rotation config-specific options: -P - Use public network IP addresses (default: $publicNetwork) @@ -93,7 +95,7 @@ command=$1 shift [[ $command = create || $command = config || $command = delete ]] || usage "Invalid command: $command" -while getopts "h?p:Pn:c:z:gG:a:d:" opt; do +while getopts "h?p:Pn:c:z:gG:a:d:b" opt; do case $opt in h | \?) usage @@ -114,6 +116,9 @@ while getopts "h?p:Pn:c:z:gG:a:d:" opt; do z) cloud_SetZone "$OPTARG" ;; + b) + leaderRotation=false + ;; g) enableGpu=true bootstrapLeaderMachineType=$gpuBootstrapLeaderMachineType @@ -179,6 +184,9 @@ ec2) ;; esac +if $leaderRotation; then + fullNodeMachineType=$bootstrapLeaderMachineType +fi # cloud_ForEachInstance [cmd] [extra args to cmd] # @@ -218,6 +226,7 @@ prepareInstancesAndWriteConfigFile() { netBasename=$prefix publicNetwork=$publicNetwork sshPrivateKey=$sshPrivateKey +leaderRotation=$leaderRotation EOF buildSshOptions @@ -369,6 +378,8 @@ Network composition: Additional fullnodes = $additionalFullNodeCount x $fullNodeMachineType Client(s) = $clientNodeCount x $clientMachineType +Leader rotation: $leaderRotation + ======================================================================================== EOF diff --git a/net/net.sh b/net/net.sh index f38acca37..70dcdee86 100755 --- a/net/net.sh +++ b/net/net.sh @@ -197,6 +197,7 @@ startBootstrapLeader() { ${#fullnodeIpList[@]} \ \"$RUST_LOG\" \ $skipSetup \ + $leaderRotation \ " ) >> "$logFile" 2>&1 || { cat "$logFile" @@ -223,6 +224,7 @@ startNode() { ${#fullnodeIpList[@]} \ \"$RUST_LOG\" \ $skipSetup \ + $leaderRotation \ " ) >> "$logFile" 2>&1 & declare pid=$! diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index ff1c04cec..6ec43089a 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -11,6 +11,7 @@ entrypointIp="$4" numNodes="$5" RUST_LOG="$6" skipSetup="$7" +leaderRotation="$8" set +x export RUST_LOG=${RUST_LOG:-solana=warn} # if RUST_LOG is unset, default to warn @@ -25,11 +26,13 @@ missing() { [[ -n $entrypointIp ]] || missing entrypointIp [[ -n $numNodes ]] || missing numNodes [[ -n $skipSetup ]] || missing skipSetup +[[ -n $leaderRotation ]] || missing leaderRotation cat > deployConfig < oom-monitor.log 2>&1 & scripts/net-stats.sh > net-stats.log 2>&1 & + maybeNoLeaderRotation= + if ! $leaderRotation; then + maybeNoLeaderRotation="--no-leader-rotation" + fi + + case $nodeType in bootstrap-leader) if [[ -e /dev/nvidia0 && -x ~/.cargo/bin/solana-fullnode-cuda ]]; then @@ -144,7 +154,7 @@ local|tar) ./multinode-demo/setup.sh -t bootstrap-leader $setupArgs fi ./multinode-demo/drone.sh > drone.log 2>&1 & - ./multinode-demo/bootstrap-leader.sh > bootstrap-leader.log 2>&1 & + ./multinode-demo/bootstrap-leader.sh $maybeNoLeaderRotation > bootstrap-leader.log 2>&1 & ln -sTf bootstrap-leader.log fullnode.log ;; fullnode) @@ -158,7 +168,7 @@ local|tar) if [[ $skipSetup != true ]]; then ./multinode-demo/setup.sh -t fullnode $setupArgs fi - ./multinode-demo/fullnode.sh "$entrypointIp":~/solana "$entrypointIp:8001" > fullnode.log 2>&1 & + ./multinode-demo/fullnode.sh $maybeNoLeaderRotation "$entrypointIp":~/solana "$entrypointIp:8001" > fullnode.log 2>&1 & ;; *) echo "Error: unknown node type: $nodeType" diff --git a/net/remote/remote-sanity.sh b/net/remote/remote-sanity.sh index 004043062..f475253fb 100755 --- a/net/remote/remote-sanity.sh +++ b/net/remote/remote-sanity.sh @@ -22,9 +22,10 @@ missing() { exit 1 } -[[ -n $deployMethod ]] || missing deployMethod -[[ -n $entrypointIp ]] || missing entrypointIp -[[ -n $numNodes ]] || missing numNodes +[[ -n $deployMethod ]] || missing deployMethod +[[ -n $entrypointIp ]] || missing entrypointIp +[[ -n $numNodes ]] || missing numNodes +[[ -n $leaderRotation ]] || missing leaderRotation ledgerVerify=true validatorSanity=true