From f5794de636d6db5a2cc4fc248904cc63b108f61c Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Sun, 9 Dec 2018 09:42:09 -0800 Subject: [PATCH] Clean up bootstrap leader terminology in comments and variable names --- net/common.sh | 18 ++++++++----- net/gce.sh | 63 ++++++++++++++++++++------------------------ net/net.sh | 73 +++++++++++++++++++++++++++++++-------------------- net/ssh.sh | 7 ++--- 4 files changed, 87 insertions(+), 74 deletions(-) diff --git a/net/common.sh b/net/common.sh index 7191d69ea..156f3509c 100644 --- a/net/common.sh +++ b/net/common.sh @@ -22,12 +22,13 @@ configFile="$netConfigDir/config" entrypointIp= publicNetwork= -bootstrapFullNodeIp= netBasename= sshPrivateKey= -clientIpList=() sshOptions=() -additionalFullNodeIps=() +fullnodeIpList=() +fullnodeIpListPrivate=() +clientIpList=() +clientIpListPrivate=() buildSshOptions() { sshOptions=( @@ -46,12 +47,17 @@ loadConfigFile() { # shellcheck source=/dev/null source "$configFile" - [[ -n "$entrypointIp" ]] || usage "Config file invalid, entrypointIp unspecified: $configFile" [[ -n "$publicNetwork" ]] || usage "Config file invalid, publicNetwork unspecified: $configFile" - [[ -n "$bootstrapFullNodeIp" ]] || usage "Config file invalid, bootstrapFullNodeIp unspecified: $configFile" [[ -n "$netBasename" ]] || usage "Config file invalid, netBasename unspecified: $configFile" [[ -n $sshPrivateKey ]] || usage "Config file invalid, sshPrivateKey unspecified: $configFile" - [[ ${#additionalFullNodeIps[@]} -gt 0 ]] || usage "Config file invalid, additionalFullNodeIps unspecified: $configFile" + [[ ${#fullnodeIpList[@]} -gt 0 ]] || usage "Config file invalid, fullnodeIpList unspecified: $configFile" + [[ ${#fullnodeIpListPrivate[@]} -gt 0 ]] || usage "Config file invalid, fullnodeIpListPrivate unspecified: $configFile" + + if $publicNetwork; then + entrypointIp=${fullnodeIpList[0]} + else + entrypointIp=${fullnodeIpListPrivate[0]} + fi buildSshOptions configureMetrics diff --git a/net/gce.sh b/net/gce.sh index 713fbff9a..39d46cd97 100755 --- a/net/gce.sh +++ b/net/gce.sh @@ -12,9 +12,9 @@ gce) # shellcheck source=net/scripts/gce-provider.sh source "$here"/scripts/gce-provider.sh - cpuBootstrapFullNodeMachineType=n1-standard-16 - gpuBootstrapFullNodeMachineType="$cpuBootstrapFullNodeMachineType --accelerator count=4,type=nvidia-tesla-k80" - bootstrapFullNodeMachineType=$cpuBootstrapFullNodeMachineType + cpuBootstrapLeaderMachineType=n1-standard-16 + gpuBootstrapLeaderMachineType="$cpuBootstrapLeaderMachineType --accelerator count=4,type=nvidia-tesla-k80" + bootstrapLeaderMachineType=$cpuBootstrapLeaderMachineType fullNodeMachineType=n1-standard-16 clientMachineType=n1-standard-16 ;; @@ -22,9 +22,9 @@ ec2) # shellcheck source=net/scripts/ec2-provider.sh source "$here"/scripts/ec2-provider.sh - cpuBootstrapFullNodeMachineType=m4.4xlarge - gpuBootstrapFullNodeMachineType=p2.xlarge - bootstrapFullNodeMachineType=$cpuBootstrapFullNodeMachineType + cpuBootstrapLeaderMachineType=m4.4xlarge + gpuBootstrapLeaderMachineType=p2.xlarge + bootstrapLeaderMachineType=$cpuBootstrapLeaderMachineType fullNodeMachineType=m4.2xlarge clientMachineType=m4.2xlarge ;; @@ -42,7 +42,7 @@ clientBootDiskSizeInGb=75 publicNetwork=false enableGpu=false -bootstrapFullNodeAddress= +bootstrapLeaderAddress= usage() { exitcode=0 @@ -64,13 +64,13 @@ Manage testnet instances collisions (default: $prefix) create-specific options: - -n [number] - Number of additional full nodes (default: $additionalFullNodeCount) + -n [number] - Number of additional fullnodes (default: $additionalFullNodeCount) -c [number] - Number of client nodes (default: $clientNodeCount) -P - Use public network IP addresses (default: $publicNetwork) -z [zone] - Zone for the nodes (default: $zone) -g - Enable GPU (default: $enableGpu) - -G - Enable GPU, and set count/type of GPUs to use (e.g $cpuBootstrapFullNodeMachineType --accelerator count=4,type=nvidia-tesla-k80) - -a [address] - Set the bootstreap full node's external IP address to this value. + -G - Enable GPU, and set count/type of GPUs to use (e.g $cpuBootstrapLeaderMachineType --accelerator count=4,type=nvidia-tesla-k80) + -a [address] - Set the bootstreap fullnode's external IP address to this value. For GCE, [address] is the "name" of the desired External IP Address. For EC2, [address] is the "allocation ID" of the desired @@ -116,14 +116,14 @@ while getopts "h?p:Pn:c:z:gG:a:d:" opt; do ;; g) enableGpu=true - bootstrapFullNodeMachineType=$gpuBootstrapFullNodeMachineType + bootstrapLeaderMachineType=$gpuBootstrapLeaderMachineType ;; G) enableGpu=true - bootstrapFullNodeMachineType="$OPTARG" + bootstrapLeaderMachineType="$OPTARG" ;; a) - bootstrapFullNodeAddress=$OPTARG + bootstrapLeaderAddress=$OPTARG ;; d) bootDiskType=$OPTARG @@ -230,13 +230,7 @@ EOF declare arrayName="$5" echo "$arrayName+=($publicIp) # $name" >> "$configFile" - if [[ $arrayName = "bootstrapFullNodeIp" ]]; then - if $publicNetwork; then - echo "entrypointIp=$publicIp" >> "$configFile" - else - echo "entrypointIp=$privateIp" >> "$configFile" - fi - fi + echo "${arrayName}Private+=($privateIp) # $name" >> "$configFile" } waitForStartupComplete() { @@ -261,10 +255,10 @@ EOF echo "$name has booted." } - echo "Looking for bootstrap fullnode instance..." - cloud_FindInstance "$prefix-bootstrap-fullnode" + echo "Looking for bootstrap leader instance..." + cloud_FindInstance "$prefix-bootstrap-leader" [[ ${#instances[@]} -eq 1 ]] || { - echo "Unable to find bootstrap fullnode" + echo "Unable to find bootstrap leader" exit 1 } @@ -296,8 +290,9 @@ EOF fi ) - echo "bootstrapFullNodeIp=()" >> "$configFile" - cloud_ForEachInstance recordInstanceIp bootstrapFullNodeIp + echo "fullnodeIpList=()" >> "$configFile" + echo "fullnodeIpListPrivate=()" >> "$configFile" + cloud_ForEachInstance recordInstanceIp fullnodeIpList cloud_ForEachInstance waitForStartupComplete echo "Looking for additional fullnode instances..." @@ -306,11 +301,11 @@ EOF echo "Unable to find additional fullnodes" exit 1 } - echo "additionalFullNodeIps=()" >> "$configFile" - cloud_ForEachInstance recordInstanceIp additionalFullNodeIps + cloud_ForEachInstance recordInstanceIp fullnodeIpList cloud_ForEachInstance waitForStartupComplete echo "clientIpList=()" >> "$configFile" + echo "clientIpListPrivate=()" >> "$configFile" echo "Looking for client bencher instances..." cloud_FindInstances "$prefix-client" [[ ${#instances[@]} -eq 0 ]] || { @@ -325,11 +320,11 @@ EOF delete() { $metricsWriteDatapoint "testnet-deploy net-delete-begin=1" - # Delete the bootstrap fullnode first to prevent unusual metrics on the dashboard + # Delete the bootstrap leader first to prevent unusual metrics on the dashboard # during shutdown (only applicable when leader rotation is disabled). # TODO: It would be better to fully cut-off metrics reporting before any # instances are deleted. - for filter in "$prefix-bootstrap-fullnode" "$prefix-"; do + for filter in "$prefix-bootstrap-leader" "$prefix-"; do echo "Searching for instances: $filter" cloud_FindInstances "$filter" @@ -370,8 +365,8 @@ create) ======================================================================================== Network composition: - Bootstrap full node = $bootstrapFullNodeMachineType (GPU=$enableGpu) - Additional full nodes = $additionalFullNodeCount x $fullNodeMachineType + Bootstrap leader = $bootstrapLeaderMachineType (GPU=$enableGpu) + Additional fullnodes = $additionalFullNodeCount x $fullNodeMachineType Client(s) = $clientNodeCount x $clientMachineType ======================================================================================== @@ -434,9 +429,9 @@ touch /.instance-startup-complete EOF - cloud_CreateInstances "$prefix" "$prefix-bootstrap-fullnode" 1 \ - "$imageName" "$bootstrapFullNodeMachineType" "$fullNodeBootDiskSizeInGb" \ - "$startupScript" "$bootstrapFullNodeAddress" "$bootDiskType" + cloud_CreateInstances "$prefix" "$prefix-bootstrap-leader" 1 \ + "$imageName" "$bootstrapLeaderMachineType" "$fullNodeBootDiskSizeInGb" \ + "$startupScript" "$bootstrapLeaderAddress" "$bootDiskType" cloud_CreateInstances "$prefix" "$prefix-fullnode" "$additionalFullNodeCount" \ "$imageName" "$fullNodeMachineType" "$fullNodeBootDiskSizeInGb" \ diff --git a/net/net.sh b/net/net.sh index 9c7f2ee01..8e3a93775 100755 --- a/net/net.sh +++ b/net/net.sh @@ -117,7 +117,6 @@ while getopts "h?S:s:T:t:o:f:" opt; do done loadConfigFile -expectedNodeCount=$((${#additionalFullNodeIps[@]} + 1)) build() { declare MAYBE_DOCKER= @@ -157,13 +156,13 @@ startCommon() { "$ipAddress":~/solana/ } -startBootstrapNode() { +startBootstrapLeader() { declare ipAddress=$1 declare logFile="$2" - echo "--- Starting bootstrap full node: $bootstrapFullNodeIp" + echo "--- Starting bootstrap leader: $ipAddress" echo "start log: $logFile" - # Deploy local binaries to bootstrap full node. Other full nodes and clients later fetch the + # Deploy local binaries to bootstrap fullnode. Other fullnodes and clients later fetch the # binaries from it ( set -x @@ -184,7 +183,14 @@ startBootstrapNode() { esac ssh "${sshOptions[@]}" -n "$ipAddress" \ - "./solana/net/remote/remote-node.sh $deployMethod bootstrap-leader $publicNetwork $entrypointIp $expectedNodeCount \"$RUST_LOG\"" + "./solana/net/remote/remote-node.sh \ + $deployMethod \ + bootstrap-leader \ + $publicNetwork \ + $entrypointIp \ + ${#fullnodeIpList[@]} \ + \"$RUST_LOG\" \ + " ) >> "$logFile" 2>&1 || { cat "$logFile" echo "^^^ +++" @@ -196,13 +202,20 @@ startNode() { declare ipAddress=$1 declare logFile="$netLogDir/fullnode-$ipAddress.log" - echo "--- Starting full node: $ipAddress" + echo "--- Starting fullnode: $ipAddress" echo "start log: $logFile" ( set -x startCommon "$ipAddress" ssh "${sshOptions[@]}" -n "$ipAddress" \ - "./solana/net/remote/remote-node.sh $deployMethod fullnode $publicNetwork $entrypointIp $expectedNodeCount \"$RUST_LOG\"" + "./solana/net/remote/remote-node.sh \ + $deployMethod \ + fullnode \ + $publicNetwork \ + $entrypointIp \ + ${#fullnodeIpList[@]} \ + \"$RUST_LOG\" \ + " ) >> "$logFile" 2>&1 & declare pid=$! ln -sfT "fullnode-$ipAddress.log" "$netLogDir/fullnode-$pid.log" @@ -227,13 +240,12 @@ startClient() { } sanity() { - declare expectedNodeCount=$((${#additionalFullNodeIps[@]} + 1)) declare ok=true echo "--- Sanity" $metricsWriteDatapoint "testnet-deploy net-sanity-begin=1" - declare host=$bootstrapFullNodeIp # TODO: maybe use ${additionalFullNodeIps[0]} ? + declare host=${fullnodeIpList[0]} ( set -x # shellcheck disable=SC2029 # remote-client.sh args are expanded on client side intentionally @@ -304,22 +316,27 @@ start() { echo "Deployment started at $(date)" $metricsWriteDatapoint "testnet-deploy net-start-begin=1" - SECONDS=0 - declare bootstrapNodeDeployTime= - startBootstrapNode "$bootstrapFullNodeIp" "$netLogDir/bootstrap-fullnode-$bootstrapFullNodeIp.log" - bootstrapNodeDeployTime=$SECONDS - $metricsWriteDatapoint "testnet-deploy net-bootnode-fullnode-started=1" + bootstrapLeader=true + for ipAddress in "${fullnodeIpList[@]}"; do + if $bootstrapLeader; then + SECONDS=0 + declare bootstrapNodeDeployTime= + startBootstrapLeader "$ipAddress" "$netLogDir/bootstrap-leader-$ipAddress.log" + bootstrapNodeDeployTime=$SECONDS + $metricsWriteDatapoint "testnet-deploy net-bootnode-leader-started=1" - SECONDS=0 - pids=() - loopCount=0 - for ipAddress in "${additionalFullNodeIps[@]}"; do - startNode "$ipAddress" + bootstrapLeader=false + SECONDS=0 + pids=() + loopCount=0 + else + startNode "$ipAddress" - # Stagger additional node start time. If too many nodes start simultaneously - # the bootstrap node gets more rsync requests from the additional nodes than - # it can handle. - ((loopCount++ % 2 == 0)) && sleep 2 + # Stagger additional node start time. If too many nodes start simultaneously + # the bootstrap node gets more rsync requests from the additional nodes than + # it can handle. + ((loopCount++ % 2 == 0)) && sleep 2 + fi done for pid in "${pids[@]}"; do @@ -348,7 +365,7 @@ start() { case $deployMethod in snap) IFS=\ read -r _ networkVersion _ < <( - ssh "${sshOptions[@]}" "$bootstrapFullNodeIp" \ + ssh "${sshOptions[@]}" "${fullnodeIpList[0]}" \ "snap info solana | grep \"^installed:\"" ) networkVersion=${networkVersion/0+git./} @@ -369,8 +386,8 @@ start() { echo echo "+++ Deployment Successful" - echo "Bootstrap full node deployment took $bootstrapNodeDeployTime seconds" - echo "Additional full node deployment (${#additionalFullNodeIps[@]} instances) took $additionalNodeDeployTime seconds" + echo "Bootstrap leader deployment took $bootstrapNodeDeployTime seconds" + echo "Additional fullnode deployment (${#fullnodeIpList[@]} instances) took $additionalNodeDeployTime seconds" echo "Client deployment (${#clientIpList[@]} instances) took $clientDeployTime seconds" echo "Network start logs in $netLogDir:" ls -l "$netLogDir" @@ -400,9 +417,7 @@ stop() { SECONDS=0 $metricsWriteDatapoint "testnet-deploy net-stop-begin=1" - stopNode "$bootstrapFullNodeIp" - - for ipAddress in "${additionalFullNodeIps[@]}" "${clientIpList[@]}"; do + for ipAddress in "${fullnodeIpList[@]}" "${clientIpList[@]}"; do stopNode "$ipAddress" done diff --git a/net/ssh.sh b/net/ssh.sh index de1bdef03..48c8f0d53 100755 --- a/net/ssh.sh +++ b/net/ssh.sh @@ -49,11 +49,8 @@ printNode() { printf " %-25s | For logs run: $0 $ip tail -f solana/$nodeType.log\n" "$0 $ip" } -echo Bootstrap full node: -printNode fullnode "$bootstrapFullNodeIp" -echo -echo Additional full nodes: -for ipAddress in "${additionalFullNodeIps[@]}"; do +echo Full nodes: +for ipAddress in "${fullnodeIpList[@]}"; do printNode fullnode "$ipAddress" done echo