`net/net.sh start --skip-setup ...` now works again (#5977)

multinode-demo/setup.sh

@@ -6,12 +6,13 @@ source "$here"/common.sh
 setup_secondary_mount
 
 set -e
-"$here"/clear-config.sh
+
+rm -rf "$SOLANA_CONFIG_DIR"/bootstrap-leader
+mkdir -p "$SOLANA_CONFIG_DIR"/bootstrap-leader
 
 # Create genesis ledger
-$solana_keygen new -o "$SOLANA_CONFIG_DIR"/mint-keypair.json
+$solana_keygen new -f -o "$SOLANA_CONFIG_DIR"/mint-keypair.json
 
-mkdir "$SOLANA_CONFIG_DIR"/bootstrap-leader
 $solana_keygen new -o "$SOLANA_CONFIG_DIR"/bootstrap-leader/identity-keypair.json
 $solana_keygen new -o "$SOLANA_CONFIG_DIR"/bootstrap-leader/vote-keypair.json
 $solana_keygen new -o "$SOLANA_CONFIG_DIR"/bootstrap-leader/stake-keypair.json

net/net.sh

@@ -22,12 +22,11 @@ Operate a configured testnet
  sanity   - Sanity check the network
  stop     - Stop the network
  restart  - Shortcut for stop then start
- update   - Live update all network nodes
  logs     - Fetch remote logs from each network node
  startnode- Start an individual node (previously stopped with stopNode)
  stopnode - Stop an individual node
 
- start/update-specific options:
+ start-specific options:
    -T [tarFilename]                   - Deploy the specified release tarball
    -t edge|beta|stable|vX.Y.Z         - Deploy the latest tarball release for the
                                         specified release channel (edge|beta|stable) or release tag
@@ -37,7 +36,7 @@ Operate a configured testnet
                                         (-t option must be supplied as well)
    -f [cargoFeatures]                 - List of |cargo --feaures=| to activate
                                         (ignored if -s or -S is specified)
-   -r                                 - Reuse existing node/ledger configuration from a
+   -r / --skip-setup                  - Reuse existing node/ledger configuration from a
                                         previous |start| (ie, don't run ./multinode-demo/setup.sh).
    -d / --debug                       - Build/deploy the testnet with debug binaries
    -D /path/to/programs               - Deploy custom programs from this location
@@ -66,7 +65,8 @@ Operate a configured testnet
    --internal-nodes-lamports NUM_LAMPORTS_PER_NODE
                                       - Amount to fund internal nodes in genesis block.
    --external-accounts-file FILE_PATH
-                                      - A YML file with a list of account pubkeys and corresponding lamport balances in genesis block for external nodes
+                                      - A YML file with a list of account pubkeys and corresponding lamport balances
+                                        in genesis block for external nodes
    --no-snapshot-fetch
                                       - If set, disables booting validators from a snapshot
    --skip-ledger-verify
@@ -81,7 +81,7 @@ Operate a configured testnet
                                         existing binaries
 
 
- sanity/start/update-specific options:
+ sanity/start-specific options:
    -F                   - Discard validator nodes that didn't bootup successfully
    -o noValidatorSanity - Skip fullnode sanity
    -o noInstallCheck    - Skip solana-install sanity
@@ -107,7 +107,6 @@ deployMethod=local
 sanityExtraArgs=
 cargoFeatures=
 skipSetup=false
-updateNodes=false
 customPrograms=
 updatePlatforms=
 nodeAddress=
@@ -160,6 +159,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --skip-ledger-verify ]]; then
       maybeSkipLedgerVerify="$1"
       shift 1
+    elif [[ $1 = --skip-setup ]]; then
+      skipSetup=true
+      shift 1
     elif [[ $1 = --deploy-update ]]; then
       updatePlatforms="$updatePlatforms $2"
       shift 2
@@ -356,12 +358,12 @@ startCommon() {
   if $skipSetup; then
     ssh "${sshOptions[@]}" "$ipAddress" "
       set -x;
-      mkdir -p ~/solana/config{,-local}
+      mkdir -p ~/solana/config;
-      rm -rf ~/config{,-local};
+      rm -rf ~/config;
-      mv ~/solana/config{,-local} ~;
+      mv ~/solana/config ~;
       rm -rf ~/solana;
       mkdir -p ~/solana ~/.cargo/bin;
-      mv ~/config{,-local} ~/solana/
+      mv ~/config ~/solana/
     "
   else
     ssh "${sshOptions[@]}" "$ipAddress" "
@@ -646,11 +648,7 @@ prepare_deploy() {
 
 deploy() {
   echo "Deployment started at $(date)"
-  if $updateNodes; then
-    $metricsWriteDatapoint "testnet-deploy net-update-begin=1"
-  else
   $metricsWriteDatapoint "testnet-deploy net-start-begin=1"
-  fi
 
   declare bootstrapLeader=true
   for nodeAddress in "${fullnodeIpList[@]}" "${blockstreamerIpList[@]}" "${replicatorIpList[@]}"; do
@@ -699,11 +697,6 @@ deploy() {
 
   annotateBlockexplorerUrl
 
-  if $updateNodes; then
-    for ipAddress in "${clientIpList[@]}"; do
-      stopNode "$ipAddress" true
-    done
-  fi
   sanity skipBlockstreamerSanity # skip sanity on blockstreamer node, it may not
                                  # have caught up to the bootstrap leader yet
 
@@ -717,11 +710,7 @@ deploy() {
   done
   clientDeployTime=$SECONDS
 
-  if $updateNodes; then
-    $metricsWriteDatapoint "testnet-deploy net-update-complete=1"
-  else
   $metricsWriteDatapoint "testnet-deploy net-start-complete=1"
-  fi
 
   declare networkVersion=unknown
   case $deployMethod in
@@ -768,7 +757,7 @@ stopNode() {
       PS4=\"$PS4\"
       set -x
       ! tmux list-sessions || tmux kill-session
-      for pid in solana/{net-stats,fd-monitor,oom-monitor}.pid; do
+      for pid in solana/{blockexplorer,net-stats,fd-monitor,oom-monitor}.pid; do
         pgid=\$(ps opgid= \$(cat \$pid) | tr -d '[:space:]')
         if [[ -n \$pgid ]]; then
           sudo kill -- -\$pgid
@@ -824,11 +813,6 @@ start)
   prepare_deploy
   deploy
   ;;
-update)
-  skipSetup=true
-  updateNodes=true
-  start
-  ;;
 sanity)
   sanity
   ;;

net/remote/remote-client.sh

@@ -50,7 +50,7 @@ esac
 case $clientToRun in
 solana-bench-tps)
   net/scripts/rsync-retry.sh -vPrc \
-    "$entrypointIp":~/solana/solana-client-accounts/bench-tps"$clientIndex".yml ./client-accounts.yml
+    "$entrypointIp":~/solana/config/bench-tps"$clientIndex".yml ./client-accounts.yml
   clientCommand="\
     solana-bench-tps \
       --entrypoint $entrypointIp:8001 \
@@ -65,7 +65,7 @@ solana-bench-tps)
 solana-bench-exchange)
   solana-keygen new -f -o bench.keypair
   net/scripts/rsync-retry.sh -vPrc \
-    "$entrypointIp":~/solana/solana-client-accounts/bench-exchange"$clientIndex".yml ./client-accounts.yml
+    "$entrypointIp":~/solana/config/bench-exchange"$clientIndex".yml ./client-accounts.yml
   clientCommand="\
     solana-bench-exchange \
       --entrypoint $entrypointIp:8001 \
@@ -97,11 +97,11 @@ echo "$(date) | $0 $*" >> client.log
 (
   sudo SOLANA_METRICS_CONFIG="$SOLANA_METRICS_CONFIG" scripts/oom-monitor.sh
 ) > oom-monitor.log 2>&1 &
-echo $! > oom-monitor.pid
+echo \$! > oom-monitor.pid
 scripts/fd-monitor.sh > fd-monitor.log 2>&1 &
-echo $! > fd-monitor.pid
+echo \$! > fd-monitor.pid
 scripts/net-stats.sh  > net-stats.log 2>&1 &
-echo $! > net-stats.pid
+echo \$! > net-stats.pid
 ! tmux list-sessions || tmux kill-session
 
 tmux new -s "$clientToRun" -d "

net/remote/remote-node.sh

@@ -107,11 +107,11 @@ cat >> ~/solana/on-reboot <<EOF
   (
     sudo SOLANA_METRICS_CONFIG="$SOLANA_METRICS_CONFIG" scripts/oom-monitor.sh
   ) > oom-monitor.log 2>&1 &
-  echo $! > oom-monitor.pid
+  echo \$! > oom-monitor.pid
   scripts/fd-monitor.sh > fd-monitor.log 2>&1 &
-  echo $! > fd-monitor.pid
+  echo \$! > fd-monitor.pid
   scripts/net-stats.sh  > net-stats.log 2>&1 &
-  echo $! > net-stats.pid
+  echo \$! > net-stats.pid
 
   if [[ -e /dev/nvidia0 && -x ~/.cargo/bin/solana-validator-cuda ]]; then
     echo Selecting solana-validator-cuda
@@ -123,15 +123,14 @@ EOF
   bootstrap-leader)
     set -x
     if [[ $skipSetup != true ]]; then
-      rm -rf ./solana-node-keys
+      multinode-demo/clear-config.sh
-      rm -rf ./solana-node-balances
+
-      mkdir ./solana-node-balances
       if [[ -n $internalNodesLamports ]]; then
-        echo "---" >> ./solana-node-balances/fullnode-balances.yml
+        echo "---" >> config/fullnode-balances.yml
         for i in $(seq 0 "$numNodes"); do
-          solana-keygen new -o ./solana-node-keys/"$i"
+          solana-keygen new -o config/fullnode-"$i"-identity.json
-          pubkey="$(solana-keygen pubkey ./solana-node-keys/"$i")"
+          pubkey="$(solana-keygen pubkey config/fullnode-"$i"-identity.json)"
-          cat >> ./solana-node-balances/fullnode-balances.yml <<EOF
+          cat >> config/fullnode-balances.yml <<EOF
 $pubkey:
   balance: $internalNodesLamports
   owner: 11111111111111111111111111111111
@@ -151,29 +150,29 @@ EOF
         fi
       done
 
-      rm -rf ./solana-client-accounts
-      mkdir ./solana-client-accounts
       for i in $(seq 0 $((numBenchTpsClients-1))); do
         # shellcheck disable=SC2086 # Do not want to quote $benchTpsExtraArgs
-        solana-bench-tps --write-client-keys ./solana-client-accounts/bench-tps"$i".yml \
+        solana-bench-tps --write-client-keys config/bench-tps"$i".yml \
           --target-lamports-per-signature "$lamports_per_signature" $benchTpsExtraArgs
         # Skip first line, as it contains header
-        tail -n +2 -q ./solana-client-accounts/bench-tps"$i".yml >> ./solana-client-accounts/client-accounts.yml
+        tail -n +2 -q config/bench-tps"$i".yml >> config/client-accounts.yml
-        echo "" >> ./solana-client-accounts/client-accounts.yml
+        echo "" >> config/client-accounts.yml
       done
       for i in $(seq 0 $((numBenchExchangeClients-1))); do
         # shellcheck disable=SC2086 # Do not want to quote $benchExchangeExtraArgs
         solana-bench-exchange --batch-size 1000 --fund-amount 20000 \
-          --write-client-keys ./solana-client-accounts/bench-exchange"$i".yml $benchExchangeExtraArgs
+          --write-client-keys config/bench-exchange"$i".yml $benchExchangeExtraArgs
-        tail -n +2 -q ./solana-client-accounts/bench-exchange"$i".yml >> ./solana-client-accounts/client-accounts.yml
+        tail -n +2 -q config/bench-exchange"$i".yml >> config/client-accounts.yml
-        echo "" >> ./solana-client-accounts/client-accounts.yml
+        echo "" >> config/client-accounts.yml
       done
-      [[ -z $externalPrimordialAccountsFile ]] || cat "$externalPrimordialAccountsFile" >> ./solana-node-balances/fullnode-balances.yml
+      if [[ -f $externalPrimordialAccountsFile ]]; then
-      if [ -f ./solana-node-balances/fullnode-balances.yml ]; then
+        cat "$externalPrimordialAccountsFile" >> config/fullnode-balances.yml
-        genesisOptions+=" --primordial-accounts-file ./solana-node-balances/fullnode-balances.yml"
       fi
-      if [ -f ./solana-client-accounts/client-accounts.yml ]; then
+      if [[ -f config/fullnode-balances.yml ]]; then
-        genesisOptions+=" --primordial-keypairs-file ./solana-client-accounts/client-accounts.yml"
+        genesisOptions+=" --primordial-accounts-file config/fullnode-balances.yml"
+      fi
+      if [[ -f config/client-accounts.yml ]]; then
+        genesisOptions+=" --primordial-keypairs-file config/client-accounts.yml"
       fi
 
       args=(
@@ -184,7 +183,7 @@ EOF
       fi
       # shellcheck disable=SC2206 # Do not want to quote $genesisOptions
       args+=($genesisOptions)
-      ./multinode-demo/setup.sh "${args[@]}"
+      multinode-demo/setup.sh "${args[@]}"
     fi
     args=(
       --gossip-port "$entrypointIp":8001
@@ -208,18 +207,20 @@ EOF
     ~/solana/on-reboot
     waitForNodeToInit
 
+    if [[ $skipSetup != true ]]; then
       solana --url http://"$entrypointIp":8899 \
         --keypair ~/solana/config/bootstrap-leader/identity-keypair.json \
         validator-info publish "$(hostname)" -n team/solana --force || true
+    fi
     ;;
   validator|blockstreamer)
     if [[ $deployMethod != skip ]]; then
       net/scripts/rsync-retry.sh -vPrc "$entrypointIp":~/.cargo/bin/ ~/.cargo/bin/
     fi
     if [[ $skipSetup != true ]]; then
-      rm -f ~/solana/fullnode-identity.json
+      multinode-demo/clear-config.sh
       [[ -z $internalNodesLamports ]] || net/scripts/rsync-retry.sh -vPrc \
-      "$entrypointIp":~/solana/solana-node-keys/"$nodeIndex" ~/solana/fullnode-identity.json
+      "$entrypointIp":~/solana/config/fullnode-"$nodeIndex"-identity.json config/fullnode-identity.json
     fi
 
     args=(
@@ -239,20 +240,16 @@ EOF
       fi
     fi
 
-    if [[ ! -f ~/solana/fullnode-identity.json ]]; then
+    if [[ ! -f config/fullnode-identity.json ]]; then
-      solana-keygen new -o ~/solana/fullnode-identity.json
+      solana-keygen new -o config/fullnode-identity.json
     fi
-    args+=(--identity ~/solana/fullnode-identity.json)
+    args+=(--identity config/fullnode-identity.json)
 
     if [[ $airdropsEnabled != true ]]; then
       args+=(--no-airdrop)
     fi
 
     set -x
-    if [[ $skipSetup != true ]]; then
-      ./multinode-demo/clear-config.sh
-    fi
-
     if [[ $nodeType = blockstreamer ]]; then
       # Sneak the mint-keypair.json from the bootstrap leader and run another drone
       # with it on the blockstreamer node.  Typically the blockstreamer node has
@@ -261,7 +258,7 @@ EOF
       scp "$entrypointIp":~/solana/config/mint-keypair.json config/
       if [[ $airdropsEnabled = true ]]; then
 cat >> ~/solana/on-reboot <<EOF
-        ./multinode-demo/drone.sh > drone.log 2>&1 &
+        multinode-demo/drone.sh > drone.log 2>&1 &
 EOF
       fi
 
@@ -278,6 +275,7 @@ EOF
       killall node || true
       export BLOCKEXPLORER_GEOIP_WHITELIST=$PWD/net/config/geoip.yml
       npx solana-blockexplorer > blockexplorer.log 2>&1 &
+      echo \$! > blockexplorer.pid
 
       # Redirect port 80 to port 5000
       sudo iptables -A INPUT -p tcp --dport 80 -j ACCEPT
@@ -301,7 +299,7 @@ EOF
     # shellcheck disable=SC2206 # Don't want to double quote $extraNodeArgs
     args+=($extraNodeArgs)
 cat >> ~/solana/on-reboot <<EOF
-    nohup ./multinode-demo/validator.sh ${args[@]} > fullnode.log 2>&1 &
+    nohup multinode-demo/validator.sh ${args[@]} > fullnode.log 2>&1 &
     pid=\$!
     oom_score_adj "\$pid" 1000
     disown
@@ -318,16 +316,18 @@ EOF
       if [[ $airdropsEnabled != true ]]; then
         args+=(--no-airdrop)
       fi
-      if [[ -f ~/solana/fullnode-identity.json ]]; then
+      if [[ -f config/fullnode-identity.json ]]; then
-        args+=(--keypair ~/solana/fullnode-identity.json)
+        args+=(--keypair config/fullnode-identity.json)
       fi
 
-      ./multinode-demo/delegate-stake.sh "${args[@]}"
+      multinode-demo/delegate-stake.sh "${args[@]}"
     fi
 
+    if [[ $skipSetup != true ]]; then
       solana --url http://"$entrypointIp":8899 \
-      --keypair ~/solana/fullnode-identity.json \
+        --keypair config/fullnode-identity.json \
         validator-info publish "$(hostname)" -n team/solana --force || true
+    fi
     ;;
   replicator)
     if [[ $deployMethod != skip ]]; then
@@ -346,7 +346,7 @@ EOF
     fi
 
 cat >> ~/solana/on-reboot <<EOF
-    nohup ./multinode-demo/replicator.sh ${args[@]} > fullnode.log 2>&1 &
+    nohup multinode-demo/replicator.sh ${args[@]} > fullnode.log 2>&1 &
     pid=\$!
     oom_score_adj "\$pid" 1000
     disown