testnet-demo: skip over validator nodes that fail to boot

ci/testnet-deploy.sh

@@ -14,6 +14,7 @@ stopNetwork=false
 skipSetup=false
 skipStart=false
 externalNode=false
+failOnValidatorBootupFailure=true
 tarChannelOrTag=edge
 delete=false
 enableGpu=false
@@ -57,6 +58,7 @@ Deploys a CD testnet
    -x                   - External node.  Default: false
    -s                   - Skip start.  Nodes will still be created or configured, but network software will not be started.
    -S                   - Stop network software without tearing down nodes.
+   -f                   - Discard validator nodes that didn't bootup successfully
 
    Note: the SOLANA_METRICS_CONFIG environment variable is used to configure
          metrics
@@ -66,7 +68,7 @@ EOF
 
 zone=()
 
-while getopts "h?p:Pn:c:t:gG:a:Dbd:rusxz:p:C:S" opt; do
+while getopts "h?p:Pn:c:t:gG:a:Dbd:rusxz:p:C:Sf" opt; do
   case $opt in
   h | \?)
     usage
@@ -127,6 +129,9 @@ while getopts "h?p:Pn:c:t:gG:a:Dbd:rusxz:p:C:S" opt; do
   x)
     externalNode=true
     ;;
+  f)
+    failOnValidatorBootupFailure=false
+    ;;
   u)
     blockstreamer=true
     ;;
@@ -224,6 +229,10 @@ if ! $skipSetup; then
     create_args+=(-x)
   fi
 
+  if ! $failOnValidatorBootupFailure; then
+    create_args+=(-f)
+  fi
+
   time net/"$cloudProvider".sh create "${create_args[@]}"
 else
   echo "--- $cloudProvider.sh config"
@@ -236,6 +245,10 @@ else
     config_args+=(-P)
   fi
 
+  if ! $failOnValidatorBootupFailure; then
+    config_args+=(-f)
+  fi
+
   time net/"$cloudProvider".sh config "${config_args[@]}"
 fi
 net/init-metrics.sh -e

ci/testnet-manager.sh

@@ -231,7 +231,7 @@ sanity() {
       ok=true
       if [[ -n $GCE_NODE_COUNT ]]; then
         NO_LEDGER_VERIFY=1 \
-          ci/testnet-sanity.sh demo-testnet-solana-com gce "${GCE_ZONES[0]}" || ok=false
+          ci/testnet-sanity.sh demo-testnet-solana-com gce "${GCE_ZONES[0]}" -f || ok=false
       else
         echo "Error: no GCE nodes"
         ok=false
@@ -374,7 +374,7 @@ deploy() {
       if [[ -n $GCE_NODE_COUNT ]]; then
         # shellcheck disable=SC2068
         ci/testnet-deploy.sh -p testnet-demo -C gce ${GCE_ZONE_ARGS[@]} \
-          -t "$CHANNEL_OR_TAG" -n "$GCE_NODE_COUNT" -c 0 -P -u \
+          -t "$CHANNEL_OR_TAG" -n "$GCE_NODE_COUNT" -c 0 -P -u -f \
           -a demo-testnet-solana-com \
           ${skipCreate:+-r} \
           ${skipStart:+-s} \