Add update-to-restart operation. Also try to update before restarting on sanity failures

ci/testnet-manager.sh

@@ -50,6 +50,8 @@ steps:
             value: "sanity-or-restart"
           - label: "Start (or restart) the network"
             value: "start"
+          - label: "Update the network software.  Restart network on failure"
+            value: "update-or-restart"
           - label: "Stop the network"
             value: "stop"
           - label: "Sanity check only"
@@ -176,6 +178,7 @@ start() {
   else
     echo "--- stop $TESTNET"
   fi
+  declare maybeReuseLedger=$2
 
   case $TESTNET in
   testnet-edge)
@@ -186,6 +189,7 @@ start() {
       RUST_LOG=solana=info \
         ci/testnet-deploy.sh edge-testnet-solana-com ec2 us-west-1a \
           -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a eipalloc-0ccd4f2239886fa94 \
+          ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-d}
     )
     ;;
@@ -197,6 +201,7 @@ start() {
         ci/testnet-deploy.sh edge-perf-testnet-solana-com ec2 us-west-2b \
           -g -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
+          ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-d}
     )
     ;;
@@ -209,6 +214,7 @@ start() {
         ci/testnet-deploy.sh beta-testnet-solana-com ec2 us-west-1a \
           -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a eipalloc-0f286cf8a0771ce35 \
           -b \
+          ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-d}
     )
     ;;
@@ -220,6 +226,7 @@ start() {
         ci/testnet-deploy.sh beta-perf-testnet-solana-com ec2 us-west-2b \
           -g -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
+          ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-d}
     )
     ;;
@@ -232,9 +239,11 @@ start() {
         ci/testnet-deploy.sh testnet-solana-com ec2 us-west-1a \
           -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a eipalloc-0fa502bf95f6f18b2 \
           -b \
+          ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-d}
         #ci/testnet-deploy.sh testnet-solana-com gce us-east1-c \
         #  -s "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a testnet-solana-com  \
+        #  ${maybeReuseLedger:+-r} \
         #  ${maybeDelete:+-d}
     )
     ;;
@@ -248,10 +257,12 @@ start() {
           -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
           -d pd-ssd \
+          ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-d}
         #ci/testnet-deploy.sh perf-testnet-solana-com ec2 us-east-1a \
         #  -g \
         #  -t "$CHANNEL_OR_TAG" -c 2 \
+        #  ${maybeReuseLedger:+-r} \
         #  ${maybeDelete:+-d}
     )
     ;;
@@ -276,13 +287,28 @@ start)
 stop)
   stop
   ;;
+update-or-restart)
+  if start "" update; then
+    echo Update successful
+  else
+    echo "+++ Update failed, restarting the network"
+    $metricsWriteDatapoint "testnet-manager update-failure=1"
+    start
+  fi
+  ;;
 sanity-or-restart)
   if sanity; then
     echo Pass
   else
-    echo "+++ Sanity failed, restarting the network"
+    echo "+++ Sanity failed, updating the network"
     $metricsWriteDatapoint "testnet-manager sanity-failure=1"
-    start
+    if start "" update; then
+      echo Update successful
+    else
+      echo "+++ Update failed, restarting the network"
+      $metricsWriteDatapoint "testnet-manager update-failure=1"
+      start
+    fi
   fi
   ;;
 esac