Add on-demand functionality to all testnets (#4003)

* Add on-demand functionality to all testnets

ci/testnet-deploy.sh

@@ -53,6 +53,9 @@ Deploys a CD testnet
    -D                   - Delete the network
    -r                   - Reuse existing node/ledger configuration from a
                           previous |start| (ie, don't run ./multinode-demo/setup.sh).
+   -x                   - External node.  Default: false
+   -s                   - Skip start.  Nodes will still be created or configured, but network software will not be started.
+   -S                   - Stop network software without tearing down nodes.
 
    Note: the SOLANA_METRICS_CONFIG environment variable is used to configure
          metrics
@@ -62,7 +65,7 @@ EOF
 
 zone=()
 
-while getopts "h?p:Pn:c:t:gG:a:Dbd:rusxz:p:C:" opt; do
+while getopts "h?p:Pn:c:t:gG:a:Dbd:rusxz:p:C:S" opt; do
   case $opt in
   h | \?)
     usage
@@ -126,6 +129,9 @@ while getopts "h?p:Pn:c:t:gG:a:Dbd:rusxz:p:C:" opt; do
   u)
     blockstreamer=true
     ;;
+  S)
+    stopNetwork=true
+    ;;
   *)
     usage "Error: unhandled option: $opt"
     ;;
@@ -162,6 +168,11 @@ for val in "${zone[@]}"; do
   zone_args+=("-z $val")
 done
 
+if $stopNetwork; then
+  skipSetup=true
+fi
+
+# Create the network
 if ! $skipSetup; then
   echo "--- $cloudProvider.sh delete"
   # shellcheck disable=SC2068
@@ -227,6 +238,12 @@ net/init-metrics.sh -e
 echo "+++ $cloudProvider.sh info"
 net/"$cloudProvider".sh info
 
+if $stopNetwork; then
+  echo --- net.sh stop
+  time net/net.sh stop
+  exit 0
+fi
+
 echo --- net.sh start
 maybeRejectExtraNodes=
 if ! $publicNetwork; then

ci/testnet-manager.sh

@@ -42,20 +42,28 @@ steps:
             value: "testnet-beta"
           - label: "testnet-beta-perf"
             value: "testnet-beta-perf"
+          - label: "testnet-demo"
+            value: "testnet-demo"
       - select: "Operation"
         key: "testnet-operation"
         default: "sanity-or-restart"
         options:
-          - label: "Sanity check.  Restart network on failure"
+          - label: "Create new testnet nodes and then start network software.  If nodes are already created, they will be deleted and then re-created."
-            value: "sanity-or-restart"
+            value: "create-and-start"
-          - label: "Start (or restart) the network"
+          - label: "Create new testnet nodes, but do not start network software.  If nodes are already created, they will be deleted and then re-created."
+            value: "create"
+          - label: "Start network software on already-created testnet nodes.  If software is already running, it will be restarted."
             value: "start"
-          - label: "Update the network software.  Restart network on failure"
+          - label: "Stop network software without deleting testnet nodes"
-            value: "update-or-restart"
-          - label: "Stop the network"
             value: "stop"
+          - label: "Update the network software.  Restart network software on failure"
+            value: "update-or-restart"
+          - label: "Sanity check.  Restart network software on failure"
+            value: "sanity-or-restart"
           - label: "Sanity check only"
             value: "sanity"
+          - label: "Delete all nodes on a testnet.  Network software will be stopped first if it is running"
+            value: "delete"
   - command: "ci/$(basename "$0")"
     agents:
       - "queue=$BUILDKITE_AGENT_META_DATA_QUEUE"
@@ -88,6 +96,10 @@ testnet|testnet-perf)
   CHANNEL_BRANCH=$STABLE_CHANNEL
   : "${TESTNET_DB_HOST:=https://clocktower-f1d56615.influxcloud.net:8086}"
   ;;
+testnet-demo)
+  CHANNEL_OR_TAG=beta
+  CHANNEL_BRANCH=$BETA_CHANNEL
+  ;;
 *)
   echo "Error: Invalid TESTNET=$TESTNET"
   exit 1
@@ -105,6 +117,7 @@ source scripts/configure-metrics.sh
 if [[ -n $TESTNET_TAG ]]; then
   CHANNEL_OR_TAG=$TESTNET_TAG
 else
+
   if [[ $BUILDKITE_BRANCH != "$CHANNEL_BRANCH" ]]; then
     (
       cat <<EOF
@@ -197,15 +210,37 @@ sanity() {
   esac
 }
 
+deploy() {
+  declare maybeCreate=$1
+  declare maybeStart=$2
+  declare maybeStop=$3
+  declare maybeDelete=$4
 
-start() {
+  # Create or recreate the nodes
-  declare maybeDelete=$1
+  if [[ -z $maybeCreate ]]; then
-  if [[ -z $maybeDelete ]]; then
+    skipCreate=skip
-    echo "--- start $TESTNET"
   else
+    skipCreate=""
+    echo "--- create $TESTNET"
+  fi
+
+  # Start or restart the network software on the nodes
+  if [[ -z $maybeStart ]]; then
+    skipStart=skip
+  else
+    skipStart=""
+    echo "--- start $TESTNET"
+  fi
+
+  # Stop the nodes
+  if [[ -n $maybeStop ]]; then
     echo "--- stop $TESTNET"
   fi
-  declare maybeReuseLedger=$2
+
+  # Delete the nodes
+  if [[ -n $maybeDelete ]]; then
+    echo "--- delete $TESTNET"
+  fi
 
   case $TESTNET in
   testnet-edge)
@@ -213,7 +248,9 @@ start() {
       set -x
       ci/testnet-deploy.sh -p edge-testnet-solana-com -C ec2 -z us-west-1a \
         -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P -a eipalloc-0ccd4f2239886fa94 \
-        ${maybeReuseLedger:+-r} \
+        ${skipCreate:+-r} \
+        ${skipStart:+-s} \
+        ${maybeStop:+-S} \
         ${maybeDelete:+-D}
     )
     ;;
@@ -226,7 +263,9 @@ start() {
         ci/testnet-deploy.sh -p edge-perf-testnet-solana-com -C ec2 -z us-west-2b \
           -g -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
-          ${maybeReuseLedger:+-r} \
+          ${skipCreate:+-r} \
+          ${skipStart:+-s} \
+          ${maybeStop:+-S} \
           ${maybeDelete:+-D}
     )
     ;;
@@ -246,19 +285,26 @@ start() {
       done
 
       if [[ -n $EC2_NODE_COUNT ]]; then
+        if [[ -n $GCE_NODE_COUNT ]] || [[ -n $skipStart ]]; then
+          maybeSkipStart="skip"
+        fi
+
         # shellcheck disable=SC2068
         ci/testnet-deploy.sh -p beta-testnet-solana-com -C ec2 ${EC2_ZONE_ARGS[@]} \
           -t "$CHANNEL_OR_TAG" -n "$EC2_NODE_COUNT" -c 0 -u -P -a eipalloc-0f286cf8a0771ce35 \
-          ${maybeReuseLedger:+-r} \
+          ${skipCreate:+-r} \
-          ${maybeDelete:+-D} \
+          ${maybeSkipStart:+-s} \
-          ${GCE_NODE_COUNT:+-s}
+          ${maybeStop:+-S} \
+          ${maybeDelete:+-D}
       fi
 
       if [[ -n $GCE_NODE_COUNT ]]; then
         # shellcheck disable=SC2068
         ci/testnet-deploy.sh -p beta-testnet-solana-com -C gce ${GCE_ZONE_ARGS[@]} \
           -t "$CHANNEL_OR_TAG" -n "$GCE_NODE_COUNT" -c 0 -P \
-          ${maybeReuseLedger:+-r} \
+          ${skipCreate:+-r} \
+          ${skipStart:+-s} \
+          ${maybeStop:+-S} \
           ${maybeDelete:+-D} \
           ${EC2_NODE_COUNT:+-x}
       fi
@@ -273,7 +319,9 @@ start() {
         ci/testnet-deploy.sh -p beta-perf-testnet-solana-com -C ec2 -z us-west-2b \
           -g -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
-          ${maybeReuseLedger:+-r} \
+          ${skipCreate:+-r} \
+          ${skipStart:+-s} \
+          ${maybeStop:+-S} \
           ${maybeDelete:+-D}
     )
     ;;
@@ -284,7 +332,9 @@ start() {
         ci/testnet-deploy.sh -p testnet-solana-com -C ec2 -z us-west-1a \
           -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P -a eipalloc-0fa502bf95f6f18b2 \
           -b \
-          ${maybeReuseLedger:+-r} \
+          ${skipCreate:+-r} \
+          ${skipStart:+-s} \
+          ${maybeStop:+-S} \
           ${maybeDelete:+-D}
         #ci/testnet-deploy.sh -p testnet-solana-com -C gce -z us-east1-c \
         #  -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a testnet-solana-com  \
@@ -303,7 +353,9 @@ start() {
           -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
           -d pd-ssd \
-          ${maybeReuseLedger:+-r} \
+          ${skipCreate:+-r} \
+          ${skipStart:+-s} \
+          ${maybeStop:+-S} \
           ${maybeDelete:+-D}
         #ci/testnet-deploy.sh -p perf-testnet-solana-com -C ec2 -z us-east-1a \
         #  -g \
@@ -312,6 +364,13 @@ start() {
         #  ${maybeDelete:+-D}
     )
     ;;
+  testnet-demo)
+    (
+      set -x
+      echo "Demo net not yet implemented!"
+      exit 1
+    )
+    ;;
   *)
     echo "Error: Invalid TESTNET=$TESTNET"
     exit 1
@@ -319,13 +378,49 @@ start() {
   esac
 }
 
+CREATED_LOCKFILE="${HOME}/${TESTNET}.is_created"
+STARTED_LOCKFILE="${HOME}/${TESTNET}.is_started"
+
+create-and-start() {
+  rm -f "${CREATED_LOCKFILE}"
+  rm -f "${STARTED_LOCKFILE}"
+  deploy create start
+  touch "${CREATED_LOCKFILE}"
+  touch "${STARTED_LOCKFILE}"
+}
+create() {
+  rm -f "${CREATED_LOCKFILE}"
+  rm -f "${STARTED_LOCKFILE}"
+  deploy create
+  touch "${CREATED_LOCKFILE}"
+}
+start() {
+  if [[ -f ${CREATED_LOCKFILE} ]]; then
+    rm -f "${STARTED_LOCKFILE}"
+    deploy "" start
+    touch "${STARTED_LOCKFILE}"
+  else
+    echo "Unable to start ${TESTNET}.  Are the nodes created?
+    Re-run ci/testnet-manager.sh with \$TESTNET_OP=create or \$TESTNET_OP=create-and-start"
+    exit 1
+  fi
+}
 stop() {
-  start delete
+  deploy "" ""
+  rm -f "${STARTED_LOCKFILE}"
+}
+delete() {
+  deploy "" "" "" delete
+  rm -f "${CREATED_LOCKFILE}"
+  rm -f "${STARTED_LOCKFILE}"
 }
 
 case $TESTNET_OP in
-sanity)
+create-and-start)
-  sanity
+  create-and-start
+  ;;
+create)
+  create
   ;;
 start)
   start
@@ -333,8 +428,14 @@ start)
 stop)
   stop
   ;;
+sanity)
+  sanity
+  ;;
+delete)
+  delete
+  ;;
 update-or-restart)
-  if start "" update; then
+  if start; then
     echo Update successful
   else
     echo "+++ Update failed, restarting the network"
@@ -352,7 +453,7 @@ sanity-or-restart)
     # TODO: Restore attempt to restart the cluster before recreating it
     #       See https://github.com/solana-labs/solana/issues/3774
     if false; then
-      if start "" update; then
+      if start; then
         echo Update successful
       else
         echo "+++ Update failed, restarting the network"
@@ -364,6 +465,10 @@ sanity-or-restart)
     fi
   fi
   ;;
+*)
+  echo "Error: Invalid TESTNET_OP=$TESTNET_OP"
+  exit 1
+  ;;
 esac
 
 echo --- fin