Add multi-region deploy functionality

ci/testnet-deploy.sh

@@ -27,13 +27,14 @@ usage() {
     echo "Error: $*"
   fi
   cat <<EOF
-usage: $0 [name] [cloud] [zone] [options...]
+usage: $0 -N name -C cloud -z zone1 [-z zone2] ... [-z zoneN] [options...]
 
 Deploys a CD testnet
 
-  name  - name of the network
-  cloud - cloud provider to use (gce, ec2)
-  zone  - cloud provider zone to deploy the network into
+  mandatory arguments:
+  -N [name]  - name of the network
+  -C [cloud] - cloud provider to use (gce, ec2)
+  -z [zone]  - cloud provider zone to deploy the network into.  Must specify at least one zone
 
   options:
    -t edge|beta|stable|vX.Y.Z  - Deploy the latest tarball release for the
@@ -59,19 +60,22 @@ EOF
   exit $exitcode
 }
 
-netName=$1
-cloudProvider=$2
-zone=$3
-[[ -n $netName ]] || usage
-[[ -n $cloudProvider ]] || usage "Cloud provider not specified"
-[[ -n $zone ]] || usage "Zone not specified"
-shift 3
+zone=()
 
-while getopts "h?p:Pn:c:t:gG:a:Dbd:rusx" opt; do
+while getopts "h?p:Pn:c:t:gG:a:Dbd:rusxz:N:C:" opt; do
   case $opt in
   h | \?)
     usage
     ;;
+  N)
+    netName=$OPTARG
+    ;;
+  C)
+    cloudProvider=$OPTARG
+    ;;
+  z)
+    zone+=("$OPTARG")
+    ;;
   P)
     publicNetwork=true
     ;;
@@ -128,6 +132,10 @@ while getopts "h?p:Pn:c:t:gG:a:Dbd:rusx" opt; do
   esac
 done
 
+[[ -n $netName ]] || usage
+[[ -n $cloudProvider ]] || usage "Cloud provider not specified"
+[[ -n $zone ]] || usage "Zone not specified"
+
 shutdown() {
   exitcode=$?
 

ci/testnet-manager.sh

@@ -202,7 +202,7 @@ start() {
       set -x
       NO_VALIDATOR_SANITY=1 \
       RUST_LOG=solana=info \
-        ci/testnet-deploy.sh edge-testnet-solana-com ec2 us-west-1a \
+        ci/testnet-deploy.sh -N edge-testnet-solana-com -C ec2 -z us-west-1a \
           -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P -a eipalloc-0ccd4f2239886fa94 \
           ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-D}
@@ -213,7 +213,7 @@ start() {
       set -x
       NO_LEDGER_VERIFY=1 \
       NO_VALIDATOR_SANITY=1 \
-        ci/testnet-deploy.sh edge-perf-testnet-solana-com ec2 us-west-2b \
+        ci/testnet-deploy.sh -N edge-perf-testnet-solana-com -C ec2 -z us-west-2b \
           -g -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
           ${maybeReuseLedger:+-r} \
@@ -223,27 +223,30 @@ start() {
   testnet-beta)
     (
       set -x
-      # Force delete the network, as two different cloud providers are being used
-      # (otherwise, a subset of the nodes stay up while first cloud is deleted,
-      #  and the leader might be in that subset)
+
+      # List of zones to deploy for each cloud provider
+      GCE_ZONES=(us-west1-b asia-east2-a europe-west4-a southamerica-east1-b us-east4-c)
+      EC2_ZONES=(sa-east-1a us-west-1a ap-northeast-2a eu-central-1a ca-central-1a)
+
+      # Build a string to pass as opts to testnet-deploy.sh: "-z zone1 -z zone2 ..."
+      for val in "${GCE_ZONES[@]}"; do
+        GCE_ZONE_ARGS="-z $val $GCE_ZONE_ARGS"
+      done
+
+      for val in "${EC2_ZONES[@]}"; do
+        EC2_ZONE_ARGS="-z $val $EC2_ZONE_ARGS"
+      done
+
       NO_VALIDATOR_SANITY=1 \
       RUST_LOG=solana=info \
-        ci/testnet-deploy.sh beta-testnet-solana-com ec2 us-west-1a \
-          -t "$CHANNEL_OR_TAG" -n 35 -c 0 -s -u -P -a eipalloc-0f286cf8a0771ce35 -D
-      NO_VALIDATOR_SANITY=1 \
-      RUST_LOG=solana=info \
-        ci/testnet-deploy.sh beta-testnet-solana-com gce us-west1-a \
-          -t "$CHANNEL_OR_TAG" -n 65 -c 0 -x -P -D
-      NO_VALIDATOR_SANITY=1 \
-      RUST_LOG=solana=info \
-        ci/testnet-deploy.sh beta-testnet-solana-com ec2 us-west-1a \
-          -t "$CHANNEL_OR_TAG" -n 35 -c 0 -s -u -P -a eipalloc-0f286cf8a0771ce35 \
+        ci/testnet-deploy.sh -N beta-testnet-solana-com -C ec2 "$EC2_ZONE_ARGS"\
+          -t "$CHANNEL_OR_TAG" -n 60 -c 0 -s -u -P -a eipalloc-0f286cf8a0771ce35 \
           ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-D}
       NO_VALIDATOR_SANITY=1 \
       RUST_LOG=solana=info \
-        ci/testnet-deploy.sh beta-testnet-solana-com gce us-west1-a \
-          -t "$CHANNEL_OR_TAG" -n 65 -c 0 -x -P \
+        ci/testnet-deploy.sh -N beta-testnet-solana-com -C gce "$GCE_ZONE_ARGS"\
+          -t "$CHANNEL_OR_TAG" -n 40 -c 0 -x -P \
           ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-D}
     )
@@ -253,7 +256,7 @@ start() {
       set -x
       NO_LEDGER_VERIFY=1 \
       NO_VALIDATOR_SANITY=1 \
-        ci/testnet-deploy.sh beta-perf-testnet-solana-com ec2 us-west-2b \
+        ci/testnet-deploy.sh -N beta-perf-testnet-solana-com -C ec2 -z us-west-2b \
           -g -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
           ${maybeReuseLedger:+-r} \
@@ -265,12 +268,12 @@ start() {
       set -x
       NO_VALIDATOR_SANITY=1 \
       RUST_LOG=solana=info \
-        ci/testnet-deploy.sh testnet-solana-com ec2 us-west-1a \
+        ci/testnet-deploy.sh -N testnet-solana-com -C ec2 -z us-west-1a \
           -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P -a eipalloc-0fa502bf95f6f18b2 \
           -b \
           ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-D}
-        #ci/testnet-deploy.sh testnet-solana-com gce us-east1-c \
+        #ci/testnet-deploy.sh -N testnet-solana-com -C gce -z us-east1-c \
         #  -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a testnet-solana-com  \
         #  ${maybeReuseLedger:+-r} \
         #  ${maybeDelete:+-D}
@@ -281,14 +284,14 @@ start() {
       set -x
       NO_LEDGER_VERIFY=1 \
       NO_VALIDATOR_SANITY=1 \
-        ci/testnet-deploy.sh perf-testnet-solana-com gce us-west1-b \
+        ci/testnet-deploy.sh -N perf-testnet-solana-com -C gce -z us-west1-b \
           -G "n1-standard-16 --accelerator count=2,type=nvidia-tesla-v100" \
           -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
           -d pd-ssd \
           ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-D}
-        #ci/testnet-deploy.sh perf-testnet-solana-com ec2 us-east-1a \
+        #ci/testnet-deploy.sh -N perf-testnet-solana-com -C ec2 -z us-east-1a \
         #  -g \
         #  -t "$CHANNEL_OR_TAG" -c 2 \
         #  ${maybeReuseLedger:+-r} \