Use a local user to avoid GCP login quota limits

ci/testnet-deploy.sh

@@ -71,7 +71,7 @@ echo "Install command: $SNAP_INSTALL_CMD"
 
 leaderName=${publicUrl//./-}
 
-vmlist=() # Each array element is the triple "class:vmName:vmZone"
+vmlist=() # Each array element is formatted as "class:vmName:vmZone:vmPublicIp"
 
 #
 # vm_foreach [cmd] [extra args to cmd]
@@ -81,6 +81,7 @@ vmlist=() # Each array element is the triple "class:vmName:vmZone"
 #           additionl arguments supplied to vm_foreach:
 #               vmName - GCP name of the VM
 #               vmZone - The GCP zone the VM is located in
+#               vmPublicIp - The public IP address of this VM
 #               vmClass - The 'class' of this VM
 #               count  - Monotonically increasing count for each
 #                        invocation of cmd, starting at 1
@@ -93,10 +94,10 @@ vm_foreach() {
 
   declare count=1
   for info in "${vmlist[@]}"; do
-    declare vmClass vmName vmZone
+    declare vmClass vmName vmZone vmPublicIp
-    IFS=: read -r vmClass vmName vmZone < <(echo "$info")
+    IFS=: read -r vmClass vmName vmZone vmPublicIp < <(echo "$info")
 
-    eval "$cmd" "$vmName" "$vmZone" "$vmClass" "$count" "$@"
+    eval "$cmd" "$vmName" "$vmZone" "$vmPublicIp" "$vmClass" "$count" "$@"
     count=$((count + 1))
   done
 }
@@ -109,6 +110,7 @@ vm_foreach() {
 #           The command will receive three arguments:
 #               vmName - GCP name of the VM
 #               vmZone - The GCP zone the VM is located in
+#               vmPublicIp - The public IP address of this VM
 #               count  - Monotonically increasing count for each
 #                        invocation of cmd, starting at 1
 #
@@ -116,12 +118,13 @@ vm_foreach() {
 _run_cmd_if_class() {
   declare vmName=$1
   declare vmZone=$2
-  declare vmClass=$3
+  declare vmPublicIp=$3
-  declare count=$4
+  declare vmClass=$4
-  declare class=$5
+  declare count=$5
-  declare cmd=$6
+  declare class=$6
+  declare cmd=$7
   if [[ $class = "$vmClass" ]]; then
-    eval "$cmd" "$vmName" "$vmZone" "$count"
+    eval "$cmd" "$vmName" "$vmZone" "$vmPublicIp" "$count"
   fi
 }
 
@@ -138,13 +141,15 @@ findVms() {
   declare class="$1"
   declare filter="$2"
   gcloud compute instances list --filter="$filter"
-  while read -r vmName vmZone status; do
+  while read -r vmName vmZone vmPublicIp status; do
     if [[ $status != RUNNING ]]; then
       echo "Warning: $vmName is not RUNNING, ignoring it."
       continue
     fi
-    vmlist+=("$class:$vmName:$vmZone")
+    vmlist+=("$class:$vmName:$vmZone:$vmPublicIp")
-  done < <(gcloud compute instances list --filter="$filter" --format 'value(name,zone,status)')
+  done < <(gcloud compute instances list \
+             --filter="$filter" \
+             --format 'value(name,zone,networkInterfaces[0].accessConfigs[0].natIP,status)')
 }
 
 echo "Leader node:"
@@ -174,31 +179,25 @@ netName=${SOLANA_NET_URL/.*/}
 gcp_vm_exec() {
   declare vmName=$1
   declare vmZone=$2
-  declare message=$3
+  declare vmPublicIp=$3
-  declare cmd=$4
+  declare message=$4
+  declare cmd=$5
 
-  echo "--- $message $vmName in zone $vmZone"
+  echo "--- $message $vmName in zone $vmZone ($vmPublicIp)"
-  gcloud compute ssh "$vmName" --zone "$vmZone" \
+  (
-    --ssh-flag="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+    set -x
-    --command="$cmd"
+    ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-}
+      testnet-deploy@"$vmPublicIp" "$cmd"
-
+  )
-gcp_login_quota_workaround() {
-  declare count=$1
-
-  if [[ $((count % 5)) = 0 ]]; then
-    #  Slow down deployment to avoid triggering GCP login
-    #  quota limits (each |ssh| counts as a login)
-    sleep 3
-  fi
 }
 
 client_start() {
   declare vmName=$1
   declare vmZone=$2
-  declare count=$3
+  declare vmPublicIp=$3
+  declare count=$4
 
-  gcp_vm_exec "$vmName" "$vmZone" \
+  gcp_vm_exec "$vmName" "$vmZone" "$vmPublicIp" \
     "Starting client $count:" \
     "\
       set -x;
@@ -208,7 +207,8 @@ client_start() {
       if [[ \$threadCount -gt 4 ]]; then threadCount=4; fi; \
       tmux kill-session -t solana; \
       tmux new -s solana -d \" \
-          set -x;
+          set -x; \
+          sudo rm /tmp/solana.log; \
           /snap/bin/solana.bench-tps $SOLANA_NET_URL $fullnode_count --loop -s 600 --sustained -t \$threadCount 2>&1 | tee /tmp/solana.log; \
           echo 'https://metrics.solana.com:8086/write?db=${INFLUX_DATABASE}&u=${INFLUX_USERNAME}&p=${INFLUX_PASSWORD}' \
             | xargs curl --max-time 5 -XPOST --data-binary 'testnet-deploy,name=$netName clientexit=1'; \
@@ -224,12 +224,13 @@ client_start() {
 client_stop() {
   declare vmName=$1
   declare vmZone=$2
-  declare count=$3
+  declare vmPublicIp=$3
+  declare count=$4
 
   (
     SECONDS=0
-    gcp_vm_exec "$vmName" "$vmZone" \
+    gcp_vm_exec "$vmName" "$vmZone" "$vmPublicIp" \
-      "Stopping client $count:" \
+      "Stopping client $vmName ($count):" \
       "\
         set -x;
         tmux list-sessions; \
@@ -257,7 +258,8 @@ fullnode_start() {
   declare class=$1
   declare vmName=$2
   declare vmZone=$3
-  declare count=$4
+  declare vmPublicIp=$4
+  declare count=$5
 
   (
     SECONDS=0
@@ -275,7 +277,7 @@ fullnode_start() {
       nodeConfig="mode=validator leader-address=$publicIp $commonNodeConfig"
     fi
 
-    gcp_vm_exec "$vmName" "$vmZone" "Starting $class $count:" \
+    gcp_vm_exec "$vmName" "$vmZone" "$vmPublicIp" "Starting $class $count:" \
       "\
         set -ex; \
         logmarker='solana deploy $(date)/$RANDOM'; \
@@ -298,7 +300,6 @@ fullnode_start() {
   done
   mv "log-$vmName.txt" "log-$pid.txt"
 
-  gcp_login_quota_workaround "$count"
   pids+=("$pid")
 }
 
@@ -313,11 +314,12 @@ validator_start() {
 fullnode_stop() {
   declare vmName=$1
   declare vmZone=$2
-  declare count=$3
+  declare vmPublicIp=$3
+  declare count=$4
 
   (
     SECONDS=0
-    gcp_vm_exec "$vmName" "$vmZone" "Shutting down" "\
+    gcp_vm_exec "$vmName" "$vmZone" "$vmPublicIp" "Shutting down" "\
       if snap list solana; then \
         sudo snap set solana mode=; \
       fi"
@@ -331,7 +333,6 @@ fullnode_stop() {
   done
   mv "log-$vmName.txt" "log-$pid.txt"
 
-  gcp_login_quota_workaround "$count"
   pids+=("$pid")
 }
 
@@ -355,12 +356,14 @@ if [[ -n $LOCAL_SNAP ]]; then
   transfer_local_snap() {
     declare vmName=$1
     declare vmZone=$2
-    declare vmClass=$3
+    declare vmPublicIp=$3
-    declare count=$4
+    declare vmClass=$4
+    declare count=$5
 
     echo "--- $vmName in zone $vmZone ($count)"
     SECONDS=0
-    gcloud compute scp --zone "$vmZone" "$LOCAL_SNAP" "$vmName":solana_local.snap
+    scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+      "$LOCAL_SNAP" testnet-deploy@"$vmPublicIp":solana_local.snap
     echo "Succeeded in ${SECONDS} seconds"
   }
   vm_foreach transfer_local_snap