Allow delay between validator booting and client start (#7297)

* Allow delay between validator booting and client start

net/net.sh

@@ -48,6 +48,9 @@ Operate a configured testnet
                                             -c bench-tps=2="--tx_count 25000"
                                         This will start 2 bench-tps clients, and supply "--tx_count 25000"
                                         to the bench-tps client.
+   --client-delay-start
+                                      - Number of seconds to wait after validators have finished starting before starting client programs
+                                        (default: $clientDelayStart)
    -n NUM_VALIDATORS                  - Number of validators to apply command to.
    --gpu-mode GPU_MODE                - Specify GPU mode to launch validators with (default: $gpuMode).
                                         MODE must be one of
@@ -90,7 +93,6 @@ Operate a configured testnet
    --operating-mode development|softlaunch
                                       - Specify whether or not to launch the cluster in "development" mode with all features enabled at epoch 0,
                                         or "softlaunch" mode with some features disabled at epoch 0 (default: development)
-
  sanity/start-specific options:
    -F                   - Discard validator nodes that didn't bootup successfully
    -o noInstallCheck    - Skip solana-install sanity
@@ -153,6 +155,7 @@ netemPartition=""
 netemConfig=""
 netemConfigFile=""
 netemCommand="add"
+clientDelayStart=0
 
 command=$1
 [[ -n $command ]] || usage
@@ -250,6 +253,9 @@ while [[ -n $1 ]]; do
           ;;
       esac
       shift 2
+    elif [[ $1 == --client-delay-start ]]; then
+      clientDelayStart=$2
+      shift 2
     else
       usage "Unknown long option: $1"
     fi
@@ -809,6 +815,9 @@ deploy() {
   sanity skipBlockstreamerSanity # skip sanity on blockstreamer node, it may not
                                  # have caught up to the bootstrap leader yet
 
+  echo "--- Sleeping $clientDelayStart seconds after validators are started before starting clients"
+  sleep "$clientDelayStart"
+
   SECONDS=0
   for ((i=0; i < "$numClients" && i < "$numClientsRequested"; i++)) do
     if [[ $i -lt "$numBenchTpsClients" ]]; then

system-test/testnet-performance/testnet-automation.sh

@@ -225,13 +225,13 @@ function launchTestnet() {
     # shellcheck disable=SC2086
     net/net.sh start -t "$CHANNEL" \
       "$maybeClientOptions" "$CLIENT_OPTIONS" $maybeStartAllowBootFailures \
-      --gpu-mode $startGpuMode
+      --gpu-mode $startGpuMode --client-delay-start $CLIENT_DELAY_START
   else
     # shellcheck disable=SC2068
     # shellcheck disable=SC2086
     net/net.sh start -T solana-release*.tar.bz2 \
       "$maybeClientOptions" "$CLIENT_OPTIONS" $maybeStartAllowBootFailures \
-      --gpu-mode $startGpuMode
+      --gpu-mode $startGpuMode --client-delay-start $CLIENT_DELAY_START
   fi
 
   execution_step "Wait ${RAMP_UP_TIME} seconds for network throughput to stabilize"
@@ -344,6 +344,8 @@ if [[ "$USE_PUBLIC_IP_ADDRESSES" = "true" ]] ; then
   maybePublicIpAddresses="-P"
 fi
 
+: "${CLIENT_DELAY_START:=0}"
+
 if [[ -z $CHANNEL ]]; then
   execution_step "Downloading tar from build artifacts"
   buildkite-agent artifact download "solana-release*.tar.bz2" .
@@ -367,6 +369,7 @@ TEST_PARAMS_TO_DISPLAY=(CLOUD_PROVIDER \
                         VALIDATOR_NODE_MACHINE_TYPE \
                         NUMBER_OF_CLIENT_NODES \
                         CLIENT_OPTIONS \
+                        CLIENT_DELAY_START \
                         TESTNET_ZONES \
                         TEST_DURATION_SECONDS \
                         USE_PUBLIC_IP_ADDRESSES \