localnet-sanity.sh now supports iterations testing

ci/localnet-sanity.sh

@@ -1,23 +1,80 @@
 #!/usr/bin/env bash
 set -e
-#
+
-# Perform a quick sanity test on a leader, drone, validator and client running
+iterations=1
-# locally on the same machine
+maybeNoLeaderRotation=
-#
+extraNodes=0
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [options...]
+
+Start a local cluster and run sanity on it
+
+  options:
+   -i [number] - Number of times to run sanity (default: $iterations)
+   -b          - Disable leader rotation
+   -x          - Add an extra fullnode (may be supplied multiple times)
+
+EOF
+  exit $exitcode
+}
 
 cd "$(dirname "$0")"/..
+
+while getopts "h?i:bx" opt; do
+  case $opt in
+  h | \?)
+    usage
+    ;;
+  i)
+    iterations=$OPTARG
+    ;;
+  b)
+    maybeNoLeaderRotation="--no-leader-rotation"
+    ;;
+  x)
+    extraNodes=$((extraNodes + 1))
+    ;;
+  *)
+    usage "Error: unhandled option: $opt"
+    ;;
+  esac
+done
+
 source ci/upload-ci-artifact.sh
 source scripts/configure-metrics.sh
 
 multinode-demo/setup.sh
 
-backgroundCommands="drone bootstrap-leader fullnode fullnode-x"
+backgroundCommands=(
-pids=()
+  "multinode-demo/drone.sh"
+  "multinode-demo/bootstrap-leader.sh $maybeNoLeaderRotation"
+  "multinode-demo/fullnode.sh $maybeNoLeaderRotation"
+)
 
-for cmd in $backgroundCommands; do
+for _ in $(seq 1 $extraNodes); do
+  backgroundCommands+=(
+    "multinode-demo/fullnode-x.sh $maybeNoLeaderRotation"
+  )
+done
+numNodes=$((2 + extraNodes))
+
+pids=()
+logs=()
+
+for cmd in "${backgroundCommands[@]}"; do
   echo "--- Start $cmd"
-  rm -f log-"$cmd".txt
+  baseCmd=$(basename "${cmd// */}" .sh)
-  multinode-demo/"$cmd".sh > log-"$cmd".txt 2>&1 &
+  declare log=log-$baseCmd.txt
+  rm -f "$log"
+  $cmd > "$log" 2>&1 &
+  logs+=("$log")
   declare pid=$!
   pids+=("$pid")
   echo "pid: $pid"
@@ -43,10 +100,9 @@ shutdown() {
   set +e
 
   echo "--- Upload artifacts"
-  for cmd in $backgroundCommands; do
+  for log in "${logs[@]}"; do
-    declare logfile=log-$cmd.txt
+    upload-ci-artifact "$log"
-    upload-ci-artifact "$logfile"
+    tail "$log"
-    tail "$logfile"
   done
 
   exit $exitcode
@@ -56,40 +112,46 @@ trap shutdown EXIT INT
 
 set -e
 
+declare iteration=1
+
 flag_error() {
-  echo Failed
+  echo "Failed (iteration: $iteration/$iterations)"
   echo "^^^ +++"
   exit 1
 }
 
-echo "--- Node count"
+while [[ $iteration -le $iterations ]]; do
-(
+  echo "--- Node count ($iteration)"
-  source multinode-demo/common.sh
+  (
-  set -x
+    source multinode-demo/common.sh
-  client_id=/tmp/client-id.json-$$
+    set -x
-  $solana_keygen -o $client_id || exit $?
+    client_id=/tmp/client-id.json-$$
-  $solana_bench_tps \
+    $solana_keygen -o $client_id || exit $?
-    --identity $client_id \
+    $solana_bench_tps \
-    --num-nodes 3 \
+      --identity $client_id \
-    --reject-extra-nodes \
+      --num-nodes $numNodes \
-    --converge-only || exit $?
+      --reject-extra-nodes \
-  rm -rf $client_id
+      --converge-only || exit $?
-) || flag_error
+    rm -rf $client_id
+  ) || flag_error
 
-echo "--- RPC API: getTransactionCount"
+  echo "--- RPC API: getTransactionCount ($iteration)"
-(
+  (
-  set -x
+    set -x
-  curl --retry 5 --retry-delay 2 --retry-connrefused \
+    curl --retry 5 --retry-delay 2 --retry-connrefused \
-    -X POST -H 'Content-Type: application/json' \
+      -X POST -H 'Content-Type: application/json' \
-    -d '{"jsonrpc":"2.0","id":1, "method":"getTransactionCount"}' \
+      -d '{"jsonrpc":"2.0","id":1, "method":"getTransactionCount"}' \
-    http://localhost:8899
+      http://localhost:8899
-) || flag_error
+  ) || flag_error
 
-echo "--- Wallet sanity"
+  echo "--- Wallet sanity ($iteration)"
-(
+  (
-  set -x
+    set -x
-  timeout 60s scripts/wallet-sanity.sh
+    timeout 60s scripts/wallet-sanity.sh
-) || flag_error
+  ) || flag_error
+
+  iteration=$((iteration + 1))
+done
 
 killBackgroundCommands
 
@@ -103,5 +165,6 @@ echo "--- Ledger verification"
 ) || flag_error
 
 echo +++
-echo Ok
+echo "Ok ($iterations iterations)"
+
 exit 0

ci/test-stable.sh

@@ -39,5 +39,5 @@ echo --- ci/localnet-sanity.sh
   set -x
   # Assume |cargo build| has populated target/debug/ successfully.
   export PATH=$PWD/target/debug:$PATH
-  USE_INSTALL=1 ci/localnet-sanity.sh
+  USE_INSTALL=1 ci/localnet-sanity.sh -x
 )