Add net/ sanity

net/net.sh

@@ -18,16 +18,21 @@ usage: $0 [start|stop]
 Operate a configured testnet
 
  start - Start the network
+ sanity - Sanity check the network
  stop  - Stop the network
 
  start-specific options:
-   -S snapFilename     - Deploy the specified Snap file
-   -s edge|beta|stable - Deploy the latest Snap on the specified Snap release channel
-   -a "setup args"     - Optional additional arguments for ./multinode-demo/setup.sh
+   -S snapFilename      - Deploy the specified Snap file
+   -s edge|beta|stable  - Deploy the latest Snap on the specified Snap release channel
+   -a "setup args"      - Optional additional arguments for ./multinode-demo/setup.sh
 
    Note: if RUST_LOG is set in the environment it will be propogated into the
          network nodes.
 
+ sanity-specific options:
+   -o noLedgerVerify    - Skip ledger verification
+   -o noValidatorSanity - Skip validatory sanity
+
  stop-specific options:
    none
 
@@ -39,13 +44,15 @@ snapChannel=
 snapFilename=
 nodeSetupArgs=
 deployMethod=local
+sanityExtraArgs=
 
 command=$1
 [[ -n $command ]] || usage
 shift
-[[ $command = start || $command = stop ]] || usage "Invalid command: $command"
+[[ $command = start || $command = sanity || $command = stop ]] ||
+  usage "Invalid command: $command"
 
-while getopts "h?S:s:a:" opt; do
+while getopts "h?S:s:a:o:" opt; do
   case $opt in
   h | \?)
     usage
@@ -69,6 +76,17 @@ while getopts "h?S:s:a:" opt; do
   a)
     nodeSetupArgs="$OPTARG"
     ;;
+  o)
+    case $OPTARG in
+    noLedgerVerify|noValidatorSanity)
+      sanityExtraArgs="$sanityExtraArgs -o $OPTARG"
+      ;;
+    *)
+      echo "Error: unknown option: $OPTARG"
+      exit 1
+      ;;
+    esac
+    ;;
   *)
     usage "Error: unhandled option: $opt"
     ;;
@@ -148,7 +166,7 @@ startValidator() {
     set -x
     ssh "${sshOptions[@]}" -f "$ipAddress" \
       "./solana/net/remote/remote_node.sh $deployMethod validator $leaderIp \"$nodeSetupArgs\" \"$RUST_LOG\""
-   ) >> "$logFile"
+  ) >> "$logFile"
 }
 
 startClient() {
@@ -160,13 +178,24 @@ startClient() {
 
   declare expectedNodeCount=$((${#validatorIpList[@]} + 1))
 
-  ssh "${sshOptions[@]}" -f "$ipAddress" \
-    "./solana/net/remote/remote_client.sh $deployMethod $leaderIp $expectedNodeCount \"$RUST_LOG\"" >> "$logFile"
+  (
+    set -x
+    ssh "${sshOptions[@]}" -f "$ipAddress" \
+      "./solana/net/remote/remote_client.sh $deployMethod $leaderIp $expectedNodeCount \"$RUST_LOG\""
+  ) >> "$logFile"
+}
+
+sanity() {
+  declare expectedNodeCount=$((${#validatorIpList[@]} + 1))
+  (
+    set -x
+    # shellcheck disable=SC2029 # remote_client.sh are expanded on client side intentionally...
+    ssh "${sshOptions[@]}" "$leaderIp" \
+      "./solana/net/remote/remote_sanity.sh $deployMethod $leaderIp $expectedNodeCount $sanityExtraArgs"
+  )
 }
 
 start() {
-  [[ $command = "start" ]] || return
-
   case $deployMethod in
   snap)
     if [[ -n $snapChannel ]]; then
@@ -202,6 +231,8 @@ start() {
   wait
   validatorDeployTime=$SECONDS
 
+  sanity
+
   SECONDS=0
   for ipAddress in "${clientIpList[@]}"; do
     startClient "$ipAddress" "$netLogDir/client-$ipAddress.log"
@@ -261,6 +292,18 @@ stop() {
   echo "Stopping nodes took $SECONDS seconds"
 }
 
-stop
-start
-
+case $command in
+start)
+  stop
+  start
+  ;;
+sanity)
+  sanity
+  ;;
+stop)
+  stop
+  ;;
+*)
+  echo "Internal error: Unknown command: $command"
+  exit 1
+esac

net/remote/remote_sanity.sh

@@ -0,0 +1,109 @@
+#!/bin/bash -e
+
+deployMethod="$1"
+netEntrypoint="$2"
+numNodes="$3"
+
+[[ -n $deployMethod ]] || exit
+[[ -n $netEntrypoint ]] || exit
+[[ -n $numNodes ]] || exit
+
+shift 3
+
+ledgerVerify=true
+validatorSanity=true
+while [[ $1 = "-o" ]]; do
+  opt="$2"
+  shift 2
+  case $opt in
+  noLedgerVerify)
+    ledgerVerify=false
+    ;;
+  noValidatorSanity)
+    validatorSanity=false
+    ;;
+  *)
+    echo "Error: unknown option: $opt"
+    exit 1
+    ;;
+  esac
+done
+
+
+cd "$(dirname "$0")"/../..
+source net/common.sh
+loadConfigFile
+
+case $deployMethod in
+snap)
+  export USE_SNAP=1
+  solana_bench_tps=/snap/bin/solana.bench-tps
+  solana_ledger_tool=/snap/bin/solana.ledger-tool
+  ledger=/var/snap/solana/current/config/ledger
+  ;;
+local)
+  PATH="$HOME"/.cargo/bin:"$PATH"
+  export USE_INSTALL=1
+
+  solana_bench_tps=multinode-demo/client.sh
+  solana_ledger_tool=solana-ledger-tool
+  ledger=config/ledger
+  netEntrypoint="$:~/solana"
+  ;;
+*)
+  echo "Unknown deployment method: $deployMethod"
+  exit 1
+esac
+
+
+echo "--- $netEntrypoint: wallet sanity"
+(
+  set -x
+  multinode-demo/test/wallet-sanity.sh "$netEntrypoint"
+)
+
+echo "--- $netEntrypoint: node count"
+(
+  set -x
+  $solana_bench_tps "$netEntrypoint" "$numNodes" -c
+)
+
+echo "--- $netEntrypoint: verify ledger"
+if $ledgerVerify; then
+  if [[ -d $ledger ]]; then
+    (
+      set -x
+      rm -rf /var/tmp/ledger-verify
+      cp -r $ledger /var/tmp/ledger-verify
+      $solana_ledger_tool --ledger /var/tmp/ledger-verify verify
+    )
+  else
+    echo "^^^ +++"
+    echo "Ledger verify skipped"
+  fi
+else
+  echo "^^^ +++"
+  echo "Ledger verify skipped (NO_LEDGER_VERIFY defined)"
+fi
+
+
+echo "--- $netEntrypoint: validator sanity"
+if $validatorSanity; then
+  (
+    ./multinode-demo/setup.sh -t validator
+    set -e pipefail
+    timeout 10s ./multinode-demo/validator.sh "$netEntrypoint" 2>&1 | tee validator.log
+  )
+  wc -l validator.log
+  if grep -C100 panic validator.log; then
+    echo "^^^ +++"
+    echo "Panic observed"
+    exit 1
+  else
+    echo "Validator log looks ok"
+  fi
+else
+  echo "^^^ +++"
+  echo "Validator sanity disabled (NO_VALIDATOR_SANITY defined)"
+fi
+