Verify transaction count as reported by the bootstrap-leader node is advancing

ci/localnet-sanity.sh

@@ -231,6 +231,8 @@ else
   verifyLedger
 fi
 startNodes
+lastTransactionCount=
+enforceTransactionCountAdvance=true
 while [[ $iteration -le $iterations ]]; do
   echo "--- Node count ($iteration)"
   (
@@ -252,7 +254,9 @@ while [[ $iteration -le $iterations ]]; do
     curl --retry 5 --retry-delay 2 --retry-connrefused \
       -X POST -H 'Content-Type: application/json' \
       -d '{"jsonrpc":"2.0","id":1, "method":"getTransactionCount"}' \
+      -o transactionCount.json \
       http://localhost:8899
+    cat transactionCount.json
   ) || flag_error
 
   echo "--- RPC API: fullnode getTransactionCount ($iteration)"
@@ -264,12 +268,34 @@ while [[ $iteration -le $iterations ]]; do
       http://localhost:18899
   ) || flag_error
 
+  # Verify transaction count as reported by the bootstrap-leader node is advancing
+  transactionCount=$(sed -e 's/{"jsonrpc":"2.0","result":\([0-9]*\),"id":1}/\1/' transactionCount.json)
+  if [[ -n $lastTransactionCount ]]; then
+    echo "--- Transaction count check: $lastTransactionCount < $transactionCount"
+    if $enforceTransactionCountAdvance; then
+      if [[ $lastTransactionCount -ge $transactionCount ]]; then
+        echo "Error: Transaction count is not advancing"
+        echo "* lastTransactionCount: $lastTransactionCount"
+        echo "* transactionCount: $transactionCount"
+        flag_error
+      fi
+    else
+      echo "enforceTransactionCountAdvance=false"
+    fi
+    enforceTransactionCountAdvance=true
+  fi
+  lastTransactionCount=$transactionCount
+
   echo "--- Wallet sanity ($iteration)"
   flag_error_if_no_leader_rotation() {
     # TODO: Stop ignoring wallet sanity failures when leader rotation is enabled
     #       once https://github.com/solana-labs/solana/issues/2474 is fixed
     if [[ -n $maybeNoLeaderRotation ]]; then
       flag_error
+    else
+      # Wallet error occurred (and was ignored) so transactionCount may not
+      # advance on the next iteration
+      enforceTransactionCountAdvance=false
     fi
   }
   (