Run buildkite iterations in parallel

ci/buildkite-stability.yml

@@ -1,4 +0,0 @@
-steps:
-  - command: "ci/iterations-localnet.sh"
-    name: "iterations-localnet"
-    timeout_in_minutes: 40

ci/iterations-localnet.sh

@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 set -e
 
-FEATURES="$1"
+testCmd="$*"
+genPipeline=false
 
 cd "$(dirname "$0")/.."
 
@@ -9,27 +10,57 @@ cd "$(dirname "$0")/.."
 rm -rf "$HOME/.config/solana"
 
 source ci/_
-ci/version-check-with-upgrade.sh stable
 export RUST_BACKTRACE=1
 export RUSTFLAGS="-D warnings"
-
-_ scripts/ulimit-n.sh
-_ cargo build --all --features="$FEATURES"
-
 export PATH=$PWD/target/debug:$PATH
 export USE_INSTALL=1
 
-# Leader rotation disabled
-_ ci/localnet-sanity.sh -b -i 128
+if [[ -n $BUILDKITE && -z $testCmd ]]; then
+  genPipeline=true
+  echo "
+steps:
+  "
+fi
 
-# Leader rotation disabled, restart all nodes periodically
-_ ci/localnet-sanity.sh -b -i 128 -k 16
+build() {
+  $genPipeline && return
+  ci/version-check-with-upgrade.sh stable
+  _ scripts/ulimit-n.sh
+  FEATURES=""
+  _ cargo build --all --features="$FEATURES"
+}
 
-# Leader rotation enabled
-_ ci/localnet-sanity.sh -i 128
+runTest() {
+  declare runTestName="$1"
+  declare runTestCmd="$2"
+  if $genPipeline; then
+    echo "
+  - command: \"$0 '$runTestCmd'\"
+    name: \"$runTestName\"
+    timeout_in_minutes: 30
+"
+    return
+  fi
 
-# Leader rotation enabled, restart all nodes periodically
-# TODO: Enable
-#_ ci/localnet-sanity.sh -i 128 -k 16
+  if [[ -n $testCmd && "$testCmd" != "$runTestCmd" ]]; then
+    echo Skipped "$runTestName"...
+    return
+  fi
+  #shellcheck disable=SC2068 # Don't want to double quote $runTestCmd
+  $runTestCmd
+}
+
+build
+
+runTest "Leader rotation off" \
+  "ci/localnet-sanity.sh -b -i 128"
+
+runTest "Leader rotation off, periodic node restart" \
+  "ci/localnet-sanity.sh -b -i 128 -k 16"
+
+runTest "Leader rotation on" \
+  "ci/localnet-sanity.sh -i 128"
+
+runTest "Leader rotation on, periodic node restart" \
+  "ci/localnet-sanity.sh -i 128 -k 16"
 
-echo --- fin