Add option to restart the cluster during iterations
This commit is contained in:
parent
da5cb0b012
commit
443d8ce7c4
|
@ -2,6 +2,7 @@
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
iterations=1
|
iterations=1
|
||||||
|
restartInterval=never
|
||||||
maybeNoLeaderRotation=
|
maybeNoLeaderRotation=
|
||||||
extraNodes=0
|
extraNodes=0
|
||||||
walletRpcEndpoint=
|
walletRpcEndpoint=
|
||||||
|
@ -19,6 +20,7 @@ Start a local cluster and run sanity on it
|
||||||
|
|
||||||
options:
|
options:
|
||||||
-i [number] - Number of times to run sanity (default: $iterations)
|
-i [number] - Number of times to run sanity (default: $iterations)
|
||||||
|
-k [number] - Restart the cluster after this number of sanity iterations (default: $restartInterval)
|
||||||
-b - Disable leader rotation
|
-b - Disable leader rotation
|
||||||
-x - Add an extra fullnode (may be supplied multiple times)
|
-x - Add an extra fullnode (may be supplied multiple times)
|
||||||
-r - Select the RPC endpoint hosted by a node that starts as
|
-r - Select the RPC endpoint hosted by a node that starts as
|
||||||
|
@ -31,7 +33,7 @@ EOF
|
||||||
|
|
||||||
cd "$(dirname "$0")"/..
|
cd "$(dirname "$0")"/..
|
||||||
|
|
||||||
while getopts "h?i:brx" opt; do
|
while getopts "h?i:k:brx" opt; do
|
||||||
case $opt in
|
case $opt in
|
||||||
h | \?)
|
h | \?)
|
||||||
usage
|
usage
|
||||||
|
@ -39,6 +41,9 @@ while getopts "h?i:brx" opt; do
|
||||||
i)
|
i)
|
||||||
iterations=$OPTARG
|
iterations=$OPTARG
|
||||||
;;
|
;;
|
||||||
|
k)
|
||||||
|
restartInterval=$OPTARG
|
||||||
|
;;
|
||||||
b)
|
b)
|
||||||
maybeNoLeaderRotation="--no-leader-rotation"
|
maybeNoLeaderRotation="--no-leader-rotation"
|
||||||
;;
|
;;
|
||||||
|
@ -57,37 +62,42 @@ done
|
||||||
source ci/upload-ci-artifact.sh
|
source ci/upload-ci-artifact.sh
|
||||||
source scripts/configure-metrics.sh
|
source scripts/configure-metrics.sh
|
||||||
|
|
||||||
multinode-demo/setup.sh
|
nodes=(
|
||||||
|
|
||||||
backgroundCommands=(
|
|
||||||
"multinode-demo/drone.sh"
|
"multinode-demo/drone.sh"
|
||||||
"multinode-demo/bootstrap-leader.sh $maybeNoLeaderRotation"
|
"multinode-demo/bootstrap-leader.sh $maybeNoLeaderRotation"
|
||||||
"multinode-demo/fullnode.sh $maybeNoLeaderRotation --rpc-port 18899"
|
"multinode-demo/fullnode.sh $maybeNoLeaderRotation --rpc-port 18899"
|
||||||
)
|
)
|
||||||
|
|
||||||
for _ in $(seq 1 $extraNodes); do
|
for _ in $(seq 1 $extraNodes); do
|
||||||
backgroundCommands+=(
|
nodes+=("multinode-demo/fullnode-x.sh $maybeNoLeaderRotation")
|
||||||
"multinode-demo/fullnode-x.sh $maybeNoLeaderRotation"
|
|
||||||
)
|
|
||||||
done
|
done
|
||||||
numNodes=$((2 + extraNodes))
|
numNodes=$((2 + extraNodes))
|
||||||
|
|
||||||
pids=()
|
pids=()
|
||||||
logs=()
|
logs=()
|
||||||
|
|
||||||
for cmd in "${backgroundCommands[@]}"; do
|
startNodes() {
|
||||||
|
declare addLogs=false
|
||||||
|
if [[ ${#logs[@]} -eq 0 ]]; then
|
||||||
|
addLogs=true
|
||||||
|
fi
|
||||||
|
for cmd in "${nodes[@]}"; do
|
||||||
echo "--- Start $cmd"
|
echo "--- Start $cmd"
|
||||||
baseCmd=$(basename "${cmd// */}" .sh)
|
baseCmd=$(basename "${cmd// */}" .sh)
|
||||||
declare log=log-$baseCmd.txt
|
declare log=log-$baseCmd.txt
|
||||||
rm -f "$log"
|
rm -f "$log"
|
||||||
$cmd > "$log" 2>&1 &
|
$cmd > "$log" 2>&1 &
|
||||||
|
if $addLogs; then
|
||||||
logs+=("$log")
|
logs+=("$log")
|
||||||
|
fi
|
||||||
declare pid=$!
|
declare pid=$!
|
||||||
pids+=("$pid")
|
pids+=("$pid")
|
||||||
echo "pid: $pid"
|
echo "pid: $pid"
|
||||||
done
|
done
|
||||||
|
}
|
||||||
|
|
||||||
killBackgroundCommands() {
|
killNodes() {
|
||||||
|
echo "--- Killing nodes"
|
||||||
set +e
|
set +e
|
||||||
for pid in "${pids[@]}"; do
|
for pid in "${pids[@]}"; do
|
||||||
if kill "$pid"; then
|
if kill "$pid"; then
|
||||||
|
@ -100,9 +110,20 @@ killBackgroundCommands() {
|
||||||
pids=()
|
pids=()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
verifyLedger() {
|
||||||
|
for ledger in bootstrap-leader fullnode; do
|
||||||
|
echo "--- $ledger ledger verification"
|
||||||
|
(
|
||||||
|
source multinode-demo/common.sh
|
||||||
|
set -x
|
||||||
|
$solana_ledger_tool --ledger "$SOLANA_CONFIG_DIR"/$ledger-ledger verify
|
||||||
|
) || flag_error
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
shutdown() {
|
shutdown() {
|
||||||
exitcode=$?
|
exitcode=$?
|
||||||
killBackgroundCommands
|
killNodes
|
||||||
|
|
||||||
set +e
|
set +e
|
||||||
|
|
||||||
|
@ -127,6 +148,8 @@ flag_error() {
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
multinode-demo/setup.sh
|
||||||
|
startNodes
|
||||||
while [[ $iteration -le $iterations ]]; do
|
while [[ $iteration -le $iterations ]]; do
|
||||||
echo "--- Node count ($iteration)"
|
echo "--- Node count ($iteration)"
|
||||||
(
|
(
|
||||||
|
@ -168,18 +191,16 @@ while [[ $iteration -le $iterations ]]; do
|
||||||
) || flag_error
|
) || flag_error
|
||||||
|
|
||||||
iteration=$((iteration + 1))
|
iteration=$((iteration + 1))
|
||||||
|
|
||||||
|
if [[ $restartInterval != never && $((iteration % restartInterval)) -eq 0 ]]; then
|
||||||
|
killNodes
|
||||||
|
verifyLedger
|
||||||
|
startNodes
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
killBackgroundCommands
|
killNodes
|
||||||
|
verifyLedger
|
||||||
echo "--- Ledger verification"
|
|
||||||
(
|
|
||||||
source multinode-demo/common.sh
|
|
||||||
set -x
|
|
||||||
cp -R "$SOLANA_CONFIG_DIR"/bootstrap-leader-ledger /tmp/ledger-$$
|
|
||||||
$solana_ledger_tool --ledger /tmp/ledger-$$ verify || exit $?
|
|
||||||
rm -rf /tmp/ledger-$$
|
|
||||||
) || flag_error
|
|
||||||
|
|
||||||
echo +++
|
echo +++
|
||||||
echo "Ok ($iterations iterations)"
|
echo "Ok ($iterations iterations)"
|
||||||
|
|
Loading…
Reference in New Issue