Wait for nodes to finish booting before running sanity checks
This commit is contained in:
parent
e84f1f6de7
commit
d1bedeae13
|
@ -77,12 +77,22 @@ source scripts/configure-metrics.sh
|
||||||
|
|
||||||
nodes=(
|
nodes=(
|
||||||
"multinode-demo/drone.sh"
|
"multinode-demo/drone.sh"
|
||||||
"multinode-demo/bootstrap-leader.sh $maybeNoLeaderRotation"
|
"multinode-demo/bootstrap-leader.sh \
|
||||||
"multinode-demo/fullnode.sh $maybeNoLeaderRotation --rpc-port 18899"
|
$maybeNoLeaderRotation \
|
||||||
|
--init-complete-file init-complete-node1.log"
|
||||||
|
"multinode-demo/fullnode.sh \
|
||||||
|
$maybeNoLeaderRotation \
|
||||||
|
--init-complete-file init-complete-node2.log \
|
||||||
|
--rpc-port 18899"
|
||||||
)
|
)
|
||||||
|
|
||||||
for i in $(seq 1 $extraNodes); do
|
for i in $(seq 1 $extraNodes); do
|
||||||
nodes+=("multinode-demo/fullnode.sh -X dyn$i $maybeNoLeaderRotation")
|
nodes+=(
|
||||||
|
"multinode-demo/fullnode.sh \
|
||||||
|
-X dyn$i \
|
||||||
|
--init-complete-file init-complete-node$((2 + i)).log \
|
||||||
|
$maybeNoLeaderRotation"
|
||||||
|
)
|
||||||
done
|
done
|
||||||
numNodes=$((2 + extraNodes))
|
numNodes=$((2 + extraNodes))
|
||||||
|
|
||||||
|
@ -111,18 +121,45 @@ startNode() {
|
||||||
echo "log: $log"
|
echo "log: $log"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
initCompleteFiles=()
|
||||||
|
waitForAllNodesToInit() {
|
||||||
|
echo "--- ${#initCompleteFiles[@]} nodes booting"
|
||||||
|
SECONDS=
|
||||||
|
for initCompleteFile in "${initCompleteFiles[@]}"; do
|
||||||
|
while [[ ! -r $initCompleteFile ]]; do
|
||||||
|
if [[ $SECONDS -ge 30 ]]; then
|
||||||
|
echo "Error: $initCompleteFile not found in $SECONDS seconds"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Waiting for $initCompleteFile ($SECONDS)..."
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
echo "Found $initCompleteFile"
|
||||||
|
done
|
||||||
|
echo "All nodes finished booting in $SECONDS seconds"
|
||||||
|
}
|
||||||
|
|
||||||
startNodes() {
|
startNodes() {
|
||||||
declare addLogs=false
|
declare addLogs=false
|
||||||
if [[ ${#logs[@]} -eq 0 ]]; then
|
if [[ ${#logs[@]} -eq 0 ]]; then
|
||||||
addLogs=true
|
addLogs=true
|
||||||
fi
|
fi
|
||||||
|
initCompleteFiles=()
|
||||||
for i in $(seq 0 $((${#nodes[@]} - 1))); do
|
for i in $(seq 0 $((${#nodes[@]} - 1))); do
|
||||||
declare cmd=${nodes[$i]}
|
declare cmd=${nodes[$i]}
|
||||||
|
|
||||||
|
if [[ "$i" -ne 0 ]]; then # 0 == drone, skip it
|
||||||
|
declare initCompleteFile="init-complete-node$i.log"
|
||||||
|
rm -f "$initCompleteFile"
|
||||||
|
initCompleteFiles+=("$initCompleteFile")
|
||||||
|
fi
|
||||||
startNode "$i" "$cmd"
|
startNode "$i" "$cmd"
|
||||||
if $addLogs; then
|
if $addLogs; then
|
||||||
logs+=("$(getNodeLogFile "$i" "$cmd")")
|
logs+=("$(getNodeLogFile "$i" "$cmd")")
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
waitForAllNodesToInit
|
||||||
}
|
}
|
||||||
|
|
||||||
killNode() {
|
killNode() {
|
||||||
|
@ -171,6 +208,10 @@ rollingNodeRestart() {
|
||||||
# node that was just stopped
|
# node that was just stopped
|
||||||
echo "(sleeping for 20 seconds)"
|
echo "(sleeping for 20 seconds)"
|
||||||
sleep 20
|
sleep 20
|
||||||
|
|
||||||
|
declare initCompleteFile="init-complete-node$i.log"
|
||||||
|
rm -f "$initCompleteFile"
|
||||||
|
initCompleteFiles+=("$initCompleteFile")
|
||||||
startNode "$i" "$cmd"
|
startNode "$i" "$cmd"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
@ -185,6 +226,8 @@ rollingNodeRestart() {
|
||||||
}
|
}
|
||||||
done
|
done
|
||||||
pids=("${newPids[@]}")
|
pids=("${newPids[@]}")
|
||||||
|
|
||||||
|
waitForAllNodesToInit
|
||||||
}
|
}
|
||||||
|
|
||||||
verifyLedger() {
|
verifyLedger() {
|
||||||
|
|
Loading…
Reference in New Issue