Implement automated partition testing (#7222)
This commit is contained in:
parent
023074650f
commit
7587656cf6
|
@ -1009,6 +1009,7 @@ logs)
|
||||||
;;
|
;;
|
||||||
netem)
|
netem)
|
||||||
if [[ -n $netemConfigFile ]]; then
|
if [[ -n $netemConfigFile ]]; then
|
||||||
|
remoteNetemConfigFile="$(basename "$netemConfigFile")"
|
||||||
if [[ $netemCommand = "add" ]]; then
|
if [[ $netemCommand = "add" ]]; then
|
||||||
for ipAddress in "${validatorIpList[@]}"; do
|
for ipAddress in "${validatorIpList[@]}"; do
|
||||||
"$here"/scp.sh "$netemConfigFile" solana@"$ipAddress":~/solana
|
"$here"/scp.sh "$netemConfigFile" solana@"$ipAddress":~/solana
|
||||||
|
@ -1016,7 +1017,7 @@ netem)
|
||||||
fi
|
fi
|
||||||
for i in "${!validatorIpList[@]}"; do
|
for i in "${!validatorIpList[@]}"; do
|
||||||
"$here"/ssh.sh solana@"${validatorIpList[$i]}" 'solana/scripts/net-shaper.sh' \
|
"$here"/ssh.sh solana@"${validatorIpList[$i]}" 'solana/scripts/net-shaper.sh' \
|
||||||
"$netemCommand" ~solana/solana/"$netemConfigFile" "${#validatorIpList[@]}" "$i"
|
"$netemCommand" ~solana/solana/"$remoteNetemConfigFile" "${#validatorIpList[@]}" "$i"
|
||||||
done
|
done
|
||||||
else
|
else
|
||||||
num_nodes=$((${#validatorIpList[@]}*netemPartition/100))
|
num_nodes=$((${#validatorIpList[@]}*netemPartition/100))
|
||||||
|
|
|
@ -0,0 +1,39 @@
|
||||||
|
{
|
||||||
|
"partitions":[
|
||||||
|
34,
|
||||||
|
33,
|
||||||
|
33
|
||||||
|
],
|
||||||
|
"interconnects":[
|
||||||
|
{
|
||||||
|
"a":0,
|
||||||
|
"b":1,
|
||||||
|
"config":"loss 15% delay 25ms"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"a":1,
|
||||||
|
"b":0,
|
||||||
|
"config":"loss 15% delay 25ms"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"a":0,
|
||||||
|
"b":2,
|
||||||
|
"config":"loss 10% delay 15ms"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"a":2,
|
||||||
|
"b":0,
|
||||||
|
"config":"loss 10% delay 15ms"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"a":2,
|
||||||
|
"b":1,
|
||||||
|
"config":"loss 5% delay 5ms"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"a":1,
|
||||||
|
"b":2,
|
||||||
|
"config":"loss 5% delay 5ms"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
|
@ -0,0 +1,19 @@
|
||||||
|
steps:
|
||||||
|
- command: "system-test/testnet-automation.sh"
|
||||||
|
label: "Colo - CPU Only - Partial Loss 3 Partitions"
|
||||||
|
env:
|
||||||
|
UPLOAD_RESULTS_TO_SLACK: "true"
|
||||||
|
CLOUD_PROVIDER: "colo"
|
||||||
|
TESTNET_TAG: "colo-perf-cpu-only"
|
||||||
|
NUMBER_OF_VALIDATOR_NODES: 4
|
||||||
|
ENABLE_GPU: "false"
|
||||||
|
NUMBER_OF_CLIENT_NODES: 2
|
||||||
|
CLIENT_OPTIONS: "bench-tps=2=--tx_count 15000 --thread-batch-sleep-ms 250"
|
||||||
|
ADDITIONAL_FLAGS: ""
|
||||||
|
APPLY_PARTITIONS: "true"
|
||||||
|
NETEM_CONFIG_FILE: "system-test/netem-configs/partial-loss-three-partitions"
|
||||||
|
PARTITION_ACTIVE_DURATION: 30
|
||||||
|
PARTITION_INACTIVE_DURATION: 30
|
||||||
|
PARTITION_ITERATION_COUNT: 5
|
||||||
|
agents:
|
||||||
|
- "queue=colo-deploy"
|
|
@ -0,0 +1,19 @@
|
||||||
|
steps:
|
||||||
|
- command: "system-test/testnet-automation.sh"
|
||||||
|
label: "Colo - CPU Only - Partial Loss 3 Partitions"
|
||||||
|
env:
|
||||||
|
UPLOAD_RESULTS_TO_SLACK: "true"
|
||||||
|
CLOUD_PROVIDER: "colo"
|
||||||
|
TESTNET_TAG: "colo-perf-cpu-only"
|
||||||
|
NUMBER_OF_VALIDATOR_NODES: 4
|
||||||
|
ENABLE_GPU: "false"
|
||||||
|
NUMBER_OF_CLIENT_NODES: 1
|
||||||
|
CLIENT_OPTIONS: "bench-tps=1=--tx_count 15000 --thread-batch-sleep-ms 250"
|
||||||
|
ADDITIONAL_FLAGS: ""
|
||||||
|
APPLY_PARTITIONS: "true"
|
||||||
|
NETEM_CONFIG_FILE: "system-test/netem-configs/partial-loss-three-partitions"
|
||||||
|
PARTITION_ACTIVE_DURATION: 60
|
||||||
|
PARTITION_INACTIVE_DURATION: 60
|
||||||
|
PARTITION_ITERATION_COUNT: 10
|
||||||
|
agents:
|
||||||
|
- "queue=colo-deploy"
|
|
@ -0,0 +1,19 @@
|
||||||
|
steps:
|
||||||
|
- command: "system-test/testnet-automation.sh"
|
||||||
|
label: "Colo - CPU Only - Partial Loss 3 Partitions"
|
||||||
|
env:
|
||||||
|
UPLOAD_RESULTS_TO_SLACK: "true"
|
||||||
|
CLOUD_PROVIDER: "colo"
|
||||||
|
TESTNET_TAG: "colo-perf-cpu-only"
|
||||||
|
NUMBER_OF_VALIDATOR_NODES: 4
|
||||||
|
ENABLE_GPU: "false"
|
||||||
|
NUMBER_OF_CLIENT_NODES: 1
|
||||||
|
CLIENT_OPTIONS: "bench-tps=1=--tx_count 15000 --thread-batch-sleep-ms 250"
|
||||||
|
ADDITIONAL_FLAGS: ""
|
||||||
|
APPLY_PARTITIONS: "true"
|
||||||
|
NETEM_CONFIG_FILE: "system-test/netem-configs/partial-loss-three-partitions"
|
||||||
|
PARTITION_ACTIVE_DURATION: 30
|
||||||
|
PARTITION_INACTIVE_DURATION: 30
|
||||||
|
PARTITION_ITERATION_COUNT: 2
|
||||||
|
agents:
|
||||||
|
- "queue=colo-deploy"
|
|
@ -0,0 +1,22 @@
|
||||||
|
steps:
|
||||||
|
- command: "system-test/testnet-automation.sh"
|
||||||
|
label: "GCE - CPU Only 5 Node - Partial Loss 3 Partitions"
|
||||||
|
env:
|
||||||
|
UPLOAD_RESULTS_TO_SLACK: "true"
|
||||||
|
CLOUD_PROVIDER: "gce"
|
||||||
|
TESTNET_TAG: "gce-perf-cpu-only"
|
||||||
|
NUMBER_OF_VALIDATOR_NODES: 5
|
||||||
|
ENABLE_GPU: "false"
|
||||||
|
VALIDATOR_NODE_MACHINE_TYPE: "--machine-type n1-standard-16"
|
||||||
|
NUMBER_OF_CLIENT_NODES: 2
|
||||||
|
CLIENT_OPTIONS: "bench-tps=2=--tx_count 15000 --thread-batch-sleep-ms 250"
|
||||||
|
TESTNET_ZONES: "us-west1-a"
|
||||||
|
USE_PUBLIC_IP_ADDRESSES: "true"
|
||||||
|
ADDITIONAL_FLAGS: ""
|
||||||
|
APPLY_PARTITIONS: "true"
|
||||||
|
NETEM_CONFIG_FILE: "system-test/netem-configs/partial-loss-three-partitions"
|
||||||
|
PARTITION_ACTIVE_DURATION: 30
|
||||||
|
PARTITION_INACTIVE_DURATION: 30
|
||||||
|
PARTITION_ITERATION_COUNT: 5
|
||||||
|
agents:
|
||||||
|
- "queue=testnet-deploy"
|
|
@ -5,7 +5,6 @@ steps:
|
||||||
UPLOAD_RESULTS_TO_SLACK: "true"
|
UPLOAD_RESULTS_TO_SLACK: "true"
|
||||||
CLOUD_PROVIDER: "ec2"
|
CLOUD_PROVIDER: "ec2"
|
||||||
TESTNET_TAG: "aws-perf-cpu-only"
|
TESTNET_TAG: "aws-perf-cpu-only"
|
||||||
RAMP_UP_TIME: 0
|
|
||||||
TEST_DURATION_SECONDS: 600
|
TEST_DURATION_SECONDS: 600
|
||||||
NUMBER_OF_VALIDATOR_NODES: 10
|
NUMBER_OF_VALIDATOR_NODES: 10
|
||||||
ENABLE_GPU: "false"
|
ENABLE_GPU: "false"
|
||||||
|
|
|
@ -5,7 +5,6 @@ steps:
|
||||||
UPLOAD_RESULTS_TO_SLACK: "true"
|
UPLOAD_RESULTS_TO_SLACK: "true"
|
||||||
CLOUD_PROVIDER: "ec2"
|
CLOUD_PROVIDER: "ec2"
|
||||||
TESTNET_TAG: "aws-perf-cpu-only"
|
TESTNET_TAG: "aws-perf-cpu-only"
|
||||||
RAMP_UP_TIME: 0
|
|
||||||
TEST_DURATION_SECONDS: 600
|
TEST_DURATION_SECONDS: 600
|
||||||
NUMBER_OF_VALIDATOR_NODES: 5
|
NUMBER_OF_VALIDATOR_NODES: 5
|
||||||
ENABLE_GPU: "false"
|
ENABLE_GPU: "false"
|
||||||
|
|
|
@ -5,7 +5,6 @@ steps:
|
||||||
UPLOAD_RESULTS_TO_SLACK: "true"
|
UPLOAD_RESULTS_TO_SLACK: "true"
|
||||||
CLOUD_PROVIDER: "azure"
|
CLOUD_PROVIDER: "azure"
|
||||||
TESTNET_TAG: "azure-perf-cpu-only"
|
TESTNET_TAG: "azure-perf-cpu-only"
|
||||||
RAMP_UP_TIME: 0
|
|
||||||
TEST_DURATION_SECONDS: 600
|
TEST_DURATION_SECONDS: 600
|
||||||
NUMBER_OF_VALIDATOR_NODES: 5
|
NUMBER_OF_VALIDATOR_NODES: 5
|
||||||
ENABLE_GPU: "false"
|
ENABLE_GPU: "false"
|
||||||
|
|
|
@ -6,7 +6,6 @@ steps:
|
||||||
CLOUD_PROVIDER: "colo"
|
CLOUD_PROVIDER: "colo"
|
||||||
TESTNET_TAG: "colo-perf-cpu-only"
|
TESTNET_TAG: "colo-perf-cpu-only"
|
||||||
ENABLE_GPU: "false"
|
ENABLE_GPU: "false"
|
||||||
RAMP_UP_TIME: 0
|
|
||||||
TEST_DURATION_SECONDS: 600
|
TEST_DURATION_SECONDS: 600
|
||||||
NUMBER_OF_VALIDATOR_NODES: 4
|
NUMBER_OF_VALIDATOR_NODES: 4
|
||||||
NUMBER_OF_CLIENT_NODES: 2
|
NUMBER_OF_CLIENT_NODES: 2
|
||||||
|
|
|
@ -6,7 +6,6 @@ steps:
|
||||||
CLOUD_PROVIDER: "colo"
|
CLOUD_PROVIDER: "colo"
|
||||||
TESTNET_TAG: "colo-perf-gpu-enabled"
|
TESTNET_TAG: "colo-perf-gpu-enabled"
|
||||||
ENABLE_GPU: "true"
|
ENABLE_GPU: "true"
|
||||||
RAMP_UP_TIME: 0
|
|
||||||
TEST_DURATION_SECONDS: 600
|
TEST_DURATION_SECONDS: 600
|
||||||
NUMBER_OF_VALIDATOR_NODES: 4
|
NUMBER_OF_VALIDATOR_NODES: 4
|
||||||
NUMBER_OF_CLIENT_NODES: 2
|
NUMBER_OF_CLIENT_NODES: 2
|
||||||
|
|
|
@ -6,7 +6,6 @@ steps:
|
||||||
CLOUD_PROVIDER: "colo"
|
CLOUD_PROVIDER: "colo"
|
||||||
TESTNET_TAG: "colo-perf-gpu-enabled"
|
TESTNET_TAG: "colo-perf-gpu-enabled"
|
||||||
ENABLE_GPU: "true"
|
ENABLE_GPU: "true"
|
||||||
RAMP_UP_TIME: 0
|
|
||||||
TEST_DURATION_SECONDS: 600
|
TEST_DURATION_SECONDS: 600
|
||||||
NUMBER_OF_VALIDATOR_NODES: 4
|
NUMBER_OF_VALIDATOR_NODES: 4
|
||||||
NUMBER_OF_CLIENT_NODES: 2
|
NUMBER_OF_CLIENT_NODES: 2
|
||||||
|
|
|
@ -5,7 +5,6 @@ steps:
|
||||||
UPLOAD_RESULTS_TO_SLACK: "true"
|
UPLOAD_RESULTS_TO_SLACK: "true"
|
||||||
CLOUD_PROVIDER: "gce"
|
CLOUD_PROVIDER: "gce"
|
||||||
TESTNET_TAG: "gce-perf-cpu-only"
|
TESTNET_TAG: "gce-perf-cpu-only"
|
||||||
RAMP_UP_TIME: 0
|
|
||||||
TEST_DURATION_SECONDS: 600
|
TEST_DURATION_SECONDS: 600
|
||||||
NUMBER_OF_VALIDATOR_NODES: 10
|
NUMBER_OF_VALIDATOR_NODES: 10
|
||||||
ENABLE_GPU: "false"
|
ENABLE_GPU: "false"
|
||||||
|
|
|
@ -5,7 +5,6 @@ steps:
|
||||||
UPLOAD_RESULTS_TO_SLACK: "true"
|
UPLOAD_RESULTS_TO_SLACK: "true"
|
||||||
CLOUD_PROVIDER: "gce"
|
CLOUD_PROVIDER: "gce"
|
||||||
TESTNET_TAG: "gce-perf-cpu-only"
|
TESTNET_TAG: "gce-perf-cpu-only"
|
||||||
RAMP_UP_TIME: 0
|
|
||||||
TEST_DURATION_SECONDS: 600
|
TEST_DURATION_SECONDS: 600
|
||||||
NUMBER_OF_VALIDATOR_NODES: 5
|
NUMBER_OF_VALIDATOR_NODES: 5
|
||||||
ENABLE_GPU: "false"
|
ENABLE_GPU: "false"
|
||||||
|
@ -14,6 +13,6 @@ steps:
|
||||||
CLIENT_OPTIONS: "bench-tps=2=--tx_count 20000 --thread-batch-sleep-ms 250"
|
CLIENT_OPTIONS: "bench-tps=2=--tx_count 20000 --thread-batch-sleep-ms 250"
|
||||||
TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
|
TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
|
||||||
USE_PUBLIC_IP_ADDRESSES: "true"
|
USE_PUBLIC_IP_ADDRESSES: "true"
|
||||||
ADDITIONAL_FLAGS: "--dedicated"
|
ADDITIONAL_FLAGS: ""
|
||||||
agents:
|
agents:
|
||||||
- "queue=gce-deploy"
|
- "queue=gce-deploy"
|
||||||
|
|
|
@ -5,7 +5,6 @@ steps:
|
||||||
UPLOAD_RESULTS_TO_SLACK: "true"
|
UPLOAD_RESULTS_TO_SLACK: "true"
|
||||||
CLOUD_PROVIDER: "gce"
|
CLOUD_PROVIDER: "gce"
|
||||||
TESTNET_TAG: "gce-perf-gpu-enabled"
|
TESTNET_TAG: "gce-perf-gpu-enabled"
|
||||||
RAMP_UP_TIME: 0
|
|
||||||
TEST_DURATION_SECONDS: 600
|
TEST_DURATION_SECONDS: 600
|
||||||
NUMBER_OF_VALIDATOR_NODES: 10
|
NUMBER_OF_VALIDATOR_NODES: 10
|
||||||
ENABLE_GPU: "true"
|
ENABLE_GPU: "true"
|
||||||
|
|
|
@ -5,7 +5,6 @@ steps:
|
||||||
UPLOAD_RESULTS_TO_SLACK: "true"
|
UPLOAD_RESULTS_TO_SLACK: "true"
|
||||||
CLOUD_PROVIDER: "gce"
|
CLOUD_PROVIDER: "gce"
|
||||||
TESTNET_TAG: "gce-perf-gpu-enabled"
|
TESTNET_TAG: "gce-perf-gpu-enabled"
|
||||||
RAMP_UP_TIME: 0
|
|
||||||
TEST_DURATION_SECONDS: 600
|
TEST_DURATION_SECONDS: 600
|
||||||
NUMBER_OF_VALIDATOR_NODES: 100
|
NUMBER_OF_VALIDATOR_NODES: 100
|
||||||
ENABLE_GPU: "true"
|
ENABLE_GPU: "true"
|
||||||
|
|
|
@ -5,7 +5,6 @@ steps:
|
||||||
UPLOAD_RESULTS_TO_SLACK: "true"
|
UPLOAD_RESULTS_TO_SLACK: "true"
|
||||||
CLOUD_PROVIDER: "gce"
|
CLOUD_PROVIDER: "gce"
|
||||||
TESTNET_TAG: "gce-perf-gpu-enabled"
|
TESTNET_TAG: "gce-perf-gpu-enabled"
|
||||||
RAMP_UP_TIME: 0
|
|
||||||
TEST_DURATION_SECONDS: 600
|
TEST_DURATION_SECONDS: 600
|
||||||
NUMBER_OF_VALIDATOR_NODES: 25
|
NUMBER_OF_VALIDATOR_NODES: 25
|
||||||
ENABLE_GPU: "true"
|
ENABLE_GPU: "true"
|
||||||
|
|
|
@ -5,7 +5,6 @@ steps:
|
||||||
UPLOAD_RESULTS_TO_SLACK: "true"
|
UPLOAD_RESULTS_TO_SLACK: "true"
|
||||||
CLOUD_PROVIDER: "gce"
|
CLOUD_PROVIDER: "gce"
|
||||||
TESTNET_TAG: "gce-perf-gpu-enabled"
|
TESTNET_TAG: "gce-perf-gpu-enabled"
|
||||||
RAMP_UP_TIME: 0
|
|
||||||
TEST_DURATION_SECONDS: 600
|
TEST_DURATION_SECONDS: 600
|
||||||
NUMBER_OF_VALIDATOR_NODES: 5
|
NUMBER_OF_VALIDATOR_NODES: 5
|
||||||
ENABLE_GPU: "true"
|
ENABLE_GPU: "true"
|
||||||
|
|
|
@ -5,7 +5,6 @@ steps:
|
||||||
UPLOAD_RESULTS_TO_SLACK: "true"
|
UPLOAD_RESULTS_TO_SLACK: "true"
|
||||||
CLOUD_PROVIDER: "gce"
|
CLOUD_PROVIDER: "gce"
|
||||||
TESTNET_TAG: "gce-perf-gpu-enabled"
|
TESTNET_TAG: "gce-perf-gpu-enabled"
|
||||||
RAMP_UP_TIME: 0
|
|
||||||
TEST_DURATION_SECONDS: 600
|
TEST_DURATION_SECONDS: 600
|
||||||
NUMBER_OF_VALIDATOR_NODES: 50
|
NUMBER_OF_VALIDATOR_NODES: 50
|
||||||
ENABLE_GPU: "true"
|
ENABLE_GPU: "true"
|
||||||
|
|
|
@ -5,7 +5,6 @@ steps:
|
||||||
UPLOAD_RESULTS_TO_SLACK: "true"
|
UPLOAD_RESULTS_TO_SLACK: "true"
|
||||||
CLOUD_PROVIDER: "gce"
|
CLOUD_PROVIDER: "gce"
|
||||||
TESTNET_TAG: "gce-stability"
|
TESTNET_TAG: "gce-stability"
|
||||||
RAMP_UP_TIME: 0
|
|
||||||
TEST_DURATION_SECONDS: 28800
|
TEST_DURATION_SECONDS: 28800
|
||||||
NUMBER_OF_VALIDATOR_NODES: 5
|
NUMBER_OF_VALIDATOR_NODES: 5
|
||||||
ENABLE_GPU: "false"
|
ENABLE_GPU: "false"
|
||||||
|
|
|
@ -82,7 +82,7 @@ Failure occured when running the following command:
|
||||||
$(eval echo "$@")"
|
$(eval echo "$@")"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
FINISH_UNIX_MSECS="$(($(date +%s%N)/1000000))"
|
TESTNET_FINISH_UNIX_MSECS="$(($(date +%s%N)/1000000))"
|
||||||
if [[ "$UPLOAD_RESULTS_TO_SLACK" = "true" ]]; then
|
if [[ "$UPLOAD_RESULTS_TO_SLACK" = "true" ]]; then
|
||||||
upload_results_to_slack
|
upload_results_to_slack
|
||||||
fi
|
fi
|
||||||
|
@ -200,8 +200,29 @@ function launchTestnet() {
|
||||||
execution_step "Waiting for bootstrap validator's stake percentage to fall below $BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD %"
|
execution_step "Waiting for bootstrap validator's stake percentage to fall below $BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD %"
|
||||||
wait_for_bootstrap_validator_stake_drop "$BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD"
|
wait_for_bootstrap_validator_stake_drop "$BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD"
|
||||||
|
|
||||||
|
if [[ -n $TEST_DURATION_SECONDS ]]; then
|
||||||
execution_step "Wait ${TEST_DURATION_SECONDS} seconds to complete test"
|
execution_step "Wait ${TEST_DURATION_SECONDS} seconds to complete test"
|
||||||
sleep "$TEST_DURATION_SECONDS"
|
sleep "$TEST_DURATION_SECONDS"
|
||||||
|
elif [[ "$APPLY_PARTITIONS" = "true" ]]; then
|
||||||
|
STATS_START_SECONDS=$SECONDS
|
||||||
|
execution_step "Wait $PARTITION_INACTIVE_DURATION before beginning to apply partitions"
|
||||||
|
sleep "$PARTITION_INACTIVE_DURATION"
|
||||||
|
for (( i=1; i<=PARTITION_ITERATION_COUNT; i++ )); do
|
||||||
|
execution_step "Partition Iteration $i of $PARTITION_ITERATION_COUNT"
|
||||||
|
execution_step "Applying netem config $NETEM_CONFIG_FILE for $PARTITION_ACTIVE_DURATION seconds"
|
||||||
|
net/net.sh netem --config-file "$NETEM_CONFIG_FILE"
|
||||||
|
sleep "$PARTITION_ACTIVE_DURATION"
|
||||||
|
|
||||||
|
execution_step "Resolving partitions for $PARTITION_INACTIVE_DURATION seconds"
|
||||||
|
net/net.sh netem --config-file "$NETEM_CONFIG_FILE" --netem-cmd cleanup
|
||||||
|
sleep "$PARTITION_INACTIVE_DURATION"
|
||||||
|
done
|
||||||
|
STATS_FINISH_SECONDS=$SECONDS
|
||||||
|
TEST_DURATION_SECONDS=$((STATS_FINISH_SECONDS - STATS_START_SECONDS))
|
||||||
|
else
|
||||||
|
# We should never get here
|
||||||
|
echo Test duration and partition config not defined
|
||||||
|
fi
|
||||||
|
|
||||||
execution_step "Collect statistics about run"
|
execution_step "Collect statistics about run"
|
||||||
declare q_mean_tps='
|
declare q_mean_tps='
|
||||||
|
@ -239,9 +260,16 @@ function launchTestnet() {
|
||||||
FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
|
FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
|
||||||
WHERE time > now() - '"$TEST_DURATION_SECONDS"'s'
|
WHERE time > now() - '"$TEST_DURATION_SECONDS"'s'
|
||||||
|
|
||||||
|
declare q_max_tower_distance_observed='
|
||||||
|
SELECT MAX("tower_distance") as "max_tower_distance" FROM (
|
||||||
|
SELECT last("slot") - last("root") as "tower_distance"
|
||||||
|
FROM "'$TESTNET_TAG'"."autogen"."tower-observed"
|
||||||
|
WHERE time > now() - '"$TEST_DURATION_SECONDS"'s
|
||||||
|
GROUP BY time(1s), host_id)'
|
||||||
|
|
||||||
curl -G "${INFLUX_HOST}/query?u=ro&p=topsecret" \
|
curl -G "${INFLUX_HOST}/query?u=ro&p=topsecret" \
|
||||||
--data-urlencode "db=${TESTNET_TAG}" \
|
--data-urlencode "db=${TESTNET_TAG}" \
|
||||||
--data-urlencode "q=$q_mean_tps;$q_max_tps;$q_mean_confirmation;$q_max_confirmation;$q_99th_confirmation" |
|
--data-urlencode "q=$q_mean_tps;$q_max_tps;$q_mean_confirmation;$q_max_confirmation;$q_99th_confirmation;$q_max_tower_distance_observed" |
|
||||||
python system-test/testnet-automation-json-parser.py >>"$RESULT_FILE"
|
python system-test/testnet-automation-json-parser.py >>"$RESULT_FILE"
|
||||||
|
|
||||||
execution_step "Writing test results to ${RESULT_FILE}"
|
execution_step "Writing test results to ${RESULT_FILE}"
|
||||||
|
@ -259,11 +287,6 @@ cd "$(dirname "$0")/.."
|
||||||
[[ -n $INFLUX_HOST ]] || INFLUX_HOST=https://metrics.solana.com:8086
|
[[ -n $INFLUX_HOST ]] || INFLUX_HOST=https://metrics.solana.com:8086
|
||||||
[[ -n $BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD ]] || BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD=66
|
[[ -n $BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD ]] || BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD=66
|
||||||
|
|
||||||
if [[ -z $TEST_DURATION_SECONDS ]] ; then
|
|
||||||
echo TEST_DURATION_SECONDS not defined
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ -z $NUMBER_OF_VALIDATOR_NODES ]]; then
|
if [[ -z $NUMBER_OF_VALIDATOR_NODES ]]; then
|
||||||
echo NUMBER_OF_VALIDATOR_NODES not defined
|
echo NUMBER_OF_VALIDATOR_NODES not defined
|
||||||
exit 1
|
exit 1
|
||||||
|
@ -309,6 +332,19 @@ fi
|
||||||
|
|
||||||
: "${CLIENT_DELAY_START:=0}"
|
: "${CLIENT_DELAY_START:=0}"
|
||||||
|
|
||||||
|
if [[ -z $APPLY_PARTITIONS ]]; then
|
||||||
|
APPLY_PARTITIONS=false
|
||||||
|
fi
|
||||||
|
if [[ "$APPLY_PARTITIONS" = "true" ]]; then
|
||||||
|
if [[ -n $TEST_DURATION_SECONDS ]]; then
|
||||||
|
echo Cannot accept TEST_DURATION_SECONDS and a parition looping config
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
elif [[ -z $TEST_DURATION_SECONDS ]]; then
|
||||||
|
echo TEST_DURATION_SECONDS not defined
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ -z $CHANNEL ]]; then
|
if [[ -z $CHANNEL ]]; then
|
||||||
execution_step "Downloading tar from build artifacts"
|
execution_step "Downloading tar from build artifacts"
|
||||||
buildkite-agent artifact download "solana-release*.tar.bz2" .
|
buildkite-agent artifact download "solana-release*.tar.bz2" .
|
||||||
|
@ -337,7 +373,13 @@ TEST_PARAMS_TO_DISPLAY=(CLOUD_PROVIDER \
|
||||||
TEST_DURATION_SECONDS \
|
TEST_DURATION_SECONDS \
|
||||||
USE_PUBLIC_IP_ADDRESSES \
|
USE_PUBLIC_IP_ADDRESSES \
|
||||||
ALLOW_BOOT_FAILURES \
|
ALLOW_BOOT_FAILURES \
|
||||||
ADDITIONAL_FLAGS)
|
ADDITIONAL_FLAGS \
|
||||||
|
APPLY_PARTITIONS \
|
||||||
|
NETEM_CONFIG_FILE \
|
||||||
|
PARTITION_ACTIVE_DURATION \
|
||||||
|
PARTITION_INACTIVE_DURATION \
|
||||||
|
PARTITION_ITERATION_COUNT \
|
||||||
|
)
|
||||||
|
|
||||||
TEST_CONFIGURATION=
|
TEST_CONFIGURATION=
|
||||||
for i in "${TEST_PARAMS_TO_DISPLAY[@]}"; do
|
for i in "${TEST_PARAMS_TO_DISPLAY[@]}"; do
|
||||||
|
@ -346,6 +388,6 @@ for i in "${TEST_PARAMS_TO_DISPLAY[@]}" ; do
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
START_UNIX_MSECS="$(($(date +%s%N)/1000000))"
|
TESTNET_START_UNIX_MSECS="$(($(date +%s%N)/1000000))"
|
||||||
|
|
||||||
launchTestnet
|
launchTestnet
|
||||||
|
|
|
@ -19,7 +19,7 @@ upload_results_to_slack() {
|
||||||
BUILDKITE_BUILD_URL="https://buildkite.com/solana-labs/"
|
BUILDKITE_BUILD_URL="https://buildkite.com/solana-labs/"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
GRAFANA_URL="https://metrics.solana.com:3000/d/testnet-${CHANNEL:-edge}/testnet-monitor-${CHANNEL:-edge}?var-testnet=${TESTNET_TAG:-testnet-automation}&from=${START_UNIX_MSECS:-0}&to=${FINISH_UNIX_MSECS:-0}"
|
GRAFANA_URL="https://metrics.solana.com:3000/d/testnet-${CHANNEL:-edge}/testnet-monitor-${CHANNEL:-edge}?var-testnet=${TESTNET_TAG:-testnet-automation}&from=${TESTNET_START_UNIX_MSECS:-0}&to=${TESTNET_FINISH_UNIX_MSECS:-0}"
|
||||||
|
|
||||||
[[ -n $RESULT_DETAILS ]] || RESULT_DETAILS="Undefined"
|
[[ -n $RESULT_DETAILS ]] || RESULT_DETAILS="Undefined"
|
||||||
[[ -n $TEST_CONFIGURATION ]] || TEST_CONFIGURATION="Undefined"
|
[[ -n $TEST_CONFIGURATION ]] || TEST_CONFIGURATION="Undefined"
|
||||||
|
|
Loading…
Reference in New Issue