blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Automated test framework can run scripts on launched clusters. Add offline stake operations test case and script. (#8510) 

automerge
This commit is contained in:
Dan Albert 2020-03-18 14:57:19 -07:00 committed by GitHub
parent 21b287ef0b
commit 90c9462dd4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 478 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
system-test/automation_utils.sh
View File

@ -62,6 +62,10 @@ function analyze_packet_loss {
function wait_for_bootstrap_validator_stake_drop {
max_stake="$1"
if [[ $max_stake -eq 100 ]]; then
return
fi
source "${REPO_ROOT}"/net/common.sh
loadConfigFile
@ -85,6 +89,68 @@ function get_slot {
ssh "${sshOptions[@]}" "${validatorIpList[0]}" '$HOME/.cargo/bin/solana slot'
}
function get_bootstrap_validator_ip_address {
source "${REPO_ROOT}"/net/common.sh
loadConfigFile
echo "${validatorIpList[0]}"
}
function collect_performance_statistics {
execution_step "Collect performance statistics about run"
declare q_mean_tps='
SELECT ROUND(MEAN("median_sum")) as "mean_tps" FROM (
SELECT MEDIAN(sum_count) AS "median_sum" FROM (
SELECT SUM("count") AS "sum_count"
FROM "'$TESTNET_TAG'"."autogen"."bank-process_transactions"
WHERE time > now() - '"$TEST_DURATION_SECONDS"'s AND count > 0
GROUP BY time(1s), host_id)
GROUP BY time(1s)
)'
declare q_max_tps='
SELECT MAX("median_sum") as "max_tps" FROM (
SELECT MEDIAN(sum_count) AS "median_sum" FROM (
SELECT SUM("count") AS "sum_count"
FROM "'$TESTNET_TAG'"."autogen"."bank-process_transactions"
WHERE time > now() - '"$TEST_DURATION_SECONDS"'s AND count > 0
GROUP BY time(1s), host_id)
GROUP BY time(1s)
)'
declare q_mean_confirmation='
SELECT round(mean("duration_ms")) as "mean_confirmation_ms"
FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
WHERE time > now() - '"$TEST_DURATION_SECONDS"'s'
declare q_max_confirmation='
SELECT round(max("duration_ms")) as "max_confirmation_ms"
FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
WHERE time > now() - '"$TEST_DURATION_SECONDS"'s'
declare q_99th_confirmation='
SELECT round(percentile("duration_ms", 99)) as "99th_percentile_confirmation_ms"
FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
WHERE time > now() - '"$TEST_DURATION_SECONDS"'s'
declare q_max_tower_distance_observed='
SELECT MAX("tower_distance") as "max_tower_distance" FROM (
SELECT last("slot") - last("root") as "tower_distance"
FROM "'$TESTNET_TAG'"."autogen"."tower-observed"
WHERE time > now() - '"$TEST_DURATION_SECONDS"'s
GROUP BY time(1s), host_id)'
declare q_last_tower_distance_observed='
SELECT MEAN("tower_distance") as "last_tower_distance" FROM (
SELECT last("slot") - last("root") as "tower_distance"
FROM "'$TESTNET_TAG'"."autogen"."tower-observed"
GROUP BY host_id)'
curl -G "${INFLUX_HOST}/query?u=ro&p=topsecret" \
--data-urlencode "db=${TESTNET_TAG}" \
--data-urlencode "q=$q_mean_tps;$q_max_tps;$q_mean_confirmation;$q_max_confirmation;$q_99th_confirmation;$q_max_tower_distance_observed;$q_last_tower_distance_observed" |
python "${REPO_ROOT}"/system-test/testnet-automation-json-parser.py >>"$RESULT_FILE"
}
function upload_results_to_slack() {
echo --- Uploading results to Slack Performance Results App
@ -166,7 +232,7 @@ function upload_results_to_slack() {
{
"type": "divider"
},
{
{
"type": "section",
"text": {
"type": "mrkdwn",

1
system-test/deprecated-testcases/colo-cpu-only-perf.yml
View File

@ -11,5 +11,6 @@ steps:
NUMBER_OF_CLIENT_NODES: 2
CLIENT_OPTIONS: "bench-tps=2=--tx_count 20000 --thread-batch-sleep-ms 250"
ADDITIONAL_FLAGS: ""
TEST_TYPE: "fixed_duration"
agents:
- "queue=colo-deploy"

1
system-test/deprecated-testcases/colo-gpu-perf-high-txcount.yml
View File

@ -11,5 +11,6 @@ steps:
NUMBER_OF_CLIENT_NODES: 2
CLIENT_OPTIONS: "bench-tps=2=--tx_count 30000 --thread-batch-sleep-ms 250"
ADDITIONAL_FLAGS: ""
TEST_TYPE: "fixed_duration"
agents:
- "queue=colo-deploy"

1
system-test/deprecated-testcases/colo-gpu-perf.yml
View File

@ -11,5 +11,6 @@ steps:
NUMBER_OF_CLIENT_NODES: 2
CLIENT_OPTIONS: "bench-tps=2=--tx_count 20000 --thread-batch-sleep-ms 250"
ADDITIONAL_FLAGS: ""
TEST_TYPE: "fixed_duration"
agents:
- "queue=colo-deploy"

1
system-test/deprecated-testcases/gce-gpu-perf-100-node.yml
View File

@ -15,5 +15,6 @@ steps:
ALLOW_BOOT_FAILURES: "true"
USE_PUBLIC_IP_ADDRESSES: "true"
ADDITIONAL_FLAGS: "--dedicated"
TEST_TYPE: "fixed_duration"
agents:
- "queue=gce-deploy"

1
system-test/partition-testcases/colo-3-partition.yml
View File

@ -15,5 +15,6 @@ steps:
PARTITION_ACTIVE_DURATION: 30
PARTITION_INACTIVE_DURATION: 30
PARTITION_ITERATION_COUNT: 5
TEST_TYPE: "partition"
agents:
- "queue=colo-deploy"

1
system-test/partition-testcases/colo-partition-long-sanity-test.yml
View File

@ -15,5 +15,6 @@ steps:
PARTITION_ACTIVE_DURATION: 60
PARTITION_INACTIVE_DURATION: 60
PARTITION_ITERATION_COUNT: 10
TEST_TYPE: "partition"
agents:
- "queue=colo-deploy"

1
system-test/partition-testcases/colo-partition-once-then-stabilize.yml
View File

@ -15,5 +15,6 @@ steps:
PARTITION_ACTIVE_DURATION: 60
PARTITION_INACTIVE_DURATION: 300
PARTITION_ITERATION_COUNT: 1
TEST_TYPE: "partition"
agents:
- "queue=colo-deploy"

1
system-test/partition-testcases/gce-5-node-3-partition.yml
View File

@ -18,5 +18,6 @@ steps:
PARTITION_ACTIVE_DURATION: 30
PARTITION_INACTIVE_DURATION: 30
PARTITION_ITERATION_COUNT: 5
TEST_TYPE: "partition"
agents:
- "queue=gce-deploy"

1
system-test/partition-testcases/gce-5-node-single-region-2-partitions.yml
View File

@ -18,5 +18,6 @@ steps:
PARTITION_ACTIVE_DURATION: 60
PARTITION_INACTIVE_DURATION: 300
PARTITION_ITERATION_COUNT: 5
TEST_TYPE: "partition"
agents:
- "queue=gce-deploy"

1
system-test/partition-testcases/gce-partition-once-then-stabilize.yml
View File

@ -18,5 +18,6 @@ steps:
PARTITION_ACTIVE_DURATION: 60
PARTITION_INACTIVE_DURATION: 300
PARTITION_ITERATION_COUNT: 1
TEST_TYPE: "partition"
agents:
- "queue=gce-deploy"

1
system-test/performance-testcases/aws-cpu-only-perf-10-node.yml
View File

@ -15,5 +15,6 @@ steps:
TESTNET_ZONES: "us-west-1a,us-west-1c,us-east-1a,eu-west-1a"
USE_PUBLIC_IP_ADDRESSES: "true"
ADDITIONAL_FLAGS: ""
TEST_TYPE: "fixed_duration"
agents:
- "queue=aws-deploy"

1
system-test/performance-testcases/aws-cpu-only-perf-5-node.yml
View File

@ -15,5 +15,6 @@ steps:
TESTNET_ZONES: "us-west-1a,us-west-1c,us-east-1a,eu-west-1a"
USE_PUBLIC_IP_ADDRESSES: "true"
ADDITIONAL_FLAGS: ""
TEST_TYPE: "fixed_duration"
agents:
- "queue=aws-deploy"

1
system-test/performance-testcases/azure-cpu-only-perf-5-node.yml
View File

@ -14,5 +14,6 @@ steps:
TESTNET_ZONES: "westus"
USE_PUBLIC_IP_ADDRESSES: "true"
ADDITIONAL_FLAGS: ""
TEST_TYPE: "fixed_duration"
agents:
- "queue=azure-deploy"

1
system-test/performance-testcases/colo-cpu-only-perf-4-val-1-client.yml
View File

@ -11,5 +11,6 @@ steps:
NUMBER_OF_CLIENT_NODES: 1
CLIENT_OPTIONS: "bench-tps=1=--tx_count 40000 --thread-batch-sleep-ms 250"
ADDITIONAL_FLAGS: ""
TEST_TYPE: "fixed_duration"
agents:
- "queue=colo-deploy"

1
system-test/performance-testcases/colo-gpu-perf-4-val-1-client.yml
View File

@ -11,5 +11,6 @@ steps:
NUMBER_OF_CLIENT_NODES: 1
CLIENT_OPTIONS: "bench-tps=1=--tx_count 40000 --thread-batch-sleep-ms 250"
ADDITIONAL_FLAGS: ""
TEST_TYPE: "fixed_duration"
agents:
- "queue=colo-deploy"

1
system-test/performance-testcases/colo-gpu-perf-high-txcount-4-val-1-client.yml
View File

@ -11,5 +11,6 @@ steps:
NUMBER_OF_CLIENT_NODES: 1
CLIENT_OPTIONS: "bench-tps=1=--tx_count 60000 --thread-batch-sleep-ms 250"
ADDITIONAL_FLAGS: ""
TEST_TYPE: "fixed_duration"
agents:
- "queue=colo-deploy"

1
system-test/performance-testcases/gce-cpu-only-perf-10-node.yml
View File

@ -14,5 +14,6 @@ steps:
TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
USE_PUBLIC_IP_ADDRESSES: "true"
ADDITIONAL_FLAGS: "--dedicated"
TEST_TYPE: "fixed_duration"
agents:
- "queue=gce-deploy"

1
system-test/performance-testcases/gce-cpu-only-perf-5-node.yml
View File

@ -14,5 +14,6 @@ steps:
TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
USE_PUBLIC_IP_ADDRESSES: "true"
ADDITIONAL_FLAGS: "--dedicated"
TEST_TYPE: "fixed_duration"
agents:
- "queue=gce-deploy"

1
system-test/performance-testcases/gce-gpu-perf-10-node.yml
View File

@ -14,5 +14,6 @@ steps:
TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
USE_PUBLIC_IP_ADDRESSES: "true"
ADDITIONAL_FLAGS: "--dedicated"
TEST_TYPE: "fixed_duration"
agents:
- "queue=gce-deploy"

1
system-test/performance-testcases/gce-gpu-perf-25-node.yml
View File

@ -14,5 +14,6 @@ steps:
TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
USE_PUBLIC_IP_ADDRESSES: "true"
ADDITIONAL_FLAGS: "--dedicated"
TEST_TYPE: "fixed_duration"
agents:
- "queue=gce-deploy"

1
system-test/performance-testcases/gce-gpu-perf-5-node.yml
View File

@ -14,5 +14,6 @@ steps:
TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
USE_PUBLIC_IP_ADDRESSES: "true"
ADDITIONAL_FLAGS: "--dedicated"
TEST_TYPE: "fixed_duration"
agents:
- "queue=gce-deploy"

1
system-test/performance-testcases/gce-gpu-perf-50-node.yml
View File

@ -15,5 +15,6 @@ steps:
ALLOW_BOOT_FAILURES: "true"
USE_PUBLIC_IP_ADDRESSES: "true"
ADDITIONAL_FLAGS: "--dedicated"
TEST_TYPE: "fixed_duration"
agents:
- "queue=gce-deploy"

5
system-test/sanity-testcases/colo-cpu-only-quick-sanity-test.yml
View File

@ -6,11 +6,12 @@ steps:
CLOUD_PROVIDER: "colo"
TESTNET_TAG: "colo-perf-cpu-only"
ENABLE_GPU: "false"
TEST_DURATION_SECONDS: 30
TEST_DURATION_SECONDS: 60
NUMBER_OF_VALIDATOR_NODES: 1
NUMBER_OF_CLIENT_NODES: 1
CLIENT_OPTIONS: "bench-tps=1=--tx_count 40000 --thread-batch-sleep-ms 250"
ADDITIONAL_FLAGS: ""
BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100
BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 99
TEST_TYPE: "fixed_duration"
agents:
- "queue=colo-deploy"

5
system-test/sanity-testcases/colo-partition-sanity-test.yml
View File

@ -5,7 +5,7 @@ steps:
UPLOAD_RESULTS_TO_SLACK: "true"
CLOUD_PROVIDER: "colo"
TESTNET_TAG: "colo-perf-cpu-only"
NUMBER_OF_VALIDATOR_NODES: 4
NUMBER_OF_VALIDATOR_NODES: 3
ENABLE_GPU: "false"
NUMBER_OF_CLIENT_NODES: 1
CLIENT_OPTIONS: "bench-tps=1=--tx_count 15000 --thread-batch-sleep-ms 250"
@ -15,6 +15,7 @@ steps:
PARTITION_ACTIVE_DURATION: 30
PARTITION_INACTIVE_DURATION: 30
PARTITION_ITERATION_COUNT: 2
BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100
BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 66
TEST_TYPE: "partition"
agents:
- "queue=colo-deploy"

1
system-test/stability-testcases/colo-long-duration-cpu-only-perf.yml
View File

@ -11,5 +11,6 @@ steps:
NUMBER_OF_CLIENT_NODES: 1
CLIENT_OPTIONS: "bench-tps=1=--tx_count 30000 --thread-batch-sleep-ms 250"
ADDITIONAL_FLAGS: ""
TEST_TYPE: "fixed_duration"
agents:
- "queue=colo-deploy"

1
system-test/stability-testcases/colo-long-duration-gpu-perf.yml
View File

@ -11,5 +11,6 @@ steps:
NUMBER_OF_CLIENT_NODES: 1
CLIENT_OPTIONS: "bench-tps=1=--tx_count 30000 --thread-batch-sleep-ms 250"
ADDITIONAL_FLAGS: ""
TEST_TYPE: "fixed_duration"
agents:
- "queue=colo-deploy"

1
system-test/stability-testcases/gce-perf-stability-5-node-single-region.yml
View File

@ -14,5 +14,6 @@ steps:
TESTNET_ZONES: "us-west1-a"
USE_PUBLIC_IP_ADDRESSES: "false"
ADDITIONAL_FLAGS: "--dedicated"
TEST_TYPE: "fixed_duration"
agents:
- "queue=gce-deploy"

1
system-test/stability-testcases/gce-stability-5-node.yml
View File

@ -13,5 +13,6 @@ steps:
TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
USE_PUBLIC_IP_ADDRESSES: "true"
ADDITIONAL_FLAGS: "--dedicated"
TEST_TYPE: "fixed_duration"
agents:
- "queue=stability-deploy"

17
system-test/stake-operations-testcases/offline_stake_colo.yml Executable file
View File

@ -0,0 +1,17 @@
steps:
- command: "system-test/testnet-automation.sh"
label: "Running Offline Stake Operations Tests on Colo"
env:
UPLOAD_RESULTS_TO_SLACK: "true"
CLOUD_PROVIDER: "colo"
ENABLE_GPU: "false"
TEST_DURATION_SECONDS: 30
NUMBER_OF_VALIDATOR_NODES: 1
NUMBER_OF_CLIENT_NODES: 0
ADDITIONAL_FLAGS: ""
BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100
SKIP_PERF_RESULTS: "true"
TEST_TYPE: "script"
CUSTOM_SCRIPT: "system-test/stake-operations-testcases/stake_test_automation.sh"
agents:
- "queue=colo-deploy"

18
system-test/stake-operations-testcases/offline_stake_gce.yml Executable file
View File

@ -0,0 +1,18 @@
steps:
- command: "system-test/testnet-automation.sh"
label: "Running Offline Stake Operations Tests on GCE"
env:
UPLOAD_RESULTS_TO_SLACK: "true"
CLOUD_PROVIDER: "gce"
ENABLE_GPU: "false"
TEST_DURATION_SECONDS: 30
NUMBER_OF_VALIDATOR_NODES: 1
VALIDATOR_NODE_MACHINE_TYPE: "--machine-type n1-standard-16"
NUMBER_OF_CLIENT_NODES: 0
ADDITIONAL_FLAGS: "--dedicated"
BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100
SKIP_PERF_RESULTS: "true"
TEST_TYPE: "script"
CUSTOM_SCRIPT: "system-test/stake-operations-testcases/stake_test_automation.sh"
agents:
- "queue=gce-deploy"

258
system-test/stake-operations-testcases/offline_stake_operations.sh Executable file
View File

@ -0,0 +1,258 @@
#!/usr/bin/env bash
# shellcheck disable=SC2086
# shellcheck disable=SC2068
# shellcheck disable=SC2206
# shellcheck disable=SC2162
# shellcheck disable=SC2178
# shellcheck disable=SC2145
# shellcheck disable=SC1090
# shellcheck disable=SC1091
source "$(dirname "$0")"/../automation_utils.sh
set -e
function get_signers_string() {
sign_only_output="$1"
signers=()
while read LINE; do
signers+=( --signer $LINE )
done <<<"$(sed -Ee $'s/^ ([a-km-zA-HJ-NP-Z1-9]{32,44}=[a-km-zA-HJ-NP-Z1-9]{64,88})$/\\1/\nt\nd' <<<"$sign_only_output")"
echo "${signers[@]}"
}
if [[ -n "$1" ]]; then
url="$1"
fi
if [[ -z "$url" ]]; then
echo Provide complete URL, ex: "$0" http://devnet.solana.com:8899
exit 1
fi
solana config set --url $url
# Create a dummy keypair file with no balance for operations that require a "client keypair file" to exist even if they don't touch it
dummy_keypair=dummy.json
solana-keygen new -o "$dummy_keypair" --no-passphrase --force --silent
solana config set --keypair $dummy_keypair
### Offline stake account creation
# The nonce account and the system account funding its creation are online
online_nonce_account_keypair=nonce_keypair.json
online_system_account_keypair=online_system_account_keypair.json
solana-keygen new -o "$online_system_account_keypair" --no-passphrase --force --silent
solana-keygen new -o "$online_nonce_account_keypair" --no-passphrase --force --silent
online_system_account_pubkey="$(solana-keygen pubkey $online_system_account_keypair)"
nonce_account_pubkey="$(solana-keygen pubkey $online_nonce_account_keypair)"
# System account funding the stake account is offline, and the auth staker and withdrawer keypairs are offline
offline_system_account_keypair=offline_system_account_keypair.json
offline_staker_keypair=offline_staker_keypair.json
offline_withdrawer_keypair=offline_withdrawer_keypair.json
offline_custodian_keypair=offline_custodian_keypair.json
solana-keygen new -o "$offline_system_account_keypair" --no-passphrase --force --silent
solana-keygen new -o "$offline_staker_keypair" --no-passphrase --force --silent
solana-keygen new -o "$offline_withdrawer_keypair" --no-passphrase --force --silent
solana-keygen new -o "$offline_custodian_keypair" --no-passphrase --force --silent
offline_system_account_pubkey="$(solana-keygen pubkey $offline_system_account_keypair)"
offline_withdrawer_pubkey="$(solana-keygen pubkey $offline_withdrawer_keypair)"
offline_staker_pubkey="$(solana-keygen pubkey $offline_staker_keypair)"
offline_custodian_pubkey="$(solana-keygen pubkey $offline_custodian_keypair)"
# Airdrop some funds to the offline account.
solana airdrop 100 $offline_system_account_pubkey
solana airdrop 2 $online_system_account_pubkey
# Create a nonce account funded by the online account, with the authority given to the offline account
solana create-nonce-account $online_nonce_account_keypair 1 --nonce-authority $offline_system_account_pubkey --keypair $online_system_account_keypair
nonce="$(solana nonce $nonce_account_pubkey)"
execution_step OFFLINE SYSTEM ACCOUNT BALANCE BEFORE CREATING STAKE ACCOUNTS
(
set -x
solana balance $offline_system_account_pubkey
)
################################
execution_step CREATE OFFLINE STAKE ACCOUNT
################################
# Create a stake account funded by the offline system account
stake_account_keypair=stake_account_keypair.json
solana-keygen new -o "$stake_account_keypair" --no-passphrase --force --silent
stake_account_address="$(solana-keygen pubkey $stake_account_keypair)"
sign_only="$(solana create-stake-account $stake_account_keypair 50 \
--sign-only --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \
--stake-authority $offline_staker_pubkey --withdraw-authority $offline_withdrawer_pubkey \
--custodian $offline_custodian_pubkey \
--lockup-epoch 999 \
--keypair $offline_system_account_keypair --url http://0.0.0.0)"
signers="$(get_signers_string "${sign_only[@]})")"
solana create-stake-account $stake_account_keypair 50 \
--blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_pubkey \
--stake-authority $offline_staker_pubkey --withdraw-authority $offline_withdrawer_pubkey \
--custodian $offline_custodian_pubkey \
--lockup-epoch 999 \
--from $offline_system_account_pubkey --fee-payer $offline_system_account_pubkey ${signers[@]}
execution_step VIEW STAKE ACCOUNT AFTER CREATION
(
set -x
solana stake-account $stake_account_address
)
execution_step VIEW OFFLINE SYSTEM ACCOUNT BALANCE AFTER CREATING FIRST STAKE ACCOUNT
(
set -x
solana balance $offline_system_account_pubkey
)
#####################
execution_step SPLIT STAKE OFFLINE
#####################
# Split the original stake account before delegating
split_stake_account_keypair=split_stake_account_keypair.json
solana-keygen new -o $split_stake_account_keypair --no-passphrase --force --silent
split_stake_account_address=$(solana-keygen pubkey $split_stake_account_keypair)
nonce="$(solana nonce $nonce_account_pubkey)"
sign_only="$(solana split-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \
--stake-authority $offline_staker_keypair $stake_account_address $split_stake_account_keypair 10 \
--keypair $offline_system_account_keypair --sign-only --url http://0.0.0.0)"
signers="$(get_signers_string "${sign_only[@]})")"
solana split-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_pubkey \
--stake-authority $offline_staker_pubkey $stake_account_address $split_stake_account_keypair 10 \
--fee-payer $offline_system_account_pubkey ${signers[@]}
execution_step VIEW ORIGINAL STAKE ACCOUNT AFTER SPLITTING
(
set -x
solana stake-account $stake_account_address
)
execution_step VIEW NEW STAKE ACCOUNT CREATED FROM SPLITTING ORIGINAL
(
set -x
solana stake-account $split_stake_account_address
)
#####################
execution_step CHANGE CUSTODIAN LOCKUP
#####################
# Set the lockup epoch to 0 to allow stake to be withdrawn
nonce="$(solana nonce $nonce_account_pubkey)"
sign_only="$(solana stake-set-lockup --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \
$split_stake_account_address --custodian $offline_custodian_keypair --lockup-epoch 0 \
--keypair $offline_system_account_keypair --sign-only --url http://0.0.0.0)"
signers="$(get_signers_string "${sign_only[@]})")"
solana stake-set-lockup --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \
$split_stake_account_address --custodian $offline_custodian_pubkey --lockup-epoch 0 \
--fee-payer $offline_system_account_pubkey ${signers[@]}
execution_step VIEW SPLIT STAKE ACCOUNT AFTER CHANGING LOCKUP
(
set -x
solana stake-account $split_stake_account_address
)
##########################
execution_step OFFLINE STAKE WITHDRAWAL
##########################
# Withdraw the lamports from the stake account that was split off and return them to the offline system account
nonce="$(solana nonce $nonce_account_pubkey)"
sign_only="$(solana withdraw-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \
$split_stake_account_address $offline_system_account_pubkey 10 \
--withdraw-authority $offline_withdrawer_keypair \
--keypair $offline_system_account_keypair --sign-only --url http://0.0.0.0)"
signers="$(get_signers_string "${sign_only[@]})")"
solana withdraw-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_pubkey \
$split_stake_account_address $offline_system_account_pubkey 10 \
--withdraw-authority $offline_withdrawer_pubkey \
--fee-payer $offline_system_account_pubkey ${signers[@]}
execution_step VIEW OFFLINE SYSTEM ACCOUNT BALANCE AFTER WITHDRAWING SPLIT STAKE
(
set -x
solana balance $offline_system_account_pubkey
)
##########################
execution_step OFFLINE STAKE DELEGATION
##########################
# Delegate stake from the original account to a vote account
vote_account_pubkey="$(awk '{if(NR==4) print $2}'<<<"$(solana show-validators)")"
nonce="$(solana nonce $nonce_account_pubkey)"
# Sign a stake delegation, assuming the authorized staker is held offline
sign_only="$(solana delegate-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \
--stake-authority $offline_staker_keypair $stake_account_address $vote_account_pubkey \
--keypair $offline_system_account_keypair --sign-only --url http://0.0.0.0)"
signers="$(get_signers_string "${sign_only[@]})")"
# Send the signed transaction on the cluster
solana delegate-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_pubkey \
--stake-authority $offline_staker_pubkey $stake_account_address $vote_account_pubkey \
--fee-payer $offline_system_account_pubkey ${signers[@]}
execution_step VIEW ORIGINAL STAKE ACCOUNT AFTER DELEGATION
(
set -x
solana stake-account $stake_account_address
)
##########################
execution_step OFFLINE STAKE DEACTIVATION
##########################
# Deactivate delegated stake
nonce="$(solana nonce $nonce_account_pubkey)"
# Sign a stake delegation, assuming the authorized staker is held offline
sign_only="$(solana deactivate-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \
--stake-authority $offline_staker_keypair $stake_account_address \
--keypair $offline_system_account_keypair --sign-only --url http://0.0.0.0)"
signers="$(get_signers_string "${sign_only[@]})")"
# Send the signed transaction on the cluster
solana deactivate-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_pubkey \
--stake-authority $offline_staker_pubkey $stake_account_address \
--fee-payer $offline_system_account_pubkey ${signers[@]}
execution_step VIEW ORIGINAL STAKE ACCOUNT AFTER DEACTIVATION
(
set -x
solana stake-account $stake_account_address
)

16
system-test/stake-operations-testcases/stake_test_automation.sh Executable file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env bash
set -ex
# shellcheck disable=SC1090
# shellcheck disable=SC1091
source "$(dirname "$0")"/../automation_utils.sh
RESULT_FILE="$1"
# Runs offline stake operations tests against a running cluster launched from the automation framework
bootstrapper_ip_address="$(get_bootstrap_validator_ip_address)"
entrypoint=http://"${bootstrapper_ip_address}":8899
PATH="$REPO_ROOT"/solana-release/bin:$PATH "$REPO_ROOT"/system-test/stake-operations-testcases/offline_stake_operations.sh "$entrypoint"
echo "Offline Stake Operations Test Succeeded" >>"$RESULT_FILE"

160
system-test/testnet-automation.sh
View File

@ -147,30 +147,36 @@ function launch_testnet() {
SLOT_COUNT_START_SECONDS=$SECONDS
execution_step "Marking beginning of slot rate test - Slot: $START_SLOT, Seconds: $SLOT_COUNT_START_SECONDS"
if [[ -n $TEST_DURATION_SECONDS ]]; then
execution_step "Wait ${TEST_DURATION_SECONDS} seconds to complete test"
sleep "$TEST_DURATION_SECONDS"
elif [[ "$APPLY_PARTITIONS" = "true" ]]; then
STATS_START_SECONDS=$SECONDS
execution_step "Wait $PARTITION_INACTIVE_DURATION before beginning to apply partitions"
sleep "$PARTITION_INACTIVE_DURATION"
for (( i=1; i<=PARTITION_ITERATION_COUNT; i++ )); do
execution_step "Partition Iteration $i of $PARTITION_ITERATION_COUNT"
execution_step "Applying netem config $NETEM_CONFIG_FILE for $PARTITION_ACTIVE_DURATION seconds"
"${REPO_ROOT}"/net/net.sh netem --config-file "$NETEM_CONFIG_FILE"
sleep "$PARTITION_ACTIVE_DURATION"
execution_step "Resolving partitions for $PARTITION_INACTIVE_DURATION seconds"
"${REPO_ROOT}"/net/net.sh netem --config-file "$NETEM_CONFIG_FILE" --netem-cmd cleanup
case $TEST_TYPE in
fixed_duration)
execution_step "Wait ${TEST_DURATION_SECONDS} seconds to complete test"
sleep "$TEST_DURATION_SECONDS"
;;
partition)
STATS_START_SECONDS=$SECONDS
execution_step "Wait $PARTITION_INACTIVE_DURATION before beginning to apply partitions"
sleep "$PARTITION_INACTIVE_DURATION"
done
STATS_FINISH_SECONDS=$SECONDS
TEST_DURATION_SECONDS=$((STATS_FINISH_SECONDS - STATS_START_SECONDS))
else
# We should never get here
echo Test duration and partition config not defined
exit 1
fi
for (( i=1; i<=PARTITION_ITERATION_COUNT; i++ )); do
execution_step "Partition Iteration $i of $PARTITION_ITERATION_COUNT"
execution_step "Applying netem config $NETEM_CONFIG_FILE for $PARTITION_ACTIVE_DURATION seconds"
"${REPO_ROOT}"/net/net.sh netem --config-file "$NETEM_CONFIG_FILE"
sleep "$PARTITION_ACTIVE_DURATION"
execution_step "Resolving partitions for $PARTITION_INACTIVE_DURATION seconds"
"${REPO_ROOT}"/net/net.sh netem --config-file "$NETEM_CONFIG_FILE" --netem-cmd cleanup
sleep "$PARTITION_INACTIVE_DURATION"
done
STATS_FINISH_SECONDS=$SECONDS
TEST_DURATION_SECONDS=$((STATS_FINISH_SECONDS - STATS_START_SECONDS))
;;
script)
execution_step "Running custom script: ${REPO_ROOT}/${CUSTOM_SCRIPT}"
"$REPO_ROOT"/"$CUSTOM_SCRIPT" "$RESULT_FILE"
;;
*)
echo "Error: Unsupported test type: $TEST_TYPE"
;;
esac
END_SLOT=$(get_slot)
SLOT_COUNT_END_SECONDS=$SECONDS
@ -179,63 +185,11 @@ function launch_testnet() {
SLOTS_PER_SECOND="$(bc <<< "scale=3; ($END_SLOT - $START_SLOT)/($SLOT_COUNT_END_SECONDS - $SLOT_COUNT_START_SECONDS)")"
execution_step "Average slot rate: $SLOTS_PER_SECOND slots/second over $((SLOT_COUNT_END_SECONDS - SLOT_COUNT_START_SECONDS)) seconds"
execution_step "Collect statistics about run"
declare q_mean_tps='
SELECT ROUND(MEAN("median_sum")) as "mean_tps" FROM (
SELECT MEDIAN(sum_count) AS "median_sum" FROM (
SELECT SUM("count") AS "sum_count"
FROM "'$TESTNET_TAG'"."autogen"."bank-process_transactions"
WHERE time > now() - '"$TEST_DURATION_SECONDS"'s AND count > 0
GROUP BY time(1s), host_id)
GROUP BY time(1s)
)'
if [[ "$SKIP_PERF_RESULTS" = "false" ]]; then
collect_performance_statistics
echo "slots_per_second: $SLOTS_PER_SECOND" >>"$RESULT_FILE"
fi
declare q_max_tps='
SELECT MAX("median_sum") as "max_tps" FROM (
SELECT MEDIAN(sum_count) AS "median_sum" FROM (
SELECT SUM("count") AS "sum_count"
FROM "'$TESTNET_TAG'"."autogen"."bank-process_transactions"
WHERE time > now() - '"$TEST_DURATION_SECONDS"'s AND count > 0
GROUP BY time(1s), host_id)
GROUP BY time(1s)
)'
declare q_mean_confirmation='
SELECT round(mean("duration_ms")) as "mean_confirmation_ms"
FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
WHERE time > now() - '"$TEST_DURATION_SECONDS"'s'
declare q_max_confirmation='
SELECT round(max("duration_ms")) as "max_confirmation_ms"
FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
WHERE time > now() - '"$TEST_DURATION_SECONDS"'s'
declare q_99th_confirmation='
SELECT round(percentile("duration_ms", 99)) as "99th_percentile_confirmation_ms"
FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
WHERE time > now() - '"$TEST_DURATION_SECONDS"'s'
declare q_max_tower_distance_observed='
SELECT MAX("tower_distance") as "max_tower_distance" FROM (
SELECT last("slot") - last("root") as "tower_distance"
FROM "'$TESTNET_TAG'"."autogen"."tower-observed"
WHERE time > now() - '"$TEST_DURATION_SECONDS"'s
GROUP BY time(1s), host_id)'
declare q_last_tower_distance_observed='
SELECT MEAN("tower_distance") as "last_tower_distance" FROM (
SELECT last("slot") - last("root") as "tower_distance"
FROM "'$TESTNET_TAG'"."autogen"."tower-observed"
GROUP BY host_id)'
curl -G "${INFLUX_HOST}/query?u=ro&p=topsecret" \
--data-urlencode "db=${TESTNET_TAG}" \
--data-urlencode "q=$q_mean_tps;$q_max_tps;$q_mean_confirmation;$q_max_confirmation;$q_99th_confirmation;$q_max_tower_distance_observed;$q_last_tower_distance_observed" |
python "${REPO_ROOT}"/system-test/testnet-automation-json-parser.py >>"$RESULT_FILE"
echo "slots_per_second: $SLOTS_PER_SECOND" >>"$RESULT_FILE"
execution_step "Writing test results to ${RESULT_FILE}"
RESULT_DETAILS=$(<"$RESULT_FILE")
upload-ci-artifact "$RESULT_FILE"
}
@ -245,9 +199,10 @@ RESULT_DETAILS=
STEP=
execution_step "Initialize Environment"
[[ -n $TESTNET_TAG ]] || TESTNET_TAG=testnet-automation
[[ -n $TESTNET_TAG ]] || TESTNET_TAG=${CLOUD_PROVIDER}-testnet-automation
[[ -n $INFLUX_HOST ]] || INFLUX_HOST=https://metrics.solana.com:8086
[[ -n $BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD ]] || BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD=66
[[ -n $SKIP_PERF_RESULTS ]] || SKIP_PERF_RESULTS=false
if [[ -z $NUMBER_OF_VALIDATOR_NODES ]]; then
echo NUMBER_OF_VALIDATOR_NODES not defined
@ -292,18 +247,41 @@ if [[ "$USE_PUBLIC_IP_ADDRESSES" = "true" ]]; then
maybePublicIpAddresses="-P"
fi
: "${CLIENT_DELAY_START:=0}"
execution_step "Checking for required parameters"
testTypeRequiredParameters=
case $TEST_TYPE in
fixed_duration)
testTypeRequiredParameters=(
TEST_DURATION_SECONDS \
)
;;
partition)
testTypeRequiredParameters=(
NETEM_CONFIG_FILE \
PARTITION_ACTIVE_DURATION \
PARTITION_INACTIVE_DURATION \
PARTITION_ITERATION_COUNT \
)
;;
script)
testTypeRequiredParameters=(
CUSTOM_SCRIPT \
)
;;
*)
echo "Error: Unsupported test type: $TEST_TYPE"
;;
esac
if [[ -z $APPLY_PARTITIONS ]]; then
APPLY_PARTITIONS=false
fi
if [[ "$APPLY_PARTITIONS" = "true" ]]; then
if [[ -n $TEST_DURATION_SECONDS ]]; then
echo Cannot accept TEST_DURATION_SECONDS and a parition looping config
exit 1
missingParameters=
for i in "${testTypeRequiredParameters[@]}"; do
if [[ -z ${!i} ]]; then
missingParameters+="${i}, "
fi
elif [[ -z $TEST_DURATION_SECONDS ]]; then
echo TEST_DURATION_SECONDS not defined
done
if [[ -n $missingParameters ]]; then
echo "Error: For test type $TEST_TYPE, the following required parameters are missing: ${missingParameters[*]}"
exit 1
fi
@ -337,6 +315,8 @@ TEST_PARAMS_TO_DISPLAY=(CLOUD_PROVIDER \
PARTITION_ACTIVE_DURATION \
PARTITION_INACTIVE_DURATION \
PARTITION_ITERATION_COUNT \
TEST_TYPE \
CUSTOM_SCRIPT \
)
TEST_CONFIGURATION=