From 107360a001b1729a3061cb0c4d081fdaf0e5c523 Mon Sep 17 00:00:00 2001 From: Dan Albert Date: Tue, 17 Dec 2019 14:58:10 -0500 Subject: [PATCH] Refactor system-test dir structure and add stability test (#7530) --- .../aws-cpu-only-perf-10-node.yml | 2 +- .../aws-cpu-only-perf-5-node.yml | 2 +- .../azure-cpu-only-perf-5-node.yml | 2 +- .../colo-cpu-only-perf.yml | 2 +- .../colo-gpu-perf-high-txcount.yml | 2 +- .../colo-gpu-perf.yml | 2 +- .../gce-cpu-only-perf-10-node.yml | 2 +- .../gce-cpu-only-perf-5-node.yml | 2 +- .../gce-gpu-perf-10-node.yml | 2 +- .../gce-gpu-perf-100-node.yml | 2 +- .../gce-gpu-perf-25-node.yml | 2 +- .../gce-gpu-perf-5-node.yml | 2 +- .../gce-gpu-perf-50-node.yml | 2 +- .../gce-stability-5-node.yml | 18 ++++++++++++++++++ .../testnet-automation-json-parser.py | 0 .../testnet-automation.sh | 6 +++--- .../upload_results_to_slack.sh | 0 17 files changed, 34 insertions(+), 16 deletions(-) rename system-test/{testnet-performance => performance-testcases}/aws-cpu-only-perf-10-node.yml (90%) rename system-test/{testnet-performance => performance-testcases}/aws-cpu-only-perf-5-node.yml (90%) rename system-test/{testnet-performance => performance-testcases}/azure-cpu-only-perf-5-node.yml (89%) rename system-test/{testnet-performance => performance-testcases}/colo-cpu-only-perf.yml (87%) rename system-test/{testnet-performance => performance-testcases}/colo-gpu-perf-high-txcount.yml (87%) rename system-test/{testnet-performance => performance-testcases}/colo-gpu-perf.yml (87%) rename system-test/{testnet-performance => performance-testcases}/gce-cpu-only-perf-10-node.yml (90%) rename system-test/{testnet-performance => performance-testcases}/gce-cpu-only-perf-5-node.yml (90%) rename system-test/{testnet-performance => performance-testcases}/gce-gpu-perf-10-node.yml (91%) rename system-test/{testnet-performance => performance-testcases}/gce-gpu-perf-100-node.yml (91%) rename system-test/{testnet-performance => performance-testcases}/gce-gpu-perf-25-node.yml (91%) rename system-test/{testnet-performance => performance-testcases}/gce-gpu-perf-5-node.yml (90%) rename system-test/{testnet-performance => performance-testcases}/gce-gpu-perf-50-node.yml (91%) create mode 100755 system-test/stability-testcases/gce-stability-5-node.yml rename system-test/{testnet-performance => }/testnet-automation-json-parser.py (100%) rename system-test/{testnet-performance => }/testnet-automation.sh (98%) rename system-test/{testnet-performance => }/upload_results_to_slack.sh (100%) diff --git a/system-test/testnet-performance/aws-cpu-only-perf-10-node.yml b/system-test/performance-testcases/aws-cpu-only-perf-10-node.yml similarity index 90% rename from system-test/testnet-performance/aws-cpu-only-perf-10-node.yml rename to system-test/performance-testcases/aws-cpu-only-perf-10-node.yml index b0a1dc2a5..979393274 100755 --- a/system-test/testnet-performance/aws-cpu-only-perf-10-node.yml +++ b/system-test/performance-testcases/aws-cpu-only-perf-10-node.yml @@ -1,5 +1,5 @@ steps: - - command: "system-test/testnet-performance/testnet-automation.sh" + - command: "system-test/testnet-automation.sh" label: "AWS - CPU Only 10 Node" env: UPLOAD_RESULTS_TO_SLACK: "true" diff --git a/system-test/testnet-performance/aws-cpu-only-perf-5-node.yml b/system-test/performance-testcases/aws-cpu-only-perf-5-node.yml similarity index 90% rename from system-test/testnet-performance/aws-cpu-only-perf-5-node.yml rename to system-test/performance-testcases/aws-cpu-only-perf-5-node.yml index 07cc31491..749c9fa12 100755 --- a/system-test/testnet-performance/aws-cpu-only-perf-5-node.yml +++ b/system-test/performance-testcases/aws-cpu-only-perf-5-node.yml @@ -1,5 +1,5 @@ steps: - - command: "system-test/testnet-performance/testnet-automation.sh" + - command: "system-test/testnet-automation.sh" label: "AWS - CPU Only 5 Node" env: UPLOAD_RESULTS_TO_SLACK: "true" diff --git a/system-test/testnet-performance/azure-cpu-only-perf-5-node.yml b/system-test/performance-testcases/azure-cpu-only-perf-5-node.yml similarity index 89% rename from system-test/testnet-performance/azure-cpu-only-perf-5-node.yml rename to system-test/performance-testcases/azure-cpu-only-perf-5-node.yml index 6a6e04658..3c3efb82e 100755 --- a/system-test/testnet-performance/azure-cpu-only-perf-5-node.yml +++ b/system-test/performance-testcases/azure-cpu-only-perf-5-node.yml @@ -1,5 +1,5 @@ steps: - - command: "system-test/testnet-performance/testnet-automation.sh" + - command: "system-test/testnet-automation.sh" label: "Azure - CPU Only 5 Node" env: UPLOAD_RESULTS_TO_SLACK: "true" diff --git a/system-test/testnet-performance/colo-cpu-only-perf.yml b/system-test/performance-testcases/colo-cpu-only-perf.yml similarity index 87% rename from system-test/testnet-performance/colo-cpu-only-perf.yml rename to system-test/performance-testcases/colo-cpu-only-perf.yml index 8cb8abc0f..e6d125daa 100755 --- a/system-test/testnet-performance/colo-cpu-only-perf.yml +++ b/system-test/performance-testcases/colo-cpu-only-perf.yml @@ -1,5 +1,5 @@ steps: - - command: "system-test/testnet-performance/testnet-automation.sh" + - command: "system-test/testnet-automation.sh" label: "COLO performance testnet CPU only" env: UPLOAD_RESULTS_TO_SLACK: "true" diff --git a/system-test/testnet-performance/colo-gpu-perf-high-txcount.yml b/system-test/performance-testcases/colo-gpu-perf-high-txcount.yml similarity index 87% rename from system-test/testnet-performance/colo-gpu-perf-high-txcount.yml rename to system-test/performance-testcases/colo-gpu-perf-high-txcount.yml index 625f57925..662e84a72 100755 --- a/system-test/testnet-performance/colo-gpu-perf-high-txcount.yml +++ b/system-test/performance-testcases/colo-gpu-perf-high-txcount.yml @@ -1,5 +1,5 @@ steps: - - command: "system-test/testnet-performance/testnet-automation.sh" + - command: "system-test/testnet-automation.sh" label: "COLO performance testnet GPU enabled - High Tx Count" env: UPLOAD_RESULTS_TO_SLACK: "true" diff --git a/system-test/testnet-performance/colo-gpu-perf.yml b/system-test/performance-testcases/colo-gpu-perf.yml similarity index 87% rename from system-test/testnet-performance/colo-gpu-perf.yml rename to system-test/performance-testcases/colo-gpu-perf.yml index a2c97830d..95b995c13 100755 --- a/system-test/testnet-performance/colo-gpu-perf.yml +++ b/system-test/performance-testcases/colo-gpu-perf.yml @@ -1,5 +1,5 @@ steps: - - command: "system-test/testnet-performance/testnet-automation.sh" + - command: "system-test/testnet-automation.sh" label: "COLO performance testnet GPU enabled" env: UPLOAD_RESULTS_TO_SLACK: "true" diff --git a/system-test/testnet-performance/gce-cpu-only-perf-10-node.yml b/system-test/performance-testcases/gce-cpu-only-perf-10-node.yml similarity index 90% rename from system-test/testnet-performance/gce-cpu-only-perf-10-node.yml rename to system-test/performance-testcases/gce-cpu-only-perf-10-node.yml index 1bc1a9695..1f90cdde7 100755 --- a/system-test/testnet-performance/gce-cpu-only-perf-10-node.yml +++ b/system-test/performance-testcases/gce-cpu-only-perf-10-node.yml @@ -1,5 +1,5 @@ steps: - - command: "system-test/testnet-performance/testnet-automation.sh" + - command: "system-test/testnet-automation.sh" label: "GCE performance testnets CPU ONLY" env: UPLOAD_RESULTS_TO_SLACK: "true" diff --git a/system-test/testnet-performance/gce-cpu-only-perf-5-node.yml b/system-test/performance-testcases/gce-cpu-only-perf-5-node.yml similarity index 90% rename from system-test/testnet-performance/gce-cpu-only-perf-5-node.yml rename to system-test/performance-testcases/gce-cpu-only-perf-5-node.yml index 1e849f3f8..7d3021f43 100755 --- a/system-test/testnet-performance/gce-cpu-only-perf-5-node.yml +++ b/system-test/performance-testcases/gce-cpu-only-perf-5-node.yml @@ -1,5 +1,5 @@ steps: - - command: "system-test/testnet-performance/testnet-automation.sh" + - command: "system-test/testnet-automation.sh" label: "GCE - CPU Only 5 Node" env: UPLOAD_RESULTS_TO_SLACK: "true" diff --git a/system-test/testnet-performance/gce-gpu-perf-10-node.yml b/system-test/performance-testcases/gce-gpu-perf-10-node.yml similarity index 91% rename from system-test/testnet-performance/gce-gpu-perf-10-node.yml rename to system-test/performance-testcases/gce-gpu-perf-10-node.yml index 8e657cdd8..77fc80bfe 100755 --- a/system-test/testnet-performance/gce-gpu-perf-10-node.yml +++ b/system-test/performance-testcases/gce-gpu-perf-10-node.yml @@ -1,5 +1,5 @@ steps: - - command: "system-test/testnet-performance/testnet-automation.sh" + - command: "system-test/testnet-automation.sh" label: "GCE - GPU Enabled 10 Nodes" env: UPLOAD_RESULTS_TO_SLACK: "true" diff --git a/system-test/testnet-performance/gce-gpu-perf-100-node.yml b/system-test/performance-testcases/gce-gpu-perf-100-node.yml similarity index 91% rename from system-test/testnet-performance/gce-gpu-perf-100-node.yml rename to system-test/performance-testcases/gce-gpu-perf-100-node.yml index f2a22b722..3179e7f31 100755 --- a/system-test/testnet-performance/gce-gpu-perf-100-node.yml +++ b/system-test/performance-testcases/gce-gpu-perf-100-node.yml @@ -1,5 +1,5 @@ steps: - - command: "system-test/testnet-performance/testnet-automation.sh" + - command: "system-test/testnet-automation.sh" label: "GCE - GPU Enabled 100 Nodes" env: UPLOAD_RESULTS_TO_SLACK: "true" diff --git a/system-test/testnet-performance/gce-gpu-perf-25-node.yml b/system-test/performance-testcases/gce-gpu-perf-25-node.yml similarity index 91% rename from system-test/testnet-performance/gce-gpu-perf-25-node.yml rename to system-test/performance-testcases/gce-gpu-perf-25-node.yml index fce409b6b..40ba27e5a 100755 --- a/system-test/testnet-performance/gce-gpu-perf-25-node.yml +++ b/system-test/performance-testcases/gce-gpu-perf-25-node.yml @@ -1,5 +1,5 @@ steps: - - command: "system-test/testnet-performance/testnet-automation.sh" + - command: "system-test/testnet-automation.sh" label: "GCE - GPU Enabled 25 Nodes" env: UPLOAD_RESULTS_TO_SLACK: "true" diff --git a/system-test/testnet-performance/gce-gpu-perf-5-node.yml b/system-test/performance-testcases/gce-gpu-perf-5-node.yml similarity index 90% rename from system-test/testnet-performance/gce-gpu-perf-5-node.yml rename to system-test/performance-testcases/gce-gpu-perf-5-node.yml index 658857cbe..185dfba2f 100755 --- a/system-test/testnet-performance/gce-gpu-perf-5-node.yml +++ b/system-test/performance-testcases/gce-gpu-perf-5-node.yml @@ -1,5 +1,5 @@ steps: - - command: "system-test/testnet-performance/testnet-automation.sh" + - command: "system-test/testnet-automation.sh" label: "GCE - GPU Enabled 5 Nodes" env: UPLOAD_RESULTS_TO_SLACK: "true" diff --git a/system-test/testnet-performance/gce-gpu-perf-50-node.yml b/system-test/performance-testcases/gce-gpu-perf-50-node.yml similarity index 91% rename from system-test/testnet-performance/gce-gpu-perf-50-node.yml rename to system-test/performance-testcases/gce-gpu-perf-50-node.yml index 35698ee3c..47dd17042 100755 --- a/system-test/testnet-performance/gce-gpu-perf-50-node.yml +++ b/system-test/performance-testcases/gce-gpu-perf-50-node.yml @@ -1,5 +1,5 @@ steps: - - command: "system-test/testnet-performance/testnet-automation.sh" + - command: "system-test/testnet-automation.sh" label: "GCE - GPU Enabled 50 Nodes" env: UPLOAD_RESULTS_TO_SLACK: "true" diff --git a/system-test/stability-testcases/gce-stability-5-node.yml b/system-test/stability-testcases/gce-stability-5-node.yml new file mode 100755 index 000000000..569055506 --- /dev/null +++ b/system-test/stability-testcases/gce-stability-5-node.yml @@ -0,0 +1,18 @@ +steps: + - command: "system-test/testnet-automation.sh" + label: "GCE - 8 hour Stability - 5 Node" + env: + UPLOAD_RESULTS_TO_SLACK: "true" + CLOUD_PROVIDER: "gce" + TESTNET_TAG: "gce-stability" + RAMP_UP_TIME: 0 + TEST_DURATION_SECONDS: 28800 + NUMBER_OF_VALIDATOR_NODES: 5 + ENABLE_GPU: "false" + VALIDATOR_NODE_MACHINE_TYPE: "--machine-type n1-standard-16" + NUMBER_OF_CLIENT_NODES: 0 + TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a" + USE_PUBLIC_IP_ADDRESSES: "true" + ADDITIONAL_FLAGS: "--dedicated" + agents: + - "queue=testnet-deploy" diff --git a/system-test/testnet-performance/testnet-automation-json-parser.py b/system-test/testnet-automation-json-parser.py similarity index 100% rename from system-test/testnet-performance/testnet-automation-json-parser.py rename to system-test/testnet-automation-json-parser.py diff --git a/system-test/testnet-performance/testnet-automation.sh b/system-test/testnet-automation.sh similarity index 98% rename from system-test/testnet-performance/testnet-automation.sh rename to system-test/testnet-automation.sh index 9212c3caf..3171a2e7b 100755 --- a/system-test/testnet-performance/testnet-automation.sh +++ b/system-test/testnet-automation.sh @@ -279,7 +279,7 @@ function launchTestnet() { curl -G "${INFLUX_HOST}/query?u=ro&p=topsecret" \ --data-urlencode "db=${TESTNET_TAG}" \ --data-urlencode "q=$q_mean_tps;$q_max_tps;$q_mean_confirmation;$q_max_confirmation;$q_99th_confirmation" | - python system-test/testnet-performance/testnet-automation-json-parser.py >>"$RESULT_FILE" + python system-test/testnet-automation-json-parser.py >>"$RESULT_FILE" execution_step "Writing test results to ${RESULT_FILE}" RESULT_DETAILS=$(<"$RESULT_FILE") @@ -290,7 +290,7 @@ RESULT_DETAILS= STEP= execution_step "Initialize Environment" -cd "$(dirname "$0")/../.." +cd "$(dirname "$0")/.." [[ -n $TESTNET_TAG ]] || TESTNET_TAG=testnet-automation [[ -n $INFLUX_HOST ]] || INFLUX_HOST=https://metrics.solana.com:8086 @@ -353,7 +353,7 @@ fi # shellcheck disable=SC1091 source ci/upload-ci-artifact.sh -source system-test/testnet-performance/upload_results_to_slack.sh +source system-test/upload_results_to_slack.sh maybeClientOptions=${CLIENT_OPTIONS:+"-c"} maybeCustomMachineType=${VALIDATOR_NODE_MACHINE_TYPE:+"--custom-machine-type"} diff --git a/system-test/testnet-performance/upload_results_to_slack.sh b/system-test/upload_results_to_slack.sh similarity index 100% rename from system-test/testnet-performance/upload_results_to_slack.sh rename to system-test/upload_results_to_slack.sh