Michael Vines
GitHub
parent
d9919b99d2
commit
f27d001b7a
|
|
@ -1,61 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
testCmd="$*"
|
||||
genPipeline=false
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Clear cached json keypair files
|
||||
rm -rf "$HOME/.config/solana"
|
||||
|
||||
source ci/_
|
||||
export RUST_BACKTRACE=1
|
||||
export RUSTFLAGS="-D warnings"
|
||||
export PATH=$PWD/target/debug:$PATH
|
||||
export USE_INSTALL=1
|
||||
|
||||
if [[ -n $BUILDKITE && -z $testCmd ]]; then
|
||||
genPipeline=true
|
||||
echo "
|
||||
steps:
|
||||
"
|
||||
fi
|
||||
|
||||
build() {
|
||||
$genPipeline && return
|
||||
source ci/rust-version.sh stable
|
||||
source scripts/ulimit-n.sh
|
||||
_ cargo +$rust_stable build
|
||||
}
|
||||
|
||||
runTest() {
|
||||
declare runTestName="$1"
|
||||
declare runTestCmd="$2"
|
||||
if $genPipeline; then
|
||||
echo "
|
||||
- command: \"$0 '$runTestCmd'\"
|
||||
name: \"$runTestName\"
|
||||
timeout_in_minutes: 45
|
||||
"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ -n $testCmd && "$testCmd" != "$runTestCmd" ]]; then
|
||||
echo Skipped "$runTestName"...
|
||||
return
|
||||
fi
|
||||
#shellcheck disable=SC2068 # Don't want to double quote $runTestCmd
|
||||
$runTestCmd
|
||||
}
|
||||
|
||||
build
|
||||
|
||||
runTest "basic" \
|
||||
"ci/localnet-sanity.sh -i 128"
|
||||
|
||||
runTest "restart" \
|
||||
"ci/localnet-sanity.sh -i 128 -k 16"
|
||||
|
||||
runTest "incremental restart, extra node" \
|
||||
"ci/localnet-sanity.sh -i 128 -k 16 -R -x"
|
||||
|
|
@ -1,429 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
cd "$(dirname "$0")"/..
|
||||
source ci/upload-ci-artifact.sh
|
||||
|
||||
zone=
|
||||
bootstrapValidatorAddress=
|
||||
bootstrapValidatorMachineType=
|
||||
clientNodeCount=0
|
||||
idleClients=false
|
||||
additionalValidatorCount=10
|
||||
publicNetwork=false
|
||||
stopNetwork=false
|
||||
reuseLedger=false
|
||||
skipCreate=false
|
||||
skipStart=false
|
||||
externalNode=false
|
||||
failOnValidatorBootupFailure=true
|
||||
tarChannelOrTag=edge
|
||||
delete=false
|
||||
enableGpu=false
|
||||
bootDiskType=""
|
||||
blockstreamer=false
|
||||
fetchLogs=true
|
||||
maybeHashesPerTick=
|
||||
maybeDisableAirdrops=
|
||||
maybeInternalNodesStakeLamports=
|
||||
maybeInternalNodesLamports=
|
||||
maybeExternalPrimordialAccountsFile=
|
||||
maybeSlotsPerEpoch=
|
||||
maybeTargetLamportsPerSignature=
|
||||
maybeSlotsPerEpoch=
|
||||
maybeLetsEncrypt=
|
||||
maybeValidatorAdditionalDiskSize=
|
||||
maybeNoSnapshot=
|
||||
maybeLimitLedgerSize=
|
||||
|
||||
usage() {
|
||||
exitcode=0
|
||||
if [[ -n "$1" ]]; then
|
||||
exitcode=1
|
||||
echo "Error: $*"
|
||||
fi
|
||||
cat <<EOF
|
||||
usage: $0 -p network-name -C cloud -z zone1 [-z zone2] ... [-z zoneN] [options...]
|
||||
|
||||
Deploys a CD testnet
|
||||
|
||||
mandatory arguments:
|
||||
-p [network-name] - name of the network
|
||||
-C [cloud] - cloud provider to use (gce, ec2)
|
||||
-z [zone] - cloud provider zone to deploy the network into. Must specify at least one zone
|
||||
|
||||
options:
|
||||
-t edge|beta|stable|vX.Y.Z - Deploy the latest tarball release for the
|
||||
specified release channel (edge|beta|stable) or release tag
|
||||
(vX.Y.Z)
|
||||
(default: $tarChannelOrTag)
|
||||
-n [number] - Number of additional validators (default: $additionalValidatorCount)
|
||||
-c [number] - Number of client bencher nodes (default: $clientNodeCount)
|
||||
-u - Include a Blockstreamer (default: $blockstreamer)
|
||||
-P - Use public network IP addresses (default: $publicNetwork)
|
||||
-G - Enable GPU, and set count/type of GPUs to use (e.g n1-standard-16 --accelerator count=2,type=nvidia-tesla-v100)
|
||||
-g - Enable GPU (default: $enableGpu)
|
||||
-a [address] - Set the bootstrap validator's external IP address to this GCE address
|
||||
-d [disk-type] - Specify a boot disk type (default None) Use pd-ssd to get ssd on GCE.
|
||||
-D - Delete the network
|
||||
-r - Reuse existing node/ledger configuration from a
|
||||
previous |start| (ie, don't run ./multinode-demo/setup.sh).
|
||||
-x - External node. Default: false
|
||||
-e - Skip create. Assume the nodes have already been created
|
||||
-s - Skip start. Nodes will still be created or configured, but network software will not be started.
|
||||
-S - Stop network software without tearing down nodes.
|
||||
-f - Discard validator nodes that didn't bootup successfully
|
||||
--no-airdrop
|
||||
- If set, disables airdrops. Nodes must be funded in genesis config when airdrops are disabled.
|
||||
--internal-nodes-stake-lamports NUM_LAMPORTS
|
||||
- Amount to stake internal nodes.
|
||||
--internal-nodes-lamports NUM_LAMPORTS
|
||||
- Amount to fund internal nodes in genesis config
|
||||
--external-accounts-file FILE_PATH
|
||||
- Path to external Primordial Accounts file, if it exists.
|
||||
--hashes-per-tick NUM_HASHES|sleep|auto
|
||||
- Override the default --hashes-per-tick for the cluster
|
||||
--lamports NUM_LAMPORTS
|
||||
- Specify the number of lamports to mint (default 500000000000000000)
|
||||
--skip-deploy-update
|
||||
- If set, will skip software update deployment
|
||||
--skip-remote-log-retrieval
|
||||
- If set, will not fetch logs from remote nodes
|
||||
--letsencrypt [dns name]
|
||||
- Attempt to generate a TLS certificate using this DNS name
|
||||
--validator-additional-disk-size-gb [number]
|
||||
- Size of additional disk in GB for all validators
|
||||
--no-snapshot-fetch
|
||||
- If set, disables booting validators from a snapshot
|
||||
|
||||
Note: the SOLANA_METRICS_CONFIG environment variable is used to configure
|
||||
metrics
|
||||
EOF
|
||||
exit $exitcode
|
||||
}
|
||||
|
||||
zone=()
|
||||
|
||||
shortArgs=()
|
||||
while [[ -n $1 ]]; do
|
||||
if [[ ${1:0:2} = -- ]]; then
|
||||
if [[ $1 = --hashes-per-tick ]]; then
|
||||
maybeHashesPerTick="$1 $2"
|
||||
shift 2
|
||||
elif [[ $1 = --slots-per-epoch ]]; then
|
||||
maybeSlotsPerEpoch="$1 $2"
|
||||
shift 2
|
||||
elif [[ $1 = --target-lamports-per-signature ]]; then
|
||||
maybeTargetLamportsPerSignature="$1 $2"
|
||||
shift 2
|
||||
elif [[ $1 = --no-airdrop ]]; then
|
||||
maybeDisableAirdrops=$1
|
||||
shift 1
|
||||
elif [[ $1 = --internal-nodes-stake-lamports ]]; then
|
||||
maybeInternalNodesStakeLamports="$1 $2"
|
||||
shift 2
|
||||
elif [[ $1 = --internal-nodes-lamports ]]; then
|
||||
maybeInternalNodesLamports="$1 $2"
|
||||
shift 2
|
||||
elif [[ $1 = --external-accounts-file ]]; then
|
||||
maybeExternalPrimordialAccountsFile="$1 $2"
|
||||
shift 2
|
||||
elif [[ $1 = --skip-remote-log-retrieval ]]; then
|
||||
fetchLogs=false
|
||||
shift 1
|
||||
elif [[ $1 = --letsencrypt ]]; then
|
||||
maybeLetsEncrypt="$1 $2"
|
||||
shift 2
|
||||
elif [[ $1 = --validator-additional-disk-size-gb ]]; then
|
||||
maybeValidatorAdditionalDiskSize="$1 $2"
|
||||
shift 2
|
||||
elif [[ $1 == --machine-type* ]]; then # Bypass quoted long args for GPUs
|
||||
shortArgs+=("$1")
|
||||
shift
|
||||
elif [[ $1 = --no-snapshot-fetch ]]; then
|
||||
maybeNoSnapshot=$1
|
||||
shift 1
|
||||
elif [[ $1 = --limit-ledger-size ]]; then
|
||||
maybeLimitLedgerSize=$1
|
||||
shift 1
|
||||
elif [[ $1 = --idle-clients ]]; then
|
||||
idleClients=true
|
||||
shift 1
|
||||
else
|
||||
usage "Unknown long option: $1"
|
||||
fi
|
||||
else
|
||||
shortArgs+=("$1")
|
||||
shift
|
||||
fi
|
||||
done
|
||||
|
||||
while getopts "h?p:Pn:c:t:gG:a:Dd:rusxz:p:C:Sfe" opt "${shortArgs[@]}"; do
|
||||
case $opt in
|
||||
h | \?)
|
||||
usage
|
||||
;;
|
||||
p)
|
||||
netName=$OPTARG
|
||||
;;
|
||||
C)
|
||||
cloudProvider=$OPTARG
|
||||
;;
|
||||
z)
|
||||
zone+=("$OPTARG")
|
||||
;;
|
||||
P)
|
||||
publicNetwork=true
|
||||
;;
|
||||
n)
|
||||
additionalValidatorCount=$OPTARG
|
||||
;;
|
||||
c)
|
||||
clientNodeCount=$OPTARG
|
||||
;;
|
||||
t)
|
||||
case $OPTARG in
|
||||
edge|beta|stable|v*)
|
||||
tarChannelOrTag=$OPTARG
|
||||
;;
|
||||
*)
|
||||
usage "Invalid release channel: $OPTARG"
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
g)
|
||||
enableGpu=true
|
||||
;;
|
||||
G)
|
||||
enableGpu=true
|
||||
bootstrapValidatorMachineType=$OPTARG
|
||||
;;
|
||||
a)
|
||||
bootstrapValidatorAddress=$OPTARG
|
||||
;;
|
||||
d)
|
||||
bootDiskType=$OPTARG
|
||||
;;
|
||||
D)
|
||||
delete=true
|
||||
;;
|
||||
r)
|
||||
reuseLedger=true
|
||||
;;
|
||||
e)
|
||||
skipCreate=true
|
||||
;;
|
||||
s)
|
||||
skipStart=true
|
||||
;;
|
||||
x)
|
||||
externalNode=true
|
||||
;;
|
||||
f)
|
||||
failOnValidatorBootupFailure=false
|
||||
;;
|
||||
u)
|
||||
blockstreamer=true
|
||||
;;
|
||||
S)
|
||||
stopNetwork=true
|
||||
;;
|
||||
*)
|
||||
usage "Unknown option: $opt"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n $netName ]] || usage
|
||||
[[ -n $cloudProvider ]] || usage "Cloud provider not specified"
|
||||
[[ -n ${zone[*]} ]] || usage "At least one zone must be specified"
|
||||
|
||||
shutdown() {
|
||||
exitcode=$?
|
||||
|
||||
set +e
|
||||
if [[ -d net/log ]]; then
|
||||
mv net/log net/log-deploy
|
||||
for logfile in net/log-deploy/*; do
|
||||
if [[ -f $logfile ]]; then
|
||||
upload-ci-artifact "$logfile"
|
||||
tail "$logfile"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
exit $exitcode
|
||||
}
|
||||
rm -rf net/{log,-deploy}
|
||||
trap shutdown EXIT INT
|
||||
|
||||
set -x
|
||||
|
||||
# Fetch reusable testnet keypairs
|
||||
if [[ ! -d net/keypairs ]]; then
|
||||
git clone git@github.com:solana-labs/testnet-keypairs.git net/keypairs
|
||||
fi
|
||||
|
||||
# Build a string to pass zone opts to $cloudProvider.sh: "-z zone1 -z zone2 ..."
|
||||
zone_args=()
|
||||
for val in "${zone[@]}"; do
|
||||
zone_args+=("-z $val")
|
||||
done
|
||||
|
||||
if $stopNetwork; then
|
||||
skipCreate=true
|
||||
fi
|
||||
|
||||
if $delete; then
|
||||
skipCreate=false
|
||||
fi
|
||||
|
||||
# Create the network
|
||||
if ! $skipCreate; then
|
||||
echo "--- $cloudProvider.sh delete"
|
||||
# shellcheck disable=SC2068
|
||||
time net/"$cloudProvider".sh delete ${zone_args[@]} -p "$netName" ${externalNode:+-x}
|
||||
if $delete; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "--- $cloudProvider.sh create"
|
||||
create_args=(
|
||||
-p "$netName"
|
||||
-c "$clientNodeCount"
|
||||
-n "$additionalValidatorCount"
|
||||
--dedicated
|
||||
--self-destruct-hours 0
|
||||
)
|
||||
|
||||
if [[ -n $bootstrapValidatorAddress ]]; then
|
||||
create_args+=(-a "$bootstrapValidatorAddress")
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2206
|
||||
create_args+=(${zone_args[@]})
|
||||
|
||||
if [[ -n $maybeLetsEncrypt ]]; then
|
||||
# shellcheck disable=SC2206 # Do not want to quote $maybeLetsEncrypt
|
||||
create_args+=($maybeLetsEncrypt)
|
||||
fi
|
||||
|
||||
if $blockstreamer; then
|
||||
create_args+=(-u)
|
||||
fi
|
||||
|
||||
if [[ -n $bootDiskType ]]; then
|
||||
create_args+=(-d "$bootDiskType")
|
||||
fi
|
||||
|
||||
if $enableGpu; then
|
||||
if [[ -z $bootstrapValidatorMachineType ]]; then
|
||||
create_args+=(-g)
|
||||
else
|
||||
create_args+=(-G "$bootstrapValidatorMachineType")
|
||||
fi
|
||||
fi
|
||||
|
||||
if $publicNetwork; then
|
||||
create_args+=(-P)
|
||||
fi
|
||||
|
||||
if $externalNode; then
|
||||
create_args+=(-x)
|
||||
fi
|
||||
|
||||
if ! $failOnValidatorBootupFailure; then
|
||||
create_args+=(-f)
|
||||
fi
|
||||
|
||||
if [[ -n $maybeValidatorAdditionalDiskSize ]]; then
|
||||
# shellcheck disable=SC2206 # Do not want to quote
|
||||
create_args+=($maybeValidatorAdditionalDiskSize)
|
||||
fi
|
||||
|
||||
time net/"$cloudProvider".sh create "${create_args[@]}"
|
||||
else
|
||||
echo "--- $cloudProvider.sh config"
|
||||
config_args=(
|
||||
-p "$netName"
|
||||
)
|
||||
# shellcheck disable=SC2206
|
||||
config_args+=(${zone_args[@]})
|
||||
if $publicNetwork; then
|
||||
config_args+=(-P)
|
||||
fi
|
||||
|
||||
if $externalNode; then
|
||||
config_args+=(-x)
|
||||
fi
|
||||
|
||||
if ! $failOnValidatorBootupFailure; then
|
||||
config_args+=(-f)
|
||||
fi
|
||||
|
||||
time net/"$cloudProvider".sh config "${config_args[@]}"
|
||||
fi
|
||||
net/init-metrics.sh -e
|
||||
|
||||
echo "+++ $cloudProvider.sh info"
|
||||
net/"$cloudProvider".sh info
|
||||
|
||||
if $stopNetwork; then
|
||||
echo --- net.sh stop
|
||||
time net/net.sh stop
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ok=true
|
||||
if ! $skipStart; then
|
||||
(
|
||||
if $skipCreate; then
|
||||
op=restart
|
||||
else
|
||||
op=start
|
||||
fi
|
||||
echo "--- net.sh $op"
|
||||
args=(
|
||||
"$op"
|
||||
-t "$tarChannelOrTag"
|
||||
)
|
||||
|
||||
if ! $publicNetwork; then
|
||||
args+=(-o rejectExtraNodes)
|
||||
fi
|
||||
if [[ -n $NO_INSTALL_CHECK ]]; then
|
||||
args+=(-o noInstallCheck)
|
||||
fi
|
||||
if $reuseLedger; then
|
||||
args+=(-r)
|
||||
fi
|
||||
|
||||
if ! $failOnValidatorBootupFailure; then
|
||||
args+=(-F)
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2206 # Do not want to quote
|
||||
args+=(
|
||||
$maybeHashesPerTick
|
||||
$maybeDisableAirdrops
|
||||
$maybeInternalNodesStakeLamports
|
||||
$maybeInternalNodesLamports
|
||||
$maybeExternalPrimordialAccountsFile
|
||||
$maybeSlotsPerEpoch
|
||||
$maybeTargetLamportsPerSignature
|
||||
$maybeNoSnapshot
|
||||
$maybeLimitLedgerSize
|
||||
)
|
||||
|
||||
if $idleClients; then
|
||||
args+=(-c "idle=$clientNodeCount=")
|
||||
fi
|
||||
|
||||
time net/net.sh "${args[@]}"
|
||||
) || ok=false
|
||||
|
||||
if $fetchLogs; then
|
||||
net/net.sh logs
|
||||
fi
|
||||
fi
|
||||
|
||||
$ok
|
||||
|
|
@ -1,401 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
cd "$(dirname "$0")"/..
|
||||
|
||||
if [[ -z $BUILDKITE ]]; then
|
||||
echo BUILDKITE not defined
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z $SOLANA_METRICS_PARTIAL_CONFIG ]]; then
|
||||
echo SOLANA_METRICS_PARTIAL_CONFIG not defined
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z $TESTNET ]]; then
|
||||
TESTNET=$(buildkite-agent meta-data get "testnet" --default "")
|
||||
fi
|
||||
|
||||
if [[ -z $TESTNET_OP ]]; then
|
||||
TESTNET_OP=$(buildkite-agent meta-data get "testnet-operation" --default "")
|
||||
fi
|
||||
|
||||
if [[ -z $TESTNET || -z $TESTNET_OP ]]; then
|
||||
(
|
||||
cat <<EOF
|
||||
steps:
|
||||
- block: "Manage Testnet"
|
||||
fields:
|
||||
- select: "Network"
|
||||
key: "testnet"
|
||||
options:
|
||||
- label: "testnet"
|
||||
value: "testnet"
|
||||
- label: "testnet-edge"
|
||||
value: "testnet-edge"
|
||||
- label: "testnet-beta"
|
||||
value: "testnet-beta"
|
||||
- select: "Operation"
|
||||
key: "testnet-operation"
|
||||
default: "sanity-or-restart"
|
||||
options:
|
||||
- label: "Create testnet and then start software. If the testnet already exists it will be deleted and re-created"
|
||||
value: "create-and-start"
|
||||
- label: "Create testnet, but do not start software. If the testnet already exists it will be deleted and re-created"
|
||||
value: "create"
|
||||
- label: "Start network software on an existing testnet. If software is already running it will be restarted"
|
||||
value: "start"
|
||||
- label: "Stop network software without deleting testnet nodes"
|
||||
value: "stop"
|
||||
- label: "Update the network software. Restart network software on failure"
|
||||
value: "update-or-restart"
|
||||
- label: "Sanity check. Restart network software on failure"
|
||||
value: "sanity-or-restart"
|
||||
- label: "Sanity check only"
|
||||
value: "sanity"
|
||||
- label: "Delete the testnet"
|
||||
value: "delete"
|
||||
- label: "Enable/unlock the testnet"
|
||||
value: "enable"
|
||||
- label: "Delete and then lock the testnet from further operation until it is re-enabled"
|
||||
value: "disable"
|
||||
- command: "ci/$(basename "$0")"
|
||||
agents:
|
||||
- "queue=$BUILDKITE_AGENT_META_DATA_QUEUE"
|
||||
EOF
|
||||
) | buildkite-agent pipeline upload
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ci/channel-info.sh
|
||||
eval "$(ci/channel-info.sh)"
|
||||
|
||||
|
||||
EC2_ZONES=(
|
||||
us-west-1a
|
||||
us-west-2a
|
||||
us-east-1a
|
||||
us-east-2a
|
||||
sa-east-1a
|
||||
eu-west-1a
|
||||
eu-west-2a
|
||||
eu-central-1a
|
||||
ap-northeast-2a
|
||||
ap-southeast-2a
|
||||
ap-south-1a
|
||||
ca-central-1a
|
||||
)
|
||||
|
||||
# GCE zones with _lots_ of quota
|
||||
GCE_ZONES=(
|
||||
us-west1-a
|
||||
us-central1-a
|
||||
us-east1-b
|
||||
europe-west4-a
|
||||
|
||||
us-west1-b
|
||||
us-central1-b
|
||||
us-east1-c
|
||||
europe-west4-b
|
||||
|
||||
us-west1-c
|
||||
us-east1-d
|
||||
europe-west4-c
|
||||
)
|
||||
|
||||
# GCE zones with enough quota for one CPU-only validator
|
||||
GCE_LOW_QUOTA_ZONES=(
|
||||
asia-east2-a
|
||||
asia-northeast1-b
|
||||
asia-northeast2-b
|
||||
asia-south1-c
|
||||
asia-southeast1-b
|
||||
australia-southeast1-b
|
||||
europe-north1-a
|
||||
europe-west2-b
|
||||
europe-west3-c
|
||||
europe-west6-a
|
||||
northamerica-northeast1-a
|
||||
southamerica-east1-b
|
||||
)
|
||||
|
||||
case $TESTNET in
|
||||
testnet-edge)
|
||||
CHANNEL_OR_TAG=edge
|
||||
CHANNEL_BRANCH=$EDGE_CHANNEL
|
||||
;;
|
||||
testnet-beta)
|
||||
CHANNEL_OR_TAG=beta
|
||||
CHANNEL_BRANCH=$BETA_CHANNEL
|
||||
;;
|
||||
testnet)
|
||||
CHANNEL_OR_TAG=$STABLE_CHANNEL_LATEST_TAG
|
||||
CHANNEL_BRANCH=$STABLE_CHANNEL
|
||||
export CLOUDSDK_CORE_PROJECT=testnet-solana-com
|
||||
;;
|
||||
*)
|
||||
echo "Error: Invalid TESTNET=$TESTNET"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
EC2_ZONE_ARGS=()
|
||||
for val in "${EC2_ZONES[@]}"; do
|
||||
EC2_ZONE_ARGS+=("-z $val")
|
||||
done
|
||||
GCE_ZONE_ARGS=()
|
||||
for val in "${GCE_ZONES[@]}"; do
|
||||
GCE_ZONE_ARGS+=("-z $val")
|
||||
done
|
||||
GCE_LOW_QUOTA_ZONE_ARGS=()
|
||||
for val in "${GCE_LOW_QUOTA_ZONES[@]}"; do
|
||||
GCE_LOW_QUOTA_ZONE_ARGS+=("-z $val")
|
||||
done
|
||||
|
||||
if [[ -z $TESTNET_DB_HOST ]]; then
|
||||
TESTNET_DB_HOST="https://metrics.solana.com:8086"
|
||||
fi
|
||||
|
||||
export SOLANA_METRICS_CONFIG="db=$TESTNET,host=$TESTNET_DB_HOST,$SOLANA_METRICS_PARTIAL_CONFIG"
|
||||
echo "SOLANA_METRICS_CONFIG: $SOLANA_METRICS_CONFIG"
|
||||
source scripts/configure-metrics.sh
|
||||
|
||||
if [[ -n $TESTNET_TAG ]]; then
|
||||
CHANNEL_OR_TAG=$TESTNET_TAG
|
||||
else
|
||||
|
||||
if [[ $CI_BRANCH != "$CHANNEL_BRANCH" ]]; then
|
||||
(
|
||||
cat <<EOF
|
||||
steps:
|
||||
- trigger: "$BUILDKITE_PIPELINE_SLUG"
|
||||
async: true
|
||||
build:
|
||||
message: "$BUILDKITE_MESSAGE"
|
||||
branch: "$CHANNEL_BRANCH"
|
||||
env:
|
||||
TESTNET: "$TESTNET"
|
||||
TESTNET_OP: "$TESTNET_OP"
|
||||
TESTNET_DB_HOST: "$TESTNET_DB_HOST"
|
||||
GCE_NODE_COUNT: "$GCE_NODE_COUNT"
|
||||
GCE_LOW_QUOTA_NODE_COUNT: "$GCE_LOW_QUOTA_NODE_COUNT"
|
||||
RUST_LOG: "$RUST_LOG"
|
||||
EOF
|
||||
) | buildkite-agent pipeline upload
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
maybe_deploy_software() {
|
||||
declare arg=$1
|
||||
declare ok=true
|
||||
(
|
||||
echo "--- net.sh restart"
|
||||
set -x
|
||||
time net/net.sh restart --skip-setup -t "$CHANNEL_OR_TAG" --skip-poh-verify "$arg"
|
||||
) || ok=false
|
||||
if ! $ok; then
|
||||
net/net.sh logs
|
||||
fi
|
||||
$ok
|
||||
}
|
||||
|
||||
sanity() {
|
||||
echo "--- sanity $TESTNET"
|
||||
case $TESTNET in
|
||||
testnet-edge)
|
||||
(
|
||||
set -x
|
||||
NO_INSTALL_CHECK=1 \
|
||||
ci/testnet-sanity.sh edge-devnet-solana-com gce -P us-west1-b
|
||||
maybe_deploy_software
|
||||
)
|
||||
;;
|
||||
testnet-beta)
|
||||
(
|
||||
set -x
|
||||
NO_INSTALL_CHECK=1 \
|
||||
ci/testnet-sanity.sh beta-devnet-solana-com gce -P us-west1-b
|
||||
maybe_deploy_software --deploy-if-newer
|
||||
)
|
||||
;;
|
||||
testnet)
|
||||
(
|
||||
set -x
|
||||
ci/testnet-sanity.sh devnet-solana-com gce -P us-west1-b
|
||||
)
|
||||
;;
|
||||
*)
|
||||
echo "Error: Invalid TESTNET=$TESTNET"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
deploy() {
|
||||
declare maybeCreate=$1
|
||||
declare maybeStart=$2
|
||||
declare maybeStop=$3
|
||||
declare maybeDelete=$4
|
||||
|
||||
echo "--- deploy \"$maybeCreate\" \"$maybeStart\" \"$maybeStop\" \"$maybeDelete\""
|
||||
|
||||
# Create or recreate the nodes
|
||||
if [[ -z $maybeCreate ]]; then
|
||||
skipCreate=skip
|
||||
else
|
||||
skipCreate=""
|
||||
fi
|
||||
|
||||
# Start or restart the network software on the nodes
|
||||
if [[ -z $maybeStart ]]; then
|
||||
skipStart=skip
|
||||
else
|
||||
skipStart=""
|
||||
fi
|
||||
|
||||
case $TESTNET in
|
||||
testnet-edge)
|
||||
(
|
||||
set -x
|
||||
ci/testnet-deploy.sh -p edge-devnet-solana-com -C gce -z us-west1-b \
|
||||
-t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P \
|
||||
-a edge-devnet-solana-com --letsencrypt edge.devnet.solana.com \
|
||||
--limit-ledger-size \
|
||||
${skipCreate:+-e} \
|
||||
${skipStart:+-s} \
|
||||
${maybeStop:+-S} \
|
||||
${maybeDelete:+-D}
|
||||
)
|
||||
;;
|
||||
testnet-beta)
|
||||
(
|
||||
set -x
|
||||
ci/testnet-deploy.sh -p beta-devnet-solana-com -C gce -z us-west1-b \
|
||||
-t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P \
|
||||
-a beta-devnet-solana-com --letsencrypt beta.devnet.solana.com \
|
||||
--limit-ledger-size \
|
||||
${skipCreate:+-e} \
|
||||
${skipStart:+-s} \
|
||||
${maybeStop:+-S} \
|
||||
${maybeDelete:+-D}
|
||||
)
|
||||
;;
|
||||
testnet)
|
||||
(
|
||||
set -x
|
||||
ci/testnet-deploy.sh -p devnet-solana-com -C gce -z us-west1-b \
|
||||
-t "$CHANNEL_OR_TAG" -n 0 -c 0 -u -P \
|
||||
-a testnet-solana-com --letsencrypt devnet.solana.com \
|
||||
--limit-ledger-size \
|
||||
${skipCreate:+-e} \
|
||||
${skipStart:+-s} \
|
||||
${maybeStop:+-S} \
|
||||
${maybeDelete:+-D}
|
||||
)
|
||||
(
|
||||
echo "--- net.sh update"
|
||||
set -x
|
||||
time net/net.sh update -t "$CHANNEL_OR_TAG" --platform linux --platform osx #--platform windows
|
||||
)
|
||||
;;
|
||||
*)
|
||||
echo "Error: Invalid TESTNET=$TESTNET"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
ENABLED_LOCKFILE="${HOME}/${TESTNET}.is_enabled"
|
||||
|
||||
create-and-start() {
|
||||
deploy create start
|
||||
}
|
||||
create() {
|
||||
deploy create
|
||||
}
|
||||
start() {
|
||||
deploy "" start
|
||||
}
|
||||
stop() {
|
||||
deploy "" ""
|
||||
}
|
||||
delete() {
|
||||
deploy "" "" "" delete
|
||||
}
|
||||
enable_testnet() {
|
||||
touch "${ENABLED_LOCKFILE}"
|
||||
echo "+++ $TESTNET now enabled"
|
||||
}
|
||||
disable_testnet() {
|
||||
rm -f "${ENABLED_LOCKFILE}"
|
||||
echo "+++ $TESTNET now disabled"
|
||||
}
|
||||
is_testnet_enabled() {
|
||||
if [[ ! -f ${ENABLED_LOCKFILE} ]]; then
|
||||
echo "+++ ${TESTNET} is currently disabled. Enable ${TESTNET} by running ci/testnet-manager.sh with \$TESTNET_OP=enable, then re-run with current settings."
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
|
||||
case $TESTNET_OP in
|
||||
enable)
|
||||
enable_testnet
|
||||
;;
|
||||
disable)
|
||||
disable_testnet
|
||||
delete
|
||||
;;
|
||||
create-and-start)
|
||||
is_testnet_enabled
|
||||
create-and-start
|
||||
;;
|
||||
create)
|
||||
is_testnet_enabled
|
||||
create
|
||||
;;
|
||||
start)
|
||||
is_testnet_enabled
|
||||
start
|
||||
;;
|
||||
stop)
|
||||
is_testnet_enabled
|
||||
stop
|
||||
;;
|
||||
sanity)
|
||||
is_testnet_enabled
|
||||
sanity
|
||||
;;
|
||||
delete)
|
||||
is_testnet_enabled
|
||||
delete
|
||||
;;
|
||||
update-or-restart)
|
||||
is_testnet_enabled
|
||||
if start; then
|
||||
echo Update successful
|
||||
else
|
||||
echo "+++ Update failed, restarting the network"
|
||||
$metricsWriteDatapoint "testnet-manager update-failure=1"
|
||||
create-and-start
|
||||
fi
|
||||
;;
|
||||
sanity-or-restart)
|
||||
is_testnet_enabled
|
||||
if sanity; then
|
||||
echo Pass
|
||||
else
|
||||
echo "+++ Sanity failed, updating the network"
|
||||
$metricsWriteDatapoint "testnet-manager sanity-failure=1"
|
||||
create-and-start
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "Error: Invalid TESTNET_OP=$TESTNET_OP"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo --- fin
|
||||
exit 0
|
||||
|
|
@ -1,78 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
source ci/upload-ci-artifact.sh
|
||||
|
||||
usage() {
|
||||
exitcode=0
|
||||
if [[ -n "$1" ]]; then
|
||||
exitcode=1
|
||||
echo "Error: $*"
|
||||
fi
|
||||
cat <<EOF
|
||||
usage: $0 [name] [cloud] [zone1] ... [zoneN]
|
||||
|
||||
Sanity check a testnet
|
||||
|
||||
name - name of the network
|
||||
cloud - cloud provider to use (gce, ec2)
|
||||
zone1 .. zoneN - cloud provider zones to check
|
||||
|
||||
Note: the SOLANA_METRICS_CONFIG environment variable is used to configure
|
||||
metrics
|
||||
EOF
|
||||
exit $exitcode
|
||||
}
|
||||
|
||||
netName=$1
|
||||
cloudProvider=$2
|
||||
[[ -n $netName ]] || usage ""
|
||||
[[ -n $cloudProvider ]] || usage "Cloud provider not specified"
|
||||
shift 2
|
||||
|
||||
maybePublicNetwork=
|
||||
if [[ $1 = -P ]]; then
|
||||
maybePublicNetwork=-P
|
||||
shift
|
||||
fi
|
||||
[[ -n $1 ]] || usage "zone1 not specified"
|
||||
|
||||
shutdown() {
|
||||
exitcode=$?
|
||||
|
||||
set +e
|
||||
if [[ -d net/log ]]; then
|
||||
mv net/log net/log-sanity
|
||||
for logfile in net/log-sanity/*; do
|
||||
if [[ -f $logfile ]]; then
|
||||
upload-ci-artifact "$logfile"
|
||||
tail "$logfile"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
exit $exitcode
|
||||
}
|
||||
rm -rf net/{log,-sanity}
|
||||
rm -f net/config/config
|
||||
trap shutdown EXIT INT
|
||||
|
||||
set -x
|
||||
for zone in "$@"; do
|
||||
echo "--- $cloudProvider config [$zone]"
|
||||
timeout 5m net/"$cloudProvider".sh config $maybePublicNetwork -n 1 -p "$netName" -z "$zone"
|
||||
net/init-metrics.sh -e
|
||||
echo "+++ $cloudProvider.sh info"
|
||||
net/"$cloudProvider".sh info
|
||||
echo "--- net.sh sanity [$cloudProvider:$zone]"
|
||||
ok=true
|
||||
timeout 5m net/net.sh sanity \
|
||||
${REJECT_EXTRA_NODES:+-o rejectExtraNodes} \
|
||||
${NO_INSTALL_CHECK:+-o noInstallCheck} \
|
||||
$zone || ok=false
|
||||
|
||||
if ! $ok; then
|
||||
net/net.sh logs
|
||||
fi
|
||||
$ok
|
||||
done
|
||||
|
|
@ -74,27 +74,6 @@ $ ./ec2.sh create -g ...
|
|||
If deploying a tarball-based network nothing further is required, as GPU presence
|
||||
is detected at runtime and the CUDA build is auto selected.
|
||||
|
||||
### How to interact with a CD testnet deployed by ci/testnet-deploy.sh
|
||||
|
||||
**AWS-Specific Extra Setup**: Follow the steps in
|
||||
`scripts/solana-user-authorized_keys.sh`, then redeploy the testnet
|
||||
before continuing in this section.
|
||||
|
||||
Taking **master-testnet-solana-com** as an example, configure your workspace for
|
||||
the testnet using:
|
||||
```bash
|
||||
$ ./gce.sh config -p master-testnet-solana-com
|
||||
```
|
||||
or
|
||||
```bash
|
||||
$ ./ec2.sh config -p master-testnet-solana-com
|
||||
```
|
||||
|
||||
Then run the following for details on how to ssh into any testnet node to access logs or otherwise inspect the node
|
||||
```bash
|
||||
$ ./ssh.sh
|
||||
```
|
||||
|
||||
### Partition testing
|
||||
|
||||
To induce the partition `net.sh netem --config-file <config file path>`
|
||||
|
|
|
|||
Loading…
Reference in New Issue