2018-11-11 08:19:04 -08:00
|
|
|
#!/usr/bin/env bash
|
|
|
|
set -e
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
here=$(dirname "$0")
|
|
|
|
# shellcheck source=net/common.sh
|
|
|
|
source "$here"/common.sh
|
|
|
|
|
2018-09-16 14:46:08 -07:00
|
|
|
cloudProvider=$(basename "$0" .sh)
|
2018-09-22 16:43:47 -07:00
|
|
|
bootDiskType=""
|
2018-09-16 14:46:08 -07:00
|
|
|
case $cloudProvider in
|
|
|
|
gce)
|
|
|
|
# shellcheck source=net/scripts/gce-provider.sh
|
|
|
|
source "$here"/scripts/gce-provider.sh
|
|
|
|
|
2019-08-04 20:36:23 -07:00
|
|
|
cpuBootstrapLeaderMachineType="--custom-cpu 12 --custom-memory 32GB --min-cpu-platform Intel%20Skylake"
|
2019-05-17 20:06:07 -07:00
|
|
|
gpuBootstrapLeaderMachineType="$cpuBootstrapLeaderMachineType --accelerator count=1,type=nvidia-tesla-p100"
|
2019-04-18 09:36:11 -07:00
|
|
|
clientMachineType="--custom-cpu 16 --custom-memory 20GB"
|
|
|
|
blockstreamerMachineType="--machine-type n1-standard-8"
|
2019-10-21 10:29:37 -07:00
|
|
|
archiverMachineType="--custom-cpu 4 --custom-memory 16GB"
|
2019-11-12 09:21:24 -08:00
|
|
|
selfDestructHours=8
|
2018-09-16 14:46:08 -07:00
|
|
|
;;
|
|
|
|
ec2)
|
|
|
|
# shellcheck source=net/scripts/ec2-provider.sh
|
|
|
|
source "$here"/scripts/ec2-provider.sh
|
|
|
|
|
2019-05-17 20:06:07 -07:00
|
|
|
cpuBootstrapLeaderMachineType=c5.2xlarge
|
|
|
|
|
|
|
|
# NOTE: At this time only the p3dn.24xlarge EC2 instance type has GPU and
|
|
|
|
# AVX-512 support. The default, p2.xlarge, does not support
|
|
|
|
# AVX-512
|
2018-12-09 09:42:09 -08:00
|
|
|
gpuBootstrapLeaderMachineType=p2.xlarge
|
2019-05-17 20:06:07 -07:00
|
|
|
clientMachineType=c5.2xlarge
|
|
|
|
blockstreamerMachineType=c5.2xlarge
|
2019-10-21 10:29:37 -07:00
|
|
|
archiverMachineType=c5.xlarge
|
2019-11-12 09:21:24 -08:00
|
|
|
selfDestructHours=0
|
2018-09-16 14:46:08 -07:00
|
|
|
;;
|
2019-04-23 15:41:45 -07:00
|
|
|
azure)
|
|
|
|
# shellcheck source=net/scripts/azure-provider.sh
|
|
|
|
source "$here"/scripts/azure-provider.sh
|
|
|
|
|
|
|
|
cpuBootstrapLeaderMachineType=Standard_D16s_v3
|
|
|
|
gpuBootstrapLeaderMachineType=Standard_NC12
|
|
|
|
clientMachineType=Standard_D16s_v3
|
|
|
|
blockstreamerMachineType=Standard_D16s_v3
|
2019-10-21 10:29:37 -07:00
|
|
|
archiverMachineType=Standard_D4s_v3
|
2019-11-12 09:21:24 -08:00
|
|
|
selfDestructHours=0
|
2019-04-23 15:41:45 -07:00
|
|
|
;;
|
2019-09-19 14:08:23 -07:00
|
|
|
colo)
|
|
|
|
# shellcheck source=net/scripts/colo-provider.sh
|
|
|
|
source "$here"/scripts/colo-provider.sh
|
|
|
|
|
|
|
|
cpuBootstrapLeaderMachineType=0
|
|
|
|
gpuBootstrapLeaderMachineType=1
|
|
|
|
clientMachineType=0
|
|
|
|
blockstreamerMachineType=0
|
2019-10-21 10:29:37 -07:00
|
|
|
archiverMachineType=0
|
2019-11-12 09:21:24 -08:00
|
|
|
selfDestructHours=0
|
2019-09-19 14:08:23 -07:00
|
|
|
;;
|
2018-09-16 14:46:08 -07:00
|
|
|
*)
|
|
|
|
echo "Error: Unknown cloud provider: $cloudProvider"
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
|
|
|
|
|
2018-09-06 08:18:05 -07:00
|
|
|
prefix=testnet-dev-${USER//[^A-Za-z0-9]/}
|
2019-10-21 20:21:21 -07:00
|
|
|
additionalValidatorCount=2
|
2019-10-15 14:37:52 -07:00
|
|
|
clientNodeCount=0
|
2019-10-21 10:29:37 -07:00
|
|
|
archiverNodeCount=0
|
2019-02-21 15:35:26 -08:00
|
|
|
blockstreamer=false
|
2019-10-21 20:21:21 -07:00
|
|
|
validatorBootDiskSizeInGb=500
|
2018-09-17 08:25:10 -07:00
|
|
|
clientBootDiskSizeInGb=75
|
2019-10-21 10:29:37 -07:00
|
|
|
archiverBootDiskSizeInGb=500
|
2019-10-21 20:21:21 -07:00
|
|
|
validatorAdditionalDiskSizeInGb=
|
2019-04-01 15:54:41 -07:00
|
|
|
externalNodes=false
|
2019-04-24 16:23:26 -07:00
|
|
|
failOnValidatorBootupFailure=true
|
2019-10-14 23:11:29 -07:00
|
|
|
preemptible=true
|
2019-10-22 16:08:49 -07:00
|
|
|
evalInfo=false
|
2018-09-16 14:46:08 -07:00
|
|
|
|
2018-09-04 22:21:58 -07:00
|
|
|
publicNetwork=false
|
2019-07-09 15:45:46 -07:00
|
|
|
letsEncryptDomainName=
|
2018-09-16 14:46:08 -07:00
|
|
|
enableGpu=false
|
2019-10-24 14:12:25 -07:00
|
|
|
customMachineType=
|
2019-02-17 09:48:27 -08:00
|
|
|
customAddress=
|
2019-03-27 15:19:55 -07:00
|
|
|
zones=()
|
|
|
|
|
|
|
|
containsZone() {
|
|
|
|
local e match="$1"
|
|
|
|
shift
|
|
|
|
for e; do [[ "$e" == "$match" ]] && return 0; done
|
|
|
|
return 1
|
|
|
|
}
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
usage() {
|
|
|
|
exitcode=0
|
|
|
|
if [[ -n "$1" ]]; then
|
|
|
|
exitcode=1
|
|
|
|
echo "Error: $*"
|
|
|
|
fi
|
|
|
|
cat <<EOF
|
|
|
|
usage: $0 [create|config|delete] [common options] [command-specific options]
|
|
|
|
|
2018-09-16 14:46:08 -07:00
|
|
|
Manage testnet instances
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
create - create a new testnet (implies 'config')
|
|
|
|
config - configure the testnet and write a config file describing it
|
|
|
|
delete - delete the testnet
|
2019-01-16 09:38:29 -08:00
|
|
|
info - display information about the currently configured testnet
|
2019-09-19 14:08:23 -07:00
|
|
|
status - display status information of all resources
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
common options:
|
2018-09-06 10:08:34 -07:00
|
|
|
-p [prefix] - Optional common prefix for instance names to avoid
|
|
|
|
collisions (default: $prefix)
|
2019-04-09 08:51:59 -07:00
|
|
|
-z [zone] - Zone(s) for the nodes (default: $(cloud_DefaultZone))
|
2019-10-21 20:21:21 -07:00
|
|
|
If specified multiple times, the validators will be evenly
|
2019-04-09 08:51:59 -07:00
|
|
|
distributed over all specified zones and
|
|
|
|
client/blockstreamer nodes will be created in the first
|
|
|
|
zone
|
|
|
|
-x - append to the existing configuration instead of creating a
|
|
|
|
new configuration
|
2019-10-10 12:57:03 -07:00
|
|
|
--allow-boot-failures
|
|
|
|
- Discard from config validator nodes that didn't bootup
|
|
|
|
successfully
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
create-specific options:
|
2019-10-21 20:21:21 -07:00
|
|
|
-n [number] - Number of additional validators (default: $additionalValidatorCount)
|
2018-09-05 09:31:50 -07:00
|
|
|
-c [number] - Number of client nodes (default: $clientNodeCount)
|
2019-10-21 10:29:37 -07:00
|
|
|
-r [number] - Number of archiver nodes (default: $archiverNodeCount)
|
2019-02-21 15:35:26 -08:00
|
|
|
-u - Include a Blockstreamer (default: $blockstreamer)
|
2018-09-04 22:21:58 -07:00
|
|
|
-P - Use public network IP addresses (default: $publicNetwork)
|
2019-10-24 14:12:25 -07:00
|
|
|
-g - Enable GPU and automatically set validator machine types to $gpuBootstrapLeaderMachineType
|
|
|
|
(default: $enableGpu)
|
|
|
|
-G - Enable GPU, and set custom GPU machine type to use
|
2019-04-18 09:36:11 -07:00
|
|
|
(e.g $gpuBootstrapLeaderMachineType)
|
2019-02-21 15:35:26 -08:00
|
|
|
-a [address] - Address to be be assigned to the Blockstreamer if present,
|
2019-10-21 20:21:21 -07:00
|
|
|
otherwise the bootstrap validator.
|
2019-02-17 09:48:27 -08:00
|
|
|
* For GCE, [address] is the "name" of the desired External
|
|
|
|
IP Address.
|
|
|
|
* For EC2, [address] is the "allocation ID" of the desired
|
|
|
|
Elastic IP.
|
2018-09-22 16:43:47 -07:00
|
|
|
-d [disk-type] - Specify a boot disk type (default None) Use pd-ssd to get ssd on GCE.
|
2019-10-24 14:12:25 -07:00
|
|
|
--letsencrypt [dns name]
|
|
|
|
- Attempt to generate a TLS certificate using this
|
|
|
|
DNS name (useful only when the -a and -P options
|
|
|
|
are also provided)
|
|
|
|
--custom-machine-type
|
|
|
|
- Set a custom machine type without assuming whether or not
|
|
|
|
GPU is enabled. Set this explicitly with --enable-gpu/-g to call out the presence of GPUs.
|
|
|
|
--enable-gpu - Use with --custom-machine-type to specify whether or not GPUs should be used/enabled
|
2019-10-21 20:21:21 -07:00
|
|
|
--validator-additional-disk-size-gb [number]
|
|
|
|
- Add an additional [number] GB SSD to all validators to store the config directory.
|
2019-07-30 22:43:47 -07:00
|
|
|
If not set, config will be written to the boot disk by default.
|
2019-07-11 15:23:32 -07:00
|
|
|
Only supported on GCE.
|
2019-10-21 20:21:21 -07:00
|
|
|
--dedicated - Use dedicated instances for additional validators
|
2019-10-14 23:11:29 -07:00
|
|
|
(by default preemptible instances are used to reduce
|
2019-10-21 10:29:37 -07:00
|
|
|
cost). Note that the bootstrap leader, archiver,
|
2019-10-14 23:11:29 -07:00
|
|
|
blockstreamer and client nodes are always dedicated.
|
2019-11-04 10:30:26 -08:00
|
|
|
--self-destruct-hours [number]
|
|
|
|
- Specify lifetime of the allocated instances in hours. 0 to
|
|
|
|
disable. Only supported on GCE. (default: $selfDestructHours)
|
2019-10-14 23:11:29 -07:00
|
|
|
|
2018-08-28 10:19:33 -07:00
|
|
|
config-specific options:
|
2018-12-09 10:03:17 -08:00
|
|
|
-P - Use public network IP addresses (default: $publicNetwork)
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
delete-specific options:
|
2018-09-07 08:57:14 -07:00
|
|
|
none
|
2018-08-28 10:19:33 -07:00
|
|
|
|
2019-01-16 09:38:29 -08:00
|
|
|
info-specific options:
|
2019-10-22 16:08:49 -07:00
|
|
|
--eval - Output in a form that can be eval-ed by a shell: eval $(gce.sh info)
|
|
|
|
|
2019-01-16 09:38:29 -08:00
|
|
|
none
|
|
|
|
|
2018-08-28 10:19:33 -07:00
|
|
|
EOF
|
|
|
|
exit $exitcode
|
|
|
|
}
|
|
|
|
|
|
|
|
command=$1
|
|
|
|
[[ -n $command ]] || usage
|
|
|
|
shift
|
2019-09-19 14:08:23 -07:00
|
|
|
[[ $command = create || $command = config || $command = info || $command = delete || $command = status ]] ||
|
2019-01-16 09:38:29 -08:00
|
|
|
usage "Invalid command: $command"
|
2018-08-28 10:19:33 -07:00
|
|
|
|
2019-07-09 15:45:46 -07:00
|
|
|
shortArgs=()
|
|
|
|
while [[ -n $1 ]]; do
|
|
|
|
if [[ ${1:0:2} = -- ]]; then
|
|
|
|
if [[ $1 = --letsencrypt ]]; then
|
|
|
|
letsEncryptDomainName="$2"
|
|
|
|
shift 2
|
2019-10-21 20:21:21 -07:00
|
|
|
elif [[ $1 = --validator-additional-disk-size-gb ]]; then
|
|
|
|
validatorAdditionalDiskSizeInGb="$2"
|
2019-07-11 15:23:32 -07:00
|
|
|
shift 2
|
2019-09-26 15:52:03 -07:00
|
|
|
elif [[ $1 == --machine-type* || $1 == --custom-cpu* ]]; then # Bypass quoted long args for GPUs
|
2019-07-12 08:38:47 -07:00
|
|
|
shortArgs+=("$1")
|
|
|
|
shift
|
2019-10-10 12:57:03 -07:00
|
|
|
elif [[ $1 == --allow-boot-failures ]]; then
|
|
|
|
failOnValidatorBootupFailure=false
|
|
|
|
shift
|
2019-10-14 23:11:29 -07:00
|
|
|
elif [[ $1 == --dedicated ]]; then
|
|
|
|
preemptible=false
|
|
|
|
shift
|
2019-10-22 16:08:49 -07:00
|
|
|
elif [[ $1 == --eval ]]; then
|
|
|
|
evalInfo=true
|
|
|
|
shift
|
2019-10-24 14:12:25 -07:00
|
|
|
elif [[ $1 == --enable-gpu ]]; then
|
|
|
|
enableGpu=true
|
|
|
|
shift
|
|
|
|
elif [[ $1 = --custom-machine-type ]]; then
|
|
|
|
customMachineType="$2"
|
|
|
|
shift 2
|
2019-11-04 10:30:26 -08:00
|
|
|
elif [[ $1 == --self-destruct-hours ]]; then
|
|
|
|
maybeTimeout=$2
|
|
|
|
if [[ $maybeTimeout =~ ^[0-9]+$ ]]; then
|
|
|
|
selfDestructHours=$maybeTimeout
|
|
|
|
else
|
|
|
|
echo " Invalid parameter ($maybeTimeout) to $1"
|
|
|
|
usage 1
|
|
|
|
fi
|
|
|
|
shift 2
|
2019-07-09 15:45:46 -07:00
|
|
|
else
|
|
|
|
usage "Unknown long option: $1"
|
|
|
|
fi
|
|
|
|
else
|
|
|
|
shortArgs+=("$1")
|
|
|
|
shift
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
|
|
|
|
while getopts "h?p:Pn:c:r:z:gG:a:d:uxf" opt "${shortArgs[@]}"; do
|
2018-08-28 10:19:33 -07:00
|
|
|
case $opt in
|
|
|
|
h | \?)
|
|
|
|
usage
|
|
|
|
;;
|
|
|
|
p)
|
2018-09-06 10:08:34 -07:00
|
|
|
[[ ${OPTARG//[^A-Za-z0-9-]/} == "$OPTARG" ]] || usage "Invalid prefix: \"$OPTARG\", alphanumeric only"
|
2018-08-28 10:19:33 -07:00
|
|
|
prefix=$OPTARG
|
|
|
|
;;
|
|
|
|
P)
|
2018-09-04 22:21:58 -07:00
|
|
|
publicNetwork=true
|
2018-08-28 10:19:33 -07:00
|
|
|
;;
|
|
|
|
n)
|
2019-10-21 20:21:21 -07:00
|
|
|
additionalValidatorCount=$OPTARG
|
2018-08-28 10:19:33 -07:00
|
|
|
;;
|
|
|
|
c)
|
|
|
|
clientNodeCount=$OPTARG
|
|
|
|
;;
|
2019-05-31 15:27:31 -07:00
|
|
|
r)
|
2019-10-21 10:29:37 -07:00
|
|
|
archiverNodeCount=$OPTARG
|
2019-05-31 15:27:31 -07:00
|
|
|
;;
|
2018-08-28 10:19:33 -07:00
|
|
|
z)
|
2019-03-27 15:19:55 -07:00
|
|
|
containsZone "$OPTARG" "${zones[@]}" || zones+=("$OPTARG")
|
2018-08-28 10:19:33 -07:00
|
|
|
;;
|
2018-09-04 08:17:41 -07:00
|
|
|
g)
|
2018-09-16 14:46:08 -07:00
|
|
|
enableGpu=true
|
2018-09-04 08:17:41 -07:00
|
|
|
;;
|
2018-09-27 11:55:56 -07:00
|
|
|
G)
|
|
|
|
enableGpu=true
|
2019-10-24 14:12:25 -07:00
|
|
|
customMachineType="$OPTARG"
|
2018-09-27 11:55:56 -07:00
|
|
|
;;
|
2018-09-05 09:31:50 -07:00
|
|
|
a)
|
2019-02-17 09:48:27 -08:00
|
|
|
customAddress=$OPTARG
|
2018-09-05 09:31:50 -07:00
|
|
|
;;
|
2018-09-22 16:43:47 -07:00
|
|
|
d)
|
|
|
|
bootDiskType=$OPTARG
|
|
|
|
;;
|
2019-02-17 09:48:27 -08:00
|
|
|
u)
|
2019-02-21 15:35:26 -08:00
|
|
|
blockstreamer=true
|
2019-02-17 09:48:27 -08:00
|
|
|
;;
|
2019-04-01 15:54:41 -07:00
|
|
|
x)
|
|
|
|
externalNodes=true
|
|
|
|
;;
|
2018-08-28 10:19:33 -07:00
|
|
|
*)
|
2018-11-07 10:03:40 -08:00
|
|
|
usage "unhandled option: $opt"
|
2018-08-28 10:19:33 -07:00
|
|
|
;;
|
|
|
|
esac
|
|
|
|
done
|
|
|
|
|
2019-10-24 14:12:25 -07:00
|
|
|
if [[ -n "$customMachineType" ]] ; then
|
|
|
|
bootstrapLeaderMachineType="$customMachineType"
|
|
|
|
elif [[ "$enableGpu" = "true" ]] ; then
|
|
|
|
bootstrapLeaderMachineType="$gpuBootstrapLeaderMachineType"
|
|
|
|
else
|
|
|
|
bootstrapLeaderMachineType="$cpuBootstrapLeaderMachineType"
|
|
|
|
fi
|
|
|
|
validatorMachineType=$bootstrapLeaderMachineType
|
|
|
|
blockstreamerMachineType=$bootstrapLeaderMachineType
|
|
|
|
|
2019-03-27 15:42:08 -07:00
|
|
|
[[ ${#zones[@]} -gt 0 ]] || zones+=("$(cloud_DefaultZone)")
|
2019-03-27 15:19:55 -07:00
|
|
|
|
2018-09-06 20:57:05 -07:00
|
|
|
[[ -z $1 ]] || usage "Unexpected argument: $1"
|
2018-11-07 18:47:52 -08:00
|
|
|
if [[ $cloudProvider = ec2 ]]; then
|
|
|
|
# EC2 keys can't be retrieved from running instances like GCE keys can so save
|
|
|
|
# EC2 keys in the user's home directory so |./ec2.sh config| can at least be
|
|
|
|
# used on the same host that ran |./ec2.sh create| .
|
|
|
|
sshPrivateKey="$HOME/.ssh/solana-net-id_$prefix"
|
|
|
|
else
|
|
|
|
sshPrivateKey="$netConfigDir/id_$prefix"
|
|
|
|
fi
|
2018-08-28 10:19:33 -07:00
|
|
|
|
2018-11-07 10:03:40 -08:00
|
|
|
case $cloudProvider in
|
|
|
|
gce)
|
|
|
|
;;
|
2019-09-19 14:08:23 -07:00
|
|
|
ec2|azure|colo)
|
2019-10-21 20:21:21 -07:00
|
|
|
if [[ -n $validatorAdditionalDiskSizeInGb ]] ; then
|
|
|
|
usage "Error: --validator-additional-disk-size-gb currently only supported with cloud provider: gce"
|
2019-07-11 15:23:32 -07:00
|
|
|
fi
|
2019-04-23 15:41:45 -07:00
|
|
|
;;
|
2018-11-07 10:03:40 -08:00
|
|
|
*)
|
|
|
|
echo "Error: Unknown cloud provider: $cloudProvider"
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
|
2018-09-16 14:46:08 -07:00
|
|
|
# cloud_ForEachInstance [cmd] [extra args to cmd]
|
|
|
|
#
|
|
|
|
# Execute a command for each element in the `instances` array
|
|
|
|
#
|
|
|
|
# cmd - The command to execute on each instance
|
|
|
|
# The command will receive arguments followed by any
|
2019-04-23 15:41:45 -07:00
|
|
|
# additional arguments supplied to cloud_ForEachInstance:
|
2018-09-16 14:46:08 -07:00
|
|
|
# name - name of the instance
|
|
|
|
# publicIp - The public IP address of this instance
|
2019-04-23 15:41:45 -07:00
|
|
|
# privateIp - The private IP address of this instance
|
2019-04-28 19:50:02 -07:00
|
|
|
# zone - Zone of this instance
|
2018-09-16 14:46:08 -07:00
|
|
|
# count - Monotonically increasing count for each
|
|
|
|
# invocation of cmd, starting at 1
|
|
|
|
# ... - Extra args to cmd..
|
|
|
|
#
|
|
|
|
#
|
|
|
|
cloud_ForEachInstance() {
|
|
|
|
declare cmd="$1"
|
|
|
|
shift
|
|
|
|
[[ -n $cmd ]] || { echo cloud_ForEachInstance: cmd not specified; exit 1; }
|
|
|
|
|
|
|
|
declare count=1
|
|
|
|
for info in "${instances[@]}"; do
|
|
|
|
declare name publicIp privateIp
|
2019-03-27 16:16:30 -07:00
|
|
|
IFS=: read -r name publicIp privateIp zone < <(echo "$info")
|
2018-09-16 14:46:08 -07:00
|
|
|
|
2019-04-28 19:50:02 -07:00
|
|
|
eval "$cmd" "$name" "$publicIp" "$privateIp" "$zone" "$count" "$@"
|
2018-09-16 14:46:08 -07:00
|
|
|
count=$((count + 1))
|
|
|
|
done
|
|
|
|
}
|
|
|
|
|
2019-04-28 19:50:02 -07:00
|
|
|
# Given a cloud provider zone, return an approximate lat,long location for the
|
|
|
|
# data center. Normal geoip lookups for cloud provider IP addresses are
|
|
|
|
# sometimes widely inaccurate.
|
|
|
|
zoneLocation() {
|
|
|
|
declare zone="$1"
|
|
|
|
case "$zone" in
|
|
|
|
us-west1*)
|
|
|
|
echo "[45.5946, -121.1787]"
|
|
|
|
;;
|
|
|
|
us-central1*)
|
|
|
|
echo "[41.2619, -95.8608]"
|
|
|
|
;;
|
|
|
|
us-east1*)
|
|
|
|
echo "[33.1960, -80.0131]"
|
|
|
|
;;
|
|
|
|
asia-east2*)
|
|
|
|
echo "[22.3193, 114.1694]"
|
|
|
|
;;
|
|
|
|
asia-northeast1*)
|
|
|
|
echo "[35.6762, 139.6503]"
|
|
|
|
;;
|
|
|
|
asia-northeast2*)
|
|
|
|
echo "[34.6937, 135.5023]"
|
|
|
|
;;
|
|
|
|
asia-south1*)
|
|
|
|
echo "[19.0760, 72.8777]"
|
|
|
|
;;
|
|
|
|
asia-southeast1*)
|
|
|
|
echo "[1.3404, 103.7090]"
|
|
|
|
;;
|
|
|
|
australia-southeast1*)
|
|
|
|
echo "[-33.8688, 151.2093]"
|
|
|
|
;;
|
|
|
|
europe-north1*)
|
|
|
|
echo "[60.5693, 27.1878]"
|
|
|
|
;;
|
|
|
|
europe-west2*)
|
|
|
|
echo "[51.5074, -0.1278]"
|
|
|
|
;;
|
|
|
|
europe-west3*)
|
|
|
|
echo "[50.1109, 8.6821]"
|
|
|
|
;;
|
|
|
|
europe-west4*)
|
|
|
|
echo "[53.4386, 6.8355]"
|
|
|
|
;;
|
|
|
|
europe-west6*)
|
|
|
|
echo "[47.3769, 8.5417]"
|
|
|
|
;;
|
|
|
|
northamerica-northeast1*)
|
|
|
|
echo "[45.5017, -73.5673]"
|
|
|
|
;;
|
|
|
|
southamerica-east1*)
|
|
|
|
echo "[-23.5505, -46.6333]"
|
|
|
|
;;
|
|
|
|
*)
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
}
|
|
|
|
|
2018-09-03 21:15:55 -07:00
|
|
|
prepareInstancesAndWriteConfigFile() {
|
2018-09-06 13:00:01 -07:00
|
|
|
$metricsWriteDatapoint "testnet-deploy net-config-begin=1"
|
2018-08-28 10:19:33 -07:00
|
|
|
|
2019-04-01 15:54:41 -07:00
|
|
|
if $externalNodes; then
|
|
|
|
echo "Appending to existing config file"
|
|
|
|
echo "externalNodeSshKey=$sshPrivateKey" >> "$configFile"
|
|
|
|
else
|
2019-04-28 19:50:02 -07:00
|
|
|
rm -f "$geoipConfigFile"
|
2019-04-01 15:54:41 -07:00
|
|
|
cat >> "$configFile" <<EOF
|
2018-09-04 22:21:58 -07:00
|
|
|
# autogenerated at $(date)
|
|
|
|
netBasename=$prefix
|
|
|
|
publicNetwork=$publicNetwork
|
2018-09-08 19:19:12 -07:00
|
|
|
sshPrivateKey=$sshPrivateKey
|
2019-07-09 15:45:46 -07:00
|
|
|
letsEncryptDomainName=$letsEncryptDomainName
|
2018-09-04 22:21:58 -07:00
|
|
|
EOF
|
2019-04-01 15:54:41 -07:00
|
|
|
fi
|
2019-04-28 19:50:02 -07:00
|
|
|
touch "$geoipConfigFile"
|
2018-09-03 21:15:55 -07:00
|
|
|
|
2018-09-08 19:19:12 -07:00
|
|
|
buildSshOptions
|
2018-08-28 10:19:33 -07:00
|
|
|
|
2019-10-14 23:11:29 -07:00
|
|
|
cloud_RestartPreemptedInstances "$prefix"
|
|
|
|
|
2019-05-11 22:54:50 -07:00
|
|
|
fetchPrivateKey() {
|
|
|
|
declare nodeName
|
|
|
|
declare nodeIp
|
|
|
|
declare nodeZone
|
|
|
|
IFS=: read -r nodeName nodeIp _ nodeZone < <(echo "${instances[0]}")
|
|
|
|
|
|
|
|
# Make sure the machine is alive or pingable
|
|
|
|
timeout_sec=90
|
|
|
|
cloud_WaitForInstanceReady "$nodeName" "$nodeIp" "$nodeZone" "$timeout_sec"
|
|
|
|
|
|
|
|
if [[ ! -r $sshPrivateKey ]]; then
|
|
|
|
echo "Fetching $sshPrivateKey from $nodeName"
|
|
|
|
|
|
|
|
# Try to scp in a couple times, sshd may not yet be up even though the
|
|
|
|
# machine can be pinged...
|
|
|
|
(
|
|
|
|
set -o pipefail
|
2019-05-17 20:06:07 -07:00
|
|
|
for i in $(seq 1 60); do
|
2019-05-11 22:54:50 -07:00
|
|
|
set -x
|
2019-09-03 17:51:16 -07:00
|
|
|
cloud_FetchFile "$nodeName" "$nodeIp" /solana-scratch/id_ecdsa "$sshPrivateKey" "$nodeZone" &&
|
|
|
|
cloud_FetchFile "$nodeName" "$nodeIp" /solana-scratch/id_ecdsa.pub "$sshPrivateKey.pub" "$nodeZone" &&
|
2019-05-11 22:54:50 -07:00
|
|
|
break
|
|
|
|
set +x
|
|
|
|
|
|
|
|
sleep 1
|
|
|
|
echo "Retry $i..."
|
|
|
|
done
|
|
|
|
)
|
|
|
|
|
|
|
|
chmod 400 "$sshPrivateKey"
|
|
|
|
ls -l "$sshPrivateKey"
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2018-08-28 10:19:33 -07:00
|
|
|
recordInstanceIp() {
|
|
|
|
declare name="$1"
|
2018-09-16 14:46:08 -07:00
|
|
|
declare publicIp="$2"
|
|
|
|
declare privateIp="$3"
|
2019-04-28 19:50:02 -07:00
|
|
|
declare zone="$4"
|
|
|
|
#declare index="$5"
|
2018-08-28 10:19:33 -07:00
|
|
|
|
2019-04-28 19:50:02 -07:00
|
|
|
declare failOnFailure="$6"
|
|
|
|
declare arrayName="$7"
|
2018-08-28 10:19:33 -07:00
|
|
|
|
2019-04-24 16:23:26 -07:00
|
|
|
if [ "$publicIp" = "TERMINATED" ] || [ "$privateIp" = "TERMINATED" ]; then
|
|
|
|
if $failOnFailure; then
|
|
|
|
exit 1
|
|
|
|
else
|
|
|
|
return 0
|
|
|
|
fi
|
|
|
|
fi
|
2018-09-08 19:19:12 -07:00
|
|
|
|
2019-04-24 16:23:26 -07:00
|
|
|
ok=true
|
2018-09-08 19:19:12 -07:00
|
|
|
echo "Waiting for $name to finish booting..."
|
|
|
|
(
|
2019-05-11 22:54:50 -07:00
|
|
|
set +e
|
|
|
|
fetchPrivateKey || exit 1
|
2019-05-17 20:06:07 -07:00
|
|
|
for i in $(seq 1 60); do
|
2019-05-11 22:54:50 -07:00
|
|
|
(
|
|
|
|
set -x
|
2019-09-03 17:51:16 -07:00
|
|
|
timeout --preserve-status --foreground 20s ssh "${sshOptions[@]}" "$publicIp" "ls -l /solana-scratch/.instance-startup-complete"
|
2019-05-11 22:54:50 -07:00
|
|
|
)
|
2018-11-07 15:11:47 -08:00
|
|
|
ret=$?
|
|
|
|
if [[ $ret -eq 0 ]]; then
|
2019-04-24 16:23:26 -07:00
|
|
|
echo "$name has booted."
|
2018-11-06 19:23:20 -08:00
|
|
|
exit 0
|
2018-09-08 19:19:12 -07:00
|
|
|
fi
|
2019-05-11 22:54:50 -07:00
|
|
|
sleep 5
|
2018-09-08 19:19:12 -07:00
|
|
|
echo "Retry $i..."
|
|
|
|
done
|
2018-11-06 19:23:20 -08:00
|
|
|
echo "$name failed to boot."
|
|
|
|
exit 1
|
2019-04-24 16:23:26 -07:00
|
|
|
) || ok=false
|
|
|
|
|
|
|
|
if ! $ok; then
|
|
|
|
if $failOnFailure; then
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
else
|
2019-04-28 19:50:02 -07:00
|
|
|
{
|
|
|
|
echo "$arrayName+=($publicIp) # $name"
|
|
|
|
echo "${arrayName}Private+=($privateIp) # $name"
|
|
|
|
echo "${arrayName}Zone+=($zone) # $name"
|
|
|
|
} >> "$configFile"
|
|
|
|
|
|
|
|
declare latlng=
|
|
|
|
latlng=$(zoneLocation "$zone")
|
|
|
|
if [[ -n $latlng ]]; then
|
|
|
|
echo "$publicIp: $latlng" >> "$geoipConfigFile"
|
|
|
|
fi
|
2019-04-24 16:23:26 -07:00
|
|
|
fi
|
2018-09-08 19:19:12 -07:00
|
|
|
}
|
|
|
|
|
2019-04-28 19:50:02 -07:00
|
|
|
if $externalNodes; then
|
|
|
|
echo "Bootstrap leader is already configured"
|
|
|
|
else
|
|
|
|
echo "Looking for bootstrap leader instance..."
|
|
|
|
cloud_FindInstance "$prefix-bootstrap-leader"
|
|
|
|
[[ ${#instances[@]} -eq 1 ]] || {
|
|
|
|
echo "Unable to find bootstrap leader"
|
|
|
|
exit 1
|
|
|
|
}
|
|
|
|
|
2019-10-21 20:21:21 -07:00
|
|
|
echo "validatorIpList=()" >> "$configFile"
|
|
|
|
echo "validatorIpListPrivate=()" >> "$configFile"
|
|
|
|
cloud_ForEachInstance recordInstanceIp true validatorIpList
|
2019-04-01 15:54:41 -07:00
|
|
|
fi
|
2018-08-28 10:19:33 -07:00
|
|
|
|
2019-10-21 20:21:21 -07:00
|
|
|
if [[ $additionalValidatorCount -gt 0 ]]; then
|
2019-10-10 17:13:59 -07:00
|
|
|
numZones=${#zones[@]}
|
2019-10-21 20:21:21 -07:00
|
|
|
if [[ $additionalValidatorCount -gt $numZones ]]; then
|
|
|
|
numNodesPerZone=$((additionalValidatorCount / numZones))
|
|
|
|
numLeftOverNodes=$((additionalValidatorCount % numZones))
|
2019-10-10 17:13:59 -07:00
|
|
|
else
|
|
|
|
numNodesPerZone=1
|
|
|
|
numLeftOverNodes=0
|
|
|
|
fi
|
|
|
|
|
|
|
|
for ((i=((numZones - 1)); i >= 0; i--)); do
|
|
|
|
zone=${zones[i]}
|
|
|
|
if [[ $i -eq 0 ]]; then
|
|
|
|
numNodesPerZone=$((numNodesPerZone + numLeftOverNodes))
|
|
|
|
fi
|
2019-10-21 20:21:21 -07:00
|
|
|
echo "Looking for additional validator instances in $zone ..."
|
|
|
|
cloud_FindInstances "$prefix-$zone-validator"
|
2019-10-10 17:13:59 -07:00
|
|
|
declare numInstances=${#instances[@]}
|
2019-10-16 13:18:39 -07:00
|
|
|
if [[ $numInstances -ge $numNodesPerZone || ( ! $failOnValidatorBootupFailure && $numInstances -gt 0 ) ]]; then
|
2019-10-21 20:21:21 -07:00
|
|
|
cloud_ForEachInstance recordInstanceIp "$failOnValidatorBootupFailure" validatorIpList
|
2019-04-29 21:38:03 -07:00
|
|
|
else
|
2019-10-21 20:21:21 -07:00
|
|
|
echo "Unable to find additional validators"
|
2019-04-29 21:38:03 -07:00
|
|
|
if $failOnValidatorBootupFailure; then
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
fi
|
2019-04-17 17:38:18 -07:00
|
|
|
done
|
|
|
|
fi
|
2018-08-28 10:19:33 -07:00
|
|
|
|
2019-05-31 15:27:31 -07:00
|
|
|
if ! $externalNodes; then
|
2019-04-01 15:54:41 -07:00
|
|
|
echo "clientIpList=()" >> "$configFile"
|
|
|
|
echo "clientIpListPrivate=()" >> "$configFile"
|
|
|
|
fi
|
2018-12-05 16:40:08 -08:00
|
|
|
echo "Looking for client bencher instances..."
|
2018-09-16 14:46:08 -07:00
|
|
|
cloud_FindInstances "$prefix-client"
|
2018-09-04 22:46:55 -07:00
|
|
|
[[ ${#instances[@]} -eq 0 ]] || {
|
2019-04-24 16:23:26 -07:00
|
|
|
cloud_ForEachInstance recordInstanceIp true clientIpList
|
2018-09-04 22:46:55 -07:00
|
|
|
}
|
2018-08-28 10:19:33 -07:00
|
|
|
|
2019-05-31 15:27:31 -07:00
|
|
|
if ! $externalNodes; then
|
2019-04-01 15:54:41 -07:00
|
|
|
echo "blockstreamerIpList=()" >> "$configFile"
|
|
|
|
echo "blockstreamerIpListPrivate=()" >> "$configFile"
|
|
|
|
fi
|
2019-02-21 15:35:26 -08:00
|
|
|
echo "Looking for blockstreamer instances..."
|
|
|
|
cloud_FindInstances "$prefix-blockstreamer"
|
2019-02-17 09:48:27 -08:00
|
|
|
[[ ${#instances[@]} -eq 0 ]] || {
|
2019-04-24 16:23:26 -07:00
|
|
|
cloud_ForEachInstance recordInstanceIp true blockstreamerIpList
|
2019-02-17 09:48:27 -08:00
|
|
|
}
|
|
|
|
|
2019-05-31 15:27:31 -07:00
|
|
|
if ! $externalNodes; then
|
2019-10-21 10:29:37 -07:00
|
|
|
echo "archiverIpList=()" >> "$configFile"
|
|
|
|
echo "archiverIpListPrivate=()" >> "$configFile"
|
2019-05-31 15:27:31 -07:00
|
|
|
fi
|
2019-10-21 10:29:37 -07:00
|
|
|
echo "Looking for archiver instances..."
|
|
|
|
cloud_FindInstances "$prefix-archiver"
|
2019-05-31 15:27:31 -07:00
|
|
|
[[ ${#instances[@]} -eq 0 ]] || {
|
2019-10-21 10:29:37 -07:00
|
|
|
cloud_ForEachInstance recordInstanceIp true archiverIpList
|
2019-05-31 15:27:31 -07:00
|
|
|
}
|
|
|
|
|
2018-08-28 10:19:33 -07:00
|
|
|
echo "Wrote $configFile"
|
2018-09-06 12:14:04 -07:00
|
|
|
$metricsWriteDatapoint "testnet-deploy net-config-complete=1"
|
2018-08-28 10:19:33 -07:00
|
|
|
}
|
|
|
|
|
2018-09-16 14:46:08 -07:00
|
|
|
delete() {
|
2018-09-06 13:00:01 -07:00
|
|
|
$metricsWriteDatapoint "testnet-deploy net-delete-begin=1"
|
2018-09-06 12:14:04 -07:00
|
|
|
|
2019-05-15 15:16:45 -07:00
|
|
|
# Filter for all nodes
|
|
|
|
filter="$prefix-"
|
2019-03-27 15:19:55 -07:00
|
|
|
|
2019-05-15 15:16:45 -07:00
|
|
|
echo "Searching for instances: $filter"
|
|
|
|
cloud_FindInstances "$filter"
|
2018-09-07 08:56:43 -07:00
|
|
|
|
2019-05-15 15:16:45 -07:00
|
|
|
if [[ ${#instances[@]} -eq 0 ]]; then
|
|
|
|
echo "No instances found matching '$filter'"
|
|
|
|
else
|
|
|
|
cloud_DeleteInstances true &
|
|
|
|
fi
|
2019-04-19 17:46:14 -07:00
|
|
|
|
|
|
|
wait
|
|
|
|
|
2019-04-01 15:54:41 -07:00
|
|
|
if $externalNodes; then
|
|
|
|
echo "Let's not delete the current configuration file"
|
|
|
|
else
|
|
|
|
rm -f "$configFile"
|
|
|
|
fi
|
2018-09-06 12:14:04 -07:00
|
|
|
|
|
|
|
$metricsWriteDatapoint "testnet-deploy net-delete-complete=1"
|
2018-09-16 14:46:08 -07:00
|
|
|
}
|
|
|
|
|
2019-10-14 09:36:20 -07:00
|
|
|
create_error_cleanup() {
|
|
|
|
declare RC=$?
|
|
|
|
if [[ "$RC" -ne 0 ]]; then
|
|
|
|
delete
|
|
|
|
fi
|
|
|
|
exit $RC
|
|
|
|
}
|
|
|
|
|
2018-09-16 14:46:08 -07:00
|
|
|
case $command in
|
|
|
|
delete)
|
|
|
|
delete
|
2018-08-28 10:19:33 -07:00
|
|
|
;;
|
|
|
|
|
|
|
|
create)
|
2019-10-21 20:21:21 -07:00
|
|
|
[[ -n $additionalValidatorCount ]] || usage "Need number of nodes"
|
2018-09-16 14:46:08 -07:00
|
|
|
|
|
|
|
delete
|
2018-08-28 10:19:33 -07:00
|
|
|
|
2018-09-06 13:00:01 -07:00
|
|
|
$metricsWriteDatapoint "testnet-deploy net-create-begin=1"
|
2018-09-06 12:14:04 -07:00
|
|
|
|
2019-10-14 09:36:20 -07:00
|
|
|
if $failOnValidatorBootupFailure; then
|
|
|
|
trap create_error_cleanup EXIT
|
|
|
|
fi
|
|
|
|
|
2018-09-08 19:19:12 -07:00
|
|
|
rm -rf "$sshPrivateKey"{,.pub}
|
2018-09-16 14:46:08 -07:00
|
|
|
|
|
|
|
# Note: using rsa because |aws ec2 import-key-pair| seems to fail for ecdsa
|
|
|
|
ssh-keygen -t rsa -N '' -f "$sshPrivateKey"
|
2018-09-08 19:19:12 -07:00
|
|
|
|
2018-09-08 10:38:32 -07:00
|
|
|
printNetworkInfo() {
|
|
|
|
cat <<EOF
|
2019-05-26 11:17:07 -07:00
|
|
|
==[ Network composition ]===============================================================
|
2018-12-09 09:42:09 -08:00
|
|
|
Bootstrap leader = $bootstrapLeaderMachineType (GPU=$enableGpu)
|
2019-10-21 20:21:21 -07:00
|
|
|
Additional validators = $additionalValidatorCount x $validatorMachineType
|
2018-09-16 14:46:08 -07:00
|
|
|
Client(s) = $clientNodeCount x $clientMachineType
|
2019-10-21 10:29:37 -07:00
|
|
|
Archivers(s) = $archiverNodeCount x $archiverMachineType
|
2019-02-21 15:35:26 -08:00
|
|
|
Blockstreamer = $blockstreamer
|
2018-09-08 10:38:32 -07:00
|
|
|
========================================================================================
|
|
|
|
|
|
|
|
EOF
|
|
|
|
}
|
|
|
|
printNetworkInfo
|
|
|
|
|
2019-05-26 11:17:07 -07:00
|
|
|
creationDate=$(date)
|
|
|
|
creationInfo() {
|
|
|
|
cat <<EOF
|
|
|
|
|
|
|
|
Instance running since: $creationDate
|
|
|
|
|
|
|
|
========================================================================================
|
|
|
|
EOF
|
|
|
|
}
|
|
|
|
|
2018-09-16 14:46:08 -07:00
|
|
|
declare startupScript="$netConfigDir"/instance-startup-script.sh
|
2018-09-08 10:38:32 -07:00
|
|
|
cat > "$startupScript" <<EOF
|
2018-11-11 08:19:04 -08:00
|
|
|
#!/usr/bin/env bash
|
2018-09-08 10:38:32 -07:00
|
|
|
# autogenerated at $(date)
|
2018-11-11 09:25:59 -08:00
|
|
|
set -ex
|
2018-09-08 10:38:32 -07:00
|
|
|
|
2019-09-10 12:30:06 -07:00
|
|
|
if [[ -f /solana-scratch/.instance-startup-complete ]]; then
|
2019-10-27 10:56:16 -07:00
|
|
|
echo reboot
|
2019-09-21 10:45:16 -07:00
|
|
|
$(
|
|
|
|
cd "$here"/scripts/
|
|
|
|
if "$enableGpu"; then
|
|
|
|
cat enable-nvidia-persistence-mode.sh
|
|
|
|
fi
|
2019-10-24 20:14:26 -07:00
|
|
|
|
|
|
|
if [[ -n $validatorAdditionalDiskSizeInGb ]]; then
|
|
|
|
cat mount-additional-disk.sh
|
|
|
|
fi
|
2019-10-28 23:16:55 -07:00
|
|
|
|
|
|
|
cat ../../scripts/ulimit-n.sh
|
2019-09-21 10:45:16 -07:00
|
|
|
)
|
2019-10-27 10:56:16 -07:00
|
|
|
if [[ -x ~solana/solana/on-reboot ]]; then
|
|
|
|
sudo -u solana ~solana/solana/on-reboot
|
|
|
|
fi
|
2019-09-21 10:45:16 -07:00
|
|
|
|
|
|
|
# Skip most setup on instance reboot
|
2019-09-10 12:30:06 -07:00
|
|
|
exit 0
|
|
|
|
fi
|
|
|
|
|
2018-09-08 10:38:32 -07:00
|
|
|
cat > /etc/motd <<EOM
|
|
|
|
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
|
|
|
|
|
|
This instance has not been fully configured.
|
2018-09-16 14:46:08 -07:00
|
|
|
|
|
|
|
See startup script log messages in /var/log/syslog for status:
|
|
|
|
$ sudo cat /var/log/syslog | egrep \\(startup-script\\|cloud-init\)
|
2018-09-08 10:38:32 -07:00
|
|
|
|
2018-09-08 19:19:12 -07:00
|
|
|
To block until setup is complete, run:
|
2019-09-03 17:51:16 -07:00
|
|
|
$ until [[ -f /solana-scratch/.instance-startup-complete ]]; do sleep 1; done
|
2018-09-08 19:19:12 -07:00
|
|
|
|
2018-09-08 10:38:32 -07:00
|
|
|
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
2019-05-26 11:17:07 -07:00
|
|
|
$(creationInfo)
|
2018-09-08 10:38:32 -07:00
|
|
|
EOM
|
|
|
|
|
2019-09-03 17:51:16 -07:00
|
|
|
# Place the generated private key at /solana-scratch/id_ecdsa so it's retrievable by anybody
|
2018-09-08 19:19:12 -07:00
|
|
|
# who is able to log into this machine
|
2019-09-12 16:11:13 -07:00
|
|
|
mkdir -m 0777 /solana-scratch
|
2019-09-03 17:51:16 -07:00
|
|
|
cat > /solana-scratch/id_ecdsa <<EOK
|
2018-09-08 19:19:12 -07:00
|
|
|
$(cat "$sshPrivateKey")
|
|
|
|
EOK
|
2019-09-03 17:51:16 -07:00
|
|
|
cat > /solana-scratch/id_ecdsa.pub <<EOK
|
2018-09-08 19:19:12 -07:00
|
|
|
$(cat "$sshPrivateKey.pub")
|
|
|
|
EOK
|
2019-09-03 17:51:16 -07:00
|
|
|
chmod 444 /solana-scratch/id_ecdsa
|
2018-09-08 19:19:12 -07:00
|
|
|
|
2018-09-08 10:38:32 -07:00
|
|
|
USER=\$(id -un)
|
2019-06-14 08:00:12 -07:00
|
|
|
export DEBIAN_FRONTEND=noninteractive
|
2018-09-08 10:38:32 -07:00
|
|
|
$(
|
|
|
|
cd "$here"/scripts/
|
|
|
|
cat \
|
|
|
|
disable-background-upgrades.sh \
|
2018-09-08 19:19:12 -07:00
|
|
|
create-solana-user.sh \
|
2019-08-15 08:32:10 -07:00
|
|
|
solana-user-authorized_keys.sh \
|
2019-08-14 13:25:56 -07:00
|
|
|
add-testnet-solana-user-authorized_keys.sh \
|
2019-10-30 16:43:16 -07:00
|
|
|
install-ag.sh \
|
2019-07-09 15:45:46 -07:00
|
|
|
install-certbot.sh \
|
2018-09-08 10:38:32 -07:00
|
|
|
install-earlyoom.sh \
|
2019-10-26 00:06:46 -07:00
|
|
|
install-iftop.sh \
|
2018-09-08 10:38:32 -07:00
|
|
|
install-libssl-compatability.sh \
|
2019-02-15 20:17:30 -08:00
|
|
|
install-nodejs.sh \
|
|
|
|
install-redis.sh \
|
2018-09-12 16:22:22 -07:00
|
|
|
install-rsync.sh \
|
2019-05-26 11:17:07 -07:00
|
|
|
localtime.sh \
|
2018-10-30 09:17:35 -07:00
|
|
|
network-config.sh \
|
2018-11-07 17:42:29 -08:00
|
|
|
remove-docker-interface.sh \
|
2018-11-06 19:23:20 -08:00
|
|
|
|
2019-09-21 10:45:16 -07:00
|
|
|
if "$enableGpu"; then
|
|
|
|
cat enable-nvidia-persistence-mode.sh
|
|
|
|
fi
|
2019-07-11 15:23:32 -07:00
|
|
|
|
2019-10-21 20:21:21 -07:00
|
|
|
if [[ -n $validatorAdditionalDiskSizeInGb ]]; then
|
2019-09-21 10:45:16 -07:00
|
|
|
cat mount-additional-disk.sh
|
|
|
|
fi
|
2019-11-04 10:30:26 -08:00
|
|
|
|
|
|
|
if [[ $selfDestructHours -gt 0 ]]; then
|
|
|
|
cat <<EOSD
|
|
|
|
|
|
|
|
# Setup GCE self-destruct
|
|
|
|
cat >/solana-scratch/gce-self-destruct.sh <<'EOS'
|
|
|
|
$(cat gce-self-destruct.sh)
|
|
|
|
EOS
|
|
|
|
EOSD
|
|
|
|
cat <<'EOSD'
|
|
|
|
|
|
|
|
# Populate terminal prompt update script
|
|
|
|
cat >/solana-scratch/gce-self-destruct-ps1.sh <<'EOS'
|
|
|
|
#!/usr/bin/env bash
|
|
|
|
source "$(dirname "$0")/gce-self-destruct.sh"
|
|
|
|
gce_self_destruct_ps1
|
|
|
|
EOS
|
|
|
|
chmod +x /solana-scratch/gce-self-destruct-ps1.sh
|
|
|
|
|
|
|
|
# Append MOTD and PS1 replacement to .profile
|
|
|
|
cat >>~solana/.profile <<'EOS'
|
|
|
|
|
|
|
|
# Print self-destruct countdown on login
|
|
|
|
source "/solana-scratch/gce-self-destruct.sh"
|
|
|
|
gce_self_destruct_motd
|
|
|
|
|
|
|
|
# Add self-destruct countdown to terminal prompt
|
|
|
|
export PS1='\[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]$(/solana-scratch/gce-self-destruct-ps1.sh):\[\033[01;34m\]\w\[\033[00m\]\$ '
|
|
|
|
EOS
|
|
|
|
EOSD
|
|
|
|
cat <<EOSD
|
|
|
|
|
|
|
|
source /solana-scratch/gce-self-destruct.sh
|
|
|
|
gce_self_destruct_setup $selfDestructHours
|
|
|
|
EOSD
|
|
|
|
fi
|
2018-09-08 10:38:32 -07:00
|
|
|
)
|
|
|
|
|
|
|
|
cat > /etc/motd <<EOM
|
2019-10-27 10:56:16 -07:00
|
|
|
See startup script log messages in /var/log/syslog for status:
|
|
|
|
$ sudo cat /var/log/syslog | egrep \\(startup-script\\|cloud-init\)
|
2018-09-08 10:38:32 -07:00
|
|
|
$(printNetworkInfo)
|
2019-05-26 11:17:07 -07:00
|
|
|
$(creationInfo)
|
2018-09-08 10:38:32 -07:00
|
|
|
EOM
|
|
|
|
|
2019-09-03 17:51:16 -07:00
|
|
|
touch /solana-scratch/.instance-startup-complete
|
2018-09-08 19:19:12 -07:00
|
|
|
|
2018-09-08 10:38:32 -07:00
|
|
|
EOF
|
|
|
|
|
2019-02-21 15:35:26 -08:00
|
|
|
if $blockstreamer; then
|
|
|
|
blockstreamerAddress=$customAddress
|
2019-02-17 09:48:27 -08:00
|
|
|
else
|
|
|
|
bootstrapLeaderAddress=$customAddress
|
|
|
|
fi
|
|
|
|
|
2019-03-27 15:19:55 -07:00
|
|
|
for zone in "${zones[@]}"; do
|
|
|
|
cloud_Initialize "$prefix" "$zone"
|
|
|
|
done
|
2019-02-18 17:43:03 -08:00
|
|
|
|
2019-04-01 15:54:41 -07:00
|
|
|
if $externalNodes; then
|
|
|
|
echo "Bootstrap leader is already configured"
|
|
|
|
else
|
|
|
|
cloud_CreateInstances "$prefix" "$prefix-bootstrap-leader" 1 \
|
2019-10-21 20:21:21 -07:00
|
|
|
"$enableGpu" "$bootstrapLeaderMachineType" "${zones[0]}" "$validatorBootDiskSizeInGb" \
|
|
|
|
"$startupScript" "$bootstrapLeaderAddress" "$bootDiskType" "$validatorAdditionalDiskSizeInGb" \
|
2019-10-14 23:11:29 -07:00
|
|
|
"never preemptible" "$sshPrivateKey"
|
2019-04-01 15:54:41 -07:00
|
|
|
fi
|
2018-09-05 09:31:50 -07:00
|
|
|
|
2019-10-21 20:21:21 -07:00
|
|
|
if [[ $additionalValidatorCount -gt 0 ]]; then
|
2019-04-17 17:38:18 -07:00
|
|
|
num_zones=${#zones[@]}
|
2019-10-21 20:21:21 -07:00
|
|
|
if [[ $additionalValidatorCount -gt $num_zones ]]; then
|
|
|
|
numNodesPerZone=$((additionalValidatorCount / num_zones))
|
|
|
|
numLeftOverNodes=$((additionalValidatorCount % num_zones))
|
2019-05-11 14:47:27 -07:00
|
|
|
else
|
|
|
|
numNodesPerZone=1
|
|
|
|
numLeftOverNodes=0
|
|
|
|
fi
|
2019-05-11 13:48:16 -07:00
|
|
|
|
|
|
|
for ((i=((num_zones - 1)); i >= 0; i--)); do
|
2019-04-17 17:38:18 -07:00
|
|
|
zone=${zones[i]}
|
2019-05-11 13:48:16 -07:00
|
|
|
if [[ $i -eq 0 ]]; then
|
2019-04-17 17:38:18 -07:00
|
|
|
numNodesPerZone=$((numNodesPerZone + numLeftOverNodes))
|
|
|
|
fi
|
2019-10-21 20:21:21 -07:00
|
|
|
cloud_CreateInstances "$prefix" "$prefix-$zone-validator" "$numNodesPerZone" \
|
|
|
|
"$enableGpu" "$validatorMachineType" "$zone" "$validatorBootDiskSizeInGb" \
|
|
|
|
"$startupScript" "" "$bootDiskType" "$validatorAdditionalDiskSizeInGb" \
|
2019-10-14 23:11:29 -07:00
|
|
|
"$preemptible" "$sshPrivateKey" &
|
2019-04-17 17:38:18 -07:00
|
|
|
done
|
2018-09-08 10:38:32 -07:00
|
|
|
|
2019-04-17 17:38:18 -07:00
|
|
|
wait
|
|
|
|
fi
|
2019-04-07 08:13:48 -07:00
|
|
|
|
2018-09-06 10:34:24 -07:00
|
|
|
if [[ $clientNodeCount -gt 0 ]]; then
|
2018-09-16 14:46:08 -07:00
|
|
|
cloud_CreateInstances "$prefix" "$prefix-client" "$clientNodeCount" \
|
2019-03-27 15:19:55 -07:00
|
|
|
"$enableGpu" "$clientMachineType" "${zones[0]}" "$clientBootDiskSizeInGb" \
|
2019-10-14 23:11:29 -07:00
|
|
|
"$startupScript" "" "$bootDiskType" "" "never preemptible" "$sshPrivateKey"
|
2018-08-28 10:19:33 -07:00
|
|
|
fi
|
2018-08-29 08:13:35 -07:00
|
|
|
|
2019-02-21 15:35:26 -08:00
|
|
|
if $blockstreamer; then
|
|
|
|
cloud_CreateInstances "$prefix" "$prefix-blockstreamer" "1" \
|
2019-10-21 20:21:21 -07:00
|
|
|
"$enableGpu" "$blockstreamerMachineType" "${zones[0]}" "$validatorBootDiskSizeInGb" \
|
2019-09-19 14:08:23 -07:00
|
|
|
"$startupScript" "$blockstreamerAddress" "$bootDiskType" "" "$sshPrivateKey"
|
2019-02-17 09:48:27 -08:00
|
|
|
fi
|
|
|
|
|
2019-10-21 10:29:37 -07:00
|
|
|
if [[ $archiverNodeCount -gt 0 ]]; then
|
|
|
|
cloud_CreateInstances "$prefix" "$prefix-archiver" "$archiverNodeCount" \
|
|
|
|
false "$archiverMachineType" "${zones[0]}" "$archiverBootDiskSizeInGb" \
|
2019-10-14 23:11:29 -07:00
|
|
|
"$startupScript" "" "" "" "never preemptible" "$sshPrivateKey"
|
2019-05-31 15:27:31 -07:00
|
|
|
fi
|
|
|
|
|
2018-09-06 12:14:04 -07:00
|
|
|
$metricsWriteDatapoint "testnet-deploy net-create-complete=1"
|
|
|
|
|
2018-09-03 21:15:55 -07:00
|
|
|
prepareInstancesAndWriteConfigFile
|
2018-08-28 10:19:33 -07:00
|
|
|
;;
|
|
|
|
|
|
|
|
config)
|
2018-09-03 21:15:55 -07:00
|
|
|
prepareInstancesAndWriteConfigFile
|
2018-08-28 10:19:33 -07:00
|
|
|
;;
|
2019-01-16 09:38:29 -08:00
|
|
|
info)
|
|
|
|
loadConfigFile
|
|
|
|
printNode() {
|
|
|
|
declare nodeType=$1
|
|
|
|
declare ip=$2
|
|
|
|
declare ipPrivate=$3
|
2019-04-28 19:50:02 -07:00
|
|
|
declare zone=$4
|
|
|
|
printf " %-16s | %-15s | %-15s | %s\n" "$nodeType" "$ip" "$ipPrivate" "$zone"
|
2019-01-16 09:38:29 -08:00
|
|
|
}
|
|
|
|
|
2019-10-25 22:57:37 -07:00
|
|
|
if $evalInfo; then
|
|
|
|
echo "NET_NUM_VALIDATORS=${#validatorIpList[@]}"
|
|
|
|
echo "NET_NUM_CLIENTS=${#clientIpList[@]}"
|
|
|
|
echo "NET_NUM_BLOCKSTREAMERS=${#blockstreamerIpList[@]}"
|
|
|
|
echo "NET_NUM_ARCHIVERS=${#archiverIpList[@]}"
|
|
|
|
else
|
2019-10-22 16:08:49 -07:00
|
|
|
printNode "Node Type" "Public IP" "Private IP" "Zone"
|
|
|
|
echo "-------------------+-----------------+-----------------+--------------"
|
|
|
|
fi
|
|
|
|
|
2019-01-16 09:38:29 -08:00
|
|
|
nodeType=bootstrap-leader
|
2019-10-21 20:21:21 -07:00
|
|
|
for i in $(seq 0 $(( ${#validatorIpList[@]} - 1)) ); do
|
|
|
|
ipAddress=${validatorIpList[$i]}
|
|
|
|
ipAddressPrivate=${validatorIpListPrivate[$i]}
|
|
|
|
zone=${validatorIpListZone[$i]}
|
2019-10-22 16:08:49 -07:00
|
|
|
if $evalInfo; then
|
|
|
|
echo "NET_VALIDATOR${i}_IP=$ipAddress"
|
|
|
|
else
|
|
|
|
printNode $nodeType "$ipAddress" "$ipAddressPrivate" "$zone"
|
|
|
|
fi
|
2019-10-21 20:21:21 -07:00
|
|
|
nodeType=validator
|
2019-01-16 09:38:29 -08:00
|
|
|
done
|
|
|
|
|
|
|
|
for i in $(seq 0 $(( ${#clientIpList[@]} - 1)) ); do
|
|
|
|
ipAddress=${clientIpList[$i]}
|
|
|
|
ipAddressPrivate=${clientIpListPrivate[$i]}
|
2019-04-28 19:50:02 -07:00
|
|
|
zone=${clientIpListZone[$i]}
|
2019-10-22 16:08:49 -07:00
|
|
|
if $evalInfo; then
|
|
|
|
echo "NET_CLIENT${i}_IP=$ipAddress"
|
|
|
|
else
|
|
|
|
printNode client "$ipAddress" "$ipAddressPrivate" "$zone"
|
|
|
|
fi
|
2019-01-16 09:38:29 -08:00
|
|
|
done
|
2019-02-17 09:48:27 -08:00
|
|
|
|
2019-02-21 15:35:26 -08:00
|
|
|
for i in $(seq 0 $(( ${#blockstreamerIpList[@]} - 1)) ); do
|
|
|
|
ipAddress=${blockstreamerIpList[$i]}
|
|
|
|
ipAddressPrivate=${blockstreamerIpListPrivate[$i]}
|
2019-04-28 19:50:02 -07:00
|
|
|
zone=${blockstreamerIpListZone[$i]}
|
2019-10-22 16:08:49 -07:00
|
|
|
if $evalInfo; then
|
|
|
|
echo "NET_BLOCKSTREAMER${i}_IP=$ipAddress"
|
|
|
|
else
|
|
|
|
printNode blockstreamer "$ipAddress" "$ipAddressPrivate" "$zone"
|
|
|
|
fi
|
2019-02-17 09:48:27 -08:00
|
|
|
done
|
2019-05-31 22:33:55 -07:00
|
|
|
|
2019-10-21 10:29:37 -07:00
|
|
|
for i in $(seq 0 $(( ${#archiverIpList[@]} - 1)) ); do
|
|
|
|
ipAddress=${archiverIpList[$i]}
|
|
|
|
ipAddressPrivate=${archiverIpListPrivate[$i]}
|
|
|
|
zone=${archiverIpListZone[$i]}
|
2019-10-22 16:08:49 -07:00
|
|
|
if $evalInfo; then
|
|
|
|
echo "NET_ARCHIVER${i}_IP=$ipAddress"
|
|
|
|
else
|
|
|
|
printNode archiver "$ipAddress" "$ipAddressPrivate" "$zone"
|
|
|
|
fi
|
2019-05-31 22:33:55 -07:00
|
|
|
done
|
2019-01-16 09:38:29 -08:00
|
|
|
;;
|
2019-09-19 14:08:23 -07:00
|
|
|
status)
|
|
|
|
cloud_StatusAll
|
|
|
|
;;
|
2018-08-28 10:19:33 -07:00
|
|
|
*)
|
|
|
|
usage "Unknown command: $command"
|
|
|
|
esac
|