2018-08-28 10:19:33 -07:00
|
|
|
#!/bin/bash -e
|
|
|
|
|
|
|
|
here=$(dirname "$0")
|
2018-09-08 10:17:53 -07:00
|
|
|
# shellcheck source=net/scripts/gcloud.sh
|
|
|
|
source "$here"/scripts/gcloud.sh
|
2018-08-28 10:19:33 -07:00
|
|
|
# shellcheck source=net/common.sh
|
|
|
|
source "$here"/common.sh
|
|
|
|
|
2018-09-06 08:18:05 -07:00
|
|
|
prefix=testnet-dev-${USER//[^A-Za-z0-9]/}
|
2018-09-07 08:03:52 -07:00
|
|
|
validatorNodeCount=5
|
2018-08-29 08:13:35 -07:00
|
|
|
clientNodeCount=1
|
2018-09-06 12:14:04 -07:00
|
|
|
leaderBootDiskSize=1TB
|
2018-09-04 08:17:41 -07:00
|
|
|
leaderMachineType=n1-standard-16
|
|
|
|
leaderAccelerator=
|
|
|
|
validatorMachineType=n1-standard-4
|
2018-09-06 12:14:04 -07:00
|
|
|
validatorBootDiskSize=$leaderBootDiskSize
|
2018-08-29 08:13:35 -07:00
|
|
|
validatorAccelerator=
|
2018-09-04 08:17:41 -07:00
|
|
|
clientMachineType=n1-standard-16
|
2018-09-06 12:14:04 -07:00
|
|
|
clientBootDiskSize=40GB
|
2018-08-29 08:13:35 -07:00
|
|
|
clientAccelerator=
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
imageName="ubuntu-16-04-cuda-9-2-new"
|
2018-09-04 22:21:58 -07:00
|
|
|
publicNetwork=false
|
2018-08-28 10:19:33 -07:00
|
|
|
zone="us-west1-b"
|
2018-09-05 09:31:50 -07:00
|
|
|
leaderAddress=
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
usage() {
|
|
|
|
exitcode=0
|
|
|
|
if [[ -n "$1" ]]; then
|
|
|
|
exitcode=1
|
|
|
|
echo "Error: $*"
|
|
|
|
fi
|
|
|
|
cat <<EOF
|
|
|
|
usage: $0 [create|config|delete] [common options] [command-specific options]
|
|
|
|
|
2018-09-03 21:15:55 -07:00
|
|
|
Configure a GCE-based testnet
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
create - create a new testnet (implies 'config')
|
|
|
|
config - configure the testnet and write a config file describing it
|
|
|
|
delete - delete the testnet
|
|
|
|
|
|
|
|
common options:
|
2018-09-06 10:08:34 -07:00
|
|
|
-p [prefix] - Optional common prefix for instance names to avoid
|
|
|
|
collisions (default: $prefix)
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
create-specific options:
|
2018-09-05 09:31:50 -07:00
|
|
|
-n [number] - Number of validator nodes (default: $validatorNodeCount)
|
|
|
|
-c [number] - Number of client nodes (default: $clientNodeCount)
|
2018-09-04 22:21:58 -07:00
|
|
|
-P - Use public network IP addresses (default: $publicNetwork)
|
2018-09-05 09:31:50 -07:00
|
|
|
-z [zone] - GCP Zone for the nodes (default: $zone)
|
|
|
|
-i [imageName] - Existing image on GCE (default: $imageName)
|
2018-09-04 08:17:41 -07:00
|
|
|
-g - Enable GPU
|
2018-09-05 09:31:50 -07:00
|
|
|
-a [address] - Set the leader node's external IP address to this GCE address
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
config-specific options:
|
|
|
|
none
|
|
|
|
|
|
|
|
delete-specific options:
|
2018-09-07 08:57:14 -07:00
|
|
|
none
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
EOF
|
|
|
|
exit $exitcode
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
command=$1
|
|
|
|
[[ -n $command ]] || usage
|
|
|
|
shift
|
|
|
|
[[ $command = create || $command = config || $command = delete ]] || usage "Invalid command: $command"
|
|
|
|
|
2018-09-07 08:57:14 -07:00
|
|
|
while getopts "h?p:Pi:n:c:z:ga:" opt; do
|
2018-08-28 10:19:33 -07:00
|
|
|
case $opt in
|
|
|
|
h | \?)
|
|
|
|
usage
|
|
|
|
;;
|
|
|
|
p)
|
2018-09-06 10:08:34 -07:00
|
|
|
[[ ${OPTARG//[^A-Za-z0-9-]/} == "$OPTARG" ]] || usage "Invalid prefix: \"$OPTARG\", alphanumeric only"
|
2018-08-28 10:19:33 -07:00
|
|
|
prefix=$OPTARG
|
|
|
|
;;
|
|
|
|
P)
|
2018-09-04 22:21:58 -07:00
|
|
|
publicNetwork=true
|
2018-08-28 10:19:33 -07:00
|
|
|
;;
|
|
|
|
i)
|
|
|
|
imageName=$OPTARG
|
|
|
|
;;
|
|
|
|
n)
|
|
|
|
validatorNodeCount=$OPTARG
|
|
|
|
;;
|
|
|
|
c)
|
|
|
|
clientNodeCount=$OPTARG
|
|
|
|
;;
|
|
|
|
z)
|
|
|
|
zone=$OPTARG
|
|
|
|
;;
|
2018-09-04 08:17:41 -07:00
|
|
|
g)
|
|
|
|
leaderAccelerator="count=4,type=nvidia-tesla-k80"
|
|
|
|
;;
|
2018-09-05 09:31:50 -07:00
|
|
|
a)
|
|
|
|
leaderAddress=$OPTARG
|
|
|
|
;;
|
2018-08-28 10:19:33 -07:00
|
|
|
*)
|
|
|
|
usage "Error: unhandled option: $opt"
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
done
|
2018-09-06 20:57:05 -07:00
|
|
|
shift $((OPTIND - 1))
|
2018-08-28 10:19:33 -07:00
|
|
|
|
2018-09-06 20:57:05 -07:00
|
|
|
[[ -z $1 ]] || usage "Unexpected argument: $1"
|
2018-09-08 19:19:12 -07:00
|
|
|
sshPrivateKey="$netConfigDir/id_$prefix"
|
2018-08-28 10:19:33 -07:00
|
|
|
|
2018-09-03 21:15:55 -07:00
|
|
|
prepareInstancesAndWriteConfigFile() {
|
2018-09-06 13:00:01 -07:00
|
|
|
$metricsWriteDatapoint "testnet-deploy net-config-begin=1"
|
2018-08-28 10:19:33 -07:00
|
|
|
|
2018-09-04 22:21:58 -07:00
|
|
|
cat >> "$configFile" <<EOF
|
|
|
|
# autogenerated at $(date)
|
|
|
|
netBasename=$prefix
|
|
|
|
publicNetwork=$publicNetwork
|
2018-09-08 19:19:12 -07:00
|
|
|
sshPrivateKey=$sshPrivateKey
|
2018-09-04 22:21:58 -07:00
|
|
|
EOF
|
2018-09-03 21:15:55 -07:00
|
|
|
|
2018-09-08 19:19:12 -07:00
|
|
|
buildSshOptions
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
recordInstanceIp() {
|
|
|
|
declare name="$1"
|
|
|
|
declare publicIp="$3"
|
|
|
|
declare privateIp="$4"
|
|
|
|
|
|
|
|
declare arrayName="$6"
|
|
|
|
|
2018-09-04 22:21:58 -07:00
|
|
|
echo "$arrayName+=($publicIp) # $name" >> "$configFile"
|
|
|
|
if [[ $arrayName = "leaderIp" ]]; then
|
|
|
|
if $publicNetwork; then
|
|
|
|
echo "entrypointIp=$publicIp" >> "$configFile"
|
|
|
|
else
|
|
|
|
echo "entrypointIp=$privateIp" >> "$configFile"
|
|
|
|
fi
|
2018-08-28 10:19:33 -07:00
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2018-09-08 19:19:12 -07:00
|
|
|
waitForStartupComplete() {
|
|
|
|
declare name="$1"
|
|
|
|
declare publicIp="$3"
|
|
|
|
|
|
|
|
echo "Waiting for $name to finish booting..."
|
|
|
|
(
|
|
|
|
for i in $(seq 1 30); do
|
|
|
|
if (set -x; ssh "${sshOptions[@]}" "$publicIp" "test -f /.gce-startup-complete"); then
|
|
|
|
break
|
|
|
|
fi
|
|
|
|
sleep 2
|
|
|
|
echo "Retry $i..."
|
|
|
|
done
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2018-09-04 12:59:19 -07:00
|
|
|
echo "Looking for leader instance..."
|
2018-08-28 10:19:33 -07:00
|
|
|
gcloud_FindInstances "name=$prefix-leader" show
|
|
|
|
[[ ${#instances[@]} -eq 1 ]] || {
|
2018-09-08 19:19:12 -07:00
|
|
|
echo "Unable to find leader"
|
2018-08-28 10:19:33 -07:00
|
|
|
exit 1
|
|
|
|
}
|
2018-09-03 21:15:55 -07:00
|
|
|
|
2018-09-08 19:19:12 -07:00
|
|
|
echo "Fetching $sshPrivateKey from $leaderName"
|
|
|
|
(
|
|
|
|
rm -rf "$sshPrivateKey"{,pub}
|
|
|
|
|
|
|
|
declare leaderName
|
|
|
|
declare leaderZone
|
|
|
|
declare leaderIp
|
|
|
|
IFS=: read -r leaderName leaderZone leaderIp _ < <(echo "${instances[0]}")
|
|
|
|
|
|
|
|
set -x
|
|
|
|
|
|
|
|
# Try to ping the machine first. There can be a delay between when the
|
|
|
|
# instance is reported as RUNNING and when it's reachable over the network
|
|
|
|
timeout 30s bash -c "set -o pipefail; until ping -c 3 $leaderIp | tr - _; do echo .; done"
|
|
|
|
|
|
|
|
# Try to scp in a couple times, sshd may not yet be up even though the
|
|
|
|
# machine can be pinged...
|
|
|
|
set -o pipefail
|
|
|
|
for i in $(seq 1 10); do
|
|
|
|
if gcloud compute scp --zone "$leaderZone" \
|
|
|
|
"$leaderName:/solana-id_ecdsa" "$sshPrivateKey"; then
|
|
|
|
break
|
|
|
|
fi
|
|
|
|
sleep 1
|
|
|
|
echo "Retry $i..."
|
|
|
|
done
|
|
|
|
|
|
|
|
chmod 400 "$sshPrivateKey"
|
|
|
|
)
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
echo "leaderIp=()" >> "$configFile"
|
|
|
|
gcloud_ForEachInstance recordInstanceIp leaderIp
|
2018-09-08 19:19:12 -07:00
|
|
|
gcloud_ForEachInstance waitForStartupComplete
|
2018-08-28 10:19:33 -07:00
|
|
|
|
2018-09-04 12:59:19 -07:00
|
|
|
echo "Looking for validator instances..."
|
2018-08-28 10:19:33 -07:00
|
|
|
gcloud_FindInstances "name~^$prefix-validator" show
|
|
|
|
[[ ${#instances[@]} -gt 0 ]] || {
|
2018-09-08 19:19:12 -07:00
|
|
|
echo "Unable to find validators"
|
2018-08-28 10:19:33 -07:00
|
|
|
exit 1
|
|
|
|
}
|
|
|
|
echo "validatorIpList=()" >> "$configFile"
|
|
|
|
gcloud_ForEachInstance recordInstanceIp validatorIpList
|
2018-09-08 19:19:12 -07:00
|
|
|
gcloud_ForEachInstance waitForStartupComplete
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
echo "clientIpList=()" >> "$configFile"
|
2018-09-04 12:59:19 -07:00
|
|
|
echo "Looking for client instances..."
|
2018-08-28 10:19:33 -07:00
|
|
|
gcloud_FindInstances "name~^$prefix-client" show
|
2018-09-04 22:46:55 -07:00
|
|
|
[[ ${#instances[@]} -eq 0 ]] || {
|
2018-08-28 10:19:33 -07:00
|
|
|
gcloud_ForEachInstance recordInstanceIp clientIpList
|
2018-09-08 19:19:12 -07:00
|
|
|
gcloud_ForEachInstance waitForStartupComplete
|
2018-09-04 22:46:55 -07:00
|
|
|
}
|
2018-08-28 10:19:33 -07:00
|
|
|
|
|
|
|
echo "Wrote $configFile"
|
2018-09-06 12:14:04 -07:00
|
|
|
$metricsWriteDatapoint "testnet-deploy net-config-complete=1"
|
2018-08-28 10:19:33 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
case $command in
|
|
|
|
delete)
|
2018-09-06 13:00:01 -07:00
|
|
|
$metricsWriteDatapoint "testnet-deploy net-delete-begin=1"
|
2018-09-06 12:14:04 -07:00
|
|
|
|
2018-09-07 08:56:43 -07:00
|
|
|
# Delete the leader node first to prevent unusual metrics on the dashboard
|
|
|
|
# during shutdown.
|
|
|
|
# TODO: It would be better to fully cut-off metrics reporting before any
|
|
|
|
# instances are deleted.
|
|
|
|
for filter in "^$prefix-leader" "^$prefix-"; do
|
|
|
|
gcloud_FindInstances "name~$filter"
|
|
|
|
|
|
|
|
if [[ ${#instances[@]} -eq 0 ]]; then
|
|
|
|
echo "No instances found matching '$filter'"
|
|
|
|
else
|
2018-09-07 08:57:14 -07:00
|
|
|
gcloud_DeleteInstances true
|
2018-09-07 08:56:43 -07:00
|
|
|
fi
|
|
|
|
done
|
2018-09-04 09:21:03 -07:00
|
|
|
rm -f "$configFile"
|
2018-09-06 12:14:04 -07:00
|
|
|
|
|
|
|
$metricsWriteDatapoint "testnet-deploy net-delete-complete=1"
|
2018-08-28 10:19:33 -07:00
|
|
|
;;
|
|
|
|
|
|
|
|
create)
|
|
|
|
[[ -n $validatorNodeCount ]] || usage "Need number of nodes"
|
|
|
|
|
2018-09-06 13:00:01 -07:00
|
|
|
$metricsWriteDatapoint "testnet-deploy net-create-begin=1"
|
2018-09-06 12:14:04 -07:00
|
|
|
|
2018-09-08 19:19:12 -07:00
|
|
|
rm -rf "$sshPrivateKey"{,.pub}
|
|
|
|
ssh-keygen -t ecdsa -N '' -f "$sshPrivateKey"
|
|
|
|
|
2018-09-08 10:38:32 -07:00
|
|
|
printNetworkInfo() {
|
|
|
|
cat <<EOF
|
|
|
|
========================================================================================
|
|
|
|
|
|
|
|
Network composition:
|
|
|
|
Leader = $leaderMachineType (GPU=${leaderAccelerator:-none})
|
|
|
|
Validators = $validatorNodeCount x $validatorMachineType (GPU=${validatorAccelerator:-none})
|
|
|
|
Client(s) = $clientNodeCount x $clientMachineType (GPU=${clientAccelerator:-none})
|
|
|
|
|
|
|
|
========================================================================================
|
|
|
|
|
|
|
|
EOF
|
|
|
|
}
|
|
|
|
printNetworkInfo
|
|
|
|
|
|
|
|
declare startupScript="$netConfigDir"/gce-startup-script.sh
|
|
|
|
cat > "$startupScript" <<EOF
|
|
|
|
#!/bin/bash -ex
|
|
|
|
# autogenerated at $(date)
|
|
|
|
|
|
|
|
cat > /etc/motd <<EOM
|
|
|
|
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
|
|
|
|
|
|
This instance has not been fully configured.
|
|
|
|
See "startup-script" log messages in /var/log/syslog for status:
|
|
|
|
$ sudo cat /var/log/syslog | grep startup-script
|
|
|
|
|
2018-09-08 19:19:12 -07:00
|
|
|
To block until setup is complete, run:
|
|
|
|
$ until [[ -f /.gce-startup-complete ]]; do sleep 1; done
|
|
|
|
|
2018-09-08 10:38:32 -07:00
|
|
|
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
|
|
EOM
|
|
|
|
|
2018-09-08 19:19:12 -07:00
|
|
|
# Place the generated private key at /solana-id_ecdsa so it's retrievable by anybody
|
|
|
|
# who is able to log into this machine
|
|
|
|
cat > /solana-id_ecdsa <<EOK
|
|
|
|
$(cat "$sshPrivateKey")
|
|
|
|
EOK
|
|
|
|
cat > /solana-id_ecdsa.pub <<EOK
|
|
|
|
$(cat "$sshPrivateKey.pub")
|
|
|
|
EOK
|
|
|
|
chmod 444 /solana-id_ecdsa
|
|
|
|
|
2018-09-08 10:38:32 -07:00
|
|
|
USER=\$(id -un)
|
|
|
|
|
|
|
|
$(
|
|
|
|
cd "$here"/scripts/
|
|
|
|
cat \
|
|
|
|
disable-background-upgrades.sh \
|
2018-09-08 19:19:12 -07:00
|
|
|
create-solana-user.sh \
|
2018-09-08 10:38:32 -07:00
|
|
|
install-earlyoom.sh \
|
|
|
|
install-libssl-compatability.sh \
|
2018-09-12 16:22:22 -07:00
|
|
|
install-rsync.sh \
|
2018-09-08 10:38:32 -07:00
|
|
|
)
|
|
|
|
|
|
|
|
cat > /etc/motd <<EOM
|
|
|
|
$(printNetworkInfo)
|
|
|
|
EOM
|
|
|
|
|
2018-09-08 19:19:12 -07:00
|
|
|
touch /.gce-startup-complete
|
|
|
|
|
2018-09-08 10:38:32 -07:00
|
|
|
EOF
|
|
|
|
|
2018-09-05 09:31:50 -07:00
|
|
|
gcloud_CreateInstances "$prefix-leader" 1 "$zone" \
|
2018-09-06 12:14:04 -07:00
|
|
|
"$imageName" "$leaderMachineType" "$leaderBootDiskSize" "$leaderAccelerator" \
|
2018-09-08 10:38:32 -07:00
|
|
|
"$startupScript" "$leaderAddress"
|
2018-09-05 09:31:50 -07:00
|
|
|
|
|
|
|
gcloud_CreateInstances "$prefix-validator" "$validatorNodeCount" "$zone" \
|
2018-09-06 12:14:04 -07:00
|
|
|
"$imageName" "$validatorMachineType" "$validatorBootDiskSize" "$validatorAccelerator" \
|
2018-09-08 10:38:32 -07:00
|
|
|
"$startupScript" ""
|
|
|
|
|
2018-09-06 10:34:24 -07:00
|
|
|
if [[ $clientNodeCount -gt 0 ]]; then
|
2018-09-05 09:31:50 -07:00
|
|
|
gcloud_CreateInstances "$prefix-client" "$clientNodeCount" "$zone" \
|
2018-09-06 12:14:04 -07:00
|
|
|
"$imageName" "$clientMachineType" "$clientBootDiskSize" "$clientAccelerator" \
|
2018-09-08 10:38:32 -07:00
|
|
|
"$startupScript" ""
|
2018-08-28 10:19:33 -07:00
|
|
|
fi
|
2018-08-29 08:13:35 -07:00
|
|
|
|
2018-09-06 12:14:04 -07:00
|
|
|
$metricsWriteDatapoint "testnet-deploy net-create-complete=1"
|
|
|
|
|
2018-09-03 21:15:55 -07:00
|
|
|
prepareInstancesAndWriteConfigFile
|
2018-08-28 10:19:33 -07:00
|
|
|
;;
|
|
|
|
|
|
|
|
config)
|
2018-09-03 21:15:55 -07:00
|
|
|
prepareInstancesAndWriteConfigFile
|
2018-08-28 10:19:33 -07:00
|
|
|
;;
|
|
|
|
*)
|
|
|
|
usage "Unknown command: $command"
|
|
|
|
esac
|