net/ can now deploy Snaps
This commit is contained in:
parent
449d7042f0
commit
fa07c49cc9
|
@ -11,25 +11,20 @@ netConfigDir="$(dirname "${BASH_SOURCE[0]}")"/config
|
||||||
netLogDir="$(dirname "${BASH_SOURCE[0]}")"/log
|
netLogDir="$(dirname "${BASH_SOURCE[0]}")"/log
|
||||||
mkdir -p "$netConfigDir" "$netLogDir"
|
mkdir -p "$netConfigDir" "$netLogDir"
|
||||||
|
|
||||||
|
# shellcheck source=scripts/configure-metrics.sh
|
||||||
|
source "$(dirname "${BASH_SOURCE[0]}")"/../scripts/configure-metrics.sh
|
||||||
|
|
||||||
configFile="$netConfigDir/config"
|
configFile="$netConfigDir/config"
|
||||||
|
|
||||||
clientIpList=()
|
clientIpList=()
|
||||||
leaderIp=
|
leaderIp=
|
||||||
|
netBasename=
|
||||||
|
sshOptions=()
|
||||||
sshPrivateKey=
|
sshPrivateKey=
|
||||||
sshUsername=
|
sshUsername=
|
||||||
sshOptions=()
|
|
||||||
validatorIpList=()
|
validatorIpList=()
|
||||||
|
|
||||||
loadConfigFile() {
|
buildSshOptions() {
|
||||||
[[ -r $configFile ]] || usage "Config file unreadable: $configFile"
|
|
||||||
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
source "$configFile"
|
|
||||||
[[ -n "$leaderIp" ]] || usage "Config file invalid, leaderIp unspecified: $configFile"
|
|
||||||
[[ ${#validatorIpList[@]} -gt 0 ]] || usage "Config file invalid, validatorIpList unspecified: $configFile"
|
|
||||||
[[ -n $sshUsername ]] || usage "Config file invalid, sshUsername unspecified: $configFile"
|
|
||||||
[[ -n $sshPrivateKey ]] || usage "Config file invalid, sshPrivateKey unspecified: $configFile"
|
|
||||||
|
|
||||||
sshOptions=(
|
sshOptions=(
|
||||||
-o "BatchMode=yes"
|
-o "BatchMode=yes"
|
||||||
-o "StrictHostKeyChecking=no"
|
-o "StrictHostKeyChecking=no"
|
||||||
|
@ -39,3 +34,18 @@ loadConfigFile() {
|
||||||
-o "LogLevel=ERROR"
|
-o "LogLevel=ERROR"
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
loadConfigFile() {
|
||||||
|
[[ -r $configFile ]] || usage "Config file unreadable: $configFile"
|
||||||
|
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "$configFile"
|
||||||
|
[[ -n "$netBasename" ]] || usage "Config file invalid, netBasename unspecified: $configFile"
|
||||||
|
[[ -n "$leaderIp" ]] || usage "Config file invalid, leaderIp unspecified: $configFile"
|
||||||
|
[[ ${#validatorIpList[@]} -gt 0 ]] || usage "Config file invalid, validatorIpList unspecified: $configFile"
|
||||||
|
[[ -n $sshUsername ]] || usage "Config file invalid, sshUsername unspecified: $configFile"
|
||||||
|
[[ -n $sshPrivateKey ]] || usage "Config file invalid, sshPrivateKey unspecified: $configFile"
|
||||||
|
|
||||||
|
buildSshOptions
|
||||||
|
configureMetrics
|
||||||
|
}
|
||||||
|
|
33
net/gce.sh
33
net/gce.sh
|
@ -29,7 +29,7 @@ usage() {
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
usage: $0 [create|config|delete] [common options] [command-specific options]
|
usage: $0 [create|config|delete] [common options] [command-specific options]
|
||||||
|
|
||||||
Manage a GCE-based testnet
|
Configure a GCE-based testnet
|
||||||
|
|
||||||
create - create a new testnet (implies 'config')
|
create - create a new testnet (implies 'config')
|
||||||
config - configure the testnet and write a config file describing it
|
config - configure the testnet and write a config file describing it
|
||||||
|
@ -92,9 +92,11 @@ while getopts "h?p:Pi:n:c:z:" opt; do
|
||||||
done
|
done
|
||||||
|
|
||||||
|
|
||||||
writeConfigFile() {
|
prepareInstancesAndWriteConfigFile() {
|
||||||
echo "# autogenerated at $(date)" >> "$configFile"
|
echo "# autogenerated at $(date)" >> "$configFile"
|
||||||
|
|
||||||
|
echo "netBasename=$prefix" >> "$configFile"
|
||||||
|
|
||||||
declare sshPrivateKey="$netConfigDir/id_$prefix"
|
declare sshPrivateKey="$netConfigDir/id_$prefix"
|
||||||
rm -rf "$sshPrivateKey"{,.pub}
|
rm -rf "$sshPrivateKey"{,.pub}
|
||||||
(
|
(
|
||||||
|
@ -117,17 +119,36 @@ writeConfigFile() {
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
prepareInstance() {
|
||||||
|
declare name="$1"
|
||||||
|
declare publicIp="$3"
|
||||||
|
|
||||||
|
# TODO: Make the following a requirement of $imageName
|
||||||
|
# instead of a manual install
|
||||||
|
ssh "${sshOptions[@]}" "$publicIp" "
|
||||||
|
set -ex;
|
||||||
|
sudo systemctl disable apt-daily.service # disable run when system boot
|
||||||
|
sudo systemctl disable apt-daily.timer # disable timer run
|
||||||
|
sudo apt-get --assume-yes install rsync libssl-dev;
|
||||||
|
mkdir -p ~/solana ~/.cargo/bin;
|
||||||
|
"
|
||||||
|
}
|
||||||
|
|
||||||
gcloud_FindInstances "name=$prefix-leader" show
|
gcloud_FindInstances "name=$prefix-leader" show
|
||||||
[[ ${#instances[@]} -eq 1 ]] || {
|
[[ ${#instances[@]} -eq 1 ]] || {
|
||||||
echo "Unable to start leader"
|
echo "Unable to start leader"
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
gcloud_FigureRemoteUsername "${instances[0]}"
|
gcloud_FigureRemoteUsername "${instances[0]}"
|
||||||
echo "sshUsername=$gcloud_username" >> "$configFile"
|
sshUsername=$gcloud_username
|
||||||
|
echo "sshUsername=$sshUsername" >> "$configFile"
|
||||||
|
buildSshOptions
|
||||||
|
|
||||||
gcloud_PrepInstancesForSsh "$gcloud_username" "$sshPrivateKey"
|
gcloud_PrepInstancesForSsh "$gcloud_username" "$sshPrivateKey"
|
||||||
|
|
||||||
echo "leaderIp=()" >> "$configFile"
|
echo "leaderIp=()" >> "$configFile"
|
||||||
gcloud_ForEachInstance recordInstanceIp leaderIp
|
gcloud_ForEachInstance recordInstanceIp leaderIp
|
||||||
|
gcloud_ForEachInstance prepareInstance
|
||||||
|
|
||||||
gcloud_FindInstances "name~^$prefix-validator" show
|
gcloud_FindInstances "name~^$prefix-validator" show
|
||||||
[[ ${#instances[@]} -gt 0 ]] || {
|
[[ ${#instances[@]} -gt 0 ]] || {
|
||||||
|
@ -137,12 +158,14 @@ writeConfigFile() {
|
||||||
echo "validatorIpList=()" >> "$configFile"
|
echo "validatorIpList=()" >> "$configFile"
|
||||||
gcloud_PrepInstancesForSsh "$gcloud_username" "$sshPrivateKey"
|
gcloud_PrepInstancesForSsh "$gcloud_username" "$sshPrivateKey"
|
||||||
gcloud_ForEachInstance recordInstanceIp validatorIpList
|
gcloud_ForEachInstance recordInstanceIp validatorIpList
|
||||||
|
gcloud_ForEachInstance prepareInstance
|
||||||
|
|
||||||
echo "clientIpList=()" >> "$configFile"
|
echo "clientIpList=()" >> "$configFile"
|
||||||
gcloud_FindInstances "name~^$prefix-client" show
|
gcloud_FindInstances "name~^$prefix-client" show
|
||||||
if [[ ${#instances[@]} -gt 0 ]]; then
|
if [[ ${#instances[@]} -gt 0 ]]; then
|
||||||
gcloud_PrepInstancesForSsh "$gcloud_username" "$sshPrivateKey"
|
gcloud_PrepInstancesForSsh "$gcloud_username" "$sshPrivateKey"
|
||||||
gcloud_ForEachInstance recordInstanceIp clientIpList
|
gcloud_ForEachInstance recordInstanceIp clientIpList
|
||||||
|
gcloud_ForEachInstance prepareInstance
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "Wrote $configFile"
|
echo "Wrote $configFile"
|
||||||
|
@ -177,11 +200,11 @@ create)
|
||||||
"$zone" "$imageName" "$clientMachineType" "$clientAccelerator"
|
"$zone" "$imageName" "$clientMachineType" "$clientAccelerator"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
writeConfigFile
|
prepareInstancesAndWriteConfigFile
|
||||||
;;
|
;;
|
||||||
|
|
||||||
config)
|
config)
|
||||||
writeConfigFile
|
prepareInstancesAndWriteConfigFile
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
usage "Unknown command: $command"
|
usage "Unknown command: $command"
|
||||||
|
|
|
@ -11,20 +11,18 @@ usage() {
|
||||||
echo "Error: $*"
|
echo "Error: $*"
|
||||||
fi
|
fi
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
usage: $0 [-d] [username] [optional database name]
|
usage: $0 [-d] [username]
|
||||||
|
|
||||||
Creates a testnet dev metrics database
|
Creates a testnet dev metrics database
|
||||||
|
|
||||||
username InfluxDB user with access to create a new database
|
username InfluxDB user with access to create a new database
|
||||||
database Uncommon. Optional database suffix to follow the mandiatory
|
|
||||||
'testnet-dev-[username]' database name prefix
|
|
||||||
|
|
||||||
-d Delete the database instead of creating it
|
-d Delete the database instead of creating it
|
||||||
|
|
||||||
EOF
|
EOF
|
||||||
exit $exitcode
|
exit $exitcode
|
||||||
}
|
}
|
||||||
|
|
||||||
|
loadConfigFile
|
||||||
|
|
||||||
delete=false
|
delete=false
|
||||||
while getopts "hd" opt; do
|
while getopts "hd" opt; do
|
||||||
|
@ -34,7 +32,7 @@ while getopts "hd" opt; do
|
||||||
exit 0
|
exit 0
|
||||||
;;
|
;;
|
||||||
d)
|
d)
|
||||||
delete=true;
|
delete=true
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
usage "Error: unhandled option: $opt"
|
usage "Error: unhandled option: $opt"
|
||||||
|
@ -45,11 +43,6 @@ shift $((OPTIND - 1))
|
||||||
|
|
||||||
username=$1
|
username=$1
|
||||||
[[ -n "$username" ]] || usage "username not specified"
|
[[ -n "$username" ]] || usage "username not specified"
|
||||||
database="testnet-dev-$username"
|
|
||||||
|
|
||||||
if [[ -n "$2" ]]; then
|
|
||||||
database="$database-$2"
|
|
||||||
fi
|
|
||||||
|
|
||||||
read -rs -p "InfluxDB password for $username: " password
|
read -rs -p "InfluxDB password for $username: " password
|
||||||
[[ -n $password ]] || { echo "Password not specified"; exit 1; }
|
[[ -n $password ]] || { echo "Password not specified"; exit 1; }
|
||||||
|
@ -62,15 +55,15 @@ query() {
|
||||||
--data-urlencode "q=$*"
|
--data-urlencode "q=$*"
|
||||||
}
|
}
|
||||||
|
|
||||||
query "DROP DATABASE \"$database\""
|
query "DROP DATABASE \"$netBasename\""
|
||||||
! $delete || exit 0
|
! $delete || exit 0
|
||||||
query "CREATE DATABASE \"$database\""
|
query "CREATE DATABASE \"$netBasename\""
|
||||||
query "ALTER RETENTION POLICY autogen ON \"$database\" DURATION 7d"
|
query "ALTER RETENTION POLICY autogen ON \"$netBasename\" DURATION 7d"
|
||||||
query "GRANT READ ON \"$database\" TO \"ro\""
|
query "GRANT READ ON \"$netBasename\" TO \"ro\""
|
||||||
query "GRANT WRITE ON \"$database\" TO \"scratch_writer\""
|
query "GRANT WRITE ON \"$netBasename\" TO \"scratch_writer\""
|
||||||
|
|
||||||
echo "export \
|
echo "export \
|
||||||
SOLANA_METRICS_CONFIG=\"db=$database,u=scratch_writer,p=topsecret\" \
|
SOLANA_METRICS_CONFIG=\"db=$netBasename,u=scratch_writer,p=topsecret\" \
|
||||||
" >> "$configFile"
|
" >> "$configFile"
|
||||||
|
|
||||||
exit 0
|
exit 0
|
||||||
|
|
141
net/net.sh
141
net/net.sh
|
@ -15,23 +15,60 @@ usage() {
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
usage: $0 [start|stop]
|
usage: $0 [start|stop]
|
||||||
|
|
||||||
Manage a multinode network
|
Operate a configured testnet
|
||||||
|
|
||||||
|
start - Start the network
|
||||||
|
stop - Stop the network
|
||||||
|
|
||||||
|
start-specific options:
|
||||||
|
-S snapFilename - Deploy the specified Snap file
|
||||||
|
-s edge|beta|stable - Deploy the latest Snap on the specified Snap release channel
|
||||||
|
-a "setup args" - Optional additional arguments for ./multinode-demo/setup.sh
|
||||||
|
|
||||||
|
Note: if RUST_LOG is set in the environment it will be propogated into the
|
||||||
|
network nodes.
|
||||||
|
|
||||||
|
stop-specific options:
|
||||||
|
none
|
||||||
|
|
||||||
start|stop - Start or stop the network
|
|
||||||
EOF
|
EOF
|
||||||
exit $exitcode
|
exit $exitcode
|
||||||
}
|
}
|
||||||
|
|
||||||
|
snapChannel=
|
||||||
|
snapFilename=
|
||||||
|
nodeSetupArgs=
|
||||||
|
deployMethod=local
|
||||||
|
|
||||||
command=$1
|
command=$1
|
||||||
[[ -n $command ]] || usage
|
[[ -n $command ]] || usage
|
||||||
shift
|
shift
|
||||||
[[ $command = start || $command = stop ]] || usage "Invalid command: $command"
|
[[ $command = start || $command = stop ]] || usage "Invalid command: $command"
|
||||||
|
|
||||||
while getopts "h?" opt; do
|
while getopts "h?S:s:a:" opt; do
|
||||||
case $opt in
|
case $opt in
|
||||||
h | \?)
|
h | \?)
|
||||||
usage
|
usage
|
||||||
;;
|
;;
|
||||||
|
S)
|
||||||
|
[[ $command = start ]] || usage "-s is only valid with the 'start' command"
|
||||||
|
snapFilename=$OPTARG
|
||||||
|
[[ -f $snapFilename ]] || usage "Snap not readable: $snapFilename"
|
||||||
|
deployMethod=snap
|
||||||
|
;;
|
||||||
|
s)
|
||||||
|
case $OPTARG in
|
||||||
|
edge|beta|stable)
|
||||||
|
snapChannel=$OPTARG
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
usage "Invalid snap channel: $OPTARG"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
;;
|
||||||
|
a)
|
||||||
|
nodeSetupArgs="$OPTARG"
|
||||||
|
;;
|
||||||
*)
|
*)
|
||||||
usage "Error: unhandled option: $opt"
|
usage "Error: unhandled option: $opt"
|
||||||
;;
|
;;
|
||||||
|
@ -64,15 +101,6 @@ common_start_setup() {
|
||||||
|
|
||||||
(
|
(
|
||||||
set -x
|
set -x
|
||||||
|
|
||||||
ssh "${sshOptions[@]}" "$ipAddress" "
|
|
||||||
set -ex;
|
|
||||||
sudo systemctl disable apt-daily.service # disable run when system boot
|
|
||||||
sudo systemctl disable apt-daily.timer # disable timer run
|
|
||||||
sudo apt-get --assume-yes install rsync libssl-dev;
|
|
||||||
mkdir -p ~/solana ~/.cargo/bin;
|
|
||||||
"
|
|
||||||
|
|
||||||
test -d "$SOLANA_ROOT"
|
test -d "$SOLANA_ROOT"
|
||||||
rsync -vPrz -e "ssh ${sshOptions[*]}" \
|
rsync -vPrz -e "ssh ${sshOptions[*]}" \
|
||||||
"$SOLANA_ROOT"/{fetch-perf-libs.sh,scripts,net,multinode-demo} \
|
"$SOLANA_ROOT"/{fetch-perf-libs.sh,scripts,net,multinode-demo} \
|
||||||
|
@ -88,11 +116,24 @@ startLeader() {
|
||||||
|
|
||||||
common_start_setup "$ipAddress" "$logFile"
|
common_start_setup "$ipAddress" "$logFile"
|
||||||
|
|
||||||
|
# Deploy local binaries to leader. Validators and clients later fetch the
|
||||||
|
# binaries from the leader.
|
||||||
(
|
(
|
||||||
set -x
|
set -x
|
||||||
rsync -vPrz -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/farf/bin/* "$ipAddress":~/.cargo/bin/
|
case $deployMethod in
|
||||||
|
snap)
|
||||||
|
rsync -vPrz -e "ssh ${sshOptions[*]}" "$snapFilename" "$ipAddress:~/solana/solana.snap"
|
||||||
|
;;
|
||||||
|
local)
|
||||||
|
rsync -vPrz -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/farf/bin/* "$ipAddress:~/.cargo/bin/"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
usage "Internal error: invalid deployMethod: $deployMethod"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
ssh "${sshOptions[@]}" -f "$ipAddress" \
|
ssh "${sshOptions[@]}" -f "$ipAddress" \
|
||||||
"./solana/net/remote/remote_leader.sh"
|
"./solana/net/remote/remote_node.sh $deployMethod leader $leaderIp \"$nodeSetupArgs\" \"$RUST_LOG\""
|
||||||
) >> "$logFile"
|
) >> "$logFile"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -106,7 +147,7 @@ startValidator() {
|
||||||
(
|
(
|
||||||
set -x
|
set -x
|
||||||
ssh "${sshOptions[@]}" -f "$ipAddress" \
|
ssh "${sshOptions[@]}" -f "$ipAddress" \
|
||||||
"./solana/net/remote/remote_validator.sh $leaderIp"
|
"./solana/net/remote/remote_node.sh $deployMethod validator $leaderIp \"$nodeSetupArgs\" \"$RUST_LOG\""
|
||||||
) >> "$logFile"
|
) >> "$logFile"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -117,35 +158,67 @@ startClient() {
|
||||||
echo "Starting client: $leaderIp"
|
echo "Starting client: $leaderIp"
|
||||||
common_start_setup "$ipAddress" "$logFile"
|
common_start_setup "$ipAddress" "$logFile"
|
||||||
|
|
||||||
|
declare expectedNodeCount=$((${#validatorIpList[@]} + 1))
|
||||||
|
|
||||||
ssh "${sshOptions[@]}" -f "$ipAddress" \
|
ssh "${sshOptions[@]}" -f "$ipAddress" \
|
||||||
"./solana/net/remote/remote_client.sh $leaderIp" >> "$logFile"
|
"./solana/net/remote/remote_client.sh $deployMethod $leaderIp $expectedNodeCount \"$RUST_LOG\"" >> "$logFile"
|
||||||
}
|
}
|
||||||
|
|
||||||
start() {
|
start() {
|
||||||
echo "Deployment started at $(date)"
|
[[ $command = "start" ]] || return
|
||||||
SECONDS=0
|
|
||||||
leaderDeployTime=
|
|
||||||
|
|
||||||
|
case $deployMethod in
|
||||||
|
snap)
|
||||||
|
if [[ -n $snapChannel ]]; then
|
||||||
|
if [[ $(uname) != Linux ]]; then
|
||||||
|
echo Error: snap channel deployment only supported in Linux
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
usage "TODO: the snap download command below is probably wrong..."
|
||||||
|
snap download --"$snapChannel" solana
|
||||||
|
snapFilename=solana.snap
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
local)
|
||||||
|
build
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
usage "Internal error: invalid deployMethod: $deployMethod"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
echo "Deployment started at $(date)"
|
||||||
|
|
||||||
|
SECONDS=0
|
||||||
|
declare leaderDeployTime=
|
||||||
|
declare networkVersion=unknown
|
||||||
startLeader "$leaderIp" "$netLogDir/leader-$leaderIp.log"
|
startLeader "$leaderIp" "$netLogDir/leader-$leaderIp.log"
|
||||||
leaderDeployTime=$SECONDS
|
leaderDeployTime=$SECONDS
|
||||||
SECONDS=0
|
|
||||||
|
|
||||||
|
SECONDS=0
|
||||||
for ipAddress in "${validatorIpList[@]}"; do
|
for ipAddress in "${validatorIpList[@]}"; do
|
||||||
startValidator "$ipAddress" "$netLogDir/validator-$ipAddress.log" &
|
startValidator "$ipAddress" "$netLogDir/validator-$ipAddress.log" &
|
||||||
done
|
done
|
||||||
|
|
||||||
wait
|
wait
|
||||||
validatorDeployTime=$SECONDS
|
validatorDeployTime=$SECONDS
|
||||||
SECONDS=0
|
|
||||||
|
|
||||||
|
SECONDS=0
|
||||||
for ipAddress in "${clientIpList[@]}"; do
|
for ipAddress in "${clientIpList[@]}"; do
|
||||||
startClient "$ipAddress" "$netLogDir/client-$ipAddress.log"
|
startClient "$ipAddress" "$netLogDir/client-$ipAddress.log"
|
||||||
done
|
done
|
||||||
|
|
||||||
clientDeployTime=$SECONDS
|
clientDeployTime=$SECONDS
|
||||||
SECONDS=0
|
|
||||||
wait
|
wait
|
||||||
|
|
||||||
|
if [[ $deployMethod = "snap" ]]; then
|
||||||
|
IFS=\ read -r _ networkVersion _ < <(
|
||||||
|
ssh "${sshOptions[@]}" "$leaderIp" \
|
||||||
|
"snap info solana | grep \"^installed:\""
|
||||||
|
)
|
||||||
|
networkVersion=${networkVersion/0+git./}
|
||||||
|
fi
|
||||||
|
|
||||||
|
$metricsWriteDatapoint "testnet-deploy,name=$netBasename start=1,version=\"$networkVersion\""
|
||||||
|
|
||||||
echo
|
echo
|
||||||
echo "================================================================="
|
echo "================================================================="
|
||||||
echo "Deployment finished at $(date)"
|
echo "Deployment finished at $(date)"
|
||||||
|
@ -165,7 +238,11 @@ stop_node() {
|
||||||
set -x
|
set -x
|
||||||
ssh "${sshOptions[@]}" "$ipAddress" "
|
ssh "${sshOptions[@]}" "$ipAddress" "
|
||||||
set -x;
|
set -x;
|
||||||
pkill -9 solana- remote_ oom-monitor
|
if snap list solana; then
|
||||||
|
sudo snap set solana mode=;
|
||||||
|
sudo snap remove solana;
|
||||||
|
fi; \
|
||||||
|
pkill -9 solana- remote_ oom-monitor;
|
||||||
"
|
"
|
||||||
) || true
|
) || true
|
||||||
}
|
}
|
||||||
|
@ -173,6 +250,8 @@ stop_node() {
|
||||||
stop() {
|
stop() {
|
||||||
SECONDS=0
|
SECONDS=0
|
||||||
|
|
||||||
|
$metricsWriteDatapoint "testnet-deploy,name=$netBasename stop=1"
|
||||||
|
|
||||||
stop_node "$leaderIp"
|
stop_node "$leaderIp"
|
||||||
|
|
||||||
for ipAddress in "${validatorIpList[@]}" "${clientIpList[@]}"; do
|
for ipAddress in "${validatorIpList[@]}" "${clientIpList[@]}"; do
|
||||||
|
@ -182,14 +261,6 @@ stop() {
|
||||||
echo "Stopping nodes took $SECONDS seconds"
|
echo "Stopping nodes took $SECONDS seconds"
|
||||||
}
|
}
|
||||||
|
|
||||||
mkdir -p log
|
stop
|
||||||
|
start
|
||||||
|
|
||||||
if [[ $command == "start" ]]; then
|
|
||||||
build
|
|
||||||
stop
|
|
||||||
start
|
|
||||||
elif [[ $command == "stop" ]]; then
|
|
||||||
stop
|
|
||||||
else
|
|
||||||
usage "Unknown command: $command"
|
|
||||||
fi
|
|
||||||
|
|
|
@ -1,18 +1,56 @@
|
||||||
#!/bin/bash -e
|
#!/bin/bash -e
|
||||||
|
|
||||||
[[ -n $1 ]] || exit
|
deployMethod="$1"
|
||||||
|
netEntrypoint="$2"
|
||||||
|
numNodes="$3"
|
||||||
|
RUST_LOG="$4"
|
||||||
|
[[ -n $deployMethod ]] || exit
|
||||||
|
[[ -n $netEntrypoint ]] || exit
|
||||||
|
[[ -n $numNodes ]] || exit
|
||||||
|
|
||||||
cd "$(dirname "$0")"/../..
|
cd "$(dirname "$0")"/../..
|
||||||
source net/common.sh
|
source net/common.sh
|
||||||
loadConfigFile
|
loadConfigFile
|
||||||
|
|
||||||
PATH="$HOME"/.cargo/bin:"$PATH"
|
threadCount=$(nproc)
|
||||||
rsync -vPrz "$1":~/.cargo/bin/solana* ~/.cargo/bin/
|
if [[ $threadCount -gt 4 ]]; then
|
||||||
|
threadCount=4
|
||||||
numNodes=1 # TODO: Pass this in
|
fi
|
||||||
|
|
||||||
./script/install-earlyoom.sh
|
./script/install-earlyoom.sh
|
||||||
|
|
||||||
|
case $deployMethod in
|
||||||
|
snap)
|
||||||
|
sudo snap install solana.snap --devmode --dangerous
|
||||||
|
rm solana.snap
|
||||||
|
|
||||||
|
sudo snap set solana metrics-config="$SOLANA_METRICS_CONFIG" rust-log="$RUST_LOG"
|
||||||
|
solana_bench_tps=/snap/bin/solana.bench-tps
|
||||||
|
;;
|
||||||
|
local)
|
||||||
|
PATH="$HOME"/.cargo/bin:"$PATH"
|
||||||
|
export USE_INSTALL=1
|
||||||
|
export RUST_LOG
|
||||||
|
|
||||||
|
rsync -vPrz "$netEntrypoint:~/.cargo/bin/solana*" ~/.cargo/bin/
|
||||||
|
solana_bench_tps=multinode-demo/client.sh
|
||||||
|
netEntrypoint="$:~/solana"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown deployment method: $deployMethod"
|
||||||
|
exit 1
|
||||||
|
esac
|
||||||
|
|
||||||
./scripts/oom-monitor.sh > oom-monitor.log 2>&1 &
|
./scripts/oom-monitor.sh > oom-monitor.log 2>&1 &
|
||||||
|
|
||||||
export USE_INSTALL=1
|
while true; do
|
||||||
multinode-demo/client.sh "$1":~/solana $numNodes --loop -s 600 --sustained > client.log 2>&1 &
|
echo "=== Client start: $(date)" >> client.log
|
||||||
|
clientCommand="$solana_bench_tps $netEntrypoint $numNodes --loop -s 600 --sustained -t threadCount"
|
||||||
|
echo "$ $clientCommand" >> client.log
|
||||||
|
|
||||||
|
$clientCommand >> client.log 2>&1
|
||||||
|
|
||||||
|
$metricsWriteDatapoint "testnet-deploy,name=$netBasename clientexit=1"
|
||||||
|
echo Error: bench-tps should never exit | tee -a client.log
|
||||||
|
done
|
||||||
|
|
||||||
|
|
|
@ -1,18 +0,0 @@
|
||||||
#!/bin/bash -e
|
|
||||||
|
|
||||||
cd "$(dirname "$0")"/../..
|
|
||||||
source net/common.sh
|
|
||||||
loadConfigFile
|
|
||||||
|
|
||||||
PATH="$HOME"/.cargo/bin:"$PATH"
|
|
||||||
|
|
||||||
./fetch-perf-libs.sh
|
|
||||||
|
|
||||||
./script/install-earlyoom.sh
|
|
||||||
./scripts/oom-monitor.sh > oom-monitor.log 2>&1 &
|
|
||||||
|
|
||||||
export USE_INSTALL=1
|
|
||||||
export SOLANA_CUDA=1
|
|
||||||
./multinode-demo/setup.sh
|
|
||||||
./multinode-demo/drone.sh > drone.log 2>&1 &
|
|
||||||
./multinode-demo/leader.sh > leader.log 2>&1 &
|
|
|
@ -0,0 +1,83 @@
|
||||||
|
#!/bin/bash -e
|
||||||
|
|
||||||
|
deployMethod="$1"
|
||||||
|
nodeType="$2"
|
||||||
|
netEntrypoint="$3"
|
||||||
|
setupArgs="$4"
|
||||||
|
RUST_LOG="$5"
|
||||||
|
|
||||||
|
[[ -n $deployMethod ]] || exit
|
||||||
|
[[ -n $nodeType ]] || exit
|
||||||
|
[[ -n $netEntrypoint ]] || exit
|
||||||
|
|
||||||
|
cd "$(dirname "$0")"/../..
|
||||||
|
source net/common.sh
|
||||||
|
loadConfigFile
|
||||||
|
|
||||||
|
./script/install-earlyoom.sh
|
||||||
|
|
||||||
|
case $deployMethod in
|
||||||
|
snap)
|
||||||
|
SECONDS=0
|
||||||
|
sudo snap install solana.snap --devmode --dangerous
|
||||||
|
rm solana.snap
|
||||||
|
|
||||||
|
commonNodeConfig="\
|
||||||
|
rust-log=$RUST_LOG \
|
||||||
|
metrics-config=$SOLANA_METRICS_CONFIG \
|
||||||
|
setup-args=$setupArgs \
|
||||||
|
enable-cuda=1 \
|
||||||
|
"
|
||||||
|
if [[ $nodeType = leader ]]; then
|
||||||
|
nodeConfig="mode=leader+drone $commonNodeConfig"
|
||||||
|
else
|
||||||
|
nodeConfig="mode=validator leader-address=$netEntrypoint $commonNodeConfig"
|
||||||
|
fi
|
||||||
|
|
||||||
|
logmarker="solana deploy $(date)/$RANDOM"
|
||||||
|
logger "$logmarker"
|
||||||
|
|
||||||
|
# shellcheck disable=SC2086 # Don't want to double quote "$nodeConfig"
|
||||||
|
sudo snap set solana $nodeConfig
|
||||||
|
snap info solana
|
||||||
|
sudo snap get solana
|
||||||
|
echo Slight delay to get more syslog output
|
||||||
|
sleep 2
|
||||||
|
sudo grep -Pzo "$logmarker(.|\\n)*" /var/log/syslog
|
||||||
|
|
||||||
|
echo "Succeeded in ${SECONDS} seconds"
|
||||||
|
;;
|
||||||
|
local)
|
||||||
|
PATH="$HOME"/.cargo/bin:"$PATH"
|
||||||
|
export USE_INSTALL=1
|
||||||
|
export SOLANA_CUDA=1
|
||||||
|
export RUST_LOG=1
|
||||||
|
|
||||||
|
./fetch-perf-libs.sh
|
||||||
|
./scripts/oom-monitor.sh > oom-monitor.log 2>&1 &
|
||||||
|
|
||||||
|
case $nodeType in
|
||||||
|
leader)
|
||||||
|
# shellcheck disable=SC2086 # Don't want to double quote "$setupArgs"
|
||||||
|
./multinode-demo/setup.sh -t leader -p $setupArgs
|
||||||
|
./multinode-demo/drone.sh > drone.log 2>&1 &
|
||||||
|
./multinode-demo/leader.sh > leader.log 2>&1 &
|
||||||
|
;;
|
||||||
|
validator)
|
||||||
|
rsync -vPrz "$netEntrypoint:~/.cargo/bin/solana*" ~/.cargo/bin/
|
||||||
|
|
||||||
|
# shellcheck disable=SC2086 # Don't want to double quote "$setupArgs"
|
||||||
|
./multinode-demo/setup.sh -t validator -p $setupArgs
|
||||||
|
./multinode-demo/validator.sh "$netEntrypoint":~/solana "$netEntrypoint" >validator.log 2>&1 &
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Error: unknown node type: $nodeType"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown deployment method: $deployMethod"
|
||||||
|
exit 1
|
||||||
|
esac
|
||||||
|
|
|
@ -1,18 +0,0 @@
|
||||||
#!/bin/bash -e
|
|
||||||
|
|
||||||
[[ -n $1 ]] || exit
|
|
||||||
|
|
||||||
cd "$(dirname "$0")"/../..
|
|
||||||
source net/common.sh
|
|
||||||
loadConfigFile
|
|
||||||
|
|
||||||
PATH="$HOME"/.cargo/bin:"$PATH"
|
|
||||||
|
|
||||||
rsync -vPrz "$1":~/.cargo/bin/solana* ~/.cargo/bin/
|
|
||||||
|
|
||||||
./script/install-earlyoom.sh
|
|
||||||
./scripts/oom-monitor.sh > oom-monitor.log 2>&1 &
|
|
||||||
|
|
||||||
export USE_INSTALL=1
|
|
||||||
./multinode-demo/setup.sh
|
|
||||||
./multinode-demo/validator.sh "$1":~/solana "$1" >validator.log 2>&1 &
|
|
|
@ -6,12 +6,18 @@
|
||||||
# Example:
|
# Example:
|
||||||
# export SOLANA_METRICS_CONFIG="host=<metrics host>,db=<database name>,u=<username>,p=<password>"
|
# export SOLANA_METRICS_CONFIG="host=<metrics host>,db=<database name>,u=<username>,p=<password>"
|
||||||
#
|
#
|
||||||
configure_metrics() {
|
# The following directive disable complaints about unused variables in this
|
||||||
|
# file:
|
||||||
|
# shellcheck disable=2034
|
||||||
|
#
|
||||||
|
metricsWriteDatapoint="$(dirname "${BASH_SOURCE[0]}")"/metrics-write-datapoint.sh
|
||||||
|
|
||||||
|
configureMetrics() {
|
||||||
[[ -n $SOLANA_METRICS_CONFIG ]] || return 0
|
[[ -n $SOLANA_METRICS_CONFIG ]] || return 0
|
||||||
|
|
||||||
declare metrics_params
|
declare metricsParams
|
||||||
IFS=',' read -r -a metrics_params <<< "$SOLANA_METRICS_CONFIG"
|
IFS=',' read -r -a metricsParams <<< "$SOLANA_METRICS_CONFIG"
|
||||||
for param in "${metrics_params[@]}"; do
|
for param in "${metricsParams[@]}"; do
|
||||||
IFS='=' read -r -a pair <<< "$param"
|
IFS='=' read -r -a pair <<< "$param"
|
||||||
if [[ ${#pair[@]} != 2 ]]; then
|
if [[ ${#pair[@]} != 2 ]]; then
|
||||||
echo Error: invalid metrics parameter: "$param" >&2
|
echo Error: invalid metrics parameter: "$param" >&2
|
||||||
|
@ -42,4 +48,4 @@ configure_metrics() {
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
configure_metrics
|
configureMetrics
|
||||||
|
|
Loading…
Reference in New Issue