Install earlyoom at gce instance startup
This commit is contained in:
parent
9f413fd656
commit
9f5c86e60c
60
net/gce.sh
60
net/gce.sh
|
@ -206,23 +206,65 @@ create)
|
||||||
|
|
||||||
$metricsWriteDatapoint "testnet-deploy net-create-begin=1"
|
$metricsWriteDatapoint "testnet-deploy net-create-begin=1"
|
||||||
|
|
||||||
echo "Network composition:"
|
printNetworkInfo() {
|
||||||
echo "Leader = $leaderMachineType (GPU=${leaderAccelerator:-none})"
|
cat <<EOF
|
||||||
echo "Validators = $validatorNodeCount x $validatorMachineType (GPU=${validatorAccelerator:-none})"
|
========================================================================================
|
||||||
echo "Client(s) = $clientNodeCount x $clientMachineType (GPU=${clientAccelerator:-none})"
|
|
||||||
echo ==================================================================
|
Network composition:
|
||||||
echo
|
Leader = $leaderMachineType (GPU=${leaderAccelerator:-none})
|
||||||
|
Validators = $validatorNodeCount x $validatorMachineType (GPU=${validatorAccelerator:-none})
|
||||||
|
Client(s) = $clientNodeCount x $clientMachineType (GPU=${clientAccelerator:-none})
|
||||||
|
|
||||||
|
========================================================================================
|
||||||
|
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
printNetworkInfo
|
||||||
|
|
||||||
|
declare startupScript="$netConfigDir"/gce-startup-script.sh
|
||||||
|
cat > "$startupScript" <<EOF
|
||||||
|
#!/bin/bash -ex
|
||||||
|
# autogenerated at $(date)
|
||||||
|
|
||||||
|
cat > /etc/motd <<EOM
|
||||||
|
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||||
|
|
||||||
|
This instance has not been fully configured.
|
||||||
|
See "startup-script" log messages in /var/log/syslog for status:
|
||||||
|
$ sudo cat /var/log/syslog | grep startup-script
|
||||||
|
|
||||||
|
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||||
|
EOM
|
||||||
|
|
||||||
|
USER=\$(id -un)
|
||||||
|
|
||||||
|
$(
|
||||||
|
cd "$here"/scripts/
|
||||||
|
cat \
|
||||||
|
disable-background-upgrades.sh \
|
||||||
|
install-earlyoom.sh \
|
||||||
|
install-rsync.sh \
|
||||||
|
install-libssl-compatability.sh \
|
||||||
|
)
|
||||||
|
|
||||||
|
cat > /etc/motd <<EOM
|
||||||
|
$(printNetworkInfo)
|
||||||
|
EOM
|
||||||
|
|
||||||
|
EOF
|
||||||
|
|
||||||
gcloud_CreateInstances "$prefix-leader" 1 "$zone" \
|
gcloud_CreateInstances "$prefix-leader" 1 "$zone" \
|
||||||
"$imageName" "$leaderMachineType" "$leaderBootDiskSize" "$leaderAccelerator" \
|
"$imageName" "$leaderMachineType" "$leaderBootDiskSize" "$leaderAccelerator" \
|
||||||
"$here/remote/remote-startup.sh" "$leaderAddress" \
|
"$startupScript" "$leaderAddress"
|
||||||
|
|
||||||
gcloud_CreateInstances "$prefix-validator" "$validatorNodeCount" "$zone" \
|
gcloud_CreateInstances "$prefix-validator" "$validatorNodeCount" "$zone" \
|
||||||
"$imageName" "$validatorMachineType" "$validatorBootDiskSize" "$validatorAccelerator" \
|
"$imageName" "$validatorMachineType" "$validatorBootDiskSize" "$validatorAccelerator" \
|
||||||
"$here/remote/remote-startup.sh" ""
|
"$startupScript" ""
|
||||||
|
|
||||||
if [[ $clientNodeCount -gt 0 ]]; then
|
if [[ $clientNodeCount -gt 0 ]]; then
|
||||||
gcloud_CreateInstances "$prefix-client" "$clientNodeCount" "$zone" \
|
gcloud_CreateInstances "$prefix-client" "$clientNodeCount" "$zone" \
|
||||||
"$imageName" "$clientMachineType" "$clientBootDiskSize" "$clientAccelerator" \
|
"$imageName" "$clientMachineType" "$clientBootDiskSize" "$clientAccelerator" \
|
||||||
"$here/remote/remote-startup.sh" ""
|
"$startupScript" ""
|
||||||
fi
|
fi
|
||||||
|
|
||||||
$metricsWriteDatapoint "testnet-deploy net-create-complete=1"
|
$metricsWriteDatapoint "testnet-deploy net-create-complete=1"
|
||||||
|
|
|
@ -25,8 +25,6 @@ if [[ $threadCount -gt 4 ]]; then
|
||||||
threadCount=4
|
threadCount=4
|
||||||
fi
|
fi
|
||||||
|
|
||||||
net/scripts/install-earlyoom.sh
|
|
||||||
|
|
||||||
case $deployMethod in
|
case $deployMethod in
|
||||||
snap)
|
snap)
|
||||||
rsync -vPrc "$entrypointIp:~/solana/solana.snap" .
|
rsync -vPrc "$entrypointIp:~/solana/solana.snap" .
|
||||||
|
|
|
@ -29,8 +29,6 @@ EOF
|
||||||
source net/common.sh
|
source net/common.sh
|
||||||
loadConfigFile
|
loadConfigFile
|
||||||
|
|
||||||
time net/scripts/install-earlyoom.sh
|
|
||||||
|
|
||||||
if [[ $publicNetwork = true ]]; then
|
if [[ $publicNetwork = true ]]; then
|
||||||
setupArgs="-p"
|
setupArgs="-p"
|
||||||
else
|
else
|
||||||
|
|
|
@ -1,46 +0,0 @@
|
||||||
#!/bin/bash -x
|
|
||||||
#
|
|
||||||
# Runs at boot on each instance as root
|
|
||||||
#
|
|
||||||
# TODO: Make the following a requirement of the Instance image
|
|
||||||
# instead of a manual install?
|
|
||||||
|
|
||||||
# Prevent background upgrades that block |apt-get|
|
|
||||||
#
|
|
||||||
# TODO: This approach is pretty uncompromising. An alternative solution that
|
|
||||||
# doesn't involve deleting system files would be welcome.
|
|
||||||
rm -rf /usr/lib/apt/apt.systemd.daily
|
|
||||||
rm -rf /usr/bin/unattended-upgrade
|
|
||||||
killall apt.systemd.daily
|
|
||||||
killall unattended-upgrade
|
|
||||||
|
|
||||||
while fuser /var/lib/dpkg/lock; do
|
|
||||||
echo Waiting for lock release...
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
|
|
||||||
|
|
||||||
# rsync setup for Snap builds
|
|
||||||
apt-get --assume-yes install rsync
|
|
||||||
cat > /etc/rsyncd.conf <<-EOF
|
|
||||||
[config]
|
|
||||||
path = /var/snap/solana/current/config
|
|
||||||
hosts allow = *
|
|
||||||
read only = true
|
|
||||||
EOF
|
|
||||||
|
|
||||||
systemctl enable rsync
|
|
||||||
systemctl start rsync
|
|
||||||
|
|
||||||
# Install libssl-dev to be compatible with binaries built on an Ubuntu machine...
|
|
||||||
apt-get --assume-yes install libssl-dev
|
|
||||||
|
|
||||||
# Install libssl1.1 to be compatible with binaries built in the
|
|
||||||
# solanalabs/rust docker image
|
|
||||||
#
|
|
||||||
# cc: https://github.com/solana-labs/solana/issues/1090
|
|
||||||
# cc: https://packages.ubuntu.com/bionic/amd64/libssl1.1/download
|
|
||||||
wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
|
|
||||||
dpkg -i libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
|
|
||||||
rm libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
|
|
||||||
|
|
|
@ -0,0 +1,21 @@
|
||||||
|
#!/bin/bash -ex
|
||||||
|
|
||||||
|
# Prevent background upgrades that block |apt-get|
|
||||||
|
#
|
||||||
|
# TODO: This approach is pretty uncompromising. An alternative solution that
|
||||||
|
# doesn't involve deleting system files would be welcome.
|
||||||
|
|
||||||
|
[[ $(uname) = Linux ]] || exit 1
|
||||||
|
[[ $USER = root ]] || exit 1
|
||||||
|
|
||||||
|
rm -rf /usr/lib/apt/apt.systemd.daily
|
||||||
|
rm -rf /usr/bin/unattended-upgrade
|
||||||
|
killall apt.systemd.daily || true
|
||||||
|
killall unattended-upgrade || true
|
||||||
|
|
||||||
|
while fuser /var/lib/dpkg/lock; do
|
||||||
|
echo Waiting for lock release...
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
|
|
@ -1,32 +1,30 @@
|
||||||
#!/bin/bash -x
|
#!/bin/bash -ex
|
||||||
#
|
#
|
||||||
# Install EarlyOOM
|
# Install EarlyOOM
|
||||||
#
|
#
|
||||||
|
|
||||||
[[ $(uname) = Linux ]] || exit 1
|
[[ $(uname) = Linux ]] || exit 1
|
||||||
|
[[ $USER = root ]] || exit 1
|
||||||
|
|
||||||
# 64 - enable signalling of processes (term, kill, oom-kill)
|
# 64 - enable signalling of processes (term, kill, oom-kill)
|
||||||
# TODO: This setting will not persist across reboots
|
# TODO: This setting will not persist across reboots
|
||||||
sysrq=$(( $(cat /proc/sys/kernel/sysrq) | 64 ))
|
sysctl -w kernel.sysrq=$(( $(cat /proc/sys/kernel/sysrq) | 64 ))
|
||||||
sudo sysctl -w kernel.sysrq=$sysrq
|
|
||||||
|
|
||||||
if command -v earlyoom; then
|
if command -v earlyoom; then
|
||||||
sudo systemctl status earlyoom
|
systemctl status earlyoom
|
||||||
exit 0
|
else
|
||||||
|
wget http://ftp.us.debian.org/debian/pool/main/e/earlyoom/earlyoom_1.1-2_amd64.deb
|
||||||
|
apt install --quiet --yes ./earlyoom_1.1-2_amd64.deb
|
||||||
|
|
||||||
|
cat > earlyoom <<OOM
|
||||||
|
# use the kernel OOM killer, trigger at 20% available RAM,
|
||||||
|
EARLYOOM_ARGS="-k -m 20"
|
||||||
|
OOM
|
||||||
|
cp earlyoom /etc/default/
|
||||||
|
rm earlyoom
|
||||||
|
|
||||||
|
systemctl stop earlyoom
|
||||||
|
systemctl enable earlyoom
|
||||||
|
systemctl start earlyoom
|
||||||
fi
|
fi
|
||||||
|
|
||||||
wget http://ftp.us.debian.org/debian/pool/main/e/earlyoom/earlyoom_1.1-2_amd64.deb
|
|
||||||
sudo apt install --quiet --yes ./earlyoom_1.1-2_amd64.deb
|
|
||||||
|
|
||||||
cat > earlyoom <<OOM
|
|
||||||
# use the kernel OOM killer, trigger at 20% available RAM,
|
|
||||||
EARLYOOM_ARGS="-k -m 20"
|
|
||||||
OOM
|
|
||||||
sudo cp earlyoom /etc/default/
|
|
||||||
rm earlyoom
|
|
||||||
|
|
||||||
sudo systemctl stop earlyoom
|
|
||||||
sudo systemctl enable earlyoom
|
|
||||||
sudo systemctl start earlyoom
|
|
||||||
|
|
||||||
exit 0
|
|
||||||
|
|
|
@ -0,0 +1,18 @@
|
||||||
|
#!/bin/bash -ex
|
||||||
|
|
||||||
|
[[ $(uname) = Linux ]] || exit 1
|
||||||
|
[[ $USER = root ]] || exit 1
|
||||||
|
|
||||||
|
# Install libssl-dev to be compatible with binaries built on an Ubuntu machine...
|
||||||
|
apt-get update
|
||||||
|
apt-get --assume-yes install libssl-dev
|
||||||
|
|
||||||
|
# Install libssl1.1 to be compatible with binaries built in the
|
||||||
|
# solanalabs/rust docker image
|
||||||
|
#
|
||||||
|
# cc: https://github.com/solana-labs/solana/issues/1090
|
||||||
|
# cc: https://packages.ubuntu.com/bionic/amd64/libssl1.1/download
|
||||||
|
wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
|
||||||
|
dpkg -i libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
|
||||||
|
rm libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
|
||||||
|
|
|
@ -0,0 +1,19 @@
|
||||||
|
#!/bin/bash -ex
|
||||||
|
#
|
||||||
|
# Rsync setup for Snap builds
|
||||||
|
#
|
||||||
|
|
||||||
|
[[ $(uname) = Linux ]] || exit 1
|
||||||
|
[[ $USER = root ]] || exit 1
|
||||||
|
|
||||||
|
apt-get --assume-yes install rsync
|
||||||
|
cat > /etc/rsyncd.conf <<-EOF
|
||||||
|
[config]
|
||||||
|
path = /var/snap/solana/current/config
|
||||||
|
hosts allow = *
|
||||||
|
read only = true
|
||||||
|
EOF
|
||||||
|
|
||||||
|
systemctl enable rsync
|
||||||
|
systemctl start rsync
|
||||||
|
|
Loading…
Reference in New Issue