Colo: Put NVMe disks to use (#6357)

automerge

ci/localnet-sanity.sh

@@ -70,6 +70,7 @@ done
 
 source ci/upload-ci-artifact.sh
 source scripts/configure-metrics.sh
+source multinode-demo/common.sh
 
 nodes=(
   "multinode-demo/drone.sh"
@@ -173,7 +174,6 @@ startNodes() {
       waitForNodeToInit "$initCompleteFile"
 
       (
-        source multinode-demo/common.sh
         set -x
         $solana_cli --keypair config/bootstrap-leader/identity-keypair.json \
           --url http://127.0.0.1:8899 get-genesis-blockhash
@@ -277,7 +277,6 @@ verifyLedger() {
   for ledger in bootstrap-leader validator; do
     echo "--- $ledger ledger verification"
     (
-      source multinode-demo/common.sh
       set -x
       $solana_ledger_tool --ledger "$SOLANA_CONFIG_DIR"/$ledger verify
     ) || flag_error
@@ -312,7 +311,7 @@ flag_error() {
 }
 
 if ! $skipSetup; then
-  multinode-demo/clear-config.sh
+  clear_config_dir "$SOLANA_CONFIG_DIR"
   multinode-demo/setup.sh
 else
   verifyLedger
@@ -322,7 +321,6 @@ lastTransactionCount=
 while [[ $iteration -le $iterations ]]; do
   echo "--- Node count ($iteration)"
   (
-    source multinode-demo/common.sh
     set -x
     client_keypair=/tmp/client-id.json-$$
     $solana_keygen new -f -o $client_keypair || exit $?

multinode-demo/clear-config.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-#
-# Clear the current cluster configuration
-#
-
-here=$(dirname "$0")
-# shellcheck source=multinode-demo/common.sh
-source "$here"/common.sh
-
-set -e
-
-(
-  set -x
-  rm -rf "${SOLANA_CONFIG_DIR:?}/" # <-- $i might be a symlink, rm the other side of it first
-  rm -rf "$SOLANA_CONFIG_DIR"
-  mkdir -p "$SOLANA_CONFIG_DIR"
-)
-
-setup_secondary_mount

multinode-demo/common.sh

@@ -7,7 +7,8 @@
 # shellcheck disable=2034
 #
 
-SOLANA_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. || exit 1; pwd)"
+# shellcheck source=net/common.sh
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. || exit 1; pwd)"/net/common.sh
 
 if [[ $(uname) != Linux ]]; then
   # Protect against unsupported configurations to prevent non-obvious errors
@@ -62,21 +63,6 @@ solana_replicator=$(solana_program replicator)
 
 export RUST_BACKTRACE=1
 
-# shellcheck source=scripts/configure-metrics.sh
-source "$SOLANA_ROOT"/scripts/configure-metrics.sh
-
-SOLANA_CONFIG_DIR=$SOLANA_ROOT/config
-
-SECONDARY_DISK_MOUNT_POINT=/mnt/extra-disk
-setup_secondary_mount() {
-  # If there is a secondary disk, symlink the config/ dir there
-  if [[ -d $SECONDARY_DISK_MOUNT_POINT ]]; then
-    mkdir -p $SECONDARY_DISK_MOUNT_POINT/config
-    rm -rf "$SOLANA_CONFIG_DIR"
-    ln -sfT $SECONDARY_DISK_MOUNT_POINT/config "$SOLANA_CONFIG_DIR"
-  fi
-}
-
 default_arg() {
   declare name=$1
   declare value=$2

net/common.sh

@@ -15,8 +15,9 @@ netConfigDir="$netDir"/config
 netLogDir="$netDir"/log
 mkdir -p "$netConfigDir" "$netLogDir"
 
+SOLANA_ROOT="$netDir"/..
 # shellcheck source=scripts/configure-metrics.sh
-source "$(dirname "${BASH_SOURCE[0]}")"/../scripts/configure-metrics.sh
+source "$SOLANA_ROOT"/scripts/configure-metrics.sh
 
 configFile="$netConfigDir/config"
 geoipConfigFile="$netConfigDir/geoip.yml"
@@ -94,3 +95,28 @@ urlencode() {
   done
 }
 
+SOLANA_CONFIG_DIR=$SOLANA_ROOT/config
+# Clear the current cluster configuration
+clear_config_dir() {
+  declare config_dir="$1"
+  (
+    set -x
+    rm -rf "${config_dir:?}/" # <-- $i might be a symlink, rm the other side of it first
+    rm -rf "$config_dir"
+    mkdir -p "$config_dir"
+  )
+
+  setup_secondary_mount
+}
+
+SECONDARY_DISK_MOUNT_POINT=/mnt/extra-disk
+setup_secondary_mount() {
+  # If there is a secondary disk, symlink the config/ dir there
+  if [[ -d $SECONDARY_DISK_MOUNT_POINT ]] && \
+    [[ -w $SECONDARY_DISK_MOUNT_POINT ]]; then
+    mkdir -p $SECONDARY_DISK_MOUNT_POINT/config
+    rm -rf "$SOLANA_CONFIG_DIR"
+    ln -sfT $SECONDARY_DISK_MOUNT_POINT/config "$SOLANA_CONFIG_DIR"
+  fi
+}
+

net/remote/remote-node.sh

@@ -61,6 +61,7 @@ airdropsEnabled=$airdropsEnabled
 EOF
 
 source net/common.sh
+source multinode-demo/common.sh
 loadConfigFile
 
 initCompleteFile=init-complete-node.log
@@ -149,7 +150,7 @@ EOF
   bootstrap-leader)
     set -x
     if [[ $skipSetup != true ]]; then
-      multinode-demo/clear-config.sh
+      clear_config_dir "$SOLANA_CONFIG_DIR"
 
       if [[ -n $internalNodesLamports ]]; then
         echo "---" >> config/fullnode-balances.yml
@@ -245,7 +246,7 @@ EOF
       net/scripts/rsync-retry.sh -vPrc "$entrypointIp":~/version.yml ~/version.yml
     fi
     if [[ $skipSetup != true ]]; then
-      multinode-demo/clear-config.sh
+      clear_config_dir "$SOLANA_CONFIG_DIR"
       [[ -z $internalNodesLamports ]] || net/scripts/rsync-retry.sh -vPrc \
       "$entrypointIp":~/solana/config/fullnode-"$nodeIndex"-identity.json config/fullnode-identity.json
     fi

net/scripts/colo-node-onfree-sh

@@ -99,6 +99,10 @@ if [ -f "$SOLANA_LOCK_FILE" ]; then
     cat > "\${HOME}/.ssh/authorized_keys" <<EOAK
 $("$__colo_here"/add-datacenter-solana-user-authorized_keys.sh 2> /dev/null)
 EOAK
+    EXTERNAL_CONFIG_DIR="${SECONDARY_DISK_MOUNT_POINT}/config/"
+    if [[ -d "\$EXTERNAL_CONFIG_DIR" ]]; then
+      rm -rf "\$EXTERNAL_CONFIG_DIR"
+    fi
     # End filesystem cleanup
     RC=true
   fi

net/scripts/colo-utils.sh

@@ -3,6 +3,10 @@
 declare -r SOLANA_LOCK_FILE="/home/solana/.solana.lock"
 
 __colo_here="$(dirname "${BASH_SOURCE[0]}")"
+
+# shellcheck source=net/common.sh
+source "$__colo_here"/../common.sh
+
 # Load colo resource specs
 export COLO_RES_N=0
 export COLO_RES_HOSTNAME=()