Add support for additional disks for config-local (#5030)

* Add support for additional disks for config-local

* Restore wrongly deleted lines

* Shellcheck

* add args in the right place dummy

* Fix nits

* typo

* var naming cleanup

* Add stub function for remaining cloud providers

ci/testnet-deploy.sh

@@ -28,6 +28,7 @@ maybeStakeNodesInGenesisBlock=
 maybeExternalPrimordialAccountsFile=
 maybeLamports=
 maybeLetsEncrypt=
+maybeFullnodeAdditionalDiskSize=
 
 usage() {
   exitcode=0
@@ -80,6 +81,8 @@ Deploys a CD testnet
                         - If set, will not fetch logs from remote nodes
    --letsencrypt [dns name]
                         - Attempt to generate a TLS certificate using this DNS name
+   --fullnode-additional-disk-size-gb [number]
+                    - Size of additional disk in GB for all fullnodes
 
    Note: the SOLANA_METRICS_CONFIG environment variable is used to configure
          metrics
@@ -113,6 +116,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --letsencrypt ]]; then
       maybeLetsEncrypt="$1 $2"
       shift 2
+    elif [[ $1 = --fullnode-additional-disk-size-gb ]]; then
+      maybeFullnodeAdditionalDiskSize="$1 $2"
+      shift 2
     else
       usage "Unknown long option: $1"
     fi
@@ -292,6 +298,11 @@ if ! $skipCreate; then
     create_args+=(-f)
   fi
 
+  if [[ -n $maybeFullnodeAdditionalDiskSize ]]; then
+    # shellcheck disable=SC2206 # Do not want to quote
+    create_args+=($maybeFullnodeAdditionalDiskSize)
+  fi
+
   time net/"$cloudProvider".sh create "${create_args[@]}"
 else
   echo "--- $cloudProvider.sh config"

multinode-demo/common.sh

@@ -70,7 +70,12 @@ source "$SOLANA_ROOT"/scripts/configure-metrics.sh
 SOLANA_RSYNC_CONFIG_DIR=$SOLANA_ROOT/config
 
 # Configuration that remains local
-SOLANA_CONFIG_DIR=$SOLANA_ROOT/config-local
+SECONDARY_DISK_MOUNT_POINT=/mnt/extra-disk
+if [[ -d $SECONDARY_DISK_MOUNT_POINT ]]; then
+  SOLANA_CONFIG_DIR=$SECONDARY_DISK_MOUNT_POINT/config-local
+else
+  SOLANA_CONFIG_DIR=$SOLANA_ROOT/config-local
+fi
 
 default_arg() {
   declare name=$1

net/gce.sh

@@ -63,6 +63,7 @@ blockstreamer=false
 fullNodeBootDiskSizeInGb=1000
 clientBootDiskSizeInGb=75
 replicatorBootDiskSizeInGb=1000
+fullNodeAdditionalDiskSizeInGb=
 externalNodes=false
 failOnValidatorBootupFailure=true
 
@@ -126,7 +127,10 @@ Manage testnet instances
    --letsencrypt [dns name] - Attempt to generate a TLS certificate using this
                               DNS name (useful only when the -a and -P options
                               are also provided)
-
+   --fullnode-additional-disk-size-gb [number]
+                    - Add an additional [number] GB SSD to all fullnodes to store the config-local directory.
+                      If not set, config-local will be written to the boot disk by default.
+                      Only supported on GCE.
  config-specific options:
    -P               - Use public network IP addresses (default: $publicNetwork)
 
@@ -152,6 +156,9 @@ while [[ -n $1 ]]; do
     if [[ $1 = --letsencrypt ]]; then
       letsEncryptDomainName="$2"
       shift 2
+    elif [[ $1 = --fullnode-additional-disk-size-gb ]]; then
+      fullNodeAdditionalDiskSizeInGb="$2"
+      shift 2
     else
       usage "Unknown long option: $1"
     fi
@@ -234,8 +241,14 @@ case $cloudProvider in
 gce)
   ;;
 ec2)
+  if [[ -n $fullNodeAdditionalDiskSizeInGb ]] ; then
+    usage "Error: --fullnode-additional-disk-size-gb currently only supported with cloud provider: gce"
+  fi
   ;;
 azure)
+  if [[ -n $fullNodeAdditionalDiskSizeInGb ]] ; then
+    usage "Error: --fullnode-additional-disk-size-gb currently only supported with cloud provider: gce"
+  fi
   ;;
 *)
   echo "Error: Unknown cloud provider: $cloudProvider"
@@ -630,6 +643,10 @@ $(
       cat enable-nvidia-persistence-mode.sh
     fi
 
+    if [[ -n $fullNodeAdditionalDiskSizeInGb ]]; then
+      cat mount-additional-disk.sh
+    fi
+
 )
 
 cat > /etc/motd <<EOM
@@ -656,7 +673,7 @@ EOF
   else
     cloud_CreateInstances "$prefix" "$prefix-bootstrap-leader" 1 \
       "$enableGpu" "$bootstrapLeaderMachineType" "${zones[0]}" "$fullNodeBootDiskSizeInGb" \
-      "$startupScript" "$bootstrapLeaderAddress" "$bootDiskType"
+      "$startupScript" "$bootstrapLeaderAddress" "$bootDiskType" "$fullNodeAdditionalDiskSizeInGb"
   fi
 
   if [[ $additionalFullNodeCount -gt 0 ]]; then
@@ -676,7 +693,7 @@ EOF
       fi
       cloud_CreateInstances "$prefix" "$prefix-$zone-fullnode" "$numNodesPerZone" \
         "$enableGpu" "$fullNodeMachineType" "$zone" "$fullNodeBootDiskSizeInGb" \
-        "$startupScript" "" "$bootDiskType" &
+        "$startupScript" "" "$bootDiskType" "$fullNodeAdditionalDiskSizeInGb" &
     done
 
     wait

net/scripts/azure-provider.sh

@@ -309,3 +309,12 @@ cloud_FetchFile() {
   cloud_GetConfigValueFromInstanceName "$instanceName" osProfile.adminUsername
   scp "${config_value}@${publicIp}:${remoteFile}" "$localFile"
 }
+
+#
+# cloud_CreateAndAttachPersistentDisk
+#
+# Not yet implemented for this cloud provider
+cloud_CreateAndAttachPersistentDisk() {
+  echo "ERROR: cloud_CreateAndAttachPersistentDisk is not yet implemented for azure"
+  exit 1
+}

net/scripts/ec2-provider.sh

@@ -381,3 +381,12 @@ cloud_FetchFile() {
       "solana@$publicIp:$remoteFile" "$localFile"
   )
 }
+
+#
+# cloud_CreateAndAttachPersistentDisk
+#
+# Not yet implemented for this cloud provider
+cloud_CreateAndAttachPersistentDisk() {
+  echo "ERROR: cloud_CreateAndAttachPersistentDisk is not yet implemented for ec2"
+  exit 1
+}

net/scripts/gce-provider.sh

@@ -126,6 +126,7 @@ cloud_CreateInstances() {
   declare optionalStartupScript="$8"
   declare optionalAddress="$9"
   declare optionalBootDiskType="${10}"
+  declare optionalAdditionalDiskSize="${11}"
 
   if $enableGpu; then
     # Custom Ubuntu 18.04 LTS image with CUDA 9.2 and CUDA 10.0 installed
@@ -198,6 +199,22 @@ cloud_CreateInstances() {
     set -x
     gcloud beta compute instances create "${nodes[@]}" "${args[@]}"
   )
+
+  if [[ -n $optionalAdditionalDiskSize ]]; then
+    if [[ $numNodes = 1 ]]; then
+      (
+        set -x
+        cloud_CreateAndAttachPersistentDisk "${namePrefix}" "$optionalAdditionalDiskSize" "pd-ssd" "$zone"
+      )
+    else
+      for node in $(seq -f "${namePrefix}%0${#numNodes}g" 1 "$numNodes"); do
+        (
+          set -x
+          cloud_CreateAndAttachPersistentDisk "${node}" "$optionalAdditionalDiskSize" "pd-ssd" "$zone"
+        )
+      done
+    fi
+  fi
 }
 
 #
@@ -256,3 +273,31 @@ cloud_FetchFile() {
     gcloud compute scp --zone "$zone" "$instanceName:$remoteFile" "$localFile"
   )
 }
+
+#
+# cloud_CreateAndAttachPersistentDisk [instanceName] [diskSize] [diskType]
+#
+# Create a persistent disk and attach it to a pre-existing VM instance.
+# Set disk to auto-delete upon instance deletion
+#
+cloud_CreateAndAttachPersistentDisk() {
+  declare instanceName="$1"
+  declare diskSize="$2"
+  declare diskType="$3"
+  declare zone="$4"
+  diskName="${instanceName}-pd"
+
+  gcloud beta compute disks create "$diskName" \
+    --size "$diskSize" \
+    --type "$diskType" \
+    --zone "$zone"
+
+  gcloud compute instances attach-disk "$instanceName" \
+    --disk "$diskName" \
+    --zone "$zone"
+
+  gcloud compute instances set-disk-auto-delete "$instanceName" \
+    --disk "$diskName" \
+    --zone "$zone" \
+    --auto-delete
+}

net/scripts/mount-additional-disk.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -x
+
+mount_point=/mnt/extra-disk
+disk=sdb
+if ! lsblk | grep -q ${disk} ; then
+  echo "${disk} does not exist"
+else
+  if mount | grep -q ${disk} ; then
+    echo "${disk} is already mounted"
+  else
+    sudo mkfs.ext4 -F /dev/"$disk"
+    sudo mkdir -p "$mount_point"
+    sudo mount /dev/"$disk" "$mount_point"
+    sudo chmod a+w "$mount_point"
+    if ! mount | grep -q ${mount_point} ; then
+      echo "${disk} failed to mount!"
+      exit 1
+    fi
+  fi
+fi