From ff599ace4d201701e570f847faec5516d737e043 Mon Sep 17 00:00:00 2001
From: Trent Nelson <trent@solana.com>
Date: Tue, 5 Jan 2021 21:31:01 -0700
Subject: [PATCH] net/: Support accounts on swap-backed tmpfs

---
 net/gce.sh                | 32 ++++++++++++++++++++++++++++++++
 net/net.sh                |  2 ++
 net/remote/remote-node.sh |  9 +++++++++
 3 files changed, 43 insertions(+)

diff --git a/net/gce.sh b/net/gce.sh
index 2a860d3e9..07dad152c 100755
--- a/net/gce.sh
+++ b/net/gce.sh
@@ -68,6 +68,7 @@ externalNodes=false
 failOnValidatorBootupFailure=true
 preemptible=true
 evalInfo=false
+tmpfsAccounts=false
 
 publicNetwork=false
 letsEncryptDomainName=
@@ -154,6 +155,7 @@ Manage testnet instances
                     - Specify validator boot disk size in gb.
    --client-machine-type [type]
                     - custom client machine type
+   --tmpfs-accounts - Put accounts directory on a swap-backed tmpfs volume
 
  config-specific options:
    -P               - Use public network IP addresses (default: $publicNetwork)
@@ -228,6 +230,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 == --reclaim-all-reservations ]]; then
       reclaimAllReservations=true
       shift
+    elif [[ $1 == --tmpfs-accounts ]]; then
+      tmpfsAccounts=true
+      shift
     else
       usage "Unknown long option: $1"
     fi
@@ -298,6 +303,11 @@ fi
 case $cloudProvider in
 gce)
   customMemoryGB="$(cloud_DefaultCustomMemoryGB)"
+  if [[ "$tmpfsAccounts" = "true" ]]; then
+    customMemoryGB=$(( customMemoryGB * 2 ))
+    cpuBootstrapLeaderMachineType+=" --local-ssd interface=nvme"
+    gpuBootstrapLeaderMachineType+=" --local-ssd interface=nvme"
+  fi
   cpuBootstrapLeaderMachineType+=" --custom-memory ${customMemoryGB}GB"
   gpuBootstrapLeaderMachineType+=" --custom-memory ${customMemoryGB}GB"
   ;;
@@ -305,6 +315,9 @@ ec2|azure|colo)
   if [[ -n $validatorAdditionalDiskSizeInGb ]] ; then
     usage "Error: --validator-additional-disk-size-gb currently only supported with cloud provider: gce"
   fi
+  if [[ "$tmpfsAccounts" = "true" ]]; then
+    usage "Error: --tmpfs-accounts only supported on cloud provider: gce"
+  fi
   ;;
 *)
   echo "Error: Unknown cloud provider: $cloudProvider"
@@ -445,6 +458,7 @@ netBasename=$prefix
 publicNetwork=$publicNetwork
 sshPrivateKey=$sshPrivateKey
 letsEncryptDomainName=$letsEncryptDomainName
+export TMPFS_ACCOUNTS=$tmpfsAccounts
 EOF
   fi
   touch "$geoipConfigFile"
@@ -833,6 +847,24 @@ $(printNetworkInfo)
 $(creationInfo)
 EOM
 
+$(
+  if [[ "$tmpfsAccounts" = "true" ]]; then
+    cat <<'EOSWAP'
+
+# Setup swap/tmpfs for accounts
+tmpfsMountPoint=/mnt/solana-accounts
+swapDevice="/dev/nvme0n1"
+swapUUID="43076c54-7840-4e59-a368-2d164f8984fb"
+mkswap --uuid "$swapUUID" "$swapDevice"
+echo "UUID=$swapUUID swap swap defaults 0 0" >> /etc/fstab
+swapon "UUID=$swapUUID"
+mkdir -p -m 0777 "$tmpfsMountPoint"
+echo "tmpfs $tmpfsMountPoint tmpfs defaults,size=300G 0 0" >> /etc/fstab
+mount "$tmpfsMountPoint"
+EOSWAP
+  fi
+)
+
 touch /solana-scratch/.instance-startup-complete
 
 EOF
diff --git a/net/net.sh b/net/net.sh
index a6b9755d6..c05455516 100755
--- a/net/net.sh
+++ b/net/net.sh
@@ -294,6 +294,7 @@ startBootstrapLeader() {
          \"$maybeWarpSlot\" \
          \"$waitForNodeInit\" \
          \"$extraPrimordialStakes\" \
+         \"$TMPFS_ACCOUNTS\" \
       "
 
   ) >> "$logFile" 2>&1 || {
@@ -365,6 +366,7 @@ startNode() {
          \"$maybeWarpSlot\" \
          \"$waitForNodeInit\" \
          \"$extraPrimordialStakes\" \
+         \"$TMPFS_ACCOUNTS\" \
       "
   ) >> "$logFile" 2>&1 &
   declare pid=$!
diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh
index b3de1a792..c468d7202 100755
--- a/net/remote/remote-node.sh
+++ b/net/remote/remote-node.sh
@@ -28,6 +28,7 @@ gpuMode="${19:-auto}"
 maybeWarpSlot="${20}"
 waitForNodeInit="${21}"
 extraPrimordialStakes="${22:=0}"
+tmpfsAccounts="${23:false}"
 set +x
 
 missing() {
@@ -275,6 +276,10 @@ EOF
       --init-complete-file "$initCompleteFile"
     )
 
+    if [[ "$tmpfsAccounts" = "true" ]]; then
+      args+=(--accounts /mnt/solana-accounts)
+    fi
+
     if [[ $airdropsEnabled = true ]]; then
 cat >> ~/solana/on-reboot <<EOF
       ./multinode-demo/faucet.sh > faucet.log 2>&1 &
@@ -392,6 +397,10 @@ EOF
       maybeSkipAccountsCreation="export SKIP_ACCOUNTS_CREATION=1"
     fi
 
+    if [[ "$tmpfsAccounts" = "true" ]]; then
+      args+=(--accounts /mnt/solana-accounts)
+    fi
+
 cat >> ~/solana/on-reboot <<EOF
     $maybeSkipAccountsCreation
     nohup multinode-demo/validator.sh ${args[@]} > validator.log.\$now 2>&1 &