net/ plumbing to manage LetsEncrypt TLS certificates (#4985)

automerge

ci/testnet-deploy.sh

@@ -27,6 +27,7 @@ maybeHashesPerTick=
 maybeStakeNodesInGenesisBlock=
 maybeExternalPrimordialAccountsFile=
 maybeLamports=
+maybeLetsEncryptDomainName=
 
 usage() {
   exitcode=0
@@ -77,6 +78,9 @@ Deploys a CD testnet
                         - If set, will skip software update deployment
    --skip-remote-log-retrieval
                         - If set, will not fetch logs from remote nodes
+   --letsencrypt [dns name]
+                        - Attempt to generate a TLS certificate using this DNS name
+
    Note: the SOLANA_METRICS_CONFIG environment variable is used to configure
          metrics
 EOF
@@ -106,6 +110,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --skip-remote-log-retrieval ]]; then
       fetchLogs=false
       shift 1
+    elif [[ $1 = --letsencrypt ]]; then
+      maybeLetsEncryptDomainName="$2"
+      shift 2
     else
       usage "Unknown long option: $1"
     fi
@@ -342,7 +349,10 @@ if ! $skipStart; then
       # shellcheck disable=SC2206 # Do not want to quote $maybeHashesPerTick
       args+=($maybeHashesPerTick)
     fi
-
+    if [[ -n $maybeLetsEncryptDomainName ]]; then
+      # shellcheck disable=SC2206 # Do not want to quote $maybeLetsEncryptDomainName
+      args+=($maybeLetsEncryptDomainName)
+    fi
     if $reuseLedger; then
       args+=(-r)
     fi
@@ -371,7 +381,6 @@ if ! $skipStart; then
       args+=($maybeLamports)
     fi
 
-    # shellcheck disable=SC2086 # Don't want to double quote the $maybeXYZ variables
     time net/net.sh "${args[@]}"
   ) || ok=false
 

ci/testnet-manager.sh

@@ -336,7 +336,8 @@ deploy() {
     (
       set -x
       ci/testnet-deploy.sh -p edge-testnet-solana-com -C ec2 -z us-west-1a \
-        -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P -a eipalloc-0ccd4f2239886fa94 \
+        -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P \
+        -a eipalloc-0ccd4f2239886fa94 --letsencrypt edge.testnet.solana.com \
         ${skipCreate:+-e} \
         ${skipStart:+-s} \
         ${maybeStop:+-S} \
@@ -362,7 +363,8 @@ deploy() {
       set -x
       NO_VALIDATOR_SANITY=1 \
         ci/testnet-deploy.sh -p beta-testnet-solana-com -C ec2 -z us-west-1a \
-          -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P -a eipalloc-0f286cf8a0771ce35 \
+          -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P \
+          -a eipalloc-0f286cf8a0771ce35 --letsencrypt beta.testnet.solana.com \
           ${skipCreate:+-e} \
           ${skipStart:+-s} \
           ${maybeStop:+-S} \
@@ -393,7 +395,8 @@ deploy() {
 
       # shellcheck disable=SC2068
       ci/testnet-deploy.sh -p testnet-solana-com -C ec2 ${EC2_ZONE_ARGS[@]} \
-        -t "$CHANNEL_OR_TAG" -n "$EC2_NODE_COUNT" -c 0 -u -P -f -a eipalloc-0fa502bf95f6f18b2 \
+        -t "$CHANNEL_OR_TAG" -n "$EC2_NODE_COUNT" -c 0 -u -P -f \
+        -a eipalloc-0fa502bf95f6f18b2 --letsencrypt testnet.solana.com \
         ${skipCreate:+-e} \
         ${maybeSkipStart:+-s} \
         ${maybeStop:+-S} \
@@ -473,7 +476,8 @@ deploy() {
       NO_VALIDATOR_SANITY=1 \
         ci/testnet-deploy.sh -p tds-solana-com -C gce ${GCE_ZONE_ARGS[0]} \
           -t "$CHANNEL_OR_TAG" -n "$GCE_NODE_COUNT" -c 1 -P -u \
-          -a tds-solana-com --hashes-per-tick auto \
+          -a tds-solana-com --letsencrypt tds.solana.com \
+          --hashes-per-tick auto \
           ${skipCreate:+-e} \
           ${skipStart:+-s} \
           ${maybeStop:+-S} \

net/common.sh

@@ -25,6 +25,7 @@ entrypointIp=
 publicNetwork=
 netBasename=
 sshPrivateKey=
+letsEncryptDomainName=
 externalNodeSshKey=
 sshOptions=()
 fullnodeIpList=()

net/gce.sh

@@ -67,6 +67,7 @@ externalNodes=false
 failOnValidatorBootupFailure=true
 
 publicNetwork=false
+letsEncryptDomainName=
 enableGpu=false
 customAddress=
 zones=()
@@ -122,6 +123,9 @@ Manage testnet instances
                       * For EC2, [address] is the "allocation ID" of the desired
                         Elastic IP.
    -d [disk-type]   - Specify a boot disk type (default None) Use pd-ssd to get ssd on GCE.
+   --letsencrypt [dns name] - Attempt to generate a TLS certificate using this
+                              DNS name (useful only when the -a and -P options
+                              are also provided)
 
  config-specific options:
    -P               - Use public network IP addresses (default: $publicNetwork)
@@ -136,14 +140,28 @@ EOF
   exit $exitcode
 }
 
-
 command=$1
 [[ -n $command ]] || usage
 shift
 [[ $command = create || $command = config || $command = info || $command = delete ]] ||
   usage "Invalid command: $command"
 
-while getopts "h?p:Pn:c:r:z:gG:a:d:uxf" opt; do
+shortArgs=()
+while [[ -n $1 ]]; do
+  if [[ ${1:0:2} = -- ]]; then
+    if [[ $1 = --letsencrypt ]]; then
+      letsEncryptDomainName="$2"
+      shift 2
+    else
+      usage "Unknown long option: $1"
+    fi
+  else
+    shortArgs+=("$1")
+    shift
+  fi
+done
+
+while getopts "h?p:Pn:c:r:z:gG:a:d:uxf" opt "${shortArgs[@]}"; do
   case $opt in
   h | \?)
     usage
@@ -199,7 +217,6 @@ while getopts "h?p:Pn:c:r:z:gG:a:d:uxf" opt; do
     ;;
   esac
 done
-shift $((OPTIND - 1))
 
 [[ ${#zones[@]} -gt 0 ]] || zones+=("$(cloud_DefaultZone)")
 
@@ -328,6 +345,7 @@ prepareInstancesAndWriteConfigFile() {
 netBasename=$prefix
 publicNetwork=$publicNetwork
 sshPrivateKey=$sshPrivateKey
+letsEncryptDomainName=$letsEncryptDomainName
 EOF
   fi
   touch "$geoipConfigFile"
@@ -598,6 +616,7 @@ $(
     disable-background-upgrades.sh \
     create-solana-user.sh \
     add-solana-user-authorized_keys.sh \
+    install-certbot.sh \
     install-earlyoom.sh \
     install-libssl-compatability.sh \
     install-nodejs.sh \

net/net.sh

@@ -372,6 +372,23 @@ startNode() {
   (
     set -x
     startCommon "$ipAddress"
+
+    if [[ $nodeType = blockstreamer ]] && [[ -n $letsEncryptDomainName ]]; then
+      #
+      # Create/renew TLS certificate
+      #
+      declare localArchive=~/letsencrypt-"$letsEncryptDomainName".tgz
+      if [[ -r "$localArchive" ]]; then
+        timeout 30s scp "${sshOptions[@]}" "$localArchive" "$ipAddress:letsencrypt.tgz"
+      fi
+      ssh "${sshOptions[@]}" -n "$ipAddress" \
+        "sudo -H /certbot-restore.sh $letsEncryptDomainName maintainers@solana.com"
+      rm -f letsencrypt.tgz
+      timeout 30s scp "${sshOptions[@]}" "$ipAddress:/letsencrypt.tgz" letsencrypt.tgz
+      test -s letsencrypt.tgz # Ensure non-empty before overwriting $localArchive
+      cp letsencrypt.tgz "$localArchive"
+    fi
+
     ssh "${sshOptions[@]}" -n "$ipAddress" \
       "./solana/net/remote/remote-node.sh \
          $deployMethod \

net/remote/remote-node.sh

@@ -223,6 +223,13 @@ local|tar)
       if [[ -z $stakeNodesInGenesisBlock ]]; then
         ./multinode-demo/drone.sh > drone.log 2>&1 &
       fi
+
+      # Grab the TLS cert generated by /certbot-restore.sh
+      if [[ -f /.cert.pem ]]; then
+        sudo install -o $UID -m 400 /.cert.pem /.key.pem .
+        ls -l .cert.pem .key.pem
+      fi
+
       export BLOCKEXPLORER_GEOIP_WHITELIST=$PWD/net/config/geoip.yml
       npm install @solana/blockexplorer@1
       npx solana-blockexplorer > blockexplorer.log 2>&1 &

net/scripts/ec2-security-group-config.json

@@ -81,7 +81,7 @@
             "FromPort": 3001,
             "IpRanges": [
                 {
-                    "Description": "blockexplorer API port",
+                    "Description": "blockexplorer http API port",
                     "CidrIp": "0.0.0.0/0"
                 }
             ],
@@ -91,7 +91,26 @@
             "Ipv6Ranges": [
                 {
                     "CidrIpv6": "::/0",
-                    "Description": "blockexplorer API port"
+                    "Description": "blockexplorer http API port"
+                }
+            ]
+        },
+        {
+            "PrefixListIds": [],
+            "FromPort": 3443,
+            "IpRanges": [
+                {
+                    "Description": "blockexplorer https API port",
+                    "CidrIp": "0.0.0.0/0"
+                }
+            ],
+            "ToPort": 3443,
+            "IpProtocol": "tcp",
+            "UserIdGroupPairs": [],
+            "Ipv6Ranges": [
+                {
+                    "CidrIpv6": "::/0",
+                    "Description": "blockexplorer https API port"
                 }
             ]
         },

net/scripts/install-certbot.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -ex
+
+[[ $(uname) = Linux ]] || exit 1
+[[ $USER = root ]] || exit 1
+
+add-apt-repository --yes ppa:certbot/certbot
+apt-get --assume-yes install certbot
+
+cat > /certbot-restore.sh <<'EOF'
+#!/usr/bin/env bash
+set -e
+
+domain=$1
+email=$2
+
+if [[ $USER != root ]]; then
+  echo "Run as root"
+  exit 1
+fi
+
+if [[ -f /.cert.pem ]]; then
+  echo "Certificate already initialized"
+  exit 0
+fi
+
+set -x
+if [[ -r letsencrypt.tgz ]]; then
+  tar -C / -zxf letsencrypt.tgz
+fi
+
+cd /
+rm -f letsencrypt.tgz
+
+maybeDryRun=
+# Uncomment during testing to avoid hitting LetsEncrypt API limits while iterating
+#maybeDryRun="--dry-run"
+
+certbot certonly --standalone -d "$domain" --email "$email" --agree-tos -n $maybeDryRun
+
+tar zcf letsencrypt.tgz /etc/letsencrypt
+ls -l letsencrypt.tgz
+
+# Copy certificates to / for easy access without knowing the value of "$domain"
+rm -f /.key.pem /.cert.pem
+cp /etc/letsencrypt/live/$domain/privkey.pem /.key.pem
+cp /etc/letsencrypt/live/$domain/cert.pem /.cert.pem
+
+EOF
+
+chmod +x /certbot-restore.sh