Deploy mac OS solana-install update package to testnets (#4595)

.buildkite/env/secrets.ejson

@@ -1,12 +1,14 @@
 {
     "_public_key": "ae29f4f7ad2fc92de70d470e411c8426d5d48db8817c9e3dae574b122192335f",
     "environment": {
-      "CODECOV_TOKEN": "EJ[1:eSGdiZR0Qi0g7qnsI+qJ5H+/ik+H2qL3ned/cBdv/SY=:jA0WqO70coUtF0iokRdgtCR/lF/lETAI:d/Wl8Tdl6xVh/B39cTf1DaQkomR7I/2vMhvxd1msJ++BjI2l3p2dFoGsXqWT+/os8VgiPg==]",
+      "CODECOV_TOKEN": "EJ[1:8iZ6baJB4fbBV+XDsrUooyGAnGL/8Ol+4Qd0zKh5YjI=:ks2/ElgxwgxqgmFcxTHANNLmj23YH74h:U4uzRONRfiQyqy6HrPQ/e7OnBUY4HkW37R0iekkF3KJ9UGnHqT1UvwgVbDqLahtDIJ4rWw==]",
-      "CRATES_IO_TOKEN": "EJ[1:eSGdiZR0Qi0g7qnsI+qJ5H+/ik+H2qL3ned/cBdv/SY=:2FaZ6k4RGH8luyNRaN6yeZUQDNAu2KwC:XeYe0tCAivYE0F9HEWM79mAI6kNbfYaqP7k7yY+SBDvs0341U9BdGZp7SErbHleS]",
+      "CRATES_IO_TOKEN": "EJ[1:8iZ6baJB4fbBV+XDsrUooyGAnGL/8Ol+4Qd0zKh5YjI=:lKMh3aLW+jyRrfS/c7yvkpB+TaPhXqLq:j0v27EbaPgwRdHZAbsM0FlAnt3r9ScQrFbWJYOAZtM3qestEiByTlKpZ0eyF/823]",
-      "GITHUB_TOKEN": "EJ[1:eSGdiZR0Qi0g7qnsI+qJ5H+/ik+H2qL3ned/cBdv/SY=:9kh4DGPiGDcUU7ejSFWg3gTW8nrOM09Q:b+GE07Wu6/bEnkDZcUtf48vTKAFphrCSt3tNNER9h6A+wZ80k499edw4pbDdl9kEvxB30fFwrLQ=]",
+      "GITHUB_TOKEN": "EJ[1:8iZ6baJB4fbBV+XDsrUooyGAnGL/8Ol+4Qd0zKh5YjI=:Ll78c3jGpYqnTwR7HJq3mNNUC7pOv9Lu:GrInO2r8MjmP5c54szkyygdsrW5KQYkDgJQUVyFEPyG8SWfchyM9Gur8RV0a+cdwuxNkHLi4U2M=]",
-      "INFLUX_DATABASE": "EJ[1:eSGdiZR0Qi0g7qnsI+qJ5H+/ik+H2qL3ned/cBdv/SY=:rCHsYi0rc7dmvr1V3wEgNoaNIyr+9ClM:omjVcOqM7vwt44kJ+As4BjJL]",
+      "INFLUX_DATABASE": "EJ[1:8iZ6baJB4fbBV+XDsrUooyGAnGL/8Ol+4Qd0zKh5YjI=:IlH/ZLTXv3SwlY3TVyAPCX2KzLRY6iG3:gGmUGSU/kCfR/mTwKONaUC/X]",
-      "INFLUX_PASSWORD": "EJ[1:eSGdiZR0Qi0g7qnsI+qJ5H+/ik+H2qL3ned/cBdv/SY=:bP5Gw1Vy66viKFKO41o2Gho998XajH/5:khkCYz2LFvkJkk7R4xY1Hfz1yU3/NENjauiUkPhXA+dmg1qOIToxEagCgIkRwyeCiYaoCR6CZyw=]",
+      "INFLUX_PASSWORD": "EJ[1:8iZ6baJB4fbBV+XDsrUooyGAnGL/8Ol+4Qd0zKh5YjI=:o2qm95GU4VrrcC4OU06jjPvCwKZy/CZF:OW2ga3kLOQJvaDEdGRJ+gn3L2ckFm8AJZtv9wj/GeUIKDH2A4uBPTHsAH9PMe6zujpuHGk3qbeg=]",
-      "INFLUX_USERNAME": "EJ[1:eSGdiZR0Qi0g7qnsI+qJ5H+/ik+H2qL3ned/cBdv/SY=:ZamCvza2W9/bZRGSkqDu55xNN04XKKhp:5jlmCOdFbpL7EFez41zCbLfk3ZZlfmhI]",
+      "INFLUX_USERNAME": "EJ[1:8iZ6baJB4fbBV+XDsrUooyGAnGL/8Ol+4Qd0zKh5YjI=:yDWW/uIHsJqOTDYskZoSx3pzoB1vztWY:2z31oTA3g0Xs9fCczGNJRcx8xf/hFCed]",
-      "SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu": "EJ[1:eSGdiZR0Qi0g7qnsI+qJ5H+/ik+H2qL3ned/cBdv/SY=:Oi2nsRxnvWnnBYsB6KwEDzLPcYgpYojU:ELbvjXkXKlgFCMES45R+fxG7Ex43WHWErjMbxZoqasxyr7GSH66hQzUWqiQSJyT4ukYrRhRC9YrsKKGkjACLU57X4EGIy9TuLgTnyBYhPnxLYStC3y/7o/MB5FCTt5wHJw3/A9p+me5+T4UmyZ7OeP21NhDUCGQcb0040VwYWS78klW2aQESJJ6wTI1xboE8/zC0vtnB/u50+LydbKEyb21r6y3OH9FYNEpSwIspWKcgpruJdQSCnDoKxP9YR1yzvk2rabss13LJNdV1Y6mQNIdP4OIFQhCs6dXT253RTl5qdZ0MruHwlp8wX4btOuYDcCoM5exr]"
+      "SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu": "EJ[1:8iZ6baJB4fbBV+XDsrUooyGAnGL/8Ol+4Qd0zKh5YjI=:RqRaHlYUvGPNFJa6gmciaYM3tRJTURUH:q78/3GTHCN3Uqx9z4nOBjPZcO1lOazNoB/mdhGRDFsnAqVd2hU8zbKkqLrZfLlGqyD8WQOFuw5oTJR9qWg6L9LcOyj3pGL8jWF2yjgZxdtNMXnkbSrCWLooWBBLT61jYQnEwg73gT8ld3Q8EVv3T+MeSMu6FnPz+0+bqQCAGgfqksP4hsUAJGzgZu+i0tNOdlT7fxnh5KJK/yFM/CKgN2sRwEjukA9hXsffyB61g2zqzTDJxCUDLbCVrCkA/bfUk7Of/t0W5t0nK1H3oyGZEc/lRMauCknDBka3Gz11dVss2QT19WQNh0u7bHVaT/U4lepX1j9Zv]",
+      "SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_apple_darwin": "EJ[1:8iZ6baJB4fbBV+XDsrUooyGAnGL/8Ol+4Qd0zKh5YjI=:wFDl3INEnA3EQDHRX40avqGe1OMoJxyy:6ncCRVRTIRuYI5o/gayeuWCudWvmKNYr8KEHAWeTq34a5bdcKInBdKhjmjX+wLHqsEwQ5gcyhcxy4Ri2mbuN6AHazfZOZlubQkGlyUOAIYO5D5jkbyIh40DAtjVzo1MD/0HsW9zdGOzqUKp5xJJeDsbR4F153jbxa7fvwF90Q4UQjYFTKAtExEmHtDGSJG48ToVwTabTV/OnISMIggDZBviIv2QWHvXgK07b2mUj34rHJywEDGN1nj5rITTDdUeRcB1x4BAMOe94kTFPSTaj/OszvYlGECt8rkKFqbm092qL+XLfiBaImqe/WJHRCnAj6Don]",
+      "SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_pc_windows_msvc": "EJ[1:8iZ6baJB4fbBV+XDsrUooyGAnGL/8Ol+4Qd0zKh5YjI=:wAh+dBuZopv6vruVOYegUcq/aBnbksT1:qIJfCfDvDWiqicMOkmbJs/0n7UJLKNmgMQaKzeQ8J7Q60YpXbtWzKVW3tS6lzlgf64m3MrPXyo1C+mWh6jkjsb18T/OfggZy1ZHM4AcsOC6/ldUkV5YtuxUQuAmd5jCuV/R7iuYY8Z66AcfAevlb+bnLpgIifdA8fh/IktOo58nZUQwZDdppAacmftsLc6Frn5Er6A6+EXpxK1nmnlmLJ4AJztqlh6X0r+JvE2O7qeoZUXrIegnkxo7Aay7I/dd8zdYpp7ICSiTEtfVN/xNIu/5QmTRU7gWoz7cPl9epq4aiEALzPOzb6KVOiRcsOg+TlFvLQ71Ik5o=]"
     }
 }

book/src/testnet-participation.md

@@ -64,11 +64,11 @@ for more detail on cluster activity.
 ##### Bootstrap with `solana-install`
 
 The `solana-install` tool can be used to easily install and upgrade the cluster
-software on Linux x86_64 systems.
+software on Linux x86_64 and mac OS systems.
 
 ```bash
-$ export SOLANA_RELEASE=v0.14.2  # skip this line to install the latest release
+$ export SOLANA_RELEASE=v0.16.0  # skip this line to install the latest release
-$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v0.14.0/install/solana-install-init.sh | sh -s
+$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v0.16.0/install/solana-install-init.sh | sh -s
 ```
 
 Alternatively build the `solana-install` program from source and run the
@@ -81,8 +81,9 @@ After a successful install, `solana-install update` may be used to easily update
 software to a newer version.
 
 ##### Download Prebuilt Binaries
-Binaries are available for Linux x86_64 systems.
+Binaries are available for Linux x86_64 and mac OS systems.
 
+###### Linux
 Download the binaries by navigating to
 [https://github.com/solana-labs/solana/releases/latest](https://github.com/solana-labs/solana/releases/latest),
 download **solana-release-x86_64-unknown-linux-gnu.tar.bz2**, then extract the
@@ -92,6 +93,17 @@ $ tar jxf solana-release-x86_64-unknown-linux-gnu.tar.bz2
 $ cd solana-release/
 $ export PATH=$PWD/bin:$PATH
 ```
+###### mac OS
+Download the binaries by navigating to
+[https://github.com/solana-labs/solana/releases/latest](https://github.com/solana-labs/solana/releases/latest),
+download **solana-release-x86_64-apple-darwin.tar.bz2**, then extract the
+archive:
+```bash
+$ tar jxf solana-release-x86_64-apple-darwin.tar.bz2
+$ cd solana-release/
+$ export PATH=$PWD/bin:$PATH
+```
+
 ##### Build From Source
 If you are unable to use the prebuilt binaries or prefer to build it yourself
 from source, navigate to

ci/testnet-deploy.sh

@@ -327,10 +327,11 @@ if ! $skipStart; then
       args+=(-F)
     fi
 
-    # shellcheck disable=SC2154 # SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu comes from .buildkite/env/
+    if $deployUpdateManifest; then
-    if $deployUpdateManifest && [[ -n $SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu ]]; then
+      rm -f update_manifest_keypair.json
-      echo "$SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu" > update_manifest_keypair.json
+      args+=(--deploy-update linux)
-      args+=(-i update_manifest_keypair.json)
+      args+=(--deploy-update osx)
+      #args+=(--deploy-update windows)
     fi
 
     # shellcheck disable=SC2086 # Don't want to double quote the $maybeXYZ variables

install/src/defaults.rs

@@ -23,8 +23,9 @@ lazy_static! {
 
 pub fn update_manifest_pubkey(target: &str) -> Option<&str> {
     match target {
-        "x86_64-apple-darwin" => None,
+        "x86_64-apple-darwin" => Some("GRUP8YUGASLdu2gBwHstFgeVH28qppfuCaTzq5Yo7wRo"), // SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_apple_darwin
         "x86_64-unknown-linux-gnu" => Some("FnKt2ES9iUJkjoprf2rL62xxBAxZLVgyA4SFexPGotFE"), // SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu
+        "x86_64-pc-windows-msvc" => Some("2Lrj5xDCHDmqwCgGwjVqAUUM84vLpj5dReYeoXL9vSXV"), // SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_pc_windows_msvc
         _ => None,
     }
 }

install/testnet-deploy.sh

@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-#
-# Convenience script to easily deploy a software update to one of the testnets
-#
-# Prerequisites:
-# 1) The default keypair should have some lamports (eg, `solana-wallet airdrop 123`)
-# 2) The file update_manifest_keypair.json should exist if this script is not
-#    run from the CI environment
-#
-set -e
-
-CHANNEL=$1
-TAG=$2
-
-if [[ -z $CHANNEL || -z $TAG ]]; then
-  echo "Usage: $0 [channel] [release tag]"
-  exit 0
-fi
-
-# Prefer possible `cargo build --all` binaries over PATH binaries
-PATH=$(cd "$(dirname "$0")/.."; echo "$PWD")/target/debug:$PATH
-
-# shellcheck disable=2154 # is referenced but not assigned
-if [[ -n $SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu ]]; then
-  echo "$SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu" > update_manifest_keypair.json
-fi
-
-case $CHANNEL in
-edge|beta)
-  URL=https://api.$CHANNEL.testnet.solana.com
-  ;;
-stable)
-  URL=https://api.testnet.solana.com
-  ;;
-localhost)
-  URL=http://localhost:8899
-  ;;
-*)
-  echo "Error: unknown channel: $CHANNEL"
-  exit 1
-esac
-
-set -x
-solana-install deploy --url "$URL" \
-  https://github.com/solana-labs/solana/releases/download/"$TAG"/solana-release-x86_64-unknown-linux-gnu.tar.bz2 \
-  update_manifest_keypair.json

net/net.sh

@@ -30,7 +30,8 @@ Operate a configured testnet
    -t edge|beta|stable|vX.Y.Z         - Deploy the latest tarball release for the
                                         specified release channel (edge|beta|stable) or release tag
                                         (vX.Y.Z)
-   -i update_manifest_keypair         - Deploy the tarball using 'solana-install deploy ...'
+   --deploy-update linux|osx|windows  - Deploy the tarball using 'solana-install deploy ...' for the
+                                        given platform (multiple platforms may be specified)
                                         (-t option must be supplied as well)
    -f [cargoFeatures]                 - List of |cargo --feaures=| to activate
                                         (ignored if -s or -S is specified)
@@ -80,8 +81,7 @@ cargoFeatures=
 skipSetup=false
 updateNodes=false
 customPrograms=
-updateManifestKeypairFile=
+updatePlatforms=
-updateDownloadUrl=
 numBenchTpsClients=0
 numBenchExchangeClients=0
 benchTpsExtraArgs=
@@ -100,6 +100,9 @@ while [[ -n $1 ]]; do
     if [[ $1 = --hashes-per-tick ]]; then
       genesisOptions="$genesisOptions $1 $2"
       shift 2
+    elif [[ $1 = --deploy-update ]]; then
+      updatePlatforms="$updatePlatforms $2"
+      shift 2
     else
       usage "Unknown long option: $1"
     fi
@@ -109,7 +112,7 @@ while [[ -n $1 ]]; do
   fi
 done
 
-while getopts "h?T:t:o:f:rD:i:c:Fn:" opt "${shortArgs[@]}"; do
+while getopts "h?T:t:o:f:rD:c:Fn:" opt "${shortArgs[@]}"; do
   case $opt in
   h | \?)
     usage
@@ -136,13 +139,6 @@ while getopts "h?T:t:o:f:rD:i:c:Fn:" opt "${shortArgs[@]}"; do
   n)
     numFullnodesRequested=$OPTARG
     ;;
-  i)
-    updateManifestKeypairFile=$OPTARG
-    if [[ ! -r $updateManifestKeypairFile ]]; then
-      echo "Error: unable to read the file $updateManifestKeypairFile"
-      exit 1
-    fi
-    ;;
   r)
     skipSetup=true
     ;;
@@ -413,26 +409,31 @@ sanity() {
 }
 
 deployUpdate() {
-  if [[ -z $updateManifestKeypairFile ]]; then
+  if [[ -z $updatePlatforms ]]; then
     return
   fi
   [[ $deployMethod = tar ]] || exit 1
-  [[ -n $updateDownloadUrl ]] || exit 1
 
   declare ok=true
   declare bootstrapLeader=${fullnodeIpList[0]}
 
-  echo "--- Deploying solana-install update: $updateDownloadUrl"
+  for updatePlatform in $updatePlatforms; do
-  (
+    echo "--- Deploying solana-install update: $updatePlatform"
-    set -x
+    (
-    timeout 30s scp "${sshOptions[@]}" \
+      set -x
-      "$updateManifestKeypairFile" "$bootstrapLeader:solana/update_manifest_keypair.json"
+
+      scripts/solana-install-update-manifest-keypair.sh "$updatePlatform"
+
+      timeout 30s scp "${sshOptions[@]}" \
+        update_manifest_keypair.json "$bootstrapLeader:solana/update_manifest_keypair.json"
+
+      # shellcheck disable=SC2029 # remote-deploy-update.sh args are expanded on client side intentionally
+      ssh "${sshOptions[@]}" "$bootstrapLeader" \
+        "./solana/net/remote/remote-deploy-update.sh $releaseChannel $updatePlatform"
+    ) || ok=false
+    $ok || exit 1
+  done
 
-    # shellcheck disable=SC2029 # remote-deploy-update.sh args are expanded on client side intentionally
-    ssh "${sshOptions[@]}" "$bootstrapLeader" \
-      "./solana/net/remote/remote-deploy-update.sh $updateDownloadUrl \"$RUST_LOG\""
-  ) || ok=false
-  $ok || exit 1
 }
 
 start() {
@@ -440,15 +441,15 @@ start() {
   tar)
     if [[ -n $releaseChannel ]]; then
       rm -f "$SOLANA_ROOT"/solana-release.tar.bz2
-      updateDownloadUrl=http://release.solana.com/"$releaseChannel"/solana-release-x86_64-unknown-linux-gnu.tar.bz2
+      declare updateDownloadUrl=http://release.solana.com/"$releaseChannel"/solana-release-x86_64-unknown-linux-gnu.tar.bz2
       (
         set -x
         curl -o "$SOLANA_ROOT"/solana-release.tar.bz2 "$updateDownloadUrl"
       )
       tarballFilename="$SOLANA_ROOT"/solana-release.tar.bz2
     else
-      if [[ -n $updateManifestKeypairFile ]]; then
+      if [[ -n $updatePlatforms ]]; then
-        echo "Error: -i argument was provided but -t was not"
+        echo "Error: --deploy-update argument was provided but -t was not"
         exit 1
       fi
     fi

net/remote/remote-deploy-update.sh

@@ -6,7 +6,8 @@ set -e
 
 cd "$(dirname "$0")"/../..
 
-updateDownloadUrl=$1
+releaseChannel=$1
+updatePlatform=$2
 
 [[ -r deployConfig ]] || {
   echo deployConfig missing
@@ -20,7 +21,9 @@ missing() {
   exit 1
 }
 
-[[ -n $updateDownloadUrl ]] || missing updateDownloadUrl
+[[ -n $releaseChannel ]] || missing releaseChannel
+[[ -n $updatePlatform ]] || missing updatePlatform
+[[ -f update_manifest_keypair.json ]] || missing update_manifest_keypair.json
 
 RUST_LOG="$2"
 export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info
@@ -31,6 +34,4 @@ loadConfigFile
 PATH="$HOME"/.cargo/bin:"$PATH"
 
 set -x
-solana-wallet --url http://127.0.0.1:8899 airdrop 42
+scripts/solana-install-deploy.sh localhost "$releaseChannel" "$updatePlatform"
-solana-install deploy "$updateDownloadUrl" update_manifest_keypair.json \
-  --url http://127.0.0.1:8899

scripts/solana-install-deploy.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+#
+# Convenience script to easily deploy a software update to a testnet
+#
+set -e
+SOLANA_ROOT="$(cd "$(dirname "$0")"/..; pwd)"
+
+URL=$1
+TAG=$2
+OS=${3:-linux}
+
+if [[ -z $URL || -z $TAG ]]; then
+  echo "Usage: $0 [edge|beta|stable|localhost|RPC URL] [edge|beta|release tag] [linux|osx|windows]"
+  exit 0
+fi
+
+if [[ ! -f update_manifest_keypair.json ]]; then
+  "$SOLANA_ROOT"/scripts/solana-install-update-manifest-keypair.sh "$OS"
+fi
+
+case $URL in
+edge|beta)
+  URL=http://$URL.testnet.solana.com:8899
+  ;;
+stable)
+  URL=http://testnet.solana.com:8899
+  ;;
+localhost)
+  URL=http://localhost:8899
+  ;;
+*)
+  ;;
+esac
+
+case $TAG in
+edge|beta)
+  DOWNLOAD_URL=http://release.solana.com/"$TAG"/solana-release-$TARGET.tar.bz2
+  ;;
+*)
+  DOWNLOAD_URL=https://github.com/solana-labs/solana/releases/download/"$TAG"/solana-release-$TARGET.tar.bz2
+  ;;
+esac
+
+# Prefer possible `cargo build` binaries over PATH binaries
+PATH="$SOLANA_ROOT"/target/debug:$PATH
+
+set -x
+balance=$(solana-wallet --url "$URL" balance)
+if [[ $balance = "0 lamports" ]]; then
+  solana-wallet --url "$URL" airdrop 42
+fi
+solana-install deploy --url "$URL" "$DOWNLOAD_URL" update_manifest_keypair.json

scripts/solana-install-update-manifest-keypair.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+#
+# Convenience script to easily deploy a software update to a testnet
+#
+# Prerequisites:
+# 1) The default keypair should have some lamports (eg, `solana-wallet airdrop 123`)
+# 2) The file update_manifest_keypair.json should exist if this script is not
+#    run from the CI environment
+#
+set -e
+
+OS=${1:-linux}
+
+case "$OS" in
+osx)
+  TARGET=x86_64-apple-darwin
+  ;;
+linux)
+  TARGET=x86_64-unknown-linux-gnu
+  ;;
+windows)
+  TARGET=x86_64-pc-windows-msvc
+  ;;
+*)
+  TARGET=unknown-unknown-unknown
+  ;;
+esac
+
+SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR="SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_${TARGET//-/_}"
+
+# shellcheck disable=2154 # is referenced but not assigned
+if [[ -z ${!SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR} ]]; then
+  echo "$SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR not defined"
+  exit 1
+fi
+
+echo "${!SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR}" > update_manifest_keypair.json
+ls -l update_manifest_keypair.json