testnet-edge/testnet-beta now update while preserving the ledger (#5979)

* Check if an update is current before deploying it again

* Add (new) update command to deploy testnet updates

* Add --deploy-if-newer flag to permit conditional net updates

ci/testnet-deploy.sh

@@ -21,7 +21,6 @@ delete=false
 enableGpu=false
 bootDiskType=""
 blockstreamer=false
-deployUpdateManifest=true
 fetchLogs=true
 maybeHashesPerTick=
 maybeDisableAirdrops=
@@ -123,9 +122,6 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --external-accounts-file ]]; then
       maybeExternalPrimordialAccountsFile="$1 $2"
       shift 2
-    elif [[ $1 = --skip-deploy-update ]]; then
-      deployUpdateManifest=false
-      shift 1
     elif [[ $1 = --skip-remote-log-retrieval ]]; then
       fetchLogs=false
       shift 1
@@ -153,7 +149,7 @@ while [[ -n $1 ]]; do
   fi
 done
 
-while getopts "h?p:Pn:c:t:gG:a:Dd:rusxz:p:C:Sfew" opt "${shortArgs[@]}"; do
+while getopts "h?p:Pn:c:t:gG:a:Dd:rusxz:p:C:Sfe" opt "${shortArgs[@]}"; do
   case $opt in
   h | \?)
     usage
@@ -223,10 +219,6 @@ while getopts "h?p:Pn:c:t:gG:a:Dd:rusxz:p:C:Sfew" opt "${shortArgs[@]}"; do
   S)
     stopNetwork=true
     ;;
-  w)
-    fetchLogs=false
-    deployUpdateManifest=false
-    ;;
   *)
     usage "Unknown option: $opt"
     ;;
@@ -365,8 +357,6 @@ ok=true
 if ! $skipStart; then
   (
     if $skipCreate; then
-      # TODO: Enable rolling updates
-      #op=update
       op=restart
     else
       op=start
@@ -391,13 +381,6 @@ if ! $skipStart; then
       args+=(-F)
     fi
 
-    if $deployUpdateManifest; then
-      rm -f update_manifest_keypair.json
-      args+=(--deploy-update linux)
-      args+=(--deploy-update osx)
-      args+=(--deploy-update windows)
-    fi
-
     # shellcheck disable=SC2206 # Do not want to quote
     args+=(
       $maybeHashesPerTick

ci/testnet-manager.sh

@@ -219,6 +219,7 @@ sanity() {
       NO_INSTALL_CHECK=1 \
       NO_VALIDATOR_SANITY=1 \
         ci/testnet-sanity.sh edge-testnet-solana-com gce us-west1-b
+      time net/net.sh restart --skip-setup --deploy-if-newer -t "$CHANNEL_OR_TAG"
     )
     ;;
   testnet-edge-perf)
@@ -235,6 +236,7 @@ sanity() {
       NO_INSTALL_CHECK=1 \
       NO_VALIDATOR_SANITY=1 \
         ci/testnet-sanity.sh beta-testnet-solana-com gce us-west1-b
+      time net/net.sh restart --skip-setup --deploy-if-newer -t "$CHANNEL_OR_TAG"
     )
     ;;
   testnet-beta-perf)
@@ -376,6 +378,7 @@ deploy() {
         ${skipStart:+-s} \
         ${maybeStop:+-S} \
         ${maybeDelete:+-D}
+      time net/net.sh update -t "$CHANNEL_OR_TAG" --platform\ {linux,osx,windows}
     )
     ;;
   testnet-perf)
@@ -405,7 +408,6 @@ deploy() {
       NO_VALIDATOR_SANITY=1 \
         ci/testnet-deploy.sh -p demo-testnet-solana-com -C gce ${GCE_ZONE_ARGS[@]} \
           -t "$CHANNEL_OR_TAG" -n "$GCE_NODE_COUNT" -c 0 -P -u -f \
-          --skip-deploy-update \
           --skip-remote-log-retrieval \
           -a demo-testnet-solana-com \
           ${skipCreate:+-e} \
@@ -418,7 +420,6 @@ deploy() {
         NO_VALIDATOR_SANITY=1 \
           ci/testnet-deploy.sh -p demo-testnet-solana-com2 -C gce ${GCE_LOW_QUOTA_ZONE_ARGS[@]} \
             -t "$CHANNEL_OR_TAG" -n "$GCE_LOW_QUOTA_NODE_COUNT" -c 0 -P -f -x \
-            --skip-deploy-update \
             --skip-remote-log-retrieval \
             ${skipCreate:+-e} \
             ${skipStart:+-s} \
@@ -540,8 +541,7 @@ deploy() {
           ${maybeInternalNodesLamports} \
           ${maybeExternalAccountsFile} \
           ${maybeLamports} \
-          ${maybeAdditionalDisk} \
-          --skip-deploy-update
+          ${maybeAdditionalDisk}
     )
     ;;
   *)

install/src/command.rs

@@ -25,6 +25,7 @@ static LOOKING_GLASS: Emoji = Emoji("🔍 ", "");
 static BULLET: Emoji = Emoji("• ", "* ");
 static SPARKLE: Emoji = Emoji("✨ ", "");
 static PACKAGE: Emoji = Emoji("📦 ", "");
+static INFORMATION: Emoji = Emoji("ℹ️  ", "");
 
 /// Creates a new process bar for processing that will take an unknown amount of time
 fn new_spinner_progress_bar() -> ProgressBar {
@@ -626,6 +627,12 @@ pub fn deploy(
     let update_manifest_keypair = read_keypair(update_manifest_keypair_file)
         .map_err(|err| format!("Unable to read {}: {}", update_manifest_keypair_file, err))?;
 
+    println_name_value("JSON RPC URL:", json_rpc_url);
+    println_name_value(
+        "Update manifest pubkey:",
+        &update_manifest_keypair.pubkey().to_string(),
+    );
+
     // Confirm the `json_rpc_url` is good and that `from_keypair` is a valid account
     let rpc_client = RpcClient::new(json_rpc_url.to_string());
     let progress_bar = new_spinner_progress_bar();
@@ -648,6 +655,18 @@ pub fn deploy(
         download_to_temp_archive(download_url, None)
             .map_err(|err| format!("Unable to download {}: {}", download_url, err))?;
 
+    if let Ok(update_manifest) = get_update_manifest(&rpc_client, &update_manifest_keypair.pubkey())
+    {
+        if temp_archive_sha256 == update_manifest.download_sha256 {
+            println!(
+                "  {}{}",
+                INFORMATION,
+                style("Update is already deployed").bold()
+            );
+            return Ok(());
+        }
+    }
+
     // Extract it and load the release version metadata
     let temp_release_dir = temp_dir.path().join("archive");
     extract_release_archive(&temp_archive, &temp_release_dir).map_err(|err| {
@@ -664,12 +683,7 @@ pub fn deploy(
         )
     })?;
 
-    println_name_value("JSON RPC URL:", json_rpc_url);
     println_name_value("Update target:", &release_target);
-    println_name_value(
-        "Update manifest pubkey:",
-        &update_manifest_keypair.pubkey().to_string(),
-    );
 
     let progress_bar = new_spinner_progress_bar();
     progress_bar.set_message(&format!("{}Deploying update...", PACKAGE));

net/net.sh

@@ -25,15 +25,13 @@ Operate a configured testnet
  logs     - Fetch remote logs from each network node
  startnode- Start an individual node (previously stopped with stopNode)
  stopnode - Stop an individual node
+ update   - Deploy a new software update to the cluster
 
  start-specific options:
    -T [tarFilename]                   - Deploy the specified release tarball
    -t edge|beta|stable|vX.Y.Z         - Deploy the latest tarball release for the
                                         specified release channel (edge|beta|stable) or release tag
                                         (vX.Y.Z)
-   --deploy-update linux|osx|windows  - Deploy the tarball using 'solana-install deploy ...' for the
-                                        given platform (multiple platforms may be specified)
-                                        (-t option must be supplied as well)
    -f [cargoFeatures]                 - List of |cargo --feaures=| to activate
                                         (ignored if -s or -S is specified)
    -r / --skip-setup                  - Reuse existing node/ledger configuration from a
@@ -80,6 +78,8 @@ Operate a configured testnet
                                       - Don't build new software, deploy the
                                         existing binaries
 
+   --deploy-if-newer                  - Only deploy if newer software is
+                                        available (requires -t or -T)
 
  sanity/start-specific options:
    -F                   - Discard validator nodes that didn't bootup successfully
@@ -93,6 +93,11 @@ Operate a configured testnet
  logs-specific options:
    none
 
+ update-specific options:
+   --platform linux|osx|windows       - Deploy the tarball using 'solana-install deploy ...' for the
+                                        given platform (multiple platforms may be specified)
+                                        (-t option must be supplied as well)
+
  startnode/stopnode-specific options:
    -i [ip address]                    - IP Address of the node to start or stop
 
@@ -104,6 +109,7 @@ EOF
 
 releaseChannel=
 deployMethod=local
+deployIfNewer=
 sanityExtraArgs=
 cargoFeatures=
 skipSetup=false
@@ -147,6 +153,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --no-snapshot-fetch ]]; then
       maybeNoSnapshot="$1"
       shift 1
+    elif [[ $1 = --deploy-if-newer ]]; then
+      deployIfNewer=1
+      shift 1
     elif [[ $1 = --no-deploy ]]; then
       deployMethod=skip
       shift 1
@@ -162,7 +171,7 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --skip-setup ]]; then
       skipSetup=true
       shift 1
-    elif [[ $1 = --deploy-update ]]; then
+    elif [[ $1 = --platform ]]; then
       updatePlatforms="$updatePlatforms $2"
       shift 2
     elif [[ $1 = --internal-nodes-stake-lamports ]]; then
@@ -395,9 +404,11 @@ startBootstrapLeader() {
     case $deployMethod in
     tar)
       rsync -vPrc -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/solana-release/bin/* "$ipAddress:~/.cargo/bin/"
+      rsync -vPrc -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/solana-release/version.yml "$ipAddress:~/"
       ;;
     local)
       rsync -vPrc -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/farf/bin/* "$ipAddress:~/.cargo/bin/"
+      ssh "${sshOptions[@]}" -n "$ipAddress" "rm -f ~/version.yml; touch ~/version.yml"
       ;;
     skip)
       ;;
@@ -552,9 +563,13 @@ sanity() {
 
 deployUpdate() {
   if [[ -z $updatePlatforms ]]; then
+    echo "No update platforms"
     return
   fi
-  [[ $deployMethod = tar ]] || exit 1
+  if [[ -z $releaseChannel ]]; then
+    echo "Release channel not specified (use -t option)"
+    exit 1
+  fi
 
   declare ok=true
   declare bootstrapLeader=${fullnodeIpList[0]}
@@ -618,11 +633,6 @@ prepare_deploy() {
           -o "$SOLANA_ROOT"/solana-release.tar.bz2 "$updateDownloadUrl"
       )
       tarballFilename="$SOLANA_ROOT"/solana-release.tar.bz2
-    else
-      if [[ -n $updatePlatforms ]]; then
-        echo "Error: --deploy-update argument was provided but -t was not"
-        exit 1
-      fi
     fi
     (
       set -x
@@ -644,6 +654,26 @@ prepare_deploy() {
     usage "Internal error: invalid deployMethod: $deployMethod"
     ;;
   esac
+
+  if [[ -n $deployIfNewer ]]; then
+    if [[ $deployMethod != tar ]]; then
+      echo "Error: --deploy-if-newer only supported for tar deployments"
+      exit 1
+    fi
+
+    echo "Fetching current software version"
+    (
+      set -x
+      rsync -vPrc -e "ssh ${sshOptions[*]}" "${fullnodeIpList[0]}":~/version.yml current-version.yml
+    )
+    cat current-version.yml
+    if ! diff -q current-version.yml "$SOLANA_ROOT"/solana-release/version.yml; then
+      echo "Cluster software version is old.  Update required"
+    else
+      echo "Cluster software version is current.  No update required"
+      exit 0
+    fi
+  fi
 }
 
 deploy() {
@@ -733,8 +763,6 @@ deploy() {
   esac
   $metricsWriteDatapoint "testnet-deploy version=\"${networkVersion:0:9}\""
 
-  deployUpdate
-
   echo
   echo "+++ Deployment Successful"
   echo "Bootstrap leader deployment took $bootstrapNodeDeployTime seconds"
@@ -819,6 +847,9 @@ sanity)
 stop)
   stop
   ;;
+update)
+  deployUpdate
+  ;;
 stopnode)
   if [[ -z $nodeAddress ]]; then
     usage "node address (-i) not specified"

net/remote/remote-node.sh

@@ -216,6 +216,7 @@ EOF
   validator|blockstreamer)
     if [[ $deployMethod != skip ]]; then
       net/scripts/rsync-retry.sh -vPrc "$entrypointIp":~/.cargo/bin/ ~/.cargo/bin/
+      net/scripts/rsync-retry.sh -vPrc "$entrypointIp":~/version.yml version.yml
     fi
     if [[ $skipSetup != true ]]; then
       multinode-demo/clear-config.sh