From fee5c6c057831d8f227f30b3c7520629d8282121 Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Thu, 19 Sep 2019 12:03:47 -0700 Subject: [PATCH] testnet-edge/testnet-beta now update while preserving the ledger (#5979) * Check if an update is current before deploying it again * Add (new) update command to deploy testnet updates * Add --deploy-if-newer flag to permit conditional net updates --- ci/testnet-deploy.sh | 19 +------------- ci/testnet-manager.sh | 8 +++--- install/src/command.rs | 24 +++++++++++++---- net/net.sh | 55 ++++++++++++++++++++++++++++++--------- net/remote/remote-node.sh | 1 + 5 files changed, 68 insertions(+), 39 deletions(-) diff --git a/ci/testnet-deploy.sh b/ci/testnet-deploy.sh index abb21f8890..4839c69c6a 100755 --- a/ci/testnet-deploy.sh +++ b/ci/testnet-deploy.sh @@ -21,7 +21,6 @@ delete=false enableGpu=false bootDiskType="" blockstreamer=false -deployUpdateManifest=true fetchLogs=true maybeHashesPerTick= maybeDisableAirdrops= @@ -123,9 +122,6 @@ while [[ -n $1 ]]; do elif [[ $1 = --external-accounts-file ]]; then maybeExternalPrimordialAccountsFile="$1 $2" shift 2 - elif [[ $1 = --skip-deploy-update ]]; then - deployUpdateManifest=false - shift 1 elif [[ $1 = --skip-remote-log-retrieval ]]; then fetchLogs=false shift 1 @@ -153,7 +149,7 @@ while [[ -n $1 ]]; do fi done -while getopts "h?p:Pn:c:t:gG:a:Dd:rusxz:p:C:Sfew" opt "${shortArgs[@]}"; do +while getopts "h?p:Pn:c:t:gG:a:Dd:rusxz:p:C:Sfe" opt "${shortArgs[@]}"; do case $opt in h | \?) usage @@ -223,10 +219,6 @@ while getopts "h?p:Pn:c:t:gG:a:Dd:rusxz:p:C:Sfew" opt "${shortArgs[@]}"; do S) stopNetwork=true ;; - w) - fetchLogs=false - deployUpdateManifest=false - ;; *) usage "Unknown option: $opt" ;; @@ -365,8 +357,6 @@ ok=true if ! $skipStart; then ( if $skipCreate; then - # TODO: Enable rolling updates - #op=update op=restart else op=start @@ -391,13 +381,6 @@ if ! $skipStart; then args+=(-F) fi - if $deployUpdateManifest; then - rm -f update_manifest_keypair.json - args+=(--deploy-update linux) - args+=(--deploy-update osx) - args+=(--deploy-update windows) - fi - # shellcheck disable=SC2206 # Do not want to quote args+=( $maybeHashesPerTick diff --git a/ci/testnet-manager.sh b/ci/testnet-manager.sh index 4bea0904c5..eb95428acc 100755 --- a/ci/testnet-manager.sh +++ b/ci/testnet-manager.sh @@ -219,6 +219,7 @@ sanity() { NO_INSTALL_CHECK=1 \ NO_VALIDATOR_SANITY=1 \ ci/testnet-sanity.sh edge-testnet-solana-com gce us-west1-b + time net/net.sh restart --skip-setup --deploy-if-newer -t "$CHANNEL_OR_TAG" ) ;; testnet-edge-perf) @@ -235,6 +236,7 @@ sanity() { NO_INSTALL_CHECK=1 \ NO_VALIDATOR_SANITY=1 \ ci/testnet-sanity.sh beta-testnet-solana-com gce us-west1-b + time net/net.sh restart --skip-setup --deploy-if-newer -t "$CHANNEL_OR_TAG" ) ;; testnet-beta-perf) @@ -376,6 +378,7 @@ deploy() { ${skipStart:+-s} \ ${maybeStop:+-S} \ ${maybeDelete:+-D} + time net/net.sh update -t "$CHANNEL_OR_TAG" --platform\ {linux,osx,windows} ) ;; testnet-perf) @@ -405,7 +408,6 @@ deploy() { NO_VALIDATOR_SANITY=1 \ ci/testnet-deploy.sh -p demo-testnet-solana-com -C gce ${GCE_ZONE_ARGS[@]} \ -t "$CHANNEL_OR_TAG" -n "$GCE_NODE_COUNT" -c 0 -P -u -f \ - --skip-deploy-update \ --skip-remote-log-retrieval \ -a demo-testnet-solana-com \ ${skipCreate:+-e} \ @@ -418,7 +420,6 @@ deploy() { NO_VALIDATOR_SANITY=1 \ ci/testnet-deploy.sh -p demo-testnet-solana-com2 -C gce ${GCE_LOW_QUOTA_ZONE_ARGS[@]} \ -t "$CHANNEL_OR_TAG" -n "$GCE_LOW_QUOTA_NODE_COUNT" -c 0 -P -f -x \ - --skip-deploy-update \ --skip-remote-log-retrieval \ ${skipCreate:+-e} \ ${skipStart:+-s} \ @@ -540,8 +541,7 @@ deploy() { ${maybeInternalNodesLamports} \ ${maybeExternalAccountsFile} \ ${maybeLamports} \ - ${maybeAdditionalDisk} \ - --skip-deploy-update + ${maybeAdditionalDisk} ) ;; *) diff --git a/install/src/command.rs b/install/src/command.rs index 91af364c7e..0c50074155 100644 --- a/install/src/command.rs +++ b/install/src/command.rs @@ -25,6 +25,7 @@ static LOOKING_GLASS: Emoji = Emoji("🔍 ", ""); static BULLET: Emoji = Emoji("• ", "* "); static SPARKLE: Emoji = Emoji("✨ ", ""); static PACKAGE: Emoji = Emoji("📦 ", ""); +static INFORMATION: Emoji = Emoji("ℹ️ ", ""); /// Creates a new process bar for processing that will take an unknown amount of time fn new_spinner_progress_bar() -> ProgressBar { @@ -626,6 +627,12 @@ pub fn deploy( let update_manifest_keypair = read_keypair(update_manifest_keypair_file) .map_err(|err| format!("Unable to read {}: {}", update_manifest_keypair_file, err))?; + println_name_value("JSON RPC URL:", json_rpc_url); + println_name_value( + "Update manifest pubkey:", + &update_manifest_keypair.pubkey().to_string(), + ); + // Confirm the `json_rpc_url` is good and that `from_keypair` is a valid account let rpc_client = RpcClient::new(json_rpc_url.to_string()); let progress_bar = new_spinner_progress_bar(); @@ -648,6 +655,18 @@ pub fn deploy( download_to_temp_archive(download_url, None) .map_err(|err| format!("Unable to download {}: {}", download_url, err))?; + if let Ok(update_manifest) = get_update_manifest(&rpc_client, &update_manifest_keypair.pubkey()) + { + if temp_archive_sha256 == update_manifest.download_sha256 { + println!( + " {}{}", + INFORMATION, + style("Update is already deployed").bold() + ); + return Ok(()); + } + } + // Extract it and load the release version metadata let temp_release_dir = temp_dir.path().join("archive"); extract_release_archive(&temp_archive, &temp_release_dir).map_err(|err| { @@ -664,12 +683,7 @@ pub fn deploy( ) })?; - println_name_value("JSON RPC URL:", json_rpc_url); println_name_value("Update target:", &release_target); - println_name_value( - "Update manifest pubkey:", - &update_manifest_keypair.pubkey().to_string(), - ); let progress_bar = new_spinner_progress_bar(); progress_bar.set_message(&format!("{}Deploying update...", PACKAGE)); diff --git a/net/net.sh b/net/net.sh index 16e165dc24..7512cb9508 100755 --- a/net/net.sh +++ b/net/net.sh @@ -25,15 +25,13 @@ Operate a configured testnet logs - Fetch remote logs from each network node startnode- Start an individual node (previously stopped with stopNode) stopnode - Stop an individual node + update - Deploy a new software update to the cluster start-specific options: -T [tarFilename] - Deploy the specified release tarball -t edge|beta|stable|vX.Y.Z - Deploy the latest tarball release for the specified release channel (edge|beta|stable) or release tag (vX.Y.Z) - --deploy-update linux|osx|windows - Deploy the tarball using 'solana-install deploy ...' for the - given platform (multiple platforms may be specified) - (-t option must be supplied as well) -f [cargoFeatures] - List of |cargo --feaures=| to activate (ignored if -s or -S is specified) -r / --skip-setup - Reuse existing node/ledger configuration from a @@ -80,6 +78,8 @@ Operate a configured testnet - Don't build new software, deploy the existing binaries + --deploy-if-newer - Only deploy if newer software is + available (requires -t or -T) sanity/start-specific options: -F - Discard validator nodes that didn't bootup successfully @@ -93,6 +93,11 @@ Operate a configured testnet logs-specific options: none + update-specific options: + --platform linux|osx|windows - Deploy the tarball using 'solana-install deploy ...' for the + given platform (multiple platforms may be specified) + (-t option must be supplied as well) + startnode/stopnode-specific options: -i [ip address] - IP Address of the node to start or stop @@ -104,6 +109,7 @@ EOF releaseChannel= deployMethod=local +deployIfNewer= sanityExtraArgs= cargoFeatures= skipSetup=false @@ -147,6 +153,9 @@ while [[ -n $1 ]]; do elif [[ $1 = --no-snapshot-fetch ]]; then maybeNoSnapshot="$1" shift 1 + elif [[ $1 = --deploy-if-newer ]]; then + deployIfNewer=1 + shift 1 elif [[ $1 = --no-deploy ]]; then deployMethod=skip shift 1 @@ -162,7 +171,7 @@ while [[ -n $1 ]]; do elif [[ $1 = --skip-setup ]]; then skipSetup=true shift 1 - elif [[ $1 = --deploy-update ]]; then + elif [[ $1 = --platform ]]; then updatePlatforms="$updatePlatforms $2" shift 2 elif [[ $1 = --internal-nodes-stake-lamports ]]; then @@ -395,9 +404,11 @@ startBootstrapLeader() { case $deployMethod in tar) rsync -vPrc -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/solana-release/bin/* "$ipAddress:~/.cargo/bin/" + rsync -vPrc -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/solana-release/version.yml "$ipAddress:~/" ;; local) rsync -vPrc -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/farf/bin/* "$ipAddress:~/.cargo/bin/" + ssh "${sshOptions[@]}" -n "$ipAddress" "rm -f ~/version.yml; touch ~/version.yml" ;; skip) ;; @@ -552,9 +563,13 @@ sanity() { deployUpdate() { if [[ -z $updatePlatforms ]]; then + echo "No update platforms" return fi - [[ $deployMethod = tar ]] || exit 1 + if [[ -z $releaseChannel ]]; then + echo "Release channel not specified (use -t option)" + exit 1 + fi declare ok=true declare bootstrapLeader=${fullnodeIpList[0]} @@ -618,11 +633,6 @@ prepare_deploy() { -o "$SOLANA_ROOT"/solana-release.tar.bz2 "$updateDownloadUrl" ) tarballFilename="$SOLANA_ROOT"/solana-release.tar.bz2 - else - if [[ -n $updatePlatforms ]]; then - echo "Error: --deploy-update argument was provided but -t was not" - exit 1 - fi fi ( set -x @@ -644,6 +654,26 @@ prepare_deploy() { usage "Internal error: invalid deployMethod: $deployMethod" ;; esac + + if [[ -n $deployIfNewer ]]; then + if [[ $deployMethod != tar ]]; then + echo "Error: --deploy-if-newer only supported for tar deployments" + exit 1 + fi + + echo "Fetching current software version" + ( + set -x + rsync -vPrc -e "ssh ${sshOptions[*]}" "${fullnodeIpList[0]}":~/version.yml current-version.yml + ) + cat current-version.yml + if ! diff -q current-version.yml "$SOLANA_ROOT"/solana-release/version.yml; then + echo "Cluster software version is old. Update required" + else + echo "Cluster software version is current. No update required" + exit 0 + fi + fi } deploy() { @@ -733,8 +763,6 @@ deploy() { esac $metricsWriteDatapoint "testnet-deploy version=\"${networkVersion:0:9}\"" - deployUpdate - echo echo "+++ Deployment Successful" echo "Bootstrap leader deployment took $bootstrapNodeDeployTime seconds" @@ -819,6 +847,9 @@ sanity) stop) stop ;; +update) + deployUpdate + ;; stopnode) if [[ -z $nodeAddress ]]; then usage "node address (-i) not specified" diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index 69b5beb18a..0a87c03664 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -216,6 +216,7 @@ EOF validator|blockstreamer) if [[ $deployMethod != skip ]]; then net/scripts/rsync-retry.sh -vPrc "$entrypointIp":~/.cargo/bin/ ~/.cargo/bin/ + net/scripts/rsync-retry.sh -vPrc "$entrypointIp":~/version.yml version.yml fi if [[ $skipSetup != true ]]; then multinode-demo/clear-config.sh