From a17843c8f6d7d8d12a1a80b61e0a9a5aaaffeae6 Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Wed, 13 Mar 2019 15:31:05 -0700 Subject: [PATCH] solana-install design proposal --- Cargo.lock | 15 +++ Cargo.toml | 1 + book/src/SUMMARY.md | 1 + install/Cargo.toml | 21 ++++ install/build.rs | 15 +++ install/src/main.rs | 101 ++++++++++++++++++ proposals/src/SUMMARY.md | 1 + proposals/src/installer.md | 213 +++++++++++++++++++++++++++++++++++++ 8 files changed, 368 insertions(+) create mode 100644 install/Cargo.toml create mode 100644 install/build.rs create mode 100644 install/src/main.rs create mode 100644 proposals/src/installer.md diff --git a/Cargo.lock b/Cargo.lock index 9453c68018..99cc64f92b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2171,6 +2171,21 @@ dependencies = [ "solana-vote-api 0.13.0", ] +[[package]] +name = "solana-install" +version = "0.13.0" +dependencies = [ + "bincode 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "bs58 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", + "solana-client 0.13.0", + "solana-logger 0.13.0", + "solana-sdk 0.13.0", +] + [[package]] name = "solana-keygen" version = "0.13.0" diff --git a/Cargo.toml b/Cargo.toml index bb69aba64b..e986f2fb76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,7 @@ members = [ "drone", "fullnode", "genesis", + "install", "keygen", "ledger-tool", "logger", diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 0e6866ab7f..7409ba604c 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -33,3 +33,4 @@ - [JSON RPC API](jsonrpc-api.md) - [JavaScript API](javascript-api.md) - [solana-wallet CLI](wallet.md) + diff --git a/install/Cargo.toml b/install/Cargo.toml new file mode 100644 index 0000000000..20a48bff49 --- /dev/null +++ b/install/Cargo.toml @@ -0,0 +1,21 @@ +[package] +authors = ["Solana Maintainers "] +edition = "2018" +name = "solana-install" +description = "The solana cluster software installer" +version = "0.13.0" +repository = "https://github.com/solana-labs/solana" +license = "Apache-2.0" +homepage = "https://solana.com/" + +[dependencies] +bincode = "1.1.2" +bs58 = "0.2.0" +clap = { version = "2.32.0"} +chrono = { version = "0.4.0", features = ["serde"] } +log = "0.4.2" +serde_json = "1.0.39" +solana-client = { path = "../client", version = "0.13.0" } +solana-logger = { path = "../logger", version = "0.13.0" } +solana-sdk = { path = "../sdk", version = "0.13.0" } + diff --git a/install/build.rs b/install/build.rs new file mode 100644 index 0000000000..bc43404b0b --- /dev/null +++ b/install/build.rs @@ -0,0 +1,15 @@ +use std::time::SystemTime; + +fn main() { + println!( + "cargo:rustc-env=TARGET={}", + std::env::var("TARGET").unwrap() + ); + println!( + "cargo:rustc-env=BUILD_SECONDS_SINCE_UNIX_EPOCH={}", + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(), + ); +} diff --git a/install/src/main.rs b/install/src/main.rs new file mode 100644 index 0000000000..6556581d97 --- /dev/null +++ b/install/src/main.rs @@ -0,0 +1,101 @@ +use clap::{crate_description, crate_name, crate_version, App, AppSettings, Arg, SubCommand}; +//use clap::{crate_description, crate_version, load_yaml, App, AppSettings}; +use std::error; +use std::time::Duration; + +const TARGET: &str = env!("TARGET"); +const BUILD_SECONDS_SINCE_UNIX_EPOCH: &str = env!("BUILD_SECONDS_SINCE_UNIX_EPOCH"); + +fn main() -> Result<(), Box> { + solana_logger::setup(); + + let matches = App::new(crate_name!()) + .about(crate_description!()) + .version(crate_version!()) + .setting(AppSettings::ArgRequiredElseHelp) + .subcommand( + SubCommand::with_name("init") + .about("initializes a new installation") + .setting(AppSettings::DisableVersion) + .arg( + Arg::with_name("json_rpc_url") + .short("u") + .long("url") + .value_name("URL") + .takes_value(true) + .default_value("https://api.testnet.solana.com/") + .help("JSON RPC URL for the solana cluster"), + ) + .arg( + Arg::with_name("update_pubkey") + .short("p") + .long("pubkey") + .value_name("PUBKEY") + .takes_value(true) + .default_value("Solana-managed update manifest") + .help("Public key of the update manifest"), + ), + ) + .subcommand( + SubCommand::with_name("info") + .about("displays information about the current installation") + .setting(AppSettings::DisableVersion) + .arg( + Arg::with_name("local_info_only") + .short("l") + .long("local") + .help( + "only display local information, don't check the cluster for new updates", + ), + ), + ) + .subcommand( + SubCommand::with_name("deploy") + .about("deploys a new update") + .setting(AppSettings::DisableVersion) + .arg( + Arg::with_name("download_url") + .index(1) + .required(true) + .help("URL to the solana release archive"), + ) + .arg( + Arg::with_name("update_manifest_keypair") + .index(2) + .required(true) + .help("Keypair file for the update manifest (/path/to/keypair.json)"), + ), + ) + .subcommand( + SubCommand::with_name("update") + .about("checks for an update, and if available downloads and applies it") + .setting(AppSettings::DisableVersion), + ) + .subcommand( + SubCommand::with_name("run") + .about("Runs a program while periodically checking and applying software updates") + .after_help("The program will be restarted upon a successful software update") + .setting(AppSettings::DisableVersion) + .arg( + Arg::with_name("program_name") + .index(1) + .required(true) + .help("program to run"), + ) + .arg( + Arg::with_name("program_arguments") + .index(2) + .multiple(true) + .help("arguments to supply to the program"), + ), + ) + .get_matches(); + + println!("TARGET={}", TARGET); + println!( + "BUILD_SECONDS_SINCE_UNIX_EPOCH={:?}", + Duration::from_secs(u64::from_str_radix(BUILD_SECONDS_SINCE_UNIX_EPOCH, 10).unwrap()) + ); + println!("{:?}", matches); + Ok(()) +} diff --git a/proposals/src/SUMMARY.md b/proposals/src/SUMMARY.md index 5dc9b92c84..8754eacce6 100644 --- a/proposals/src/SUMMARY.md +++ b/proposals/src/SUMMARY.md @@ -23,6 +23,7 @@ - [References](ed_references.md) - [Cluster Test Framework](cluster-test-framework.md) - [Testing Programs](testing-programs.md) + - [Cluster Software Installation and Updates](installer.md) - [Implemented Design Proposals](implemented-proposals.md) - [Leader-to-Leader Transition](leader-leader-transition.md) diff --git a/proposals/src/installer.md b/proposals/src/installer.md new file mode 100644 index 0000000000..ac8c8eef09 --- /dev/null +++ b/proposals/src/installer.md @@ -0,0 +1,213 @@ +## Cluster Software Installation and Updates +Currently users are required to build the solana cluster software themselves +from the git repository and manually update it, which is error prone and +inconvenient. + +This document proposes an easy to use software install and updater that can be +used to deploy pre-built binaries for supported platforms. Users may elect to +use binaries supplied by Solana or any other party they trust. Deployment of +updates is managed using an on-chain update manifest program. + +### Motivating Examples +#### Fetch and run a pre-built installer + +With a well-known release URL, a pre-built binary can be obtained for supported +platforms: + +```bash +$ curl -o solana-install https://github.com/solana-labs/solana/releases/download/v1.2.3/solana-install-x86_64-apple-darwin +$ chmod +x ./solana-install +$ ./solana-install --help +``` + +Note: future enhancements should include building wrapper in the style of +https://rustup.rs/ to make bootstrapping an install even easier. + +#### Build and run the installer from source + +If a pre-built binary is not available for a given platform, building the +installer from source is always an option: +```bash +$ git clone https://github.com/solana-labs/solana.git +$ cd solana/install +$ cargo run -- --help +``` + +#### Deploy a new update to a cluster +Given a solana release tarball (as created by `ci/publish-tarball.sh`) that has already been uploaded to a publicly accessible URL, +the following commands will deploy the update: +```bash +$ solana-keygen -o update-manifest.json # <-- only generated once, the public key is shared with users +$ solana-install deploy http://example.com/path/to/solana-release.tar.bz2 update-manifest.json +``` + +Note: Supporting IPFS download URLs in the future would be attractive. + +#### Run a validator node that auto updates itself + +```bash +$ solana-install init --pubkey 92DMonmBYXwEMHJ99c9ceRSpAmk9v6i3RdvDdXaVcrfj # <-- pubkey is obtained from whoever is deploying the updates +$ export PATH=~/.local/share/solana-install/bin:$PATH +$ solana-keygen ... # <-- runs the latest solana-keygen +$ solana-install run solana-fullnode ... # <-- runs a fullnode, restarting it as necesary when an update is applied +``` + +### Update Manifest on-chain program +The Update Manifest program is used to advertise the deployment of new release tarballs +on a solana cluster. Each instance of this program describes a logical update +channel for a given target triple (eg, `x86_64-apple-darwin`). The public key +of each program instance is well-known between the entity deploying new updates +and users consuming those updates. + +The update tarball itself is hosted elsewhere, off-chain and can be fetched from +the specified `download_url`. + +```rust,ignore +use solana_sdk::signature::Signature; + +/// Information required to download and apply a given update +pub struct UpdateManifest { + pub target: String, // Target triple (TARGET) + pub commit: String, // git sha1 of this update, must match the commit sha1 in the release tar.bz2 + pub timestamp_secs: u64, // When the release was deployed (seconds since UNIX EPOC) + pub download_url: String, // Download URL to the release tar.bz2 + pub download_signature: Signature, // Signature of the release tar.bz2 file, verify with the Account public key +} + +/// Userdata of an Update Manifest program Account. +pub struct SignedUpdateManifest { + pub manifest: UpdateManifest, + pub manifest_signature: Signature, // Signature of UpdateInfo, verify with the Account public key +} +``` + +Note that the `manifest` field itself contains a corresponding signature +(`manifest_signature`) to guard against man-in-the-middle attacks between the +`solana-install` tool and the solana cluster RPC API. + +To guard against rollback attacks, `solana-install` will refuse to install an +update with an older `timestamp_secs` than what is currently installed. + +### solana-install Tool + +The `solana-install` tool is used by the user to install and update their cluster software. + +It manages the following files and directories in the user's home directory: +* `~/.config/solana/updater.json` - user configuration and information about currently installed software version +* `~/.local/share/solana-install/bin` - a symlink to the current release. eg, `~/.local/share/solana-update/-/bin` +* `~/.local/share/solana-install/-/` - contents of the release +* `~/.local/share/solana-install/-.tmp/` - temporary directory used while downloading a new release + +#### Command-line Interface +```bash +$ solana-install --help +solana-install 0.13.0 +The solana cluster software installer + +USAGE: + solana-install [SUBCOMMAND] + +FLAGS: + -h, --help Prints help information + +SUBCOMMANDS: + deploy deploys a new update + help Prints this message or the help of the given subcommand(s) + info displays information about the current installation + init initializes a new installation + run Runs a program while periodically checking and applying software updates. The program will be + restarted upon a successful software update + update checks for an update, and if available downloads and applies it +``` + +##### init +```bash +$ solana-install init --help +solana-install-init +initializes a new installation + +USAGE: + solana-install init [OPTIONS] + +FLAGS: + -h, --help Prints help information + +OPTIONS: + -u, --url JSON RPC URL for the solana cluster [default: https://api.testnet.solana.com/] + -p, --pubkey Public key of the update manifest [default: Solana-managed update manifest] +``` + +##### info +```bash +$ solana-install info --help +solana-install-info +displays information about the current installation + +USAGE: + solana-install info [FLAGS] + +FLAGS: + -h, --help Prints help information + -l only display local information, don't check the cluster for new updates +``` + +##### deploy +```bash +$ solana-install deploy --help +solana-install-deploy +deploys a new update + +USAGE: + solana-install deploy + +FLAGS: + -h, --help Prints help information + +ARGS: + URL to the solana release archive + Keypair file for the update manifest (/path/to/keypair.json) +``` + +##### update +```bash +$ solana-install update --help +solana-install-update +checks for an update, and if available downloads and applies it + +USAGE: + solana-install update + +FLAGS: + -h, --help Prints help information +``` + +##### run +```bash +$ solana-install run --help +solana-install-run +Runs a program while periodically checking and applying software updates. The program will be restarted upon a +successful software update + +USAGE: + solana-install run [program_arguments]... + +FLAGS: + -h, --help Prints help information + +ARGS: + program to run + ... arguments to supply to the program +``` + +### Release Archive Contents +A release archive is expected to be a tar file compressed with +bzip2 with the following internal structure: + +* `/version.yml` - a simple YAML file containing the fields (1) `"commit"` - the git + sha1 for this release, and (2) `"target"` - the target tuple. Any additional + fields are ignored. +* `/bin/` -- directory containing available programs in the release. + `solana-install` will symlink this directory to + `~/.local/share/solana-install/bin` for use by the `PATH` environment + variable. +* `...` -- any additional files and directories are permitted