From bd7e269280b6dfe34bf621bc84315410d4103f66 Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Tue, 30 Jul 2019 22:43:47 -0700 Subject: [PATCH] Kill rsync (#5336) automerge --- .gitignore | 3 - Cargo.lock | 2 + book/src/testnet-participation.md | 4 +- ci/localnet-sanity.sh | 2 +- core/Cargo.toml | 2 + core/src/rpc_service.rs | 67 ++- core/src/validator.rs | 1 + multinode-demo/bootstrap-leader.sh | 31 +- multinode-demo/clear-config.sh | 12 +- multinode-demo/common.sh | 14 +- multinode-demo/fullnode.sh | 479 --------------------- multinode-demo/replicator.sh | 2 +- multinode-demo/setup.sh | 28 +- multinode-demo/validator.sh | 419 +++++++++++++++++- net/gce.sh | 4 +- net/remote/remote-deploy-update.sh | 2 +- net/remote/remote-node.sh | 8 +- net/remote/remote-sanity.sh | 4 +- net/scripts/ec2-security-group-config.json | 19 - net/scripts/install-rsync.sh | 10 - run.sh | 2 +- 21 files changed, 539 insertions(+), 576 deletions(-) delete mode 100755 multinode-demo/fullnode.sh diff --git a/.gitignore b/.gitignore index 8b0e2238d..120aa3125 100644 --- a/.gitignore +++ b/.gitignore @@ -11,10 +11,7 @@ **/*.rs.bk .cargo -# node config that is rsynced /config/ -# node config that remains local -/config-local/ # log files *.log diff --git a/Cargo.lock b/Cargo.lock index 22de543b8..1b1899864 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3118,6 +3118,8 @@ dependencies = [ "sys-info 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-fs 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/book/src/testnet-participation.md b/book/src/testnet-participation.md index 1423c2498..e94165cb8 100644 --- a/book/src/testnet-participation.md +++ b/book/src/testnet-participation.md @@ -217,9 +217,9 @@ $ solana-wallet show-vote-account 2ozWvfaXQd1X6uKh8jERoRGApDqSqcEy6fF1oN13LL2G The vote pubkey for the validator can also be found by running: ```bash # If this is a `solana-install`-installation run: -$ solana-keygen pubkey ~/.local/share/solana/install/active_release/config-local/validator-vote-keypair.json +$ solana-keygen pubkey ~/.local/share/solana/install/active_release/config/validator-vote-keypair.json # Otherwise run: -$ solana-keygen pubkey ./config-local/validator-vote-keypair.json +$ solana-keygen pubkey ./config/validator-vote-keypair.json ``` diff --git a/ci/localnet-sanity.sh b/ci/localnet-sanity.sh index 5de02e9df..4b63077bf 100755 --- a/ci/localnet-sanity.sh +++ b/ci/localnet-sanity.sh @@ -265,7 +265,7 @@ verifyLedger() { ( source multinode-demo/common.sh set -x - $solana_ledger_tool --ledger "$SOLANA_CONFIG_DIR"/$ledger-ledger verify + $solana_ledger_tool --ledger "$SOLANA_CONFIG_DIR"/$ledger/ledger verify ) || flag_error done } diff --git a/core/Cargo.toml b/core/Cargo.toml index 8d9c9481b..8bea7f59e 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -75,6 +75,8 @@ solana-librapay-api = { path = "../programs/librapay_api", version = "0.18.0-pre sys-info = "0.5.7" tokio = "0.1" tokio-codec = "0.1" +tokio-fs = "0.1" +tokio-io = "0.1" untrusted = "0.7.0" # reed-solomon-erasure's simd_c feature fails to build for x86_64-pc-windows-msvc, use pure-rust diff --git a/core/src/rpc_service.rs b/core/src/rpc_service.rs index 641f42156..f5884ecec 100644 --- a/core/src/rpc_service.rs +++ b/core/src/rpc_service.rs @@ -6,12 +6,17 @@ use crate::rpc::*; use crate::service::Service; use crate::storage_stage::StorageState; use jsonrpc_core::MetaIoHandler; -use jsonrpc_http_server::{hyper, AccessControlAllowOrigin, DomainsValidation, ServerBuilder}; +use jsonrpc_http_server::{ + hyper, AccessControlAllowOrigin, DomainsValidation, RequestMiddleware, RequestMiddlewareAction, + ServerBuilder, +}; use std::net::SocketAddr; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, RwLock}; use std::thread::{self, sleep, Builder, JoinHandle}; use std::time::Duration; +use tokio::prelude::Future; pub struct JsonRpcService { thread_hdl: JoinHandle<()>, @@ -20,6 +25,61 @@ pub struct JsonRpcService { pub request_processor: Arc>, // Used only by test_rpc_new()... } +#[derive(Default)] +struct RpcRequestMiddleware { + ledger_path: PathBuf, +} +impl RpcRequestMiddleware { + pub fn new(ledger_path: PathBuf) -> Self { + Self { ledger_path } + } + + fn not_found() -> hyper::Response { + hyper::Response::builder() + .status(hyper::StatusCode::NOT_FOUND) + .body(hyper::Body::empty()) + .unwrap() + } + + fn internal_server_error() -> hyper::Response { + hyper::Response::builder() + .status(hyper::StatusCode::INTERNAL_SERVER_ERROR) + .body(hyper::Body::empty()) + .unwrap() + } + + fn get(&self, filename: &str) -> RequestMiddlewareAction { + let filename = self.ledger_path.join(filename); + RequestMiddlewareAction::Respond { + should_validate_hosts: true, + response: Box::new( + tokio_fs::file::File::open(filename) + .and_then(|file| { + let buf: Vec = Vec::new(); + tokio_io::io::read_to_end(file, buf) + .and_then(|item| Ok(hyper::Response::new(item.1.into()))) + .or_else(|_| Ok(RpcRequestMiddleware::internal_server_error())) + }) + .or_else(|_| Ok(RpcRequestMiddleware::not_found())), + ), + } + } +} + +impl RequestMiddleware for RpcRequestMiddleware { + fn on_request(&self, request: hyper::Request) -> RequestMiddlewareAction { + trace!("request uri: {}", request.uri()); + match request.uri().path() { + "/snapshot.tgz" => self.get("snapshot.tgz"), + "/genesis.tgz" => self.get("genesis.tgz"), + _ => RequestMiddlewareAction::Proceed { + should_continue_on_invalid_cors: false, + request, + }, + } + } +} + impl JsonRpcService { pub fn new( cluster_info: &Arc>, @@ -27,6 +87,7 @@ impl JsonRpcService { storage_state: StorageState, config: JsonRpcConfig, bank_forks: Arc>, + ledger_path: &Path, exit: &Arc, ) -> Self { info!("rpc bound to {:?}", rpc_addr); @@ -41,6 +102,7 @@ impl JsonRpcService { let cluster_info = cluster_info.clone(); let exit_ = exit.clone(); + let ledger_path = ledger_path.to_path_buf(); let thread_hdl = Builder::new() .name("solana-jsonrpc".to_string()) @@ -57,11 +119,13 @@ impl JsonRpcService { .cors(DomainsValidation::AllowOnly(vec![ AccessControlAllowOrigin::Any, ])) + .request_middleware(RpcRequestMiddleware::new(ledger_path)) .start_http(&rpc_addr); if let Err(e) = server { warn!("JSON RPC service unavailable error: {:?}. \nAlso, check that port {} is not already in use by another application", e, rpc_addr.port()); return; } + while !exit_.load(Ordering::Relaxed) { sleep(Duration::from_millis(100)); } @@ -116,6 +180,7 @@ mod tests { StorageState::default(), JsonRpcConfig::default(), bank_forks, + &PathBuf::from("farf"), &exit, ); let thread = rpc_service.thread_hdl.thread(); diff --git a/core/src/validator.rs b/core/src/validator.rs index 42104ebd1..9aff3c208 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -176,6 +176,7 @@ impl Validator { storage_state.clone(), config.rpc_config.clone(), bank_forks.clone(), + ledger_path, &exit, )) }; diff --git a/multinode-demo/bootstrap-leader.sh b/multinode-demo/bootstrap-leader.sh index ee550b4fd..31cb6285b 100755 --- a/multinode-demo/bootstrap-leader.sh +++ b/multinode-demo/bootstrap-leader.sh @@ -35,13 +35,6 @@ while [[ -n $1 ]]; do fi done -if [[ ! -d "$SOLANA_RSYNC_CONFIG_DIR"/ledger ]]; then - echo "$SOLANA_RSYNC_CONFIG_DIR/ledger does not exist" - echo - echo "Please run: $here/setup.sh" - exit 1 -fi - if [[ -z $CI ]]; then # Skip in CI # shellcheck source=scripts/tune-system.sh source "$here"/../scripts/tune-system.sh @@ -50,23 +43,25 @@ fi setup_secondary_mount # These keypairs are created by ./setup.sh and included in the genesis block -identity_keypair=$SOLANA_CONFIG_DIR/bootstrap-leader-keypair.json -vote_keypair="$SOLANA_CONFIG_DIR"/bootstrap-leader-vote-keypair.json -storage_keypair=$SOLANA_CONFIG_DIR/bootstrap-leader-storage-keypair.json +identity_keypair=$SOLANA_CONFIG_DIR/bootstrap-leader/identity-keypair.json +vote_keypair="$SOLANA_CONFIG_DIR"/bootstrap-leader/vote-keypair.json +storage_keypair=$SOLANA_CONFIG_DIR/bootstrap-leader/storage-keypair.json -ledger_config_dir="$SOLANA_CONFIG_DIR"/bootstrap-leader-ledger -[[ -d "$ledger_config_dir" ]] || ( - set -x - cp -a "$SOLANA_RSYNC_CONFIG_DIR"/ledger/ "$ledger_config_dir" -) +ledger_dir="$SOLANA_CONFIG_DIR"/bootstrap-leader/ledger +[[ -d "$ledger_dir" ]] || { + echo "$ledger_dir does not exist" + echo + echo "Please run: $here/setup.sh" + exit 1 +} args+=( - --accounts "$SOLANA_CONFIG_DIR"/bootstrap-leader-accounts + --accounts "$SOLANA_CONFIG_DIR"/bootstrap-leader/accounts --enable-rpc-exit --identity "$identity_keypair" - --ledger "$ledger_config_dir" + --ledger "$ledger_dir" --rpc-port 8899 - --snapshot-path "$SOLANA_CONFIG_DIR"/bootstrap-leader-snapshots + --snapshot-path "$SOLANA_CONFIG_DIR"/bootstrap-leader/snapshots --storage-keypair "$storage_keypair" --voting-keypair "$vote_keypair" --rpc-drone-address 127.0.0.1:9900 diff --git a/multinode-demo/clear-config.sh b/multinode-demo/clear-config.sh index f6bd4b143..79a6532e8 100755 --- a/multinode-demo/clear-config.sh +++ b/multinode-demo/clear-config.sh @@ -9,11 +9,11 @@ source "$here"/common.sh set -e -for i in "$SOLANA_RSYNC_CONFIG_DIR" "$SOLANA_CONFIG_DIR"; do - echo "Cleaning $i" - rm -rvf "${i:?}/" # <-- $i might be a symlink, rm the other side of it first - rm -rvf "$i" - mkdir -p "$i" -done +( + set -x + rm -rf "${SOLANA_CONFIG_DIR:?}/" # <-- $i might be a symlink, rm the other side of it first + rm -rf "$SOLANA_CONFIG_DIR" + mkdir -p "$SOLANA_CONFIG_DIR" +) setup_secondary_mount diff --git a/multinode-demo/common.sh b/multinode-demo/common.sh index bec76498b..b1a0de590 100644 --- a/multinode-demo/common.sh +++ b/multinode-demo/common.sh @@ -9,8 +9,6 @@ SOLANA_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. || exit 1; pwd)" -rsync=rsync - if [[ $(uname) != Linux ]]; then # Protect against unsupported configurations to prevent non-obvious errors # later. Arguably these should be fatal errors but for now prefer tolerance. @@ -65,19 +63,15 @@ export RUST_BACKTRACE=1 # shellcheck source=scripts/configure-metrics.sh source "$SOLANA_ROOT"/scripts/configure-metrics.sh -# The directory on the cluster entrypoint that is rsynced by other full nodes -SOLANA_RSYNC_CONFIG_DIR=$SOLANA_ROOT/config - -# Configuration that remains local -SOLANA_CONFIG_DIR=$SOLANA_ROOT/config-local +SOLANA_CONFIG_DIR=$SOLANA_ROOT/config SECONDARY_DISK_MOUNT_POINT=/mnt/extra-disk setup_secondary_mount() { - # If there is a secondary disk, symlink the config-local dir there + # If there is a secondary disk, symlink the config/ dir there if [[ -d $SECONDARY_DISK_MOUNT_POINT ]]; then - mkdir -p $SECONDARY_DISK_MOUNT_POINT/config-local + mkdir -p $SECONDARY_DISK_MOUNT_POINT/config rm -rf "$SOLANA_CONFIG_DIR" - ln -sfT $SECONDARY_DISK_MOUNT_POINT/config-local "$SOLANA_CONFIG_DIR" + ln -sfT $SECONDARY_DISK_MOUNT_POINT/config "$SOLANA_CONFIG_DIR" fi } diff --git a/multinode-demo/fullnode.sh b/multinode-demo/fullnode.sh deleted file mode 100755 index e8cdc37a3..000000000 --- a/multinode-demo/fullnode.sh +++ /dev/null @@ -1,479 +0,0 @@ -#!/usr/bin/env bash -# -# Start a validator -# -here=$(dirname "$0") -# shellcheck source=multinode-demo/common.sh -source "$here"/common.sh - -fullnode_usage() { - if [[ -n $1 ]]; then - echo "$*" - echo - fi - cat </dev/null 2>&1 -} - -set -e -PS4="$(basename "$0"): " - -pid= -kill_fullnode() { - # Note: do not echo anything from this function to ensure $pid is actually - # killed when stdout/stderr are redirected - set +ex - if [[ -n $pid ]]; then - declare _pid=$pid - pid= - kill "$_pid" || true - wait "$_pid" || true - fi - exit -} -trap 'kill_fullnode' INT TERM ERR - -if ((reset_ledger)); then - echo "Resetting ledger..." - ( - set -x - rm -rf "$state_dir" - rm -rf "$ledger_config_dir" - ) - if [[ -d "$SOLANA_RSYNC_CONFIG_DIR"/ledger/ ]]; then - cp -a "$SOLANA_RSYNC_CONFIG_DIR"/ledger/ "$ledger_config_dir" - fi -fi - -while true; do - if new_genesis_block; then - # If the genesis block has changed remove the now stale ledger and start all - # over again - ( - set -x - rm -rf "$ledger_config_dir" "$state_dir" "$configured_flag" - ) - fi - - if [[ ! -d "$ledger_config_dir" ]]; then - ( - cd "$SOLANA_RSYNC_CONFIG_DIR" - - echo "Rsyncing genesis ledger from ${rsync_entrypoint_url:?}..." - SECONDS= - while ! $rsync -Pr "${rsync_entrypoint_url:?}"/config/ledger .; do - echo "Genesis ledger rsync failed" - sleep 5 - done - echo "Fetched genesis ledger in $SECONDS seconds" - - if ((boot_from_snapshot)); then - SECONDS= - echo "Rsyncing state snapshot ${rsync_entrypoint_url:?}..." - if ! $rsync -P "${rsync_entrypoint_url:?}"/config/state.tgz .; then - echo "State snapshot rsync failed" - rm -f "$SOLANA_RSYNC_CONFIG_DIR"/state.tgz - exit - fi - echo "Fetched snapshot in $SECONDS seconds" - - SECONDS= - mkdir -p "$state_dir" - ( - set -x - tar -C "$state_dir" -zxf "$SOLANA_RSYNC_CONFIG_DIR"/state.tgz - ) - echo "Extracted snapshot in $SECONDS seconds" - fi - ) - ( - set -x - cp -a "$SOLANA_RSYNC_CONFIG_DIR"/ledger/ "$ledger_config_dir" - ) - fi - - vote_pubkey=$($solana_keygen pubkey "$voting_keypair_path") - stake_pubkey=$($solana_keygen pubkey "$stake_keypair_path") - storage_pubkey=$($solana_keygen pubkey "$storage_keypair_path") - - if ((stake_lamports)); then - setup_validator_accounts "${entrypoint_address%:*}" \ - "$node_lamports" \ - "$stake_lamports" - fi - - cat </dev/null 2>&1 +} + +set -e +PS4="$(basename "$0"): " + +pid= +kill_node() { + # Note: do not echo anything from this function to ensure $pid is actually + # killed when stdout/stderr are redirected + set +ex + if [[ -n $pid ]]; then + declare _pid=$pid + pid= + kill "$_pid" || true + wait "$_pid" || true + fi + exit +} +kill_node_and_exit() { + kill_node + exit +} +trap 'kill_node_and_exit' INT TERM ERR + +if ((reset_ledger)); then + echo "Resetting ledger..." + ( + set -x + rm -rf "$state_dir" + rm -rf "$ledger_config_dir" + ) +fi + +while true; do + if new_genesis_block; then + # If the genesis block has changed remove the now stale ledger and start all + # over again + ( + set -x + rm -rf "$ledger_config_dir" "$state_dir" "$configured_flag" + ) + fi + + if [[ ! -f "$ledger_config_dir"/.ok ]]; then + echo "Fetching ledger from $rpc_url/genesis.tgz..." + SECONDS= + mkdir -p "$ledger_config_dir" + while ! curl -f "$rpc_url"/genesis.tgz -o "$ledger_config_dir"/genesis.tgz; do + echo "Genesis ledger fetch failed" + sleep 5 + done + echo "Fetched genesis ledger in $SECONDS seconds" + + ( + set -x + cd "$ledger_config_dir" + tar -zxf genesis.tgz + touch .ok + ) + + ( + if ((boot_from_snapshot)); then + SECONDS= + + echo "Fetching state snapshot $rpc_url/snapshot.tgz..." + mkdir -p "$state_dir" + if ! curl -f "$rpc_url"/snapshot.tgz -o "$state_dir"/snapshot.tgz; then + echo "State snapshot fetch failed" + rm -f "$state_dir"/snapshot.tgz + exit 0 # None fatal + fi + echo "Fetched snapshot in $SECONDS seconds" + + SECONDS= + ( + set -x + cd "$state_dir" + tar -zxf snapshot.tgz + rm snapshot.tgz + ) + echo "Extracted snapshot in $SECONDS seconds" + fi + ) + fi + + vote_pubkey=$($solana_keygen pubkey "$voting_keypair_path") + stake_pubkey=$($solana_keygen pubkey "$stake_keypair_path") + storage_pubkey=$($solana_keygen pubkey "$storage_keypair_path") + + if ((stake_lamports)); then + setup_validator_accounts "$node_lamports" "$stake_lamports" + fi + + cat < drone.log 2>&1 & fi diff --git a/net/remote/remote-sanity.sh b/net/remote/remote-sanity.sh index f2bb1b8cf..6d62fc8cd 100755 --- a/net/remote/remote-sanity.sh +++ b/net/remote/remote-sanity.sh @@ -82,8 +82,8 @@ local|tar|skip) solana_keygen=solana-keygen solana_ledger_tool=solana-ledger-tool - ledger=config-local/bootstrap-leader-ledger - client_id=config-local/client-id.json + ledger=config/bootstrap-leader/ledger + client_id=config/client-id.json ;; *) echo "Unknown deployment method: $deployMethod" diff --git a/net/scripts/ec2-security-group-config.json b/net/scripts/ec2-security-group-config.json index 1faa4e5f8..311cedf3b 100644 --- a/net/scripts/ec2-security-group-config.json +++ b/net/scripts/ec2-security-group-config.json @@ -57,25 +57,6 @@ } ] }, - { - "PrefixListIds": [], - "FromPort": 873, - "IpRanges": [ - { - "Description": "rsync", - "CidrIp": "0.0.0.0/0" - } - ], - "ToPort": 873, - "IpProtocol": "tcp", - "UserIdGroupPairs": [], - "Ipv6Ranges": [ - { - "CidrIpv6": "::/0", - "Description": "rsync" - } - ] - }, { "PrefixListIds": [], "FromPort": 3001, diff --git a/net/scripts/install-rsync.sh b/net/scripts/install-rsync.sh index 7a6419554..e7ef4ddf2 100755 --- a/net/scripts/install-rsync.sh +++ b/net/scripts/install-rsync.sh @@ -8,13 +8,3 @@ set -ex [[ $USER = root ]] || exit 1 apt-get --assume-yes install rsync -cat > /etc/rsyncd.conf <<-EOF -[config] -path = /var/snap/solana/current/config -hosts allow = * -read only = true -EOF - -systemctl enable rsync -systemctl start rsync - diff --git a/run.sh b/run.sh index 9f990bf90..ce8817fc3 100755 --- a/run.sh +++ b/run.sh @@ -37,7 +37,7 @@ done export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info export RUST_BACKTRACE=1 -dataDir=$PWD/config-local/"$(basename "$0" .sh)" +dataDir=$PWD/config/"$(basename "$0" .sh)" ledgerDir=$PWD/config/ledger set -x