From 7ee0c9e68a50f8908c2cc2493e3f7282cb19c2f0 Mon Sep 17 00:00:00 2001 From: sakridge Date: Wed, 4 Nov 2020 10:32:27 -0800 Subject: [PATCH] Add non-progress bar download status (#13370) --- download-utils/src/lib.rs | 67 +++++++++++++++++++++------- local-cluster/tests/local_cluster.rs | 1 + validator/src/main.rs | 52 +++++++++++++-------- 3 files changed, 86 insertions(+), 34 deletions(-) diff --git a/download-utils/src/lib.rs b/download-utils/src/lib.rs index 183994118..40d903e4c 100644 --- a/download-utils/src/lib.rs +++ b/download-utils/src/lib.rs @@ -23,7 +23,11 @@ fn new_spinner_progress_bar() -> ProgressBar { progress_bar } -pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> { +pub fn download_file( + url: &str, + destination_file: &Path, + use_progress_bar: bool, +) -> Result<(), String> { if destination_file.is_file() { return Err(format!("{:?} already exists", destination_file)); } @@ -34,7 +38,9 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> { let temp_destination_file = destination_file.with_extension(".tmp"); let progress_bar = new_spinner_progress_bar(); - progress_bar.set_message(&format!("{}Downloading {}...", TRUCK, url)); + if use_progress_bar { + progress_bar.set_message(&format!("{}Downloading {}...", TRUCK, url)); + } let response = reqwest::blocking::Client::new() .get(url) @@ -53,28 +59,51 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> { .and_then(|content_length| content_length.parse().ok()) .unwrap_or(0) }; - progress_bar.set_length(download_size); - progress_bar.set_style( - ProgressStyle::default_bar() - .template(&format!( - "{}{}Downloading {} {}", - "{spinner:.green} ", - TRUCK, - url, - "[{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})" - )) - .progress_chars("=> "), - ); + + if use_progress_bar { + progress_bar.set_length(download_size); + progress_bar.set_style( + ProgressStyle::default_bar() + .template(&format!( + "{}{}Downloading {} {}", + "{spinner:.green} ", + TRUCK, + url, + "[{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})" + )) + .progress_chars("=> "), + ); + } else { + info!("Downloading {} bytes from {}", download_size, url); + } struct DownloadProgress { progress_bar: ProgressBar, response: R, + last_print: Instant, + current_bytes: usize, + download_size: f32, + use_progress_bar: bool, } impl Read for DownloadProgress { fn read(&mut self, buf: &mut [u8]) -> io::Result { self.response.read(buf).map(|n| { - self.progress_bar.inc(n as u64); + if self.use_progress_bar { + self.progress_bar.inc(n as u64); + } else { + self.current_bytes += n; + if self.last_print.elapsed().as_secs() > 5 { + let bytes_f32 = self.current_bytes as f32; + info!( + "downloaded {} bytes {:.1}% {:.1} bytes/s", + self.current_bytes, + 100f32 * (bytes_f32 / self.download_size), + bytes_f32 / self.last_print.elapsed().as_secs_f32(), + ); + self.last_print = Instant::now(); + } + } n }) } @@ -83,6 +112,10 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> { let mut source = DownloadProgress { progress_bar, response, + last_print: Instant::now(), + current_bytes: 0, + download_size: (download_size as f32).max(1f32), + use_progress_bar, }; File::create(&temp_destination_file) @@ -110,6 +143,7 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> { pub fn download_genesis_if_missing( rpc_addr: &SocketAddr, genesis_package: &Path, + use_progress_bar: bool, ) -> Result { if !genesis_package.exists() { let tmp_genesis_path = genesis_package.parent().unwrap().join("tmp-genesis"); @@ -119,6 +153,7 @@ pub fn download_genesis_if_missing( download_file( &format!("http://{}/{}", rpc_addr, "genesis.tar.bz2"), &tmp_genesis_package, + use_progress_bar, )?; Ok(tmp_genesis_package) @@ -131,6 +166,7 @@ pub fn download_snapshot( rpc_addr: &SocketAddr, ledger_path: &Path, desired_snapshot_hash: (Slot, Hash), + use_progress_bar: bool, ) -> Result<(), String> { // Remove all snapshot not matching the desired hash let snapshot_packages = snapshot_utils::get_snapshot_archives(ledger_path); @@ -171,6 +207,7 @@ pub fn download_snapshot( .unwrap() ), &desired_snapshot_package, + use_progress_bar, ) .is_ok() { diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index df2789e0b..d2536829c 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -1042,6 +1042,7 @@ fn test_snapshot_download() { &cluster.entry_point_info.rpc, &validator_archive_path, archive_snapshot_hash, + false, ) .unwrap(); diff --git a/validator/src/main.rs b/validator/src/main.rs index afb459876..5298fcd98 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -447,6 +447,7 @@ fn download_then_check_genesis_hash( expected_genesis_hash: Option, max_genesis_archive_unpacked_size: u64, no_genesis_fetch: bool, + use_progress_bar: bool, ) -> Result { if no_genesis_fetch { let genesis_config = load_local_genesis(ledger_path, expected_genesis_hash)?; @@ -454,26 +455,27 @@ fn download_then_check_genesis_hash( } let genesis_package = ledger_path.join("genesis.tar.bz2"); - let genesis_config = - if let Ok(tmp_genesis_package) = download_genesis_if_missing(rpc_addr, &genesis_package) { - unpack_genesis_archive( - &tmp_genesis_package, - &ledger_path, - max_genesis_archive_unpacked_size, - ) - .map_err(|err| format!("Failed to unpack downloaded genesis config: {}", err))?; + let genesis_config = if let Ok(tmp_genesis_package) = + download_genesis_if_missing(rpc_addr, &genesis_package, use_progress_bar) + { + unpack_genesis_archive( + &tmp_genesis_package, + &ledger_path, + max_genesis_archive_unpacked_size, + ) + .map_err(|err| format!("Failed to unpack downloaded genesis config: {}", err))?; - let downloaded_genesis = GenesisConfig::load(&ledger_path) - .map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?; + let downloaded_genesis = GenesisConfig::load(&ledger_path) + .map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?; - check_genesis_hash(&downloaded_genesis, expected_genesis_hash)?; - std::fs::rename(tmp_genesis_package, genesis_package) - .map_err(|err| format!("Unable to rename: {:?}", err))?; + check_genesis_hash(&downloaded_genesis, expected_genesis_hash)?; + std::fs::rename(tmp_genesis_package, genesis_package) + .map_err(|err| format!("Unable to rename: {:?}", err))?; - downloaded_genesis - } else { - load_local_genesis(ledger_path, expected_genesis_hash)? - }; + downloaded_genesis + } else { + load_local_genesis(ledger_path, expected_genesis_hash)? + }; Ok(genesis_config.hash()) } @@ -626,6 +628,7 @@ impl Default for RpcBootstrapConfig { } } +#[allow(clippy::too_many_arguments)] fn rpc_bootstrap( node: &Node, identity_keypair: &Arc, @@ -636,6 +639,7 @@ fn rpc_bootstrap( validator_config: &mut ValidatorConfig, bootstrap_config: RpcBootstrapConfig, no_port_check: bool, + use_progress_bar: bool, ) { if !no_port_check { verify_reachable_ports(&node, cluster_entrypoint, &validator_config); @@ -693,6 +697,7 @@ fn rpc_bootstrap( validator_config.expected_genesis_hash, bootstrap_config.max_genesis_archive_unpacked_size, bootstrap_config.no_genesis_fetch, + use_progress_bar, ); if let Ok(genesis_hash) = genesis_hash { @@ -726,8 +731,12 @@ fn rpc_bootstrap( let (_cluster_info, gossip_exit_flag, gossip_service) = gossip.take().unwrap(); gossip_exit_flag.store(true, Ordering::Relaxed); - let ret = - download_snapshot(&rpc_contact_info.rpc, &ledger_path, snapshot_hash); + let ret = download_snapshot( + &rpc_contact_info.rpc, + &ledger_path, + snapshot_hash, + use_progress_bar, + ); gossip_service.join().unwrap(); ret }) @@ -782,6 +791,7 @@ fn rpc_bootstrap( } } +#[allow(clippy::too_many_arguments)] fn create_validator( node: Node, identity_keypair: &Arc, @@ -792,6 +802,7 @@ fn create_validator( mut validator_config: ValidatorConfig, rpc_bootstrap_config: RpcBootstrapConfig, no_port_check: bool, + use_progress_bar: bool, ) -> Validator { if validator_config.cuda { solana_perf::perf_libs::init_cuda(); @@ -810,6 +821,7 @@ fn create_validator( &mut validator_config, rpc_bootstrap_config, no_port_check, + use_progress_bar, ); } @@ -1627,6 +1639,7 @@ pub fn main() { Some(logfile) } }; + let use_progress_bar = logfile.is_none(); let _logger_thread = start_logger(logfile); // Default to RUST_BACKTRACE=1 for more informative validator logs @@ -1717,6 +1730,7 @@ pub fn main() { validator_config, rpc_bootstrap_config, no_port_check, + use_progress_bar, ); if let Some(filename) = init_complete_file {