project-serum
/
solana
mirror of https://github.com/project-serum/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add non-progress bar download status (#13370)

This commit is contained in:
sakridge 2020-11-04 10:32:27 -08:00 committed by GitHub
parent bb63e9486d
commit 7ee0c9e68a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 86 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
download-utils/src/lib.rs
View File

@ -23,7 +23,11 @@ fn new_spinner_progress_bar() -> ProgressBar {
progress_bar
}
pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
pub fn download_file(
url: &str,
destination_file: &Path,
use_progress_bar: bool,
) -> Result<(), String> {
if destination_file.is_file() {
return Err(format!("{:?} already exists", destination_file));
}
@ -34,7 +38,9 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
let temp_destination_file = destination_file.with_extension(".tmp");
let progress_bar = new_spinner_progress_bar();
progress_bar.set_message(&format!("{}Downloading {}...", TRUCK, url));
if use_progress_bar {
progress_bar.set_message(&format!("{}Downloading {}...", TRUCK, url));
}
let response = reqwest::blocking::Client::new()
.get(url)
@ -53,28 +59,51 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
.and_then(|content_length| content_length.parse().ok())
.unwrap_or(0)
};
progress_bar.set_length(download_size);
progress_bar.set_style(
ProgressStyle::default_bar()
.template(&format!(
"{}{}Downloading {} {}",
"{spinner:.green} ",
TRUCK,
url,
"[{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})"
))
.progress_chars("=> "),
);
if use_progress_bar {
progress_bar.set_length(download_size);
progress_bar.set_style(
ProgressStyle::default_bar()
.template(&format!(
"{}{}Downloading {} {}",
"{spinner:.green} ",
TRUCK,
url,
"[{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})"
))
.progress_chars("=> "),
);
} else {
info!("Downloading {} bytes from {}", download_size, url);
}
struct DownloadProgress<R> {
progress_bar: ProgressBar,
response: R,
last_print: Instant,
current_bytes: usize,
download_size: f32,
use_progress_bar: bool,
}
impl<R: Read> Read for DownloadProgress<R> {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
self.response.read(buf).map(|n| {
self.progress_bar.inc(n as u64);
if self.use_progress_bar {
self.progress_bar.inc(n as u64);
} else {
self.current_bytes += n;
if self.last_print.elapsed().as_secs() > 5 {
let bytes_f32 = self.current_bytes as f32;
info!(
"downloaded {} bytes {:.1}% {:.1} bytes/s",
self.current_bytes,
100f32 * (bytes_f32 / self.download_size),
bytes_f32 / self.last_print.elapsed().as_secs_f32(),
);
self.last_print = Instant::now();
}
}
n
})
}
@ -83,6 +112,10 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
let mut source = DownloadProgress {
progress_bar,
response,
last_print: Instant::now(),
current_bytes: 0,
download_size: (download_size as f32).max(1f32),
use_progress_bar,
};
File::create(&temp_destination_file)
@ -110,6 +143,7 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
pub fn download_genesis_if_missing(
rpc_addr: &SocketAddr,
genesis_package: &Path,
use_progress_bar: bool,
) -> Result<PathBuf, String> {
if !genesis_package.exists() {
let tmp_genesis_path = genesis_package.parent().unwrap().join("tmp-genesis");
@ -119,6 +153,7 @@ pub fn download_genesis_if_missing(
download_file(
&format!("http://{}/{}", rpc_addr, "genesis.tar.bz2"),
&tmp_genesis_package,
use_progress_bar,
)?;
Ok(tmp_genesis_package)
@ -131,6 +166,7 @@ pub fn download_snapshot(
rpc_addr: &SocketAddr,
ledger_path: &Path,
desired_snapshot_hash: (Slot, Hash),
use_progress_bar: bool,
) -> Result<(), String> {
// Remove all snapshot not matching the desired hash
let snapshot_packages = snapshot_utils::get_snapshot_archives(ledger_path);
@ -171,6 +207,7 @@ pub fn download_snapshot(
.unwrap()
),
&desired_snapshot_package,
use_progress_bar,
)
.is_ok()
{

1
local-cluster/tests/local_cluster.rs
View File

@ -1042,6 +1042,7 @@ fn test_snapshot_download() {
&cluster.entry_point_info.rpc,
&validator_archive_path,
archive_snapshot_hash,
false,
)
.unwrap();

52
validator/src/main.rs
View File

@ -447,6 +447,7 @@ fn download_then_check_genesis_hash(
expected_genesis_hash: Option<Hash>,
max_genesis_archive_unpacked_size: u64,
no_genesis_fetch: bool,
use_progress_bar: bool,
) -> Result<Hash, String> {
if no_genesis_fetch {
let genesis_config = load_local_genesis(ledger_path, expected_genesis_hash)?;
@ -454,26 +455,27 @@ fn download_then_check_genesis_hash(
}
let genesis_package = ledger_path.join("genesis.tar.bz2");
let genesis_config =
if let Ok(tmp_genesis_package) = download_genesis_if_missing(rpc_addr, &genesis_package) {
unpack_genesis_archive(
&tmp_genesis_package,
&ledger_path,
max_genesis_archive_unpacked_size,
)
.map_err(|err| format!("Failed to unpack downloaded genesis config: {}", err))?;
let genesis_config = if let Ok(tmp_genesis_package) =
download_genesis_if_missing(rpc_addr, &genesis_package, use_progress_bar)
{
unpack_genesis_archive(
&tmp_genesis_package,
&ledger_path,
max_genesis_archive_unpacked_size,
)
.map_err(|err| format!("Failed to unpack downloaded genesis config: {}", err))?;
let downloaded_genesis = GenesisConfig::load(&ledger_path)
.map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?;
let downloaded_genesis = GenesisConfig::load(&ledger_path)
.map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?;
check_genesis_hash(&downloaded_genesis, expected_genesis_hash)?;
std::fs::rename(tmp_genesis_package, genesis_package)
.map_err(|err| format!("Unable to rename: {:?}", err))?;
check_genesis_hash(&downloaded_genesis, expected_genesis_hash)?;
std::fs::rename(tmp_genesis_package, genesis_package)
.map_err(|err| format!("Unable to rename: {:?}", err))?;
downloaded_genesis
} else {
load_local_genesis(ledger_path, expected_genesis_hash)?
};
downloaded_genesis
} else {
load_local_genesis(ledger_path, expected_genesis_hash)?
};
Ok(genesis_config.hash())
}
@ -626,6 +628,7 @@ impl Default for RpcBootstrapConfig {
}
}
#[allow(clippy::too_many_arguments)]
fn rpc_bootstrap(
node: &Node,
identity_keypair: &Arc<Keypair>,
@ -636,6 +639,7 @@ fn rpc_bootstrap(
validator_config: &mut ValidatorConfig,
bootstrap_config: RpcBootstrapConfig,
no_port_check: bool,
use_progress_bar: bool,
) {
if !no_port_check {
verify_reachable_ports(&node, cluster_entrypoint, &validator_config);
@ -693,6 +697,7 @@ fn rpc_bootstrap(
validator_config.expected_genesis_hash,
bootstrap_config.max_genesis_archive_unpacked_size,
bootstrap_config.no_genesis_fetch,
use_progress_bar,
);
if let Ok(genesis_hash) = genesis_hash {
@ -726,8 +731,12 @@ fn rpc_bootstrap(
let (_cluster_info, gossip_exit_flag, gossip_service) =
gossip.take().unwrap();
gossip_exit_flag.store(true, Ordering::Relaxed);
let ret =
download_snapshot(&rpc_contact_info.rpc, &ledger_path, snapshot_hash);
let ret = download_snapshot(
&rpc_contact_info.rpc,
&ledger_path,
snapshot_hash,
use_progress_bar,
);
gossip_service.join().unwrap();
ret
})
@ -782,6 +791,7 @@ fn rpc_bootstrap(
}
}
#[allow(clippy::too_many_arguments)]
fn create_validator(
node: Node,
identity_keypair: &Arc<Keypair>,
@ -792,6 +802,7 @@ fn create_validator(
mut validator_config: ValidatorConfig,
rpc_bootstrap_config: RpcBootstrapConfig,
no_port_check: bool,
use_progress_bar: bool,
) -> Validator {
if validator_config.cuda {
solana_perf::perf_libs::init_cuda();
@ -810,6 +821,7 @@ fn create_validator(
&mut validator_config,
rpc_bootstrap_config,
no_port_check,
use_progress_bar,
);
}
@ -1627,6 +1639,7 @@ pub fn main() {
Some(logfile)
}
};
let use_progress_bar = logfile.is_none();
let _logger_thread = start_logger(logfile);
// Default to RUST_BACKTRACE=1 for more informative validator logs
@ -1717,6 +1730,7 @@ pub fn main() {
validator_config,
rpc_bootstrap_config,
no_port_check,
use_progress_bar,
);
if let Some(filename) = init_complete_file {