Add non-progress bar download status (#13370)
This commit is contained in:
parent
bb63e9486d
commit
7ee0c9e68a
|
@ -23,7 +23,11 @@ fn new_spinner_progress_bar() -> ProgressBar {
|
|||
progress_bar
|
||||
}
|
||||
|
||||
pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
|
||||
pub fn download_file(
|
||||
url: &str,
|
||||
destination_file: &Path,
|
||||
use_progress_bar: bool,
|
||||
) -> Result<(), String> {
|
||||
if destination_file.is_file() {
|
||||
return Err(format!("{:?} already exists", destination_file));
|
||||
}
|
||||
|
@ -34,7 +38,9 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
|
|||
let temp_destination_file = destination_file.with_extension(".tmp");
|
||||
|
||||
let progress_bar = new_spinner_progress_bar();
|
||||
progress_bar.set_message(&format!("{}Downloading {}...", TRUCK, url));
|
||||
if use_progress_bar {
|
||||
progress_bar.set_message(&format!("{}Downloading {}...", TRUCK, url));
|
||||
}
|
||||
|
||||
let response = reqwest::blocking::Client::new()
|
||||
.get(url)
|
||||
|
@ -53,28 +59,51 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
|
|||
.and_then(|content_length| content_length.parse().ok())
|
||||
.unwrap_or(0)
|
||||
};
|
||||
progress_bar.set_length(download_size);
|
||||
progress_bar.set_style(
|
||||
ProgressStyle::default_bar()
|
||||
.template(&format!(
|
||||
"{}{}Downloading {} {}",
|
||||
"{spinner:.green} ",
|
||||
TRUCK,
|
||||
url,
|
||||
"[{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})"
|
||||
))
|
||||
.progress_chars("=> "),
|
||||
);
|
||||
|
||||
if use_progress_bar {
|
||||
progress_bar.set_length(download_size);
|
||||
progress_bar.set_style(
|
||||
ProgressStyle::default_bar()
|
||||
.template(&format!(
|
||||
"{}{}Downloading {} {}",
|
||||
"{spinner:.green} ",
|
||||
TRUCK,
|
||||
url,
|
||||
"[{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})"
|
||||
))
|
||||
.progress_chars("=> "),
|
||||
);
|
||||
} else {
|
||||
info!("Downloading {} bytes from {}", download_size, url);
|
||||
}
|
||||
|
||||
struct DownloadProgress<R> {
|
||||
progress_bar: ProgressBar,
|
||||
response: R,
|
||||
last_print: Instant,
|
||||
current_bytes: usize,
|
||||
download_size: f32,
|
||||
use_progress_bar: bool,
|
||||
}
|
||||
|
||||
impl<R: Read> Read for DownloadProgress<R> {
|
||||
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
||||
self.response.read(buf).map(|n| {
|
||||
self.progress_bar.inc(n as u64);
|
||||
if self.use_progress_bar {
|
||||
self.progress_bar.inc(n as u64);
|
||||
} else {
|
||||
self.current_bytes += n;
|
||||
if self.last_print.elapsed().as_secs() > 5 {
|
||||
let bytes_f32 = self.current_bytes as f32;
|
||||
info!(
|
||||
"downloaded {} bytes {:.1}% {:.1} bytes/s",
|
||||
self.current_bytes,
|
||||
100f32 * (bytes_f32 / self.download_size),
|
||||
bytes_f32 / self.last_print.elapsed().as_secs_f32(),
|
||||
);
|
||||
self.last_print = Instant::now();
|
||||
}
|
||||
}
|
||||
n
|
||||
})
|
||||
}
|
||||
|
@ -83,6 +112,10 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
|
|||
let mut source = DownloadProgress {
|
||||
progress_bar,
|
||||
response,
|
||||
last_print: Instant::now(),
|
||||
current_bytes: 0,
|
||||
download_size: (download_size as f32).max(1f32),
|
||||
use_progress_bar,
|
||||
};
|
||||
|
||||
File::create(&temp_destination_file)
|
||||
|
@ -110,6 +143,7 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
|
|||
pub fn download_genesis_if_missing(
|
||||
rpc_addr: &SocketAddr,
|
||||
genesis_package: &Path,
|
||||
use_progress_bar: bool,
|
||||
) -> Result<PathBuf, String> {
|
||||
if !genesis_package.exists() {
|
||||
let tmp_genesis_path = genesis_package.parent().unwrap().join("tmp-genesis");
|
||||
|
@ -119,6 +153,7 @@ pub fn download_genesis_if_missing(
|
|||
download_file(
|
||||
&format!("http://{}/{}", rpc_addr, "genesis.tar.bz2"),
|
||||
&tmp_genesis_package,
|
||||
use_progress_bar,
|
||||
)?;
|
||||
|
||||
Ok(tmp_genesis_package)
|
||||
|
@ -131,6 +166,7 @@ pub fn download_snapshot(
|
|||
rpc_addr: &SocketAddr,
|
||||
ledger_path: &Path,
|
||||
desired_snapshot_hash: (Slot, Hash),
|
||||
use_progress_bar: bool,
|
||||
) -> Result<(), String> {
|
||||
// Remove all snapshot not matching the desired hash
|
||||
let snapshot_packages = snapshot_utils::get_snapshot_archives(ledger_path);
|
||||
|
@ -171,6 +207,7 @@ pub fn download_snapshot(
|
|||
.unwrap()
|
||||
),
|
||||
&desired_snapshot_package,
|
||||
use_progress_bar,
|
||||
)
|
||||
.is_ok()
|
||||
{
|
||||
|
|
|
@ -1042,6 +1042,7 @@ fn test_snapshot_download() {
|
|||
&cluster.entry_point_info.rpc,
|
||||
&validator_archive_path,
|
||||
archive_snapshot_hash,
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
|
|
|
@ -447,6 +447,7 @@ fn download_then_check_genesis_hash(
|
|||
expected_genesis_hash: Option<Hash>,
|
||||
max_genesis_archive_unpacked_size: u64,
|
||||
no_genesis_fetch: bool,
|
||||
use_progress_bar: bool,
|
||||
) -> Result<Hash, String> {
|
||||
if no_genesis_fetch {
|
||||
let genesis_config = load_local_genesis(ledger_path, expected_genesis_hash)?;
|
||||
|
@ -454,26 +455,27 @@ fn download_then_check_genesis_hash(
|
|||
}
|
||||
|
||||
let genesis_package = ledger_path.join("genesis.tar.bz2");
|
||||
let genesis_config =
|
||||
if let Ok(tmp_genesis_package) = download_genesis_if_missing(rpc_addr, &genesis_package) {
|
||||
unpack_genesis_archive(
|
||||
&tmp_genesis_package,
|
||||
&ledger_path,
|
||||
max_genesis_archive_unpacked_size,
|
||||
)
|
||||
.map_err(|err| format!("Failed to unpack downloaded genesis config: {}", err))?;
|
||||
let genesis_config = if let Ok(tmp_genesis_package) =
|
||||
download_genesis_if_missing(rpc_addr, &genesis_package, use_progress_bar)
|
||||
{
|
||||
unpack_genesis_archive(
|
||||
&tmp_genesis_package,
|
||||
&ledger_path,
|
||||
max_genesis_archive_unpacked_size,
|
||||
)
|
||||
.map_err(|err| format!("Failed to unpack downloaded genesis config: {}", err))?;
|
||||
|
||||
let downloaded_genesis = GenesisConfig::load(&ledger_path)
|
||||
.map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?;
|
||||
let downloaded_genesis = GenesisConfig::load(&ledger_path)
|
||||
.map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?;
|
||||
|
||||
check_genesis_hash(&downloaded_genesis, expected_genesis_hash)?;
|
||||
std::fs::rename(tmp_genesis_package, genesis_package)
|
||||
.map_err(|err| format!("Unable to rename: {:?}", err))?;
|
||||
check_genesis_hash(&downloaded_genesis, expected_genesis_hash)?;
|
||||
std::fs::rename(tmp_genesis_package, genesis_package)
|
||||
.map_err(|err| format!("Unable to rename: {:?}", err))?;
|
||||
|
||||
downloaded_genesis
|
||||
} else {
|
||||
load_local_genesis(ledger_path, expected_genesis_hash)?
|
||||
};
|
||||
downloaded_genesis
|
||||
} else {
|
||||
load_local_genesis(ledger_path, expected_genesis_hash)?
|
||||
};
|
||||
|
||||
Ok(genesis_config.hash())
|
||||
}
|
||||
|
@ -626,6 +628,7 @@ impl Default for RpcBootstrapConfig {
|
|||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn rpc_bootstrap(
|
||||
node: &Node,
|
||||
identity_keypair: &Arc<Keypair>,
|
||||
|
@ -636,6 +639,7 @@ fn rpc_bootstrap(
|
|||
validator_config: &mut ValidatorConfig,
|
||||
bootstrap_config: RpcBootstrapConfig,
|
||||
no_port_check: bool,
|
||||
use_progress_bar: bool,
|
||||
) {
|
||||
if !no_port_check {
|
||||
verify_reachable_ports(&node, cluster_entrypoint, &validator_config);
|
||||
|
@ -693,6 +697,7 @@ fn rpc_bootstrap(
|
|||
validator_config.expected_genesis_hash,
|
||||
bootstrap_config.max_genesis_archive_unpacked_size,
|
||||
bootstrap_config.no_genesis_fetch,
|
||||
use_progress_bar,
|
||||
);
|
||||
|
||||
if let Ok(genesis_hash) = genesis_hash {
|
||||
|
@ -726,8 +731,12 @@ fn rpc_bootstrap(
|
|||
let (_cluster_info, gossip_exit_flag, gossip_service) =
|
||||
gossip.take().unwrap();
|
||||
gossip_exit_flag.store(true, Ordering::Relaxed);
|
||||
let ret =
|
||||
download_snapshot(&rpc_contact_info.rpc, &ledger_path, snapshot_hash);
|
||||
let ret = download_snapshot(
|
||||
&rpc_contact_info.rpc,
|
||||
&ledger_path,
|
||||
snapshot_hash,
|
||||
use_progress_bar,
|
||||
);
|
||||
gossip_service.join().unwrap();
|
||||
ret
|
||||
})
|
||||
|
@ -782,6 +791,7 @@ fn rpc_bootstrap(
|
|||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn create_validator(
|
||||
node: Node,
|
||||
identity_keypair: &Arc<Keypair>,
|
||||
|
@ -792,6 +802,7 @@ fn create_validator(
|
|||
mut validator_config: ValidatorConfig,
|
||||
rpc_bootstrap_config: RpcBootstrapConfig,
|
||||
no_port_check: bool,
|
||||
use_progress_bar: bool,
|
||||
) -> Validator {
|
||||
if validator_config.cuda {
|
||||
solana_perf::perf_libs::init_cuda();
|
||||
|
@ -810,6 +821,7 @@ fn create_validator(
|
|||
&mut validator_config,
|
||||
rpc_bootstrap_config,
|
||||
no_port_check,
|
||||
use_progress_bar,
|
||||
);
|
||||
}
|
||||
|
||||
|
@ -1627,6 +1639,7 @@ pub fn main() {
|
|||
Some(logfile)
|
||||
}
|
||||
};
|
||||
let use_progress_bar = logfile.is_none();
|
||||
let _logger_thread = start_logger(logfile);
|
||||
|
||||
// Default to RUST_BACKTRACE=1 for more informative validator logs
|
||||
|
@ -1717,6 +1730,7 @@ pub fn main() {
|
|||
validator_config,
|
||||
rpc_bootstrap_config,
|
||||
no_port_check,
|
||||
use_progress_bar,
|
||||
);
|
||||
|
||||
if let Some(filename) = init_complete_file {
|
||||
|
|
Loading…
Reference in New Issue