Add more logging while unpacking snapshots (#10266)
This commit is contained in:
parent
7d42d529af
commit
e3b444834f
|
@ -77,6 +77,8 @@ where
|
|||
let mut total_size: u64 = 0;
|
||||
let mut total_count: u64 = 0;
|
||||
|
||||
let mut total_entries = 0;
|
||||
let mut last_log_update = Instant::now();
|
||||
for entry in archive.entries()? {
|
||||
let mut entry = entry?;
|
||||
let path = entry.path()?;
|
||||
|
@ -110,8 +112,15 @@ where
|
|||
|
||||
// unpack_in does its own sanitization
|
||||
// ref: https://docs.rs/tar/*/tar/struct.Entry.html#method.unpack_in
|
||||
check_unpack_result(entry.unpack_in(&unpack_dir)?, path_str)?
|
||||
check_unpack_result(entry.unpack_in(&unpack_dir)?, path_str)?;
|
||||
total_entries += 1;
|
||||
let now = Instant::now();
|
||||
if now.duration_since(last_log_update).as_secs() >= 10 {
|
||||
info!("unpacked {} entries so far...", total_entries);
|
||||
last_log_update = now;
|
||||
}
|
||||
}
|
||||
info!("unpacked {} entries total", total_entries);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
@ -17,6 +17,7 @@ use solana_runtime::{
|
|||
use solana_sdk::{clock::Slot, genesis_config::GenesisConfig, hash::Hash, pubkey::Pubkey};
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
fmt,
|
||||
fs::{self, File},
|
||||
io::{BufReader, BufWriter, Error as IOError, ErrorKind, Read, Seek, SeekFrom, Write},
|
||||
path::{Path, PathBuf},
|
||||
|
@ -295,7 +296,7 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()
|
|||
|
||||
pub fn get_snapshot_paths<P: AsRef<Path>>(snapshot_path: P) -> Vec<SlotSnapshotPaths>
|
||||
where
|
||||
P: std::fmt::Debug,
|
||||
P: fmt::Debug,
|
||||
{
|
||||
match fs::read_dir(&snapshot_path) {
|
||||
Ok(paths) => {
|
||||
|
|
|
@ -43,6 +43,7 @@ use std::{
|
|||
path::{Path, PathBuf},
|
||||
sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
|
||||
sync::{Arc, Mutex, RwLock},
|
||||
time::Instant,
|
||||
};
|
||||
use tempfile::TempDir;
|
||||
|
||||
|
@ -1816,7 +1817,15 @@ impl AccountsDB {
|
|||
let mut slots: Vec<Slot> = storage.0.keys().cloned().collect();
|
||||
slots.sort();
|
||||
let mut accounts_index = self.accounts_index.write().unwrap();
|
||||
for slot in slots.iter() {
|
||||
|
||||
let mut last_log_update = Instant::now();
|
||||
for (index, slot) in slots.iter().enumerate() {
|
||||
let now = Instant::now();
|
||||
if now.duration_since(last_log_update).as_secs() >= 10 {
|
||||
info!("generating index: {}/{} slots...", index, slots.len());
|
||||
last_log_update = now;
|
||||
}
|
||||
|
||||
let accumulator: Vec<HashMap<Pubkey, Vec<(u64, AccountInfo)>>> = self
|
||||
.scan_account_storage(
|
||||
*slot,
|
||||
|
|
|
@ -26,6 +26,7 @@ use {
|
|||
path::{Path, PathBuf},
|
||||
result::Result,
|
||||
sync::{atomic::Ordering, Arc},
|
||||
time::Instant,
|
||||
},
|
||||
};
|
||||
|
||||
|
@ -212,10 +213,20 @@ where
|
|||
map
|
||||
};
|
||||
|
||||
let mut last_log_update = Instant::now();
|
||||
let mut remaining_slots_to_process = storage.len();
|
||||
|
||||
// Remap the deserialized AppendVec paths to point to correct local paths
|
||||
let mut storage = storage
|
||||
.into_iter()
|
||||
.map(|(slot, mut slot_storage)| {
|
||||
let now = Instant::now();
|
||||
if now.duration_since(last_log_update).as_secs() >= 10 {
|
||||
info!("{} slots remaining...", remaining_slots_to_process);
|
||||
last_log_update = now;
|
||||
}
|
||||
remaining_slots_to_process -= 1;
|
||||
|
||||
let mut new_slot_storage = HashMap::new();
|
||||
for (id, storage_entry) in slot_storage.drain() {
|
||||
let path_index = thread_rng().gen_range(0, accounts_db.paths.len());
|
||||
|
|
Loading…
Reference in New Issue