2021-12-03 09:00:31 -08:00
|
|
|
use {
|
|
|
|
crate::{
|
|
|
|
blockstore::Blockstore,
|
|
|
|
blockstore_processor::{
|
2022-03-05 08:34:26 -08:00
|
|
|
self, BlockstoreProcessorError, CacheBlockMetaSender, ProcessOptions,
|
|
|
|
TransactionStatusSender,
|
2021-12-03 09:00:31 -08:00
|
|
|
},
|
2023-05-10 16:20:51 -07:00
|
|
|
entry_notifier_service::EntryNotifierSender,
|
2021-12-03 09:00:31 -08:00
|
|
|
leader_schedule_cache::LeaderScheduleCache,
|
2023-06-27 11:26:40 -07:00
|
|
|
use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup},
|
2020-02-20 18:53:26 -08:00
|
|
|
},
|
2021-12-03 09:00:31 -08:00
|
|
|
log::*,
|
|
|
|
solana_runtime::{
|
2022-08-04 13:44:31 -07:00
|
|
|
accounts_background_service::AbsRequestSender,
|
2021-12-03 09:00:31 -08:00
|
|
|
accounts_update_notifier_interface::AccountsUpdateNotifier,
|
|
|
|
bank_forks::BankForks,
|
2023-06-26 13:31:56 -07:00
|
|
|
snapshot_archive_info::{
|
|
|
|
FullSnapshotArchiveInfo, IncrementalSnapshotArchiveInfo, SnapshotArchiveInfoGetter,
|
|
|
|
},
|
2021-12-03 09:00:31 -08:00
|
|
|
snapshot_config::SnapshotConfig,
|
|
|
|
snapshot_hash::{FullSnapshotHash, IncrementalSnapshotHash, StartingSnapshotHashes},
|
|
|
|
snapshot_utils,
|
|
|
|
},
|
2022-03-07 02:23:22 -08:00
|
|
|
solana_sdk::genesis_config::GenesisConfig,
|
2022-03-04 01:52:22 -08:00
|
|
|
std::{
|
|
|
|
path::PathBuf,
|
|
|
|
process, result,
|
2022-09-12 11:51:12 -07:00
|
|
|
sync::{atomic::AtomicBool, Arc, RwLock},
|
2022-03-04 01:52:22 -08:00
|
|
|
},
|
2021-09-07 13:43:43 -07:00
|
|
|
};
|
2020-02-20 18:53:26 -08:00
|
|
|
|
|
|
|
pub type LoadResult = result::Result<
|
2021-09-07 13:43:43 -07:00
|
|
|
(
|
2022-03-04 01:52:22 -08:00
|
|
|
Arc<RwLock<BankForks>>,
|
2021-09-07 13:43:43 -07:00
|
|
|
LeaderScheduleCache,
|
2021-10-08 13:14:56 -07:00
|
|
|
Option<StartingSnapshotHashes>,
|
2021-09-07 13:43:43 -07:00
|
|
|
),
|
2020-02-20 18:53:26 -08:00
|
|
|
BlockstoreProcessorError,
|
|
|
|
>;
|
|
|
|
|
2022-03-07 02:23:22 -08:00
|
|
|
/// Load the banks via genesis or a snapshot then processes all full blocks in blockstore
|
2021-05-27 10:00:27 -07:00
|
|
|
///
|
|
|
|
/// If a snapshot config is given, and a snapshot is found, it will be loaded. Otherwise, load
|
|
|
|
/// from genesis.
|
2021-09-30 14:26:17 -07:00
|
|
|
#[allow(clippy::too_many_arguments)]
|
2019-11-04 18:10:06 -08:00
|
|
|
pub fn load(
|
2019-11-08 20:56:57 -08:00
|
|
|
genesis_config: &GenesisConfig,
|
2020-01-13 13:13:52 -08:00
|
|
|
blockstore: &Blockstore,
|
2019-12-05 18:41:29 -08:00
|
|
|
account_paths: Vec<PathBuf>,
|
2020-12-21 21:33:37 -08:00
|
|
|
shrink_paths: Option<Vec<PathBuf>>,
|
2019-11-04 18:10:06 -08:00
|
|
|
snapshot_config: Option<&SnapshotConfig>,
|
|
|
|
process_options: ProcessOptions,
|
2021-05-10 12:14:56 -07:00
|
|
|
transaction_status_sender: Option<&TransactionStatusSender>,
|
2021-05-26 21:16:16 -07:00
|
|
|
cache_block_meta_sender: Option<&CacheBlockMetaSender>,
|
2023-05-10 16:20:51 -07:00
|
|
|
entry_notification_sender: Option<&EntryNotifierSender>,
|
2021-09-30 14:26:17 -07:00
|
|
|
accounts_update_notifier: Option<AccountsUpdateNotifier>,
|
2023-05-31 10:36:44 -07:00
|
|
|
exit: Arc<AtomicBool>,
|
2020-02-20 18:53:26 -08:00
|
|
|
) -> LoadResult {
|
2022-03-18 12:43:20 -07:00
|
|
|
let (bank_forks, leader_schedule_cache, starting_snapshot_hashes, ..) = load_bank_forks(
|
|
|
|
genesis_config,
|
|
|
|
blockstore,
|
|
|
|
account_paths,
|
|
|
|
shrink_paths,
|
|
|
|
snapshot_config,
|
|
|
|
&process_options,
|
|
|
|
cache_block_meta_sender,
|
2023-05-10 16:20:51 -07:00
|
|
|
entry_notification_sender,
|
2022-03-18 12:43:20 -07:00
|
|
|
accounts_update_notifier,
|
2022-09-12 11:51:12 -07:00
|
|
|
exit,
|
2022-03-18 12:43:20 -07:00
|
|
|
);
|
2022-03-07 02:23:22 -08:00
|
|
|
|
|
|
|
blockstore_processor::process_blockstore_from_root(
|
|
|
|
blockstore,
|
2022-03-04 01:52:22 -08:00
|
|
|
&bank_forks,
|
2022-03-14 09:16:12 -07:00
|
|
|
&leader_schedule_cache,
|
2022-03-07 02:23:22 -08:00
|
|
|
&process_options,
|
|
|
|
transaction_status_sender,
|
|
|
|
cache_block_meta_sender,
|
2023-05-10 16:20:51 -07:00
|
|
|
entry_notification_sender,
|
2022-08-04 13:44:31 -07:00
|
|
|
&AbsRequestSender::default(),
|
2022-03-07 02:23:22 -08:00
|
|
|
)
|
2022-03-14 09:16:12 -07:00
|
|
|
.map(|_| (bank_forks, leader_schedule_cache, starting_snapshot_hashes))
|
2022-03-07 02:23:22 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
|
|
pub fn load_bank_forks(
|
|
|
|
genesis_config: &GenesisConfig,
|
|
|
|
blockstore: &Blockstore,
|
|
|
|
account_paths: Vec<PathBuf>,
|
|
|
|
shrink_paths: Option<Vec<PathBuf>>,
|
|
|
|
snapshot_config: Option<&SnapshotConfig>,
|
|
|
|
process_options: &ProcessOptions,
|
|
|
|
cache_block_meta_sender: Option<&CacheBlockMetaSender>,
|
2023-05-10 16:20:51 -07:00
|
|
|
entry_notification_sender: Option<&EntryNotifierSender>,
|
2022-03-07 02:23:22 -08:00
|
|
|
accounts_update_notifier: Option<AccountsUpdateNotifier>,
|
2023-05-31 10:36:44 -07:00
|
|
|
exit: Arc<AtomicBool>,
|
2022-03-14 09:16:12 -07:00
|
|
|
) -> (
|
2022-03-04 01:52:22 -08:00
|
|
|
Arc<RwLock<BankForks>>,
|
2022-03-14 09:16:12 -07:00
|
|
|
LeaderScheduleCache,
|
|
|
|
Option<StartingSnapshotHashes>,
|
|
|
|
) {
|
2023-06-26 13:31:56 -07:00
|
|
|
fn get_snapshots_to_load(
|
|
|
|
snapshot_config: Option<&SnapshotConfig>,
|
|
|
|
) -> Option<(
|
|
|
|
FullSnapshotArchiveInfo,
|
|
|
|
Option<IncrementalSnapshotArchiveInfo>,
|
|
|
|
)> {
|
|
|
|
let Some(snapshot_config) = snapshot_config else {
|
|
|
|
info!("Snapshots disabled; will load from genesis");
|
|
|
|
return None;
|
|
|
|
};
|
2019-11-04 18:10:06 -08:00
|
|
|
|
2023-06-26 13:31:56 -07:00
|
|
|
let Some(full_snapshot_archive_info) =
|
|
|
|
snapshot_utils::get_highest_full_snapshot_archive_info(
|
2023-07-06 20:45:29 -07:00
|
|
|
&snapshot_config.full_snapshot_archives_dir,
|
2023-06-26 13:31:56 -07:00
|
|
|
)
|
|
|
|
else {
|
|
|
|
warn!(
|
|
|
|
"No snapshot package found in directory: {}; will load from genesis",
|
|
|
|
snapshot_config.full_snapshot_archives_dir.display()
|
2022-06-21 11:06:37 -07:00
|
|
|
);
|
2023-06-26 13:31:56 -07:00
|
|
|
return None;
|
|
|
|
};
|
2019-11-04 18:10:06 -08:00
|
|
|
|
2023-06-26 13:31:56 -07:00
|
|
|
let incremental_snapshot_archive_info =
|
|
|
|
snapshot_utils::get_highest_incremental_snapshot_archive_info(
|
|
|
|
&snapshot_config.incremental_snapshot_archives_dir,
|
|
|
|
full_snapshot_archive_info.slot(),
|
|
|
|
);
|
2022-04-11 11:10:09 -07:00
|
|
|
|
2023-06-26 13:31:56 -07:00
|
|
|
Some((
|
|
|
|
full_snapshot_archive_info,
|
|
|
|
incremental_snapshot_archive_info,
|
|
|
|
))
|
|
|
|
}
|
2021-10-11 10:46:27 -07:00
|
|
|
|
2023-06-26 13:31:56 -07:00
|
|
|
let (bank_forks, starting_snapshot_hashes) =
|
|
|
|
if let Some((full_snapshot_archive_info, incremental_snapshot_archive_info)) =
|
|
|
|
get_snapshots_to_load(snapshot_config)
|
|
|
|
{
|
|
|
|
// SAFETY: Having snapshots to load ensures a snapshot config
|
|
|
|
let snapshot_config = snapshot_config.unwrap();
|
|
|
|
info!(
|
|
|
|
"Initializing bank snapshots dir: {}",
|
|
|
|
snapshot_config.bank_snapshots_dir.display()
|
|
|
|
);
|
|
|
|
std::fs::create_dir_all(&snapshot_config.bank_snapshots_dir)
|
|
|
|
.expect("create bank snapshots dir");
|
|
|
|
let (bank_forks, starting_snapshot_hashes) = bank_forks_from_snapshot(
|
|
|
|
full_snapshot_archive_info,
|
|
|
|
incremental_snapshot_archive_info,
|
|
|
|
genesis_config,
|
|
|
|
account_paths,
|
|
|
|
shrink_paths,
|
|
|
|
snapshot_config,
|
|
|
|
process_options,
|
|
|
|
accounts_update_notifier,
|
|
|
|
exit,
|
|
|
|
);
|
|
|
|
(bank_forks, Some(starting_snapshot_hashes))
|
|
|
|
} else {
|
|
|
|
let maybe_filler_accounts = process_options
|
|
|
|
.accounts_db_config
|
|
|
|
.as_ref()
|
|
|
|
.map(|config| config.filler_accounts_config.count > 0);
|
2022-07-15 12:29:56 -07:00
|
|
|
|
2023-06-26 13:31:56 -07:00
|
|
|
if let Some(true) = maybe_filler_accounts {
|
|
|
|
panic!("filler accounts specified, but not loading from snapshot");
|
|
|
|
}
|
|
|
|
|
|
|
|
info!("Processing ledger from genesis");
|
|
|
|
let bank_forks = blockstore_processor::process_blockstore_for_bank_0(
|
|
|
|
genesis_config,
|
|
|
|
blockstore,
|
|
|
|
account_paths,
|
|
|
|
process_options,
|
|
|
|
cache_block_meta_sender,
|
|
|
|
entry_notification_sender,
|
|
|
|
accounts_update_notifier,
|
|
|
|
exit,
|
|
|
|
);
|
|
|
|
bank_forks
|
|
|
|
.read()
|
|
|
|
.unwrap()
|
|
|
|
.root_bank()
|
|
|
|
.set_startup_verification_complete();
|
|
|
|
|
|
|
|
(bank_forks, None)
|
|
|
|
};
|
2022-03-14 09:16:12 -07:00
|
|
|
|
2022-03-18 12:43:20 -07:00
|
|
|
let mut leader_schedule_cache =
|
|
|
|
LeaderScheduleCache::new_from_bank(&bank_forks.read().unwrap().root_bank());
|
|
|
|
if process_options.full_leader_cache {
|
|
|
|
leader_schedule_cache.set_max_schedules(std::usize::MAX);
|
|
|
|
}
|
2022-03-15 19:54:08 -07:00
|
|
|
|
|
|
|
if let Some(ref new_hard_forks) = process_options.new_hard_forks {
|
2022-03-04 01:52:22 -08:00
|
|
|
let root_bank = bank_forks.read().unwrap().root_bank();
|
2023-06-20 21:44:43 -07:00
|
|
|
new_hard_forks
|
|
|
|
.iter()
|
|
|
|
.for_each(|hard_fork_slot| root_bank.register_hard_fork(*hard_fork_slot));
|
2022-03-15 19:54:08 -07:00
|
|
|
}
|
|
|
|
|
2022-03-18 12:43:20 -07:00
|
|
|
(bank_forks, leader_schedule_cache, starting_snapshot_hashes)
|
2019-11-04 18:10:06 -08:00
|
|
|
}
|
2021-05-27 10:00:27 -07:00
|
|
|
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
2022-03-05 06:19:54 -08:00
|
|
|
fn bank_forks_from_snapshot(
|
2023-06-26 13:31:56 -07:00
|
|
|
full_snapshot_archive_info: FullSnapshotArchiveInfo,
|
|
|
|
incremental_snapshot_archive_info: Option<IncrementalSnapshotArchiveInfo>,
|
2021-05-27 10:00:27 -07:00
|
|
|
genesis_config: &GenesisConfig,
|
|
|
|
account_paths: Vec<PathBuf>,
|
|
|
|
shrink_paths: Option<Vec<PathBuf>>,
|
|
|
|
snapshot_config: &SnapshotConfig,
|
2022-03-05 06:19:54 -08:00
|
|
|
process_options: &ProcessOptions,
|
2021-09-30 14:26:17 -07:00
|
|
|
accounts_update_notifier: Option<AccountsUpdateNotifier>,
|
2023-05-31 10:36:44 -07:00
|
|
|
exit: Arc<AtomicBool>,
|
2023-06-16 11:38:03 -07:00
|
|
|
) -> (Arc<RwLock<BankForks>>, StartingSnapshotHashes) {
|
2021-05-27 10:00:27 -07:00
|
|
|
// Fail hard here if snapshot fails to load, don't silently continue
|
|
|
|
if account_paths.is_empty() {
|
|
|
|
error!("Account paths not present when booting from snapshot");
|
|
|
|
process::exit(1);
|
|
|
|
}
|
|
|
|
|
2023-06-27 11:26:40 -07:00
|
|
|
let latest_snapshot_archive_slot = std::cmp::max(
|
|
|
|
full_snapshot_archive_info.slot(),
|
|
|
|
incremental_snapshot_archive_info
|
|
|
|
.as_ref()
|
|
|
|
.map(SnapshotArchiveInfoGetter::slot)
|
|
|
|
.unwrap_or(0),
|
|
|
|
);
|
|
|
|
let latest_bank_snapshot =
|
|
|
|
snapshot_utils::get_highest_bank_snapshot_post(&snapshot_config.bank_snapshots_dir);
|
2023-06-26 09:44:25 -07:00
|
|
|
|
2023-06-27 11:26:40 -07:00
|
|
|
let will_startup_from_snapshot_archives = match process_options.use_snapshot_archives_at_startup
|
|
|
|
{
|
|
|
|
UseSnapshotArchivesAtStartup::Always => true,
|
|
|
|
UseSnapshotArchivesAtStartup::Never => false,
|
|
|
|
UseSnapshotArchivesAtStartup::WhenNewest => latest_bank_snapshot
|
|
|
|
.as_ref()
|
|
|
|
.map(|bank_snapshot| latest_snapshot_archive_slot > bank_snapshot.slot)
|
|
|
|
.unwrap_or(true),
|
|
|
|
};
|
2023-06-26 09:44:25 -07:00
|
|
|
|
2023-06-27 11:26:40 -07:00
|
|
|
let bank = if will_startup_from_snapshot_archives {
|
|
|
|
// Given that we are going to boot from an archive, the append vecs held in the snapshot dirs for fast-boot should
|
|
|
|
// be released. They will be released by the account_background_service anyway. But in the case of the account_paths
|
|
|
|
// using memory-mounted file system, they are not released early enough to give space for the new append-vecs from
|
|
|
|
// the archives, causing the out-of-memory problem. So, purge the snapshot dirs upfront before loading from the archive.
|
|
|
|
snapshot_utils::purge_old_bank_snapshots(&snapshot_config.bank_snapshots_dir, 0, None);
|
2023-06-26 09:44:25 -07:00
|
|
|
|
2023-06-27 11:26:40 -07:00
|
|
|
let (bank, _) = snapshot_utils::bank_from_snapshot_archives(
|
|
|
|
&account_paths,
|
|
|
|
&snapshot_config.bank_snapshots_dir,
|
|
|
|
&full_snapshot_archive_info,
|
|
|
|
incremental_snapshot_archive_info.as_ref(),
|
|
|
|
genesis_config,
|
|
|
|
&process_options.runtime_config,
|
|
|
|
process_options.debug_keys.clone(),
|
|
|
|
None,
|
|
|
|
process_options.account_indexes.clone(),
|
|
|
|
process_options.limit_load_slot_count_from_snapshot,
|
|
|
|
process_options.shrink_ratio,
|
|
|
|
process_options.accounts_db_test_hash_calculation,
|
|
|
|
process_options.accounts_db_skip_shrink,
|
|
|
|
process_options.verify_index,
|
|
|
|
process_options.accounts_db_config.clone(),
|
|
|
|
accounts_update_notifier,
|
|
|
|
exit,
|
|
|
|
)
|
|
|
|
.expect("load bank from snapshot archives");
|
|
|
|
bank
|
|
|
|
} else {
|
|
|
|
let Some(bank_snapshot) = latest_bank_snapshot else {
|
|
|
|
error!(
|
|
|
|
"There is no local state to startup from. Ensure --{} is *not* set to \"{}\" and restart.",
|
|
|
|
use_snapshot_archives_at_startup::cli::LONG_ARG,
|
|
|
|
UseSnapshotArchivesAtStartup::Never.to_string(),
|
2023-06-26 13:31:56 -07:00
|
|
|
);
|
2023-06-27 11:26:40 -07:00
|
|
|
process::exit(1);
|
|
|
|
};
|
2023-06-26 13:31:56 -07:00
|
|
|
|
2023-06-27 11:26:40 -07:00
|
|
|
// If a newer snapshot archive was downloaded, it is possible that its slot is
|
|
|
|
// higher than the local bank we will load. Did the user intend for this?
|
|
|
|
if bank_snapshot.slot < latest_snapshot_archive_slot {
|
|
|
|
assert_eq!(
|
|
|
|
process_options.use_snapshot_archives_at_startup,
|
|
|
|
UseSnapshotArchivesAtStartup::Never,
|
|
|
|
);
|
|
|
|
warn!(
|
|
|
|
"Starting up from local state at slot {}, which is *older* than \
|
|
|
|
the latest snapshot archive at slot {}. If this is not desired, \
|
|
|
|
change the --{} CLI option to *not* \"{}\" and restart.",
|
|
|
|
bank_snapshot.slot,
|
|
|
|
latest_snapshot_archive_slot,
|
|
|
|
use_snapshot_archives_at_startup::cli::LONG_ARG,
|
|
|
|
UseSnapshotArchivesAtStartup::Never.to_string(),
|
|
|
|
);
|
2023-06-26 13:31:56 -07:00
|
|
|
}
|
2023-06-27 11:26:40 -07:00
|
|
|
|
|
|
|
let (bank, _) = snapshot_utils::bank_from_snapshot_dir(
|
|
|
|
&account_paths,
|
|
|
|
&bank_snapshot,
|
|
|
|
genesis_config,
|
|
|
|
&process_options.runtime_config,
|
|
|
|
process_options.debug_keys.clone(),
|
|
|
|
None,
|
|
|
|
process_options.account_indexes.clone(),
|
|
|
|
process_options.limit_load_slot_count_from_snapshot,
|
|
|
|
process_options.shrink_ratio,
|
|
|
|
process_options.verify_index,
|
|
|
|
process_options.accounts_db_config.clone(),
|
|
|
|
accounts_update_notifier,
|
|
|
|
exit,
|
|
|
|
)
|
|
|
|
.expect("load bank from local state");
|
|
|
|
bank
|
2023-06-26 13:31:56 -07:00
|
|
|
};
|
2021-05-27 10:00:27 -07:00
|
|
|
|
2021-08-06 18:16:06 -07:00
|
|
|
if let Some(shrink_paths) = shrink_paths {
|
2023-06-26 13:31:56 -07:00
|
|
|
bank.set_shrink_paths(shrink_paths);
|
2021-05-27 10:00:27 -07:00
|
|
|
}
|
|
|
|
|
2023-06-15 19:54:32 -07:00
|
|
|
let full_snapshot_hash = FullSnapshotHash((
|
|
|
|
full_snapshot_archive_info.slot(),
|
|
|
|
*full_snapshot_archive_info.hash(),
|
|
|
|
));
|
|
|
|
let incremental_snapshot_hash =
|
|
|
|
incremental_snapshot_archive_info.map(|incremental_snapshot_archive_info| {
|
|
|
|
IncrementalSnapshotHash((
|
|
|
|
incremental_snapshot_archive_info.slot(),
|
|
|
|
*incremental_snapshot_archive_info.hash(),
|
|
|
|
))
|
|
|
|
});
|
|
|
|
let starting_snapshot_hashes = StartingSnapshotHashes {
|
|
|
|
full: full_snapshot_hash,
|
|
|
|
incremental: incremental_snapshot_hash,
|
|
|
|
};
|
|
|
|
|
2022-03-05 06:19:54 -08:00
|
|
|
(
|
2023-06-26 13:31:56 -07:00
|
|
|
Arc::new(RwLock::new(BankForks::new(bank))),
|
2023-06-16 11:38:03 -07:00
|
|
|
starting_snapshot_hashes,
|
2021-05-27 10:00:27 -07:00
|
|
|
)
|
|
|
|
}
|