2019-10-19 12:09:45 -07:00
|
|
|
// Long-running bank_forks tests
|
2021-02-16 13:48:20 -08:00
|
|
|
#![allow(clippy::integer_arithmetic)]
|
2019-10-19 12:09:45 -07:00
|
|
|
|
2020-06-18 22:38:37 -07:00
|
|
|
macro_rules! DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS {
|
2020-08-16 22:22:16 -07:00
|
|
|
($x:ident, $y:ident, $z:ident) => {
|
2020-06-18 22:38:37 -07:00
|
|
|
#[allow(non_snake_case)]
|
2020-08-16 22:22:16 -07:00
|
|
|
mod $z {
|
2020-06-18 22:38:37 -07:00
|
|
|
use super::*;
|
|
|
|
|
|
|
|
const SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::$x;
|
2020-09-08 07:55:09 -07:00
|
|
|
const CLUSTER_TYPE: ClusterType = ClusterType::$y;
|
2020-06-18 22:38:37 -07:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_bank_forks_status_cache_snapshot_n() {
|
2020-09-08 07:55:09 -07:00
|
|
|
run_test_bank_forks_status_cache_snapshot_n(SNAPSHOT_VERSION, CLUSTER_TYPE)
|
2020-06-18 22:38:37 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_bank_forks_snapshot_n() {
|
2020-09-08 07:55:09 -07:00
|
|
|
run_test_bank_forks_snapshot_n(SNAPSHOT_VERSION, CLUSTER_TYPE)
|
2020-06-18 22:38:37 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_concurrent_snapshot_packaging() {
|
2020-09-08 07:55:09 -07:00
|
|
|
run_test_concurrent_snapshot_packaging(SNAPSHOT_VERSION, CLUSTER_TYPE)
|
2020-06-18 22:38:37 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_slots_to_snapshot() {
|
2020-09-08 07:55:09 -07:00
|
|
|
run_test_slots_to_snapshot(SNAPSHOT_VERSION, CLUSTER_TYPE)
|
2020-06-18 22:38:37 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2019-10-19 12:09:45 -07:00
|
|
|
#[cfg(test)]
|
|
|
|
mod tests {
|
|
|
|
use bincode::serialize_into;
|
2020-09-28 16:04:46 -07:00
|
|
|
use crossbeam_channel::unbounded;
|
2019-10-19 12:09:45 -07:00
|
|
|
use fs_extra::dir::CopyOptions;
|
|
|
|
use itertools::Itertools;
|
2020-09-28 16:04:46 -07:00
|
|
|
use solana_core::{
|
2021-01-11 10:21:15 -08:00
|
|
|
cluster_info::ClusterInfo,
|
|
|
|
contact_info::ContactInfo,
|
|
|
|
snapshot_packager_service::{PendingSnapshotPackage, SnapshotPackagerService},
|
2020-09-28 16:04:46 -07:00
|
|
|
};
|
2019-11-02 00:38:30 -07:00
|
|
|
use solana_runtime::{
|
2021-02-18 23:42:09 -08:00
|
|
|
accounts_background_service::{AbsRequestSender, SnapshotRequestHandler},
|
2021-02-05 11:48:55 -08:00
|
|
|
accounts_db,
|
2021-05-10 07:22:48 -07:00
|
|
|
accounts_index::AccountSecondaryIndexes,
|
2020-02-10 03:11:37 -08:00
|
|
|
bank::{Bank, BankSlotDelta},
|
2021-01-07 22:45:42 -08:00
|
|
|
bank_forks::{ArchiveFormat, BankForks, SnapshotConfig},
|
2020-06-17 08:27:03 -07:00
|
|
|
genesis_utils::{create_genesis_config, GenesisConfigInfo},
|
|
|
|
snapshot_utils,
|
2021-05-12 10:32:27 -07:00
|
|
|
snapshot_utils::{SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
|
2020-02-10 03:11:37 -08:00
|
|
|
status_cache::MAX_CACHE_ENTRIES,
|
2019-11-02 00:38:30 -07:00
|
|
|
};
|
|
|
|
use solana_sdk::{
|
|
|
|
clock::Slot,
|
2020-09-08 07:55:09 -07:00
|
|
|
genesis_config::{ClusterType, GenesisConfig},
|
2019-11-02 00:38:30 -07:00
|
|
|
hash::hashv,
|
|
|
|
pubkey::Pubkey,
|
2020-02-20 13:28:55 -08:00
|
|
|
signature::{Keypair, Signer},
|
2020-02-10 03:11:37 -08:00
|
|
|
system_transaction,
|
2019-11-02 00:38:30 -07:00
|
|
|
};
|
2020-12-31 18:06:03 -08:00
|
|
|
use std::{
|
2021-01-11 10:21:15 -08:00
|
|
|
collections::HashSet,
|
|
|
|
fs,
|
|
|
|
path::PathBuf,
|
|
|
|
sync::{
|
|
|
|
atomic::{AtomicBool, Ordering},
|
|
|
|
mpsc::channel,
|
|
|
|
Arc,
|
|
|
|
},
|
|
|
|
time::Duration,
|
2020-12-31 18:06:03 -08:00
|
|
|
};
|
2019-10-19 12:09:45 -07:00
|
|
|
use tempfile::TempDir;
|
|
|
|
|
2020-08-16 22:22:16 -07:00
|
|
|
DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0, Development, V1_2_0_Development);
|
2020-09-08 07:55:09 -07:00
|
|
|
DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0, Devnet, V1_2_0_Devnet);
|
|
|
|
DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0, Testnet, V1_2_0_Testnet);
|
|
|
|
DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0, MainnetBeta, V1_2_0_MainnetBeta);
|
2020-06-18 22:38:37 -07:00
|
|
|
|
2019-10-19 12:09:45 -07:00
|
|
|
struct SnapshotTestConfig {
|
|
|
|
accounts_dir: TempDir,
|
|
|
|
snapshot_dir: TempDir,
|
|
|
|
_snapshot_output_path: TempDir,
|
|
|
|
snapshot_config: SnapshotConfig,
|
|
|
|
bank_forks: BankForks,
|
2019-11-08 20:56:57 -08:00
|
|
|
genesis_config_info: GenesisConfigInfo,
|
2019-10-19 12:09:45 -07:00
|
|
|
}
|
|
|
|
|
2020-06-18 22:38:37 -07:00
|
|
|
impl SnapshotTestConfig {
|
|
|
|
fn new(
|
|
|
|
snapshot_version: SnapshotVersion,
|
2020-09-08 07:55:09 -07:00
|
|
|
cluster_type: ClusterType,
|
2020-06-18 22:38:37 -07:00
|
|
|
snapshot_interval_slots: u64,
|
|
|
|
) -> SnapshotTestConfig {
|
|
|
|
let accounts_dir = TempDir::new().unwrap();
|
|
|
|
let snapshot_dir = TempDir::new().unwrap();
|
|
|
|
let snapshot_output_path = TempDir::new().unwrap();
|
2020-08-16 22:22:16 -07:00
|
|
|
let mut genesis_config_info = create_genesis_config(10_000);
|
2020-09-08 07:55:09 -07:00
|
|
|
genesis_config_info.genesis_config.cluster_type = cluster_type;
|
2020-06-18 22:38:37 -07:00
|
|
|
let bank0 = Bank::new_with_paths(
|
|
|
|
&genesis_config_info.genesis_config,
|
|
|
|
vec![accounts_dir.path().to_path_buf()],
|
|
|
|
&[],
|
2020-09-23 18:46:42 -07:00
|
|
|
None,
|
2020-09-24 12:23:09 -07:00
|
|
|
None,
|
2021-05-10 07:22:48 -07:00
|
|
|
AccountSecondaryIndexes::default(),
|
2021-01-11 17:00:23 -08:00
|
|
|
false,
|
2020-06-18 22:38:37 -07:00
|
|
|
);
|
|
|
|
bank0.freeze();
|
|
|
|
let mut bank_forks = BankForks::new(bank0);
|
|
|
|
bank_forks.accounts_hash_interval_slots = snapshot_interval_slots;
|
|
|
|
|
|
|
|
let snapshot_config = SnapshotConfig {
|
|
|
|
snapshot_interval_slots,
|
|
|
|
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
|
|
|
|
snapshot_path: PathBuf::from(snapshot_dir.path()),
|
2021-01-07 22:45:42 -08:00
|
|
|
archive_format: ArchiveFormat::TarBzip2,
|
2020-06-18 22:38:37 -07:00
|
|
|
snapshot_version,
|
2021-05-12 10:32:27 -07:00
|
|
|
maximum_snapshots_to_retain: DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
|
2020-06-18 22:38:37 -07:00
|
|
|
};
|
|
|
|
bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
|
|
|
|
SnapshotTestConfig {
|
|
|
|
accounts_dir,
|
|
|
|
snapshot_dir,
|
|
|
|
_snapshot_output_path: snapshot_output_path,
|
|
|
|
snapshot_config,
|
|
|
|
bank_forks,
|
|
|
|
genesis_config_info,
|
|
|
|
}
|
2019-10-19 12:09:45 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-02-26 18:04:18 -08:00
|
|
|
fn restore_from_snapshot(
|
|
|
|
old_bank_forks: &BankForks,
|
|
|
|
old_last_slot: Slot,
|
2020-07-13 07:00:59 -07:00
|
|
|
old_genesis_config: &GenesisConfig,
|
2020-06-18 22:38:37 -07:00
|
|
|
account_paths: &[PathBuf],
|
2020-02-26 18:04:18 -08:00
|
|
|
) {
|
2019-10-19 12:09:45 -07:00
|
|
|
let (snapshot_path, snapshot_package_output_path) = old_bank_forks
|
|
|
|
.snapshot_config
|
|
|
|
.as_ref()
|
|
|
|
.map(|c| (&c.snapshot_path, &c.snapshot_package_output_path))
|
|
|
|
.unwrap();
|
|
|
|
|
2020-02-26 18:04:18 -08:00
|
|
|
let old_last_bank = old_bank_forks.get(old_last_slot).unwrap();
|
|
|
|
|
2019-10-19 12:09:45 -07:00
|
|
|
let deserialized_bank = snapshot_utils::bank_from_archive(
|
2019-12-05 18:41:29 -08:00
|
|
|
&account_paths,
|
2020-03-22 11:10:04 -07:00
|
|
|
&[],
|
2019-10-19 12:09:45 -07:00
|
|
|
&old_bank_forks
|
|
|
|
.snapshot_config
|
|
|
|
.as_ref()
|
|
|
|
.unwrap()
|
|
|
|
.snapshot_path,
|
2020-02-26 18:04:18 -08:00
|
|
|
snapshot_utils::get_snapshot_archive_path(
|
2021-02-04 07:00:33 -08:00
|
|
|
snapshot_package_output_path.to_path_buf(),
|
2020-02-26 18:04:18 -08:00
|
|
|
&(old_last_bank.slot(), old_last_bank.get_accounts_hash()),
|
2021-01-21 18:34:51 -08:00
|
|
|
ArchiveFormat::TarBzip2,
|
2020-02-26 18:04:18 -08:00
|
|
|
),
|
2021-01-07 22:45:42 -08:00
|
|
|
ArchiveFormat::TarBzip2,
|
2020-07-13 07:00:59 -07:00
|
|
|
old_genesis_config,
|
2020-09-23 18:46:42 -07:00
|
|
|
None,
|
2020-09-24 12:23:09 -07:00
|
|
|
None,
|
2021-05-10 07:22:48 -07:00
|
|
|
AccountSecondaryIndexes::default(),
|
2021-01-11 17:00:23 -08:00
|
|
|
false,
|
2019-10-19 12:09:45 -07:00
|
|
|
)
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
let bank = old_bank_forks
|
2021-02-05 10:00:45 -08:00
|
|
|
.get(deserialized_bank.slot())
|
2019-10-19 12:09:45 -07:00
|
|
|
.unwrap()
|
|
|
|
.clone();
|
2020-10-20 09:05:45 -07:00
|
|
|
assert_eq!(*bank, deserialized_bank);
|
2019-10-19 12:09:45 -07:00
|
|
|
|
|
|
|
let slot_snapshot_paths = snapshot_utils::get_snapshot_paths(&snapshot_path);
|
|
|
|
|
|
|
|
for p in slot_snapshot_paths {
|
|
|
|
snapshot_utils::remove_snapshot(p.slot, &snapshot_path).unwrap();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// creates banks up to "last_slot" and runs the input function `f` on each bank created
|
|
|
|
// also marks each bank as root and generates snapshots
|
|
|
|
// finally tries to restore from the last bank's snapshot and compares the restored bank to the
|
|
|
|
// `last_slot` bank
|
2020-06-18 22:38:37 -07:00
|
|
|
fn run_bank_forks_snapshot_n<F>(
|
|
|
|
snapshot_version: SnapshotVersion,
|
2020-09-08 07:55:09 -07:00
|
|
|
cluster_type: ClusterType,
|
2020-06-18 22:38:37 -07:00
|
|
|
last_slot: Slot,
|
|
|
|
f: F,
|
|
|
|
set_root_interval: u64,
|
|
|
|
) where
|
2019-10-19 12:09:45 -07:00
|
|
|
F: Fn(&mut Bank, &Keypair),
|
|
|
|
{
|
|
|
|
solana_logger::setup();
|
|
|
|
// Set up snapshotting config
|
2020-09-08 07:55:09 -07:00
|
|
|
let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, cluster_type, 1);
|
2019-10-19 12:09:45 -07:00
|
|
|
|
|
|
|
let bank_forks = &mut snapshot_test_config.bank_forks;
|
2019-11-08 20:56:57 -08:00
|
|
|
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
|
2019-10-19 12:09:45 -07:00
|
|
|
|
2020-09-28 16:04:46 -07:00
|
|
|
let (s, snapshot_request_receiver) = unbounded();
|
|
|
|
let (accounts_package_sender, _r) = channel();
|
2021-02-18 23:42:09 -08:00
|
|
|
let request_sender = AbsRequestSender::new(Some(s));
|
2020-09-28 16:04:46 -07:00
|
|
|
let snapshot_request_handler = SnapshotRequestHandler {
|
|
|
|
snapshot_config: snapshot_test_config.snapshot_config.clone(),
|
|
|
|
snapshot_request_receiver,
|
|
|
|
accounts_package_sender,
|
|
|
|
};
|
2019-10-19 12:09:45 -07:00
|
|
|
for slot in 0..last_slot {
|
|
|
|
let mut bank = Bank::new_from_parent(&bank_forks[slot], &Pubkey::default(), slot + 1);
|
|
|
|
f(&mut bank, &mint_keypair);
|
|
|
|
let bank = bank_forks.insert(bank);
|
|
|
|
// Set root to make sure we don't end up with too many account storage entries
|
|
|
|
// and to allow snapshotting of bank and the purging logic on status_cache to
|
|
|
|
// kick in
|
|
|
|
if slot % set_root_interval == 0 || slot == last_slot - 1 {
|
2020-09-28 16:04:46 -07:00
|
|
|
// set_root should send a snapshot request
|
2020-12-12 17:22:34 -08:00
|
|
|
bank_forks.set_root(bank.slot(), &request_sender, None);
|
2021-02-04 07:00:33 -08:00
|
|
|
bank.update_accounts_hash();
|
2021-02-16 13:13:48 -08:00
|
|
|
snapshot_request_handler.handle_snapshot_requests(false, false, false);
|
2019-10-19 12:09:45 -07:00
|
|
|
}
|
|
|
|
}
|
2020-07-13 07:00:59 -07:00
|
|
|
|
2019-10-19 12:09:45 -07:00
|
|
|
// Generate a snapshot package for last bank
|
|
|
|
let last_bank = bank_forks.get(last_slot).unwrap();
|
2020-06-18 22:38:37 -07:00
|
|
|
let snapshot_config = &snapshot_test_config.snapshot_config;
|
|
|
|
let snapshot_path = &snapshot_config.snapshot_path;
|
|
|
|
let last_slot_snapshot_path = snapshot_utils::get_snapshot_paths(snapshot_path)
|
|
|
|
.pop()
|
|
|
|
.expect("no snapshots found in path");
|
2019-10-19 12:09:45 -07:00
|
|
|
let snapshot_package = snapshot_utils::package_snapshot(
|
|
|
|
last_bank,
|
2020-06-18 22:38:37 -07:00
|
|
|
&last_slot_snapshot_path,
|
|
|
|
snapshot_path,
|
2020-09-28 16:04:46 -07:00
|
|
|
last_bank.src.slot_deltas(&last_bank.src.roots()),
|
2020-02-26 18:04:18 -08:00
|
|
|
&snapshot_config.snapshot_package_output_path,
|
2020-06-18 22:38:37 -07:00
|
|
|
last_bank.get_snapshot_storages(),
|
2021-01-07 22:45:42 -08:00
|
|
|
ArchiveFormat::TarBzip2,
|
2020-06-18 22:38:37 -07:00
|
|
|
snapshot_version,
|
2021-02-04 07:00:33 -08:00
|
|
|
None,
|
2019-10-19 12:09:45 -07:00
|
|
|
)
|
|
|
|
.unwrap();
|
2021-02-05 11:48:55 -08:00
|
|
|
let snapshot_package = snapshot_utils::process_accounts_package_pre(
|
|
|
|
snapshot_package,
|
|
|
|
Some(&last_bank.get_thread_pool()),
|
|
|
|
);
|
2021-05-12 10:32:27 -07:00
|
|
|
snapshot_utils::archive_snapshot_package(
|
|
|
|
&snapshot_package,
|
|
|
|
DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
|
|
|
|
)
|
|
|
|
.unwrap();
|
2019-10-19 12:09:45 -07:00
|
|
|
|
2020-07-13 07:00:59 -07:00
|
|
|
// Restore bank from snapshot
|
|
|
|
let account_paths = &[snapshot_test_config.accounts_dir.path().to_path_buf()];
|
|
|
|
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
|
|
|
|
restore_from_snapshot(bank_forks, last_slot, genesis_config, account_paths);
|
2019-10-19 12:09:45 -07:00
|
|
|
}
|
|
|
|
|
2020-08-16 22:22:16 -07:00
|
|
|
fn run_test_bank_forks_snapshot_n(
|
|
|
|
snapshot_version: SnapshotVersion,
|
2020-09-08 07:55:09 -07:00
|
|
|
cluster_type: ClusterType,
|
2020-08-16 22:22:16 -07:00
|
|
|
) {
|
2020-06-17 20:54:52 -07:00
|
|
|
// create banks up to slot 4 and create 1 new account in each bank. test that bank 4 snapshots
|
2019-10-19 12:09:45 -07:00
|
|
|
// and restores correctly
|
|
|
|
run_bank_forks_snapshot_n(
|
2020-06-18 22:38:37 -07:00
|
|
|
snapshot_version,
|
2020-09-08 07:55:09 -07:00
|
|
|
cluster_type,
|
2019-10-19 12:09:45 -07:00
|
|
|
4,
|
|
|
|
|bank, mint_keypair| {
|
|
|
|
let key1 = Keypair::new().pubkey();
|
2019-10-23 22:01:22 -07:00
|
|
|
let tx =
|
|
|
|
system_transaction::transfer(&mint_keypair, &key1, 1, bank.last_blockhash());
|
2019-10-19 12:09:45 -07:00
|
|
|
assert_eq!(bank.process_transaction(&tx), Ok(()));
|
2019-11-22 18:22:28 -08:00
|
|
|
|
|
|
|
let key2 = Keypair::new().pubkey();
|
|
|
|
let tx =
|
|
|
|
system_transaction::transfer(&mint_keypair, &key2, 0, bank.last_blockhash());
|
|
|
|
assert_eq!(bank.process_transaction(&tx), Ok(()));
|
|
|
|
|
2019-10-19 12:09:45 -07:00
|
|
|
bank.freeze();
|
|
|
|
},
|
|
|
|
1,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
fn goto_end_of_slot(bank: &mut Bank) {
|
|
|
|
let mut tick_hash = bank.last_blockhash();
|
|
|
|
loop {
|
|
|
|
tick_hash = hashv(&[&tick_hash.as_ref(), &[42]]);
|
|
|
|
bank.register_tick(&tick_hash);
|
|
|
|
if tick_hash == bank.last_blockhash() {
|
|
|
|
bank.freeze();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-08-16 22:22:16 -07:00
|
|
|
fn run_test_concurrent_snapshot_packaging(
|
|
|
|
snapshot_version: SnapshotVersion,
|
2020-09-08 07:55:09 -07:00
|
|
|
cluster_type: ClusterType,
|
2020-08-16 22:22:16 -07:00
|
|
|
) {
|
2019-10-19 12:09:45 -07:00
|
|
|
solana_logger::setup();
|
|
|
|
|
|
|
|
// Set up snapshotting config
|
2020-09-08 07:55:09 -07:00
|
|
|
let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, cluster_type, 1);
|
2019-10-19 12:09:45 -07:00
|
|
|
|
|
|
|
let bank_forks = &mut snapshot_test_config.bank_forks;
|
|
|
|
let snapshots_dir = &snapshot_test_config.snapshot_dir;
|
|
|
|
let snapshot_config = &snapshot_test_config.snapshot_config;
|
2020-06-18 22:38:37 -07:00
|
|
|
let snapshot_path = &snapshot_config.snapshot_path;
|
|
|
|
let snapshot_package_output_path = &snapshot_config.snapshot_package_output_path;
|
2019-11-08 20:56:57 -08:00
|
|
|
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
|
|
|
|
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
|
2019-10-19 12:09:45 -07:00
|
|
|
|
|
|
|
// Take snapshot of zeroth bank
|
|
|
|
let bank0 = bank_forks.get(0).unwrap();
|
2020-06-18 22:38:37 -07:00
|
|
|
let storages = bank0.get_snapshot_storages();
|
|
|
|
snapshot_utils::add_snapshot(snapshot_path, bank0, &storages, snapshot_version).unwrap();
|
2019-10-19 12:09:45 -07:00
|
|
|
|
|
|
|
// Set up snapshotting channels
|
|
|
|
let (sender, receiver) = channel();
|
|
|
|
let (fake_sender, _fake_receiver) = channel();
|
|
|
|
|
|
|
|
// Create next MAX_CACHE_ENTRIES + 2 banks and snapshots. Every bank will get snapshotted
|
|
|
|
// and the snapshot purging logic will run on every snapshot taken. This means the three
|
|
|
|
// (including snapshot for bank0 created above) earliest snapshots will get purged by the
|
|
|
|
// time this loop is done.
|
|
|
|
|
|
|
|
// Also, make a saved copy of the state of the snapshot for a bank with
|
|
|
|
// bank.slot == saved_slot, so we can use it for a correctness check later.
|
|
|
|
let saved_snapshots_dir = TempDir::new().unwrap();
|
|
|
|
let saved_accounts_dir = TempDir::new().unwrap();
|
|
|
|
let saved_slot = 4;
|
2020-02-26 20:28:53 -08:00
|
|
|
let mut saved_archive_path = None;
|
|
|
|
|
2020-11-30 18:16:59 -08:00
|
|
|
for forks in 0..snapshot_utils::MAX_SNAPSHOTS + 2 {
|
2019-10-19 12:09:45 -07:00
|
|
|
let bank = Bank::new_from_parent(
|
|
|
|
&bank_forks[forks as u64],
|
|
|
|
&Pubkey::default(),
|
|
|
|
(forks + 1) as u64,
|
|
|
|
);
|
|
|
|
let slot = bank.slot();
|
|
|
|
let key1 = Keypair::new().pubkey();
|
2019-11-08 20:56:57 -08:00
|
|
|
let tx = system_transaction::transfer(&mint_keypair, &key1, 1, genesis_config.hash());
|
2019-10-19 12:09:45 -07:00
|
|
|
assert_eq!(bank.process_transaction(&tx), Ok(()));
|
2019-12-05 14:27:46 -08:00
|
|
|
bank.squash();
|
2020-02-26 20:28:53 -08:00
|
|
|
let accounts_hash = bank.update_accounts_hash();
|
2019-10-19 12:09:45 -07:00
|
|
|
|
|
|
|
let package_sender = {
|
|
|
|
if slot == saved_slot as u64 {
|
|
|
|
// Only send one package on the real sender so that the packaging service
|
|
|
|
// doesn't take forever to run the packaging logic on all MAX_CACHE_ENTRIES
|
|
|
|
// later
|
|
|
|
&sender
|
|
|
|
} else {
|
|
|
|
&fake_sender
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2020-09-28 16:04:46 -07:00
|
|
|
snapshot_utils::snapshot_bank(
|
|
|
|
&bank,
|
|
|
|
vec![],
|
|
|
|
&package_sender,
|
|
|
|
&snapshot_path,
|
|
|
|
&snapshot_package_output_path,
|
|
|
|
snapshot_config.snapshot_version,
|
2021-01-07 22:45:42 -08:00
|
|
|
&snapshot_config.archive_format,
|
2021-02-04 07:00:33 -08:00
|
|
|
None,
|
2020-09-28 16:04:46 -07:00
|
|
|
)
|
|
|
|
.unwrap();
|
2019-10-19 12:09:45 -07:00
|
|
|
|
2020-09-28 16:04:46 -07:00
|
|
|
bank_forks.insert(bank);
|
2019-10-19 12:09:45 -07:00
|
|
|
if slot == saved_slot as u64 {
|
2021-02-11 16:03:51 -08:00
|
|
|
// Find the relevant snapshot storages
|
|
|
|
let snapshot_storage_files: HashSet<_> = bank_forks[slot]
|
|
|
|
.get_snapshot_storages()
|
|
|
|
.into_iter()
|
|
|
|
.flatten()
|
|
|
|
.map(|s| s.get_path())
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
// Only save off the files returned by `get_snapshot_storages`. This is because
|
|
|
|
// some of the storage entries in the accounts directory may be filtered out by
|
|
|
|
// `get_snapshot_storages()` and will not be included in the snapshot. Ultimately,
|
|
|
|
// this means copying naitvely everything in `accounts_dir` to the `saved_accounts_dir`
|
|
|
|
// will lead to test failure by mismatch when `saved_accounts_dir` is compared to
|
|
|
|
// the unpacked snapshot later in this test's call to `verify_snapshot_archive()`.
|
|
|
|
for file in snapshot_storage_files {
|
|
|
|
fs::copy(
|
|
|
|
&file,
|
|
|
|
&saved_accounts_dir.path().join(file.file_name().unwrap()),
|
|
|
|
)
|
|
|
|
.unwrap();
|
|
|
|
}
|
2020-06-21 00:12:21 -07:00
|
|
|
let last_snapshot_path = fs::read_dir(snapshot_path)
|
2019-10-19 12:09:45 -07:00
|
|
|
.unwrap()
|
|
|
|
.filter_map(|entry| {
|
|
|
|
let e = entry.unwrap();
|
|
|
|
let file_path = e.path();
|
|
|
|
let file_name = file_path.file_name().unwrap();
|
|
|
|
file_name
|
|
|
|
.to_str()
|
|
|
|
.map(|s| s.parse::<u64>().ok().map(|_| file_path.clone()))
|
|
|
|
.unwrap_or(None)
|
|
|
|
})
|
|
|
|
.sorted()
|
2020-06-18 22:38:37 -07:00
|
|
|
.last()
|
|
|
|
.unwrap();
|
2019-10-19 12:09:45 -07:00
|
|
|
// only save off the snapshot of this slot, we don't need the others.
|
2021-02-11 16:03:51 -08:00
|
|
|
let options = CopyOptions::new();
|
2020-06-21 00:12:21 -07:00
|
|
|
fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap();
|
2020-02-26 20:28:53 -08:00
|
|
|
|
|
|
|
saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path(
|
2021-02-04 07:00:33 -08:00
|
|
|
snapshot_package_output_path.to_path_buf(),
|
2020-02-26 20:28:53 -08:00
|
|
|
&(slot, accounts_hash),
|
2021-01-21 18:34:51 -08:00
|
|
|
ArchiveFormat::TarBzip2,
|
2020-02-26 20:28:53 -08:00
|
|
|
));
|
2019-10-19 12:09:45 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Purge all the outdated snapshots, including the ones needed to generate the package
|
|
|
|
// currently sitting in the channel
|
2020-09-28 16:04:46 -07:00
|
|
|
snapshot_utils::purge_old_snapshots(&snapshot_path);
|
2020-06-21 00:12:21 -07:00
|
|
|
assert!(snapshot_utils::get_snapshot_paths(&snapshots_dir)
|
|
|
|
.into_iter()
|
|
|
|
.map(|path| path.slot)
|
2020-11-30 18:16:59 -08:00
|
|
|
.eq(3..=snapshot_utils::MAX_SNAPSHOTS as u64 + 2));
|
2019-10-19 12:09:45 -07:00
|
|
|
|
|
|
|
// Create a SnapshotPackagerService to create tarballs from all the pending
|
|
|
|
// SnapshotPackage's on the channel. By the time this service starts, we have already
|
|
|
|
// purged the first two snapshots, which are needed by every snapshot other than
|
|
|
|
// the last two snapshots. However, the packaging service should still be able to
|
|
|
|
// correctly construct the earlier snapshots because the SnapshotPackage's on the
|
|
|
|
// channel hold hard links to these deleted snapshots. We verify this is the case below.
|
|
|
|
let exit = Arc::new(AtomicBool::new(false));
|
2020-02-20 11:46:13 -08:00
|
|
|
|
2020-04-21 12:54:45 -07:00
|
|
|
let cluster_info = Arc::new(ClusterInfo::new_with_invalid_keypair(ContactInfo::default()));
|
2020-02-20 11:46:13 -08:00
|
|
|
|
2021-01-11 10:21:15 -08:00
|
|
|
let pending_snapshot_package = PendingSnapshotPackage::default();
|
|
|
|
let snapshot_packager_service = SnapshotPackagerService::new(
|
|
|
|
pending_snapshot_package.clone(),
|
|
|
|
None,
|
|
|
|
&exit,
|
|
|
|
&cluster_info,
|
2021-05-12 10:32:27 -07:00
|
|
|
DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
|
2021-01-11 10:21:15 -08:00
|
|
|
);
|
|
|
|
|
2021-02-05 11:48:55 -08:00
|
|
|
let thread_pool = accounts_db::make_min_priority_thread_pool();
|
|
|
|
|
2021-01-11 10:21:15 -08:00
|
|
|
let _package_receiver = std::thread::Builder::new()
|
|
|
|
.name("package-receiver".to_string())
|
|
|
|
.spawn(move || {
|
|
|
|
while let Ok(mut snapshot_package) = receiver.recv() {
|
|
|
|
// Only package the latest
|
|
|
|
while let Ok(new_snapshot_package) = receiver.try_recv() {
|
|
|
|
snapshot_package = new_snapshot_package;
|
|
|
|
}
|
|
|
|
|
2021-02-04 07:00:33 -08:00
|
|
|
let snapshot_package =
|
|
|
|
solana_runtime::snapshot_utils::process_accounts_package_pre(
|
|
|
|
snapshot_package,
|
2021-02-05 11:48:55 -08:00
|
|
|
Some(&thread_pool),
|
2021-02-04 07:00:33 -08:00
|
|
|
);
|
2021-01-11 10:21:15 -08:00
|
|
|
*pending_snapshot_package.lock().unwrap() = Some(snapshot_package);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Wait until the package is consumed by SnapshotPackagerService
|
|
|
|
while pending_snapshot_package.lock().unwrap().is_some() {
|
|
|
|
std::thread::sleep(Duration::from_millis(100));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Shutdown SnapshotPackagerService
|
|
|
|
exit.store(true, Ordering::Relaxed);
|
|
|
|
})
|
|
|
|
.unwrap();
|
2019-10-19 12:09:45 -07:00
|
|
|
|
2021-01-11 10:21:15 -08:00
|
|
|
// Close the channel so that the package receiver will exit after reading all the
|
2019-10-19 12:09:45 -07:00
|
|
|
// packages off the channel
|
|
|
|
drop(sender);
|
|
|
|
|
|
|
|
// Wait for service to finish
|
|
|
|
snapshot_packager_service
|
|
|
|
.join()
|
|
|
|
.expect("SnapshotPackagerService exited with error");
|
|
|
|
|
2020-02-26 20:28:53 -08:00
|
|
|
// Check the archive we cached the state for earlier was generated correctly
|
2019-10-19 12:09:45 -07:00
|
|
|
|
2020-06-17 20:54:52 -07:00
|
|
|
// before we compare, stick an empty status_cache in this dir so that the package comparison works
|
2019-10-19 12:09:45 -07:00
|
|
|
// This is needed since the status_cache is added by the packager and is not collected from
|
|
|
|
// the source dir for snapshots
|
2020-01-09 16:49:36 -08:00
|
|
|
snapshot_utils::serialize_snapshot_data_file(
|
|
|
|
&saved_snapshots_dir
|
|
|
|
.path()
|
|
|
|
.join(snapshot_utils::SNAPSHOT_STATUS_CACHE_FILE_NAME),
|
|
|
|
|stream| {
|
2020-06-18 22:38:37 -07:00
|
|
|
serialize_into(stream, &[] as &[BankSlotDelta])?;
|
2020-01-09 16:49:36 -08:00
|
|
|
Ok(())
|
|
|
|
},
|
|
|
|
)
|
|
|
|
.unwrap();
|
2019-10-19 12:09:45 -07:00
|
|
|
|
2020-01-23 10:20:37 -08:00
|
|
|
snapshot_utils::verify_snapshot_archive(
|
2020-02-26 20:28:53 -08:00
|
|
|
saved_archive_path.unwrap(),
|
2019-10-19 12:09:45 -07:00
|
|
|
saved_snapshots_dir.path(),
|
2021-02-11 16:03:51 -08:00
|
|
|
saved_accounts_dir.path(),
|
2021-01-07 22:45:42 -08:00
|
|
|
ArchiveFormat::TarBzip2,
|
2019-10-19 12:09:45 -07:00
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2020-09-08 07:55:09 -07:00
|
|
|
fn run_test_slots_to_snapshot(snapshot_version: SnapshotVersion, cluster_type: ClusterType) {
|
2019-10-19 12:09:45 -07:00
|
|
|
solana_logger::setup();
|
2020-01-03 09:58:52 -08:00
|
|
|
let num_set_roots = MAX_CACHE_ENTRIES * 2;
|
|
|
|
|
|
|
|
for add_root_interval in &[1, 3, 9] {
|
2020-09-28 16:04:46 -07:00
|
|
|
let (snapshot_sender, _snapshot_receiver) = unbounded();
|
2019-10-19 12:09:45 -07:00
|
|
|
// Make sure this test never clears bank.slots_since_snapshot
|
2020-06-18 22:38:37 -07:00
|
|
|
let mut snapshot_test_config = SnapshotTestConfig::new(
|
|
|
|
snapshot_version,
|
2020-09-08 07:55:09 -07:00
|
|
|
cluster_type,
|
2020-06-18 22:38:37 -07:00
|
|
|
(*add_root_interval * num_set_roots * 2) as u64,
|
|
|
|
);
|
2019-10-19 12:09:45 -07:00
|
|
|
let mut current_bank = snapshot_test_config.bank_forks[0].clone();
|
2021-02-18 23:42:09 -08:00
|
|
|
let request_sender = AbsRequestSender::new(Some(snapshot_sender));
|
2019-10-19 12:09:45 -07:00
|
|
|
for _ in 0..num_set_roots {
|
2020-01-03 09:58:52 -08:00
|
|
|
for _ in 0..*add_root_interval {
|
2019-10-19 12:09:45 -07:00
|
|
|
let new_slot = current_bank.slot() + 1;
|
|
|
|
let new_bank =
|
|
|
|
Bank::new_from_parent(¤t_bank, &Pubkey::default(), new_slot);
|
|
|
|
snapshot_test_config.bank_forks.insert(new_bank);
|
|
|
|
current_bank = snapshot_test_config.bank_forks[new_slot].clone();
|
|
|
|
}
|
2020-04-24 15:49:57 -07:00
|
|
|
snapshot_test_config.bank_forks.set_root(
|
|
|
|
current_bank.slot(),
|
2020-12-12 17:22:34 -08:00
|
|
|
&request_sender,
|
2020-04-24 15:49:57 -07:00
|
|
|
None,
|
|
|
|
);
|
2019-10-19 12:09:45 -07:00
|
|
|
}
|
|
|
|
|
2020-01-03 09:58:52 -08:00
|
|
|
let num_old_slots = num_set_roots * *add_root_interval - MAX_CACHE_ENTRIES + 1;
|
2020-06-18 22:38:37 -07:00
|
|
|
let expected_slots_to_snapshot =
|
|
|
|
num_old_slots as u64..=num_set_roots as u64 * *add_root_interval as u64;
|
2019-10-19 12:09:45 -07:00
|
|
|
|
2020-06-18 22:38:37 -07:00
|
|
|
let slots_to_snapshot = snapshot_test_config
|
2019-10-19 12:09:45 -07:00
|
|
|
.bank_forks
|
|
|
|
.get(snapshot_test_config.bank_forks.root())
|
2020-06-18 22:38:37 -07:00
|
|
|
.unwrap()
|
|
|
|
.src
|
|
|
|
.roots();
|
2020-06-21 00:12:21 -07:00
|
|
|
assert!(slots_to_snapshot.into_iter().eq(expected_slots_to_snapshot));
|
2019-10-19 12:09:45 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-08-16 22:22:16 -07:00
|
|
|
fn run_test_bank_forks_status_cache_snapshot_n(
|
|
|
|
snapshot_version: SnapshotVersion,
|
2020-09-08 07:55:09 -07:00
|
|
|
cluster_type: ClusterType,
|
2020-08-16 22:22:16 -07:00
|
|
|
) {
|
2020-06-17 20:54:52 -07:00
|
|
|
// create banks up to slot (MAX_CACHE_ENTRIES * 2) + 1 while transferring 1 lamport into 2 different accounts each time
|
2019-10-19 12:09:45 -07:00
|
|
|
// this is done to ensure the AccountStorageEntries keep getting cleaned up as the root moves
|
|
|
|
// ahead. Also tests the status_cache purge and status cache snapshotting.
|
|
|
|
// Makes sure that the last bank is restored correctly
|
|
|
|
let key1 = Keypair::new().pubkey();
|
|
|
|
let key2 = Keypair::new().pubkey();
|
|
|
|
for set_root_interval in &[1, 4] {
|
|
|
|
run_bank_forks_snapshot_n(
|
2020-06-18 22:38:37 -07:00
|
|
|
snapshot_version,
|
2020-09-08 07:55:09 -07:00
|
|
|
cluster_type,
|
2019-10-19 12:09:45 -07:00
|
|
|
(MAX_CACHE_ENTRIES * 2 + 1) as u64,
|
|
|
|
|bank, mint_keypair| {
|
|
|
|
let tx = system_transaction::transfer(
|
|
|
|
&mint_keypair,
|
|
|
|
&key1,
|
|
|
|
1,
|
|
|
|
bank.parent().unwrap().last_blockhash(),
|
|
|
|
);
|
|
|
|
assert_eq!(bank.process_transaction(&tx), Ok(()));
|
|
|
|
let tx = system_transaction::transfer(
|
|
|
|
&mint_keypair,
|
|
|
|
&key2,
|
|
|
|
1,
|
|
|
|
bank.parent().unwrap().last_blockhash(),
|
|
|
|
);
|
|
|
|
assert_eq!(bank.process_transaction(&tx), Ok(()));
|
|
|
|
goto_end_of_slot(bank);
|
|
|
|
},
|
|
|
|
*set_root_interval,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|