Decouple accounts hash calculation from snapshot hash (#9507)
This commit is contained in:
parent
425b4fe6dd
commit
66abe45ea1
|
@ -6,8 +6,8 @@
|
||||||
|
|
||||||
use crate::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES};
|
use crate::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES};
|
||||||
use solana_ledger::{
|
use solana_ledger::{
|
||||||
snapshot_package::SnapshotPackage, snapshot_package::SnapshotPackageReceiver,
|
snapshot_package::AccountsPackage, snapshot_package::AccountsPackageReceiver,
|
||||||
snapshot_package::SnapshotPackageSender,
|
snapshot_package::AccountsPackageSender,
|
||||||
};
|
};
|
||||||
use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
|
use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
|
@ -27,13 +27,14 @@ pub struct AccountsHashVerifier {
|
||||||
|
|
||||||
impl AccountsHashVerifier {
|
impl AccountsHashVerifier {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
snapshot_package_receiver: SnapshotPackageReceiver,
|
accounts_package_receiver: AccountsPackageReceiver,
|
||||||
snapshot_package_sender: Option<SnapshotPackageSender>,
|
accounts_package_sender: Option<AccountsPackageSender>,
|
||||||
exit: &Arc<AtomicBool>,
|
exit: &Arc<AtomicBool>,
|
||||||
cluster_info: &Arc<RwLock<ClusterInfo>>,
|
cluster_info: &Arc<RwLock<ClusterInfo>>,
|
||||||
trusted_validators: Option<HashSet<Pubkey>>,
|
trusted_validators: Option<HashSet<Pubkey>>,
|
||||||
halt_on_trusted_validators_accounts_hash_mismatch: bool,
|
halt_on_trusted_validators_accounts_hash_mismatch: bool,
|
||||||
fault_injection_rate_slots: u64,
|
fault_injection_rate_slots: u64,
|
||||||
|
snapshot_interval_slots: u64,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let exit = exit.clone();
|
let exit = exit.clone();
|
||||||
let cluster_info = cluster_info.clone();
|
let cluster_info = cluster_info.clone();
|
||||||
|
@ -46,17 +47,18 @@ impl AccountsHashVerifier {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
match snapshot_package_receiver.recv_timeout(Duration::from_secs(1)) {
|
match accounts_package_receiver.recv_timeout(Duration::from_secs(1)) {
|
||||||
Ok(snapshot_package) => {
|
Ok(accounts_package) => {
|
||||||
Self::process_snapshot(
|
Self::process_accounts_package(
|
||||||
snapshot_package,
|
accounts_package,
|
||||||
&cluster_info,
|
&cluster_info,
|
||||||
&trusted_validators,
|
&trusted_validators,
|
||||||
halt_on_trusted_validators_accounts_hash_mismatch,
|
halt_on_trusted_validators_accounts_hash_mismatch,
|
||||||
&snapshot_package_sender,
|
&accounts_package_sender,
|
||||||
&mut hashes,
|
&mut hashes,
|
||||||
&exit,
|
&exit,
|
||||||
fault_injection_rate_slots,
|
fault_injection_rate_slots,
|
||||||
|
snapshot_interval_slots,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
Err(RecvTimeoutError::Disconnected) => break,
|
Err(RecvTimeoutError::Disconnected) => break,
|
||||||
|
@ -70,28 +72,29 @@ impl AccountsHashVerifier {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn process_snapshot(
|
fn process_accounts_package(
|
||||||
snapshot_package: SnapshotPackage,
|
accounts_package: AccountsPackage,
|
||||||
cluster_info: &Arc<RwLock<ClusterInfo>>,
|
cluster_info: &Arc<RwLock<ClusterInfo>>,
|
||||||
trusted_validators: &Option<HashSet<Pubkey>>,
|
trusted_validators: &Option<HashSet<Pubkey>>,
|
||||||
halt_on_trusted_validator_accounts_hash_mismatch: bool,
|
halt_on_trusted_validator_accounts_hash_mismatch: bool,
|
||||||
snapshot_package_sender: &Option<SnapshotPackageSender>,
|
accounts_package_sender: &Option<AccountsPackageSender>,
|
||||||
hashes: &mut Vec<(Slot, Hash)>,
|
hashes: &mut Vec<(Slot, Hash)>,
|
||||||
exit: &Arc<AtomicBool>,
|
exit: &Arc<AtomicBool>,
|
||||||
fault_injection_rate_slots: u64,
|
fault_injection_rate_slots: u64,
|
||||||
|
snapshot_interval_slots: u64,
|
||||||
) {
|
) {
|
||||||
if fault_injection_rate_slots != 0
|
if fault_injection_rate_slots != 0
|
||||||
&& snapshot_package.root % fault_injection_rate_slots == 0
|
&& accounts_package.root % fault_injection_rate_slots == 0
|
||||||
{
|
{
|
||||||
// For testing, publish an invalid hash to gossip.
|
// For testing, publish an invalid hash to gossip.
|
||||||
use rand::{thread_rng, Rng};
|
use rand::{thread_rng, Rng};
|
||||||
use solana_sdk::hash::extend_and_hash;
|
use solana_sdk::hash::extend_and_hash;
|
||||||
warn!("inserting fault at slot: {}", snapshot_package.root);
|
warn!("inserting fault at slot: {}", accounts_package.root);
|
||||||
let rand = thread_rng().gen_range(0, 10);
|
let rand = thread_rng().gen_range(0, 10);
|
||||||
let hash = extend_and_hash(&snapshot_package.hash, &[rand]);
|
let hash = extend_and_hash(&accounts_package.hash, &[rand]);
|
||||||
hashes.push((snapshot_package.root, hash));
|
hashes.push((accounts_package.root, hash));
|
||||||
} else {
|
} else {
|
||||||
hashes.push((snapshot_package.root, snapshot_package.hash));
|
hashes.push((accounts_package.root, accounts_package.hash));
|
||||||
}
|
}
|
||||||
|
|
||||||
while hashes.len() > MAX_SNAPSHOT_HASHES {
|
while hashes.len() > MAX_SNAPSHOT_HASHES {
|
||||||
|
@ -107,8 +110,11 @@ impl AccountsHashVerifier {
|
||||||
exit.store(true, Ordering::Relaxed);
|
exit.store(true, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if let Some(sender) = snapshot_package_sender.as_ref() {
|
|
||||||
if sender.send(snapshot_package).is_err() {}
|
if accounts_package.block_height % snapshot_interval_slots == 0 {
|
||||||
|
if let Some(sender) = accounts_package_sender.as_ref() {
|
||||||
|
if sender.send(accounts_package).is_err() {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
cluster_info
|
cluster_info
|
||||||
|
@ -225,8 +231,9 @@ mod tests {
|
||||||
let mut hashes = vec![];
|
let mut hashes = vec![];
|
||||||
for i in 0..MAX_SNAPSHOT_HASHES + 1 {
|
for i in 0..MAX_SNAPSHOT_HASHES + 1 {
|
||||||
let snapshot_links = TempDir::new().unwrap();
|
let snapshot_links = TempDir::new().unwrap();
|
||||||
let snapshot_package = SnapshotPackage {
|
let accounts_package = AccountsPackage {
|
||||||
hash: hash(&[i as u8]),
|
hash: hash(&[i as u8]),
|
||||||
|
block_height: 100 + i as u64,
|
||||||
root: 100 + i as u64,
|
root: 100 + i as u64,
|
||||||
slot_deltas: vec![],
|
slot_deltas: vec![],
|
||||||
snapshot_links,
|
snapshot_links,
|
||||||
|
@ -235,8 +242,8 @@ mod tests {
|
||||||
compression: CompressionType::Bzip2,
|
compression: CompressionType::Bzip2,
|
||||||
};
|
};
|
||||||
|
|
||||||
AccountsHashVerifier::process_snapshot(
|
AccountsHashVerifier::process_accounts_package(
|
||||||
snapshot_package,
|
accounts_package,
|
||||||
&cluster_info,
|
&cluster_info,
|
||||||
&Some(trusted_validators.clone()),
|
&Some(trusted_validators.clone()),
|
||||||
false,
|
false,
|
||||||
|
@ -244,6 +251,7 @@ mod tests {
|
||||||
&mut hashes,
|
&mut hashes,
|
||||||
&exit,
|
&exit,
|
||||||
0,
|
0,
|
||||||
|
100,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
let cluster_info_r = cluster_info.read().unwrap();
|
let cluster_info_r = cluster_info.read().unwrap();
|
||||||
|
|
|
@ -20,7 +20,7 @@ use solana_ledger::{
|
||||||
blockstore_processor::{self, BlockstoreProcessorError, TransactionStatusSender},
|
blockstore_processor::{self, BlockstoreProcessorError, TransactionStatusSender},
|
||||||
entry::VerifyRecyclers,
|
entry::VerifyRecyclers,
|
||||||
leader_schedule_cache::LeaderScheduleCache,
|
leader_schedule_cache::LeaderScheduleCache,
|
||||||
snapshot_package::SnapshotPackageSender,
|
snapshot_package::AccountsPackageSender,
|
||||||
};
|
};
|
||||||
use solana_measure::thread_mem_usage;
|
use solana_measure::thread_mem_usage;
|
||||||
use solana_metrics::inc_new_counter_info;
|
use solana_metrics::inc_new_counter_info;
|
||||||
|
@ -97,7 +97,7 @@ pub struct ReplayStageConfig {
|
||||||
pub subscriptions: Arc<RpcSubscriptions>,
|
pub subscriptions: Arc<RpcSubscriptions>,
|
||||||
pub leader_schedule_cache: Arc<LeaderScheduleCache>,
|
pub leader_schedule_cache: Arc<LeaderScheduleCache>,
|
||||||
pub latest_root_senders: Vec<Sender<Slot>>,
|
pub latest_root_senders: Vec<Sender<Slot>>,
|
||||||
pub accounts_hash_sender: Option<SnapshotPackageSender>,
|
pub accounts_hash_sender: Option<AccountsPackageSender>,
|
||||||
pub block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
|
pub block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
|
||||||
pub transaction_status_sender: Option<TransactionStatusSender>,
|
pub transaction_status_sender: Option<TransactionStatusSender>,
|
||||||
pub rewards_recorder_sender: Option<RewardsRecorderSender>,
|
pub rewards_recorder_sender: Option<RewardsRecorderSender>,
|
||||||
|
@ -690,7 +690,7 @@ impl ReplayStage {
|
||||||
leader_schedule_cache: &Arc<LeaderScheduleCache>,
|
leader_schedule_cache: &Arc<LeaderScheduleCache>,
|
||||||
root_bank_sender: &Sender<Vec<Arc<Bank>>>,
|
root_bank_sender: &Sender<Vec<Arc<Bank>>>,
|
||||||
lockouts_sender: &Sender<CommitmentAggregationData>,
|
lockouts_sender: &Sender<CommitmentAggregationData>,
|
||||||
accounts_hash_sender: &Option<SnapshotPackageSender>,
|
accounts_hash_sender: &Option<AccountsPackageSender>,
|
||||||
latest_root_senders: &[Sender<Slot>],
|
latest_root_senders: &[Sender<Slot>],
|
||||||
all_pubkeys: &mut HashSet<Rc<Pubkey>>,
|
all_pubkeys: &mut HashSet<Rc<Pubkey>>,
|
||||||
subscriptions: &Arc<RpcSubscriptions>,
|
subscriptions: &Arc<RpcSubscriptions>,
|
||||||
|
@ -1485,7 +1485,7 @@ impl ReplayStage {
|
||||||
new_root: u64,
|
new_root: u64,
|
||||||
bank_forks: &RwLock<BankForks>,
|
bank_forks: &RwLock<BankForks>,
|
||||||
progress: &mut ProgressMap,
|
progress: &mut ProgressMap,
|
||||||
accounts_hash_sender: &Option<SnapshotPackageSender>,
|
accounts_hash_sender: &Option<AccountsPackageSender>,
|
||||||
all_pubkeys: &mut HashSet<Rc<Pubkey>>,
|
all_pubkeys: &mut HashSet<Rc<Pubkey>>,
|
||||||
) {
|
) {
|
||||||
let old_epoch = bank_forks.read().unwrap().root_bank().epoch();
|
let old_epoch = bank_forks.read().unwrap().root_bank().epoch();
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
use crate::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES};
|
use crate::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES};
|
||||||
use solana_ledger::{snapshot_package::SnapshotPackageReceiver, snapshot_utils};
|
use solana_ledger::{snapshot_package::AccountsPackageReceiver, snapshot_utils};
|
||||||
use solana_sdk::{clock::Slot, hash::Hash};
|
use solana_sdk::{clock::Slot, hash::Hash};
|
||||||
use std::{
|
use std::{
|
||||||
sync::{
|
sync::{
|
||||||
|
@ -17,7 +17,7 @@ pub struct SnapshotPackagerService {
|
||||||
|
|
||||||
impl SnapshotPackagerService {
|
impl SnapshotPackagerService {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
snapshot_package_receiver: SnapshotPackageReceiver,
|
snapshot_package_receiver: AccountsPackageReceiver,
|
||||||
starting_snapshot_hash: Option<(Slot, Hash)>,
|
starting_snapshot_hash: Option<(Slot, Hash)>,
|
||||||
exit: &Arc<AtomicBool>,
|
exit: &Arc<AtomicBool>,
|
||||||
cluster_info: &Arc<RwLock<ClusterInfo>>,
|
cluster_info: &Arc<RwLock<ClusterInfo>>,
|
||||||
|
@ -86,7 +86,7 @@ mod tests {
|
||||||
use bincode::serialize_into;
|
use bincode::serialize_into;
|
||||||
use solana_ledger::bank_forks::CompressionType;
|
use solana_ledger::bank_forks::CompressionType;
|
||||||
use solana_ledger::{
|
use solana_ledger::{
|
||||||
snapshot_package::SnapshotPackage,
|
snapshot_package::AccountsPackage,
|
||||||
snapshot_utils::{self, SNAPSHOT_STATUS_CACHE_FILE_NAME},
|
snapshot_utils::{self, SNAPSHOT_STATUS_CACHE_FILE_NAME},
|
||||||
};
|
};
|
||||||
use solana_runtime::{
|
use solana_runtime::{
|
||||||
|
@ -172,7 +172,8 @@ mod tests {
|
||||||
&(42, Hash::default()),
|
&(42, Hash::default()),
|
||||||
&CompressionType::Bzip2,
|
&CompressionType::Bzip2,
|
||||||
);
|
);
|
||||||
let snapshot_package = SnapshotPackage::new(
|
let snapshot_package = AccountsPackage::new(
|
||||||
|
5,
|
||||||
5,
|
5,
|
||||||
vec![],
|
vec![],
|
||||||
link_snapshots_dir,
|
link_snapshots_dir,
|
||||||
|
|
|
@ -26,7 +26,7 @@ use solana_ledger::{
|
||||||
bank_forks::BankForks,
|
bank_forks::BankForks,
|
||||||
blockstore::{Blockstore, CompletedSlotsReceiver},
|
blockstore::{Blockstore, CompletedSlotsReceiver},
|
||||||
blockstore_processor::TransactionStatusSender,
|
blockstore_processor::TransactionStatusSender,
|
||||||
snapshot_package::SnapshotPackageSender,
|
snapshot_package::AccountsPackageSender,
|
||||||
};
|
};
|
||||||
use solana_sdk::{
|
use solana_sdk::{
|
||||||
pubkey::Pubkey,
|
pubkey::Pubkey,
|
||||||
|
@ -98,7 +98,7 @@ impl Tvu {
|
||||||
cfg: Option<Arc<AtomicBool>>,
|
cfg: Option<Arc<AtomicBool>>,
|
||||||
transaction_status_sender: Option<TransactionStatusSender>,
|
transaction_status_sender: Option<TransactionStatusSender>,
|
||||||
rewards_recorder_sender: Option<RewardsRecorderSender>,
|
rewards_recorder_sender: Option<RewardsRecorderSender>,
|
||||||
snapshot_package_sender: Option<SnapshotPackageSender>,
|
snapshot_package_sender: Option<AccountsPackageSender>,
|
||||||
vote_tracker: Arc<VoteTracker>,
|
vote_tracker: Arc<VoteTracker>,
|
||||||
retransmit_slots_sender: RetransmitSlotsSender,
|
retransmit_slots_sender: RetransmitSlotsSender,
|
||||||
tvu_config: TvuConfig,
|
tvu_config: TvuConfig,
|
||||||
|
@ -165,6 +165,14 @@ impl Tvu {
|
||||||
|
|
||||||
let (ledger_cleanup_slot_sender, ledger_cleanup_slot_receiver) = channel();
|
let (ledger_cleanup_slot_sender, ledger_cleanup_slot_receiver) = channel();
|
||||||
|
|
||||||
|
let snapshot_interval_slots = {
|
||||||
|
if let Some(config) = bank_forks.read().unwrap().snapshot_config() {
|
||||||
|
config.snapshot_interval_slots
|
||||||
|
} else {
|
||||||
|
std::u64::MAX
|
||||||
|
}
|
||||||
|
};
|
||||||
|
info!("snapshot_interval_slots: {}", snapshot_interval_slots);
|
||||||
let (accounts_hash_sender, accounts_hash_receiver) = channel();
|
let (accounts_hash_sender, accounts_hash_receiver) = channel();
|
||||||
let accounts_hash_verifier = AccountsHashVerifier::new(
|
let accounts_hash_verifier = AccountsHashVerifier::new(
|
||||||
accounts_hash_receiver,
|
accounts_hash_receiver,
|
||||||
|
@ -174,6 +182,7 @@ impl Tvu {
|
||||||
tvu_config.trusted_validators.clone(),
|
tvu_config.trusted_validators.clone(),
|
||||||
tvu_config.halt_on_trusted_validators_accounts_hash_mismatch,
|
tvu_config.halt_on_trusted_validators_accounts_hash_mismatch,
|
||||||
tvu_config.accounts_hash_fault_injection_slots,
|
tvu_config.accounts_hash_fault_injection_slots,
|
||||||
|
snapshot_interval_slots,
|
||||||
);
|
);
|
||||||
|
|
||||||
let replay_stage_config = ReplayStageConfig {
|
let replay_stage_config = ReplayStageConfig {
|
||||||
|
|
|
@ -81,6 +81,7 @@ pub struct ValidatorConfig {
|
||||||
pub accounts_hash_fault_injection_slots: u64, // 0 = no fault injection
|
pub accounts_hash_fault_injection_slots: u64, // 0 = no fault injection
|
||||||
pub frozen_accounts: Vec<Pubkey>,
|
pub frozen_accounts: Vec<Pubkey>,
|
||||||
pub no_rocksdb_compaction: bool,
|
pub no_rocksdb_compaction: bool,
|
||||||
|
pub accounts_hash_interval_slots: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for ValidatorConfig {
|
impl Default for ValidatorConfig {
|
||||||
|
@ -107,6 +108,7 @@ impl Default for ValidatorConfig {
|
||||||
accounts_hash_fault_injection_slots: 0,
|
accounts_hash_fault_injection_slots: 0,
|
||||||
frozen_accounts: vec![],
|
frozen_accounts: vec![],
|
||||||
no_rocksdb_compaction: false,
|
no_rocksdb_compaction: false,
|
||||||
|
accounts_hash_interval_slots: std::u64::MAX,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -622,6 +624,7 @@ fn new_banks_from_blockstore(
|
||||||
leader_schedule_cache.set_fixed_leader_schedule(config.fixed_leader_schedule.clone());
|
leader_schedule_cache.set_fixed_leader_schedule(config.fixed_leader_schedule.clone());
|
||||||
|
|
||||||
bank_forks.set_snapshot_config(config.snapshot_config.clone());
|
bank_forks.set_snapshot_config(config.snapshot_config.clone());
|
||||||
|
bank_forks.set_accounts_hash_interval_slots(config.accounts_hash_interval_slots);
|
||||||
|
|
||||||
(
|
(
|
||||||
genesis_config,
|
genesis_config,
|
||||||
|
|
|
@ -38,7 +38,7 @@ mod tests {
|
||||||
genesis_config_info: GenesisConfigInfo,
|
genesis_config_info: GenesisConfigInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn setup_snapshot_test(snapshot_interval_slots: usize) -> SnapshotTestConfig {
|
fn setup_snapshot_test(snapshot_interval_slots: u64) -> SnapshotTestConfig {
|
||||||
let accounts_dir = TempDir::new().unwrap();
|
let accounts_dir = TempDir::new().unwrap();
|
||||||
let snapshot_dir = TempDir::new().unwrap();
|
let snapshot_dir = TempDir::new().unwrap();
|
||||||
let snapshot_output_path = TempDir::new().unwrap();
|
let snapshot_output_path = TempDir::new().unwrap();
|
||||||
|
@ -50,6 +50,7 @@ mod tests {
|
||||||
);
|
);
|
||||||
bank0.freeze();
|
bank0.freeze();
|
||||||
let mut bank_forks = BankForks::new(0, bank0);
|
let mut bank_forks = BankForks::new(0, bank0);
|
||||||
|
bank_forks.accounts_hash_interval_slots = snapshot_interval_slots;
|
||||||
|
|
||||||
let snapshot_config = SnapshotConfig {
|
let snapshot_config = SnapshotConfig {
|
||||||
snapshot_interval_slots,
|
snapshot_interval_slots,
|
||||||
|
@ -265,7 +266,7 @@ mod tests {
|
||||||
};
|
};
|
||||||
|
|
||||||
bank_forks
|
bank_forks
|
||||||
.generate_snapshot(slot, &vec![], &package_sender)
|
.generate_accounts_package(slot, &vec![], &package_sender)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
if slot == saved_slot as u64 {
|
if slot == saved_slot as u64 {
|
||||||
|
@ -371,7 +372,7 @@ mod tests {
|
||||||
let (snapshot_sender, _snapshot_receiver) = channel();
|
let (snapshot_sender, _snapshot_receiver) = channel();
|
||||||
// Make sure this test never clears bank.slots_since_snapshot
|
// Make sure this test never clears bank.slots_since_snapshot
|
||||||
let mut snapshot_test_config =
|
let mut snapshot_test_config =
|
||||||
setup_snapshot_test(add_root_interval * num_set_roots * 2);
|
setup_snapshot_test((add_root_interval * num_set_roots * 2) as u64);
|
||||||
let mut current_bank = snapshot_test_config.bank_forks[0].clone();
|
let mut current_bank = snapshot_test_config.bank_forks[0].clone();
|
||||||
let snapshot_sender = Some(snapshot_sender);
|
let snapshot_sender = Some(snapshot_sender);
|
||||||
for _ in 0..num_set_roots {
|
for _ in 0..num_set_roots {
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
//! The `bank_forks` module implments BankForks a DAG of checkpointed Banks
|
//! The `bank_forks` module implments BankForks a DAG of checkpointed Banks
|
||||||
|
|
||||||
use crate::snapshot_package::{SnapshotPackageSendError, SnapshotPackageSender};
|
use crate::snapshot_package::{AccountsPackageSendError, AccountsPackageSender};
|
||||||
use crate::snapshot_utils::{self, SnapshotError};
|
use crate::snapshot_utils::{self, SnapshotError};
|
||||||
use log::*;
|
use log::*;
|
||||||
use solana_measure::measure::Measure;
|
use solana_measure::measure::Measure;
|
||||||
|
@ -27,7 +27,7 @@ pub enum CompressionType {
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
pub struct SnapshotConfig {
|
pub struct SnapshotConfig {
|
||||||
// Generate a new snapshot every this many slots
|
// Generate a new snapshot every this many slots
|
||||||
pub snapshot_interval_slots: usize,
|
pub snapshot_interval_slots: u64,
|
||||||
|
|
||||||
// Where to store the latest packaged snapshot
|
// Where to store the latest packaged snapshot
|
||||||
pub snapshot_package_output_path: PathBuf,
|
pub snapshot_package_output_path: PathBuf,
|
||||||
|
@ -43,8 +43,8 @@ pub enum BankForksError {
|
||||||
#[error("snapshot error")]
|
#[error("snapshot error")]
|
||||||
SnapshotError(#[from] SnapshotError),
|
SnapshotError(#[from] SnapshotError),
|
||||||
|
|
||||||
#[error("snapshot package send error")]
|
#[error("accounts package send error")]
|
||||||
SnapshotPackageSendError(#[from] SnapshotPackageSendError),
|
AccountsPackageSendError(#[from] AccountsPackageSendError),
|
||||||
}
|
}
|
||||||
type Result<T> = std::result::Result<T, BankForksError>;
|
type Result<T> = std::result::Result<T, BankForksError>;
|
||||||
|
|
||||||
|
@ -54,6 +54,9 @@ pub struct BankForks {
|
||||||
root: Slot,
|
root: Slot,
|
||||||
pub snapshot_config: Option<SnapshotConfig>,
|
pub snapshot_config: Option<SnapshotConfig>,
|
||||||
last_snapshot_slot: Slot,
|
last_snapshot_slot: Slot,
|
||||||
|
|
||||||
|
pub accounts_hash_interval_slots: Slot,
|
||||||
|
last_accounts_hash_slot: Slot,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Index<u64> for BankForks {
|
impl Index<u64> for BankForks {
|
||||||
|
@ -74,6 +77,8 @@ impl BankForks {
|
||||||
root: 0,
|
root: 0,
|
||||||
snapshot_config: None,
|
snapshot_config: None,
|
||||||
last_snapshot_slot: bank_slot,
|
last_snapshot_slot: bank_slot,
|
||||||
|
accounts_hash_interval_slots: std::u64::MAX,
|
||||||
|
last_accounts_hash_slot: bank_slot,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -159,6 +164,8 @@ impl BankForks {
|
||||||
working_bank,
|
working_bank,
|
||||||
snapshot_config: None,
|
snapshot_config: None,
|
||||||
last_snapshot_slot: root,
|
last_snapshot_slot: root,
|
||||||
|
accounts_hash_interval_slots: std::u64::MAX,
|
||||||
|
last_accounts_hash_slot: root,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -178,7 +185,7 @@ impl BankForks {
|
||||||
pub fn set_root(
|
pub fn set_root(
|
||||||
&mut self,
|
&mut self,
|
||||||
root: Slot,
|
root: Slot,
|
||||||
snapshot_package_sender: &Option<SnapshotPackageSender>,
|
accounts_package_sender: &Option<AccountsPackageSender>,
|
||||||
) {
|
) {
|
||||||
let old_epoch = self.root_bank().epoch();
|
let old_epoch = self.root_bank().epoch();
|
||||||
self.root = root;
|
self.root = root;
|
||||||
|
@ -209,43 +216,46 @@ impl BankForks {
|
||||||
.last()
|
.last()
|
||||||
.map(|bank| bank.transaction_count())
|
.map(|bank| bank.transaction_count())
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
// Generate each snapshot at a fixed interval
|
// Calculate the accounts hash at a fixed interval
|
||||||
let mut is_root_bank_squashed = false;
|
let mut is_root_bank_squashed = false;
|
||||||
if self.snapshot_config.is_some() && snapshot_package_sender.is_some() {
|
let mut banks = vec![root_bank];
|
||||||
let config = self.snapshot_config.as_ref().unwrap();
|
let parents = root_bank.parents();
|
||||||
let mut banks = vec![root_bank];
|
banks.extend(parents.iter());
|
||||||
let parents = root_bank.parents();
|
for bank in banks.iter() {
|
||||||
banks.extend(parents.iter());
|
let bank_slot = bank.slot();
|
||||||
for bank in banks.iter() {
|
if bank.block_height() % self.accounts_hash_interval_slots == 0
|
||||||
let bank_slot = bank.slot();
|
&& bank_slot > self.last_accounts_hash_slot
|
||||||
if bank.block_height() % (config.snapshot_interval_slots as u64) == 0 {
|
{
|
||||||
// Generate a snapshot if snapshots are configured and it's been an appropriate number
|
self.last_accounts_hash_slot = bank_slot;
|
||||||
// of banks since the last snapshot
|
bank.squash();
|
||||||
if bank_slot > self.last_snapshot_slot {
|
is_root_bank_squashed = bank_slot == root;
|
||||||
bank.squash();
|
|
||||||
is_root_bank_squashed = bank_slot == root;
|
|
||||||
let mut snapshot_time = Measure::start("total-snapshot-ms");
|
|
||||||
let r = self.generate_snapshot(
|
|
||||||
bank_slot,
|
|
||||||
&bank.src.roots(),
|
|
||||||
snapshot_package_sender.as_ref().unwrap(),
|
|
||||||
);
|
|
||||||
if r.is_err() {
|
|
||||||
warn!(
|
|
||||||
"Error generating snapshot for bank: {}, err: {:?}",
|
|
||||||
bank_slot, r
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
self.last_snapshot_slot = bank_slot;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cleanup outdated snapshots
|
bank.clean_accounts();
|
||||||
self.purge_old_snapshots();
|
bank.update_accounts_hash();
|
||||||
snapshot_time.stop();
|
|
||||||
inc_new_counter_info!("total-snapshot-ms", snapshot_time.as_ms() as usize);
|
if self.snapshot_config.is_some() && accounts_package_sender.is_some() {
|
||||||
|
// Generate an accounts package
|
||||||
|
let mut snapshot_time = Measure::start("total-snapshot-ms");
|
||||||
|
let r = self.generate_accounts_package(
|
||||||
|
bank_slot,
|
||||||
|
&bank.src.roots(),
|
||||||
|
accounts_package_sender.as_ref().unwrap(),
|
||||||
|
);
|
||||||
|
if r.is_err() {
|
||||||
|
warn!(
|
||||||
|
"Error generating snapshot for bank: {}, err: {:?}",
|
||||||
|
bank_slot, r
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
self.last_snapshot_slot = bank_slot;
|
||||||
}
|
}
|
||||||
break;
|
|
||||||
|
// Cleanup outdated snapshots
|
||||||
|
self.purge_old_snapshots();
|
||||||
|
snapshot_time.stop();
|
||||||
|
inc_new_counter_info!("total-snapshot-ms", snapshot_time.as_ms() as usize);
|
||||||
}
|
}
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !is_root_bank_squashed {
|
if !is_root_bank_squashed {
|
||||||
|
@ -282,11 +292,11 @@ impl BankForks {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn generate_snapshot(
|
pub fn generate_accounts_package(
|
||||||
&self,
|
&self,
|
||||||
root: Slot,
|
root: Slot,
|
||||||
slots_to_snapshot: &[Slot],
|
slots_to_snapshot: &[Slot],
|
||||||
snapshot_package_sender: &SnapshotPackageSender,
|
accounts_package_sender: &AccountsPackageSender,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let config = self.snapshot_config.as_ref().unwrap();
|
let config = self.snapshot_config.as_ref().unwrap();
|
||||||
|
|
||||||
|
@ -319,8 +329,7 @@ impl BankForks {
|
||||||
config.compression.clone(),
|
config.compression.clone(),
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
// Send the package to the packaging thread
|
accounts_package_sender.send(package)?;
|
||||||
snapshot_package_sender.send(package)?;
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -338,6 +347,10 @@ impl BankForks {
|
||||||
pub fn snapshot_config(&self) -> &Option<SnapshotConfig> {
|
pub fn snapshot_config(&self) -> &Option<SnapshotConfig> {
|
||||||
&self.snapshot_config
|
&self.snapshot_config
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn set_accounts_hash_interval_slots(&mut self, accounts_interval_slots: u64) {
|
||||||
|
self.accounts_hash_interval_slots = accounts_interval_slots;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
|
@ -8,13 +8,14 @@ use std::{
|
||||||
};
|
};
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
pub type SnapshotPackageSender = Sender<SnapshotPackage>;
|
pub type AccountsPackageSender = Sender<AccountsPackage>;
|
||||||
pub type SnapshotPackageReceiver = Receiver<SnapshotPackage>;
|
pub type AccountsPackageReceiver = Receiver<AccountsPackage>;
|
||||||
pub type SnapshotPackageSendError = SendError<SnapshotPackage>;
|
pub type AccountsPackageSendError = SendError<AccountsPackage>;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct SnapshotPackage {
|
pub struct AccountsPackage {
|
||||||
pub root: Slot,
|
pub root: Slot,
|
||||||
|
pub block_height: Slot,
|
||||||
pub slot_deltas: Vec<BankSlotDelta>,
|
pub slot_deltas: Vec<BankSlotDelta>,
|
||||||
pub snapshot_links: TempDir,
|
pub snapshot_links: TempDir,
|
||||||
pub storages: SnapshotStorages,
|
pub storages: SnapshotStorages,
|
||||||
|
@ -23,9 +24,10 @@ pub struct SnapshotPackage {
|
||||||
pub compression: CompressionType,
|
pub compression: CompressionType,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SnapshotPackage {
|
impl AccountsPackage {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
root: Slot,
|
root: Slot,
|
||||||
|
block_height: u64,
|
||||||
slot_deltas: Vec<BankSlotDelta>,
|
slot_deltas: Vec<BankSlotDelta>,
|
||||||
snapshot_links: TempDir,
|
snapshot_links: TempDir,
|
||||||
storages: SnapshotStorages,
|
storages: SnapshotStorages,
|
||||||
|
@ -35,6 +37,7 @@ impl SnapshotPackage {
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
root,
|
root,
|
||||||
|
block_height,
|
||||||
slot_deltas,
|
slot_deltas,
|
||||||
snapshot_links,
|
snapshot_links,
|
||||||
storages,
|
storages,
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
use crate::bank_forks::CompressionType;
|
use crate::bank_forks::CompressionType;
|
||||||
use crate::hardened_unpack::{unpack_snapshot, UnpackError};
|
use crate::hardened_unpack::{unpack_snapshot, UnpackError};
|
||||||
use crate::snapshot_package::SnapshotPackage;
|
use crate::snapshot_package::AccountsPackage;
|
||||||
use bincode::serialize_into;
|
use bincode::serialize_into;
|
||||||
use bzip2::bufread::BzDecoder;
|
use bzip2::bufread::BzDecoder;
|
||||||
use flate2::read::GzDecoder;
|
use flate2::read::GzDecoder;
|
||||||
|
@ -93,7 +93,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
|
||||||
snapshot_package_output_path: P,
|
snapshot_package_output_path: P,
|
||||||
snapshot_storages: SnapshotStorages,
|
snapshot_storages: SnapshotStorages,
|
||||||
compression: CompressionType,
|
compression: CompressionType,
|
||||||
) -> Result<SnapshotPackage> {
|
) -> Result<AccountsPackage> {
|
||||||
// Hard link all the snapshots we need for this package
|
// Hard link all the snapshots we need for this package
|
||||||
let snapshot_hard_links_dir = tempfile::tempdir_in(snapshot_path)?;
|
let snapshot_hard_links_dir = tempfile::tempdir_in(snapshot_path)?;
|
||||||
|
|
||||||
|
@ -104,8 +104,8 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
|
||||||
snapshot_storages.len()
|
snapshot_storages.len()
|
||||||
);
|
);
|
||||||
|
|
||||||
// Any errors from this point on will cause the above SnapshotPackage to drop, clearing
|
// Any errors from this point on will cause the above AccountsPackage to drop, clearing
|
||||||
// any temporary state created for the SnapshotPackage (like the snapshot_hard_links_dir)
|
// any temporary state created for the AccountsPackage (like the snapshot_hard_links_dir)
|
||||||
snapshot_files.copy_snapshot_directory(snapshot_hard_links_dir.path())?;
|
snapshot_files.copy_snapshot_directory(snapshot_hard_links_dir.path())?;
|
||||||
|
|
||||||
let snapshot_package_output_file = get_snapshot_archive_path(
|
let snapshot_package_output_file = get_snapshot_archive_path(
|
||||||
|
@ -114,8 +114,9 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
|
||||||
&compression,
|
&compression,
|
||||||
);
|
);
|
||||||
|
|
||||||
let package = SnapshotPackage::new(
|
let package = AccountsPackage::new(
|
||||||
bank.slot(),
|
bank.slot(),
|
||||||
|
bank.block_height(),
|
||||||
bank.src.slot_deltas(slots_to_snapshot),
|
bank.src.slot_deltas(slots_to_snapshot),
|
||||||
snapshot_hard_links_dir,
|
snapshot_hard_links_dir,
|
||||||
snapshot_storages,
|
snapshot_storages,
|
||||||
|
@ -136,7 +137,7 @@ fn get_compression_ext(compression: &CompressionType) -> (&'static str, &'static
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn archive_snapshot_package(snapshot_package: &SnapshotPackage) -> Result<()> {
|
pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()> {
|
||||||
info!(
|
info!(
|
||||||
"Generating snapshot archive for slot {}",
|
"Generating snapshot archive for slot {}",
|
||||||
snapshot_package.root
|
snapshot_package.root
|
||||||
|
@ -354,8 +355,6 @@ pub fn add_snapshot<P: AsRef<Path>>(
|
||||||
bank: &Bank,
|
bank: &Bank,
|
||||||
snapshot_storages: &[SnapshotStorage],
|
snapshot_storages: &[SnapshotStorage],
|
||||||
) -> Result<SlotSnapshotPaths> {
|
) -> Result<SlotSnapshotPaths> {
|
||||||
bank.clean_accounts();
|
|
||||||
bank.update_accounts_hash();
|
|
||||||
let slot = bank.slot();
|
let slot = bank.slot();
|
||||||
// snapshot_path/slot
|
// snapshot_path/slot
|
||||||
let slot_snapshot_dir = get_bank_snapshot_dir(snapshot_path, slot);
|
let slot_snapshot_dir = get_bank_snapshot_dir(snapshot_path, slot);
|
||||||
|
|
|
@ -1260,7 +1260,7 @@ struct SnapshotValidatorConfig {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn setup_snapshot_validator_config(
|
fn setup_snapshot_validator_config(
|
||||||
snapshot_interval_slots: usize,
|
snapshot_interval_slots: u64,
|
||||||
num_account_paths: usize,
|
num_account_paths: usize,
|
||||||
) -> SnapshotValidatorConfig {
|
) -> SnapshotValidatorConfig {
|
||||||
// Create the snapshot config
|
// Create the snapshot config
|
||||||
|
@ -1281,6 +1281,7 @@ fn setup_snapshot_validator_config(
|
||||||
validator_config.rpc_config.enable_validator_exit = true;
|
validator_config.rpc_config.enable_validator_exit = true;
|
||||||
validator_config.snapshot_config = Some(snapshot_config);
|
validator_config.snapshot_config = Some(snapshot_config);
|
||||||
validator_config.account_paths = account_storage_paths;
|
validator_config.account_paths = account_storage_paths;
|
||||||
|
validator_config.accounts_hash_interval_slots = snapshot_interval_slots;
|
||||||
|
|
||||||
SnapshotValidatorConfig {
|
SnapshotValidatorConfig {
|
||||||
_snapshot_dir: snapshot_dir,
|
_snapshot_dir: snapshot_dir,
|
||||||
|
|
|
@ -399,6 +399,15 @@ fn download_then_check_genesis_hash(
|
||||||
Ok(genesis_config.hash())
|
Ok(genesis_config.hash())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_snapshot_config_invalid(
|
||||||
|
snapshot_interval_slots: u64,
|
||||||
|
accounts_hash_interval_slots: u64,
|
||||||
|
) -> bool {
|
||||||
|
snapshot_interval_slots != 0
|
||||||
|
&& (snapshot_interval_slots < accounts_hash_interval_slots
|
||||||
|
|| snapshot_interval_slots % accounts_hash_interval_slots != 0)
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(clippy::cognitive_complexity)]
|
#[allow(clippy::cognitive_complexity)]
|
||||||
pub fn main() {
|
pub fn main() {
|
||||||
let default_dynamic_port_range =
|
let default_dynamic_port_range =
|
||||||
|
@ -609,6 +618,14 @@ pub fn main() {
|
||||||
.default_value("100")
|
.default_value("100")
|
||||||
.help("Number of slots between generating snapshots, 0 to disable snapshots"),
|
.help("Number of slots between generating snapshots, 0 to disable snapshots"),
|
||||||
)
|
)
|
||||||
|
.arg(
|
||||||
|
clap::Arg::with_name("accounts_hash_interval_slots")
|
||||||
|
.long("accounts-hash-slots")
|
||||||
|
.value_name("ACCOUNTS_HASH_INTERVAL_SLOTS")
|
||||||
|
.takes_value(true)
|
||||||
|
.default_value("100")
|
||||||
|
.help("Number of slots between generating accounts hash."),
|
||||||
|
)
|
||||||
.arg(
|
.arg(
|
||||||
clap::Arg::with_name("limit_ledger_size")
|
clap::Arg::with_name("limit_ledger_size")
|
||||||
.long("limit-ledger-size")
|
.long("limit-ledger-size")
|
||||||
|
@ -850,7 +867,7 @@ pub fn main() {
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let snapshot_interval_slots = value_t_or_exit!(matches, "snapshot_interval_slots", usize);
|
let snapshot_interval_slots = value_t_or_exit!(matches, "snapshot_interval_slots", u64);
|
||||||
let snapshot_path = ledger_path.clone().join("snapshot");
|
let snapshot_path = ledger_path.clone().join("snapshot");
|
||||||
fs::create_dir_all(&snapshot_path).unwrap_or_else(|err| {
|
fs::create_dir_all(&snapshot_path).unwrap_or_else(|err| {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
|
@ -874,13 +891,30 @@ pub fn main() {
|
||||||
snapshot_interval_slots: if snapshot_interval_slots > 0 {
|
snapshot_interval_slots: if snapshot_interval_slots > 0 {
|
||||||
snapshot_interval_slots
|
snapshot_interval_slots
|
||||||
} else {
|
} else {
|
||||||
std::usize::MAX
|
std::u64::MAX
|
||||||
},
|
},
|
||||||
snapshot_path,
|
snapshot_path,
|
||||||
snapshot_package_output_path: ledger_path.clone(),
|
snapshot_package_output_path: ledger_path.clone(),
|
||||||
compression: snapshot_compression,
|
compression: snapshot_compression,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
validator_config.accounts_hash_interval_slots =
|
||||||
|
value_t_or_exit!(matches, "accounts_hash_interval_slots", u64);
|
||||||
|
if validator_config.accounts_hash_interval_slots == 0 {
|
||||||
|
eprintln!("Accounts hash interval should not be 0.");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
if is_snapshot_config_invalid(
|
||||||
|
snapshot_interval_slots,
|
||||||
|
validator_config.accounts_hash_interval_slots,
|
||||||
|
) {
|
||||||
|
eprintln!("Invalid snapshot interval provided ({}), must be a multiple of accounts_hash_interval_slots ({})",
|
||||||
|
snapshot_interval_slots,
|
||||||
|
validator_config.accounts_hash_interval_slots,
|
||||||
|
);
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
if matches.is_present("limit_ledger_size") {
|
if matches.is_present("limit_ledger_size") {
|
||||||
let limit_ledger_size = value_t_or_exit!(matches, "limit_ledger_size", u64);
|
let limit_ledger_size = value_t_or_exit!(matches, "limit_ledger_size", u64);
|
||||||
if limit_ledger_size < DEFAULT_MIN_MAX_LEDGER_SHREDS {
|
if limit_ledger_size < DEFAULT_MIN_MAX_LEDGER_SHREDS {
|
||||||
|
@ -1188,3 +1222,17 @@ pub fn main() {
|
||||||
validator.join().expect("validator exit");
|
validator.join().expect("validator exit");
|
||||||
info!("Validator exiting..");
|
info!("Validator exiting..");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_interval_check() {
|
||||||
|
assert!(!is_snapshot_config_invalid(0, 100));
|
||||||
|
assert!(is_snapshot_config_invalid(1, 100));
|
||||||
|
assert!(is_snapshot_config_invalid(230, 100));
|
||||||
|
assert!(!is_snapshot_config_invalid(500, 100));
|
||||||
|
assert!(!is_snapshot_config_invalid(5, 5));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue