From 89a3e4f91e8b7add9a8fd6375649fc551885dfb5 Mon Sep 17 00:00:00 2001 From: Brooks Prumo Date: Thu, 1 Jul 2021 08:55:26 -0500 Subject: [PATCH] Move SnapshotConfig into its own module (#18331) Also move ArchiveFormat to snapshot_utils, and do not reexport SnapshotVersion. --- core/src/accounts_hash_verifier.rs | 3 +- core/src/snapshot_packager_service.rs | 3 +- core/src/test_validator.rs | 4 +-- core/src/tvu.rs | 3 +- core/src/validator.rs | 3 +- core/tests/snapshots.rs | 6 ++-- download-utils/src/lib.rs | 2 +- ledger-tool/src/main.rs | 6 ++-- ledger/src/bank_forks_utils.rs | 5 ++-- local-cluster/tests/local_cluster.rs | 4 +-- rpc/src/rpc.rs | 3 +- rpc/src/rpc_service.rs | 7 ++--- runtime/src/accounts_background_service.rs | 3 +- runtime/src/bank_forks.rs | 32 +--------------------- runtime/src/lib.rs | 1 + runtime/src/snapshot_config.rs | 26 ++++++++++++++++++ runtime/src/snapshot_package.rs | 3 +- runtime/src/snapshot_utils.rs | 10 ++++++- validator/src/main.rs | 7 +++-- 19 files changed, 70 insertions(+), 61 deletions(-) create mode 100644 runtime/src/snapshot_config.rs diff --git a/core/src/accounts_hash_verifier.rs b/core/src/accounts_hash_verifier.rs index 58a3c8331..fa4dc7131 100644 --- a/core/src/accounts_hash_verifier.rs +++ b/core/src/accounts_hash_verifier.rs @@ -217,8 +217,7 @@ impl AccountsHashVerifier { mod tests { use super::*; use solana_gossip::{cluster_info::make_accounts_hashes_message, contact_info::ContactInfo}; - use solana_runtime::bank_forks::ArchiveFormat; - use solana_runtime::snapshot_utils::SnapshotVersion; + use solana_runtime::snapshot_utils::{ArchiveFormat, SnapshotVersion}; use solana_sdk::{ hash::hash, signature::{Keypair, Signer}, diff --git a/core/src/snapshot_packager_service.rs b/core/src/snapshot_packager_service.rs index a2439708f..ee582e564 100644 --- a/core/src/snapshot_packager_service.rs +++ b/core/src/snapshot_packager_service.rs @@ -78,9 +78,8 @@ mod tests { use solana_runtime::{ accounts_db::AccountStorageEntry, bank::BankSlotDelta, - bank_forks::ArchiveFormat, snapshot_package::AccountsPackage, - snapshot_utils::{self, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME}, + snapshot_utils::{self, ArchiveFormat, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME}, }; use solana_sdk::hash::Hash; use std::{ diff --git a/core/src/test_validator.rs b/core/src/test_validator.rs index 10fefb1d8..e84ac0f23 100644 --- a/core/src/test_validator.rs +++ b/core/src/test_validator.rs @@ -6,10 +6,10 @@ use { solana_net_utils::PortRange, solana_rpc::rpc::JsonRpcConfig, solana_runtime::{ - bank_forks::{ArchiveFormat, SnapshotConfig, SnapshotVersion}, genesis_utils::create_genesis_config_with_leader_ex, hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, - snapshot_utils::DEFAULT_MAX_SNAPSHOTS_TO_RETAIN, + snapshot_config::SnapshotConfig, + snapshot_utils::{ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN}, }, solana_sdk::{ account::{Account, AccountSharedData}, diff --git a/core/src/tvu.rs b/core/src/tvu.rs index 30177b1d3..5329d7dbf 100644 --- a/core/src/tvu.rs +++ b/core/src/tvu.rs @@ -38,8 +38,9 @@ use solana_runtime::{ AbsRequestHandler, AbsRequestSender, AccountsBackgroundService, SnapshotRequestHandler, }, accounts_db::AccountShrinkThreshold, - bank_forks::{BankForks, SnapshotConfig}, + bank_forks::BankForks, commitment::BlockCommitmentCache, + snapshot_config::SnapshotConfig, vote_sender_types::ReplayVoteSender, }; use solana_sdk::{pubkey::Pubkey, signature::Keypair}; diff --git a/core/src/validator.rs b/core/src/validator.rs index 85ff17568..9a9cd5224 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -57,9 +57,10 @@ use solana_runtime::{ accounts_db::AccountShrinkThreshold, accounts_index::AccountSecondaryIndexes, bank::Bank, - bank_forks::{BankForks, SnapshotConfig}, + bank_forks::BankForks, commitment::BlockCommitmentCache, hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, + snapshot_config::SnapshotConfig, }; use solana_sdk::{ clock::Slot, diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index 0a3ed2f34..17b4efd67 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -46,10 +46,10 @@ mod tests { accounts_db, accounts_index::AccountSecondaryIndexes, bank::{Bank, BankSlotDelta}, - bank_forks::{ArchiveFormat, BankForks, SnapshotConfig}, + bank_forks::BankForks, genesis_utils::{create_genesis_config, GenesisConfigInfo}, - snapshot_utils, - snapshot_utils::{SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN}, + snapshot_config::SnapshotConfig, + snapshot_utils::{self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN}, status_cache::MAX_CACHE_ENTRIES, }; use solana_sdk::{ diff --git a/download-utils/src/lib.rs b/download-utils/src/lib.rs index ec870fc9e..b911822f0 100644 --- a/download-utils/src/lib.rs +++ b/download-utils/src/lib.rs @@ -2,7 +2,7 @@ use console::Emoji; use indicatif::{ProgressBar, ProgressStyle}; use log::*; -use solana_runtime::{bank_forks::ArchiveFormat, snapshot_utils}; +use solana_runtime::{snapshot_utils, snapshot_utils::ArchiveFormat}; use solana_sdk::{clock::Slot, genesis_config::DEFAULT_GENESIS_ARCHIVE, hash::Hash}; use std::fs::{self, File}; use std::io; diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 8a99672bb..3be0f925f 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -27,10 +27,10 @@ use solana_ledger::{ }; use solana_runtime::{ bank::{Bank, RewardCalculationEvent}, - bank_forks::{ArchiveFormat, BankForks, SnapshotConfig}, + bank_forks::BankForks, hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, - snapshot_utils, - snapshot_utils::{SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN}, + snapshot_config::SnapshotConfig, + snapshot_utils::{self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN}, }; use solana_sdk::{ account::{AccountSharedData, ReadableAccount, WritableAccount}, diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index 9a8d5321c..cbe46b0c9 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -9,8 +9,9 @@ use crate::{ }; use log::*; use solana_runtime::{ - bank_forks::{ArchiveFormat, BankForks, SnapshotConfig}, - snapshot_utils, + bank_forks::BankForks, + snapshot_config::SnapshotConfig, + snapshot_utils::{self, ArchiveFormat}, }; use solana_sdk::{clock::Slot, genesis_config::GenesisConfig, hash::Hash}; use std::{fs, path::PathBuf, process, result}; diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index f5c7631e9..2db009940 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -37,8 +37,8 @@ use solana_local_cluster::{ validator_configs::*, }; use solana_runtime::{ - bank_forks::{ArchiveFormat, SnapshotConfig}, - snapshot_utils, + snapshot_config::SnapshotConfig, + snapshot_utils::{self, ArchiveFormat}, }; use solana_sdk::{ account::AccountSharedData, diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index e0db17e43..126f7aed8 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -44,10 +44,11 @@ use { accounts::AccountAddressFilter, accounts_index::{AccountIndex, AccountSecondaryIndexes, IndexKey}, bank::Bank, - bank_forks::{BankForks, SnapshotConfig}, + bank_forks::BankForks, commitment::{BlockCommitmentArray, BlockCommitmentCache, CommitmentSlots}, inline_spl_token_v2_0::{SPL_TOKEN_ACCOUNT_MINT_OFFSET, SPL_TOKEN_ACCOUNT_OWNER_OFFSET}, non_circulating_supply::calculate_non_circulating_supply, + snapshot_config::SnapshotConfig, snapshot_utils::get_highest_snapshot_archive_path, }, solana_sdk::{ diff --git a/rpc/src/rpc_service.rs b/rpc/src/rpc_service.rs index 98c67a37e..ac2ddb46f 100644 --- a/rpc/src/rpc_service.rs +++ b/rpc/src/rpc_service.rs @@ -23,8 +23,7 @@ use { solana_metrics::inc_new_counter_info, solana_poh::poh_recorder::PohRecorder, solana_runtime::{ - bank_forks::{BankForks, SnapshotConfig}, - commitment::BlockCommitmentCache, + bank_forks::BankForks, commitment::BlockCommitmentCache, snapshot_config::SnapshotConfig, snapshot_utils, }, solana_sdk::{ @@ -490,8 +489,8 @@ mod tests { get_tmp_ledger_path, }, solana_runtime::{ - bank::Bank, bank_forks::ArchiveFormat, snapshot_utils::SnapshotVersion, - snapshot_utils::DEFAULT_MAX_SNAPSHOTS_TO_RETAIN, + bank::Bank, + snapshot_utils::{ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN}, }, solana_sdk::{ genesis_config::{ClusterType, DEFAULT_GENESIS_ARCHIVE}, diff --git a/runtime/src/accounts_background_service.rs b/runtime/src/accounts_background_service.rs index 2a2992dd8..05b2ace71 100644 --- a/runtime/src/accounts_background_service.rs +++ b/runtime/src/accounts_background_service.rs @@ -4,7 +4,8 @@ use crate::{ bank::{Bank, BankSlotDelta, DropCallback}, - bank_forks::{BankForks, SnapshotConfig}, + bank_forks::BankForks, + snapshot_config::SnapshotConfig, snapshot_package::AccountsPackageSender, snapshot_utils, }; diff --git a/runtime/src/bank_forks.rs b/runtime/src/bank_forks.rs index 012f7244f..fa69b014b 100644 --- a/runtime/src/bank_forks.rs +++ b/runtime/src/bank_forks.rs @@ -3,6 +3,7 @@ use crate::{ accounts_background_service::{AbsRequestSender, SnapshotRequest}, bank::Bank, + snapshot_config::SnapshotConfig, }; use log::*; use solana_metrics::inc_new_counter_info; @@ -10,41 +11,10 @@ use solana_sdk::{clock::Slot, hash::Hash, timing}; use std::{ collections::{hash_map::Entry, HashMap, HashSet}, ops::Index, - path::PathBuf, sync::Arc, time::Instant, }; -pub use crate::snapshot_utils::SnapshotVersion; - -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub enum ArchiveFormat { - TarBzip2, - TarGzip, - TarZstd, - Tar, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct SnapshotConfig { - // Generate a new snapshot every this many slots - pub snapshot_interval_slots: u64, - - // Where to store the latest packaged snapshot - pub snapshot_package_output_path: PathBuf, - - // Where to place the snapshots for recent slots - pub snapshot_path: PathBuf, - - pub archive_format: ArchiveFormat, - - // Snapshot version to generate - pub snapshot_version: SnapshotVersion, - - // Maximum number of snapshots to retain - pub maximum_snapshots_to_retain: usize, -} - pub struct BankForks { banks: HashMap>, descendants: HashMap>, diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index e958261dc..6b4cbb3dd 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -33,6 +33,7 @@ mod read_only_accounts_cache; pub mod rent_collector; pub mod secondary_index; pub mod serde_snapshot; +pub mod snapshot_config; pub mod snapshot_package; pub mod snapshot_runtime_info; pub mod snapshot_utils; diff --git a/runtime/src/snapshot_config.rs b/runtime/src/snapshot_config.rs new file mode 100644 index 000000000..5eeb2e066 --- /dev/null +++ b/runtime/src/snapshot_config.rs @@ -0,0 +1,26 @@ +use crate::snapshot_utils::ArchiveFormat; +use crate::snapshot_utils::SnapshotVersion; +use solana_sdk::clock::Slot; +use std::path::PathBuf; + +/// Snapshot configuration and runtime information +#[derive(Clone, Debug)] +pub struct SnapshotConfig { + /// Generate a new snapshot every this many slots + pub snapshot_interval_slots: Slot, + + /// Where to store the latest packaged snapshot + pub snapshot_package_output_path: PathBuf, + + /// Where to place the snapshots for recent slots + pub snapshot_path: PathBuf, + + /// The archive format to use for snapshots + pub archive_format: ArchiveFormat, + + /// Snapshot version to generate + pub snapshot_version: SnapshotVersion, + + /// Maximum number of snapshots to retain + pub maximum_snapshots_to_retain: usize, +} diff --git a/runtime/src/snapshot_package.rs b/runtime/src/snapshot_package.rs index 8d03154fd..cc00e19a9 100644 --- a/runtime/src/snapshot_package.rs +++ b/runtime/src/snapshot_package.rs @@ -1,5 +1,4 @@ -use crate::bank_forks::ArchiveFormat; -use crate::snapshot_utils::SnapshotVersion; +use crate::snapshot_utils::{ArchiveFormat, SnapshotVersion}; use crate::{accounts_db::SnapshotStorages, bank::BankSlotDelta}; use solana_sdk::clock::Slot; use solana_sdk::genesis_config::ClusterType; diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index eb0551ff8..60f14dcac 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -3,7 +3,6 @@ use { accounts_db::{AccountShrinkThreshold, AccountsDb}, accounts_index::AccountSecondaryIndexes, bank::{Bank, BankSlotDelta, Builtins}, - bank_forks::ArchiveFormat, hardened_unpack::{unpack_snapshot, ParallelSelector, UnpackError, UnpackedAppendVecMap}, serde_snapshot::{ bank_from_stream, bank_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages, @@ -103,6 +102,15 @@ impl SnapshotVersion { } } +/// The different archive formats used for snapshots +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum ArchiveFormat { + TarBzip2, + TarGzip, + TarZstd, + Tar, +} + #[derive(PartialEq, Eq, Debug)] pub struct SlotSnapshotPaths { pub slot: Slot, diff --git a/validator/src/main.rs b/validator/src/main.rs index 1e8323022..7948c1fca 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -46,9 +46,12 @@ use { accounts_index::{ AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude, }, - bank_forks::{ArchiveFormat, SnapshotConfig, SnapshotVersion}, hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, - snapshot_utils::{get_highest_snapshot_archive_path, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN}, + snapshot_config::SnapshotConfig, + snapshot_utils::{ + get_highest_snapshot_archive_path, ArchiveFormat, SnapshotVersion, + DEFAULT_MAX_SNAPSHOTS_TO_RETAIN, + }, }, solana_sdk::{ clock::{Slot, DEFAULT_S_PER_SLOT},